chore(release): 1.32.0 [skip ci]

Replace literal string matching with ipaddr.js parsing
so equivalent encodings of 169.254.169.254
(::ffff:169.254.169.254, ::ffff:a9fe:a9fe,fully-expanded forms)
and fd00:ec2::254 are all rejected.

CONTRIBUTING.md

@@ -30,7 +30,14 @@ We are committed to providing a welcoming environment for everyone. Disrespectfu
 
 ## Before You Start
 
-**Open an issue first.** Before writing any code, please [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction.
+**Open an issue first.** Before writing any code for a non-trivial change, you must [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. **Pull requests submitted without a corresponding issue may be closed at the maintainers' discretion.**
+
+**Trivial fixes are exempt** and may be submitted directly as a PR. Examples:
+- Typo and grammar corrections
+- Documentation clarifications
+- Small one-line bug fixes with an obvious cause
+
+If you're not sure whether your change qualifies as trivial, open an issue first.
 
 When opening an issue:
 - Use a clear, descriptive title
@@ -149,7 +156,7 @@ This project uses [Semantic Versioning](https://semver.org/). Versions are manag
 2. Open a pull request against the `dev` branch of this repository
 3. In the PR description:
    - Summarize what your changes do and why
-   - Reference the related issue (e.g., `Closes #123`)
+   - Reference the related issue (e.g., `Closes #123`) — required for non-trivial changes
    - Note any relevant testing steps or environment details
 4. Be responsive to feedback — maintainers may request changes. Pull requests with no activity for an extended period may be closed.
 

Dockerfile

@@ -1,7 +1,14 @@
 FROM node:22-slim AS base
 
 # Install bash & curl for entrypoint script compatibility, graphicsmagick for pdf2pic, and vips-dev & build-base for sharp 
-RUN apt-get update && apt-get install -y bash curl graphicsmagick libvips-dev build-essential
+RUN apt-get update && apt-get install -y \
+      bash \
+      curl \
+      graphicsmagick \
+      libvips-dev \
+      build-essential \
+      pciutils \
+      && rm -rf /var/lib/apt/lists/*
 
 # All deps stage
 FROM base AS deps
@@ -27,6 +34,31 @@ FROM base
 ARG VERSION=dev
 ARG BUILD_DATE
 ARG VCS_REF
+ARG TARGETARCH
+
+# go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays
+# in sync with parseDryRunOutput().
+ARG PMTILES_VERSION=1.30.2
+# Upstream releases don't ship a checksums file, so pin per-arch SHA256 here.
+# When bumping PMTILES_VERSION, regenerate these with:
+#   curl -fsSL <release-url> | sha256sum
+ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c
+ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa
+RUN set -eux; \
+    case "${TARGETARCH:-amd64}" in \
+      amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \
+      arm64) PMTILES_ARCH=arm64;  PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \
+      *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
+    esac; \
+    TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \
+    cd /tmp; \
+    curl -fsSL -o "$TARBALL" \
+      "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \
+    echo "${PMTILES_SHA256}  ${TARBALL}" | sha256sum -c -; \
+    tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \
+    rm -f "$TARBALL"; \
+    chmod +x /usr/local/bin/pmtiles; \
+    /usr/local/bin/pmtiles version
 
 # Labels
 LABEL org.opencontainers.image.title="Project N.O.M.A.D" \
@@ -43,8 +75,10 @@ ENV NODE_ENV=production
 WORKDIR /app
 COPY --from=production-deps /app/node_modules /app/node_modules
 COPY --from=build /app/build /app
-# Copy root package.json for version info
-COPY package.json /app/version.json
+# Generate version.json from the VERSION build-arg so the image tag is the
+# single source of truth (previously copied root package.json, which drifted
+# from the tag when semantic-release did not commit the bump back).
+RUN echo "{\"version\":\"${VERSION}\"}" > /app/version.json
 
 # Copy docs and README for access within the container
 COPY admin/docs /app/docs

FAQ.md

@@ -20,7 +20,9 @@ Long answer: Custom storage paths, mount points, and external drives (like iSCSI
 
 ## Can I run NOMAD on MAC, WSL2, or a non-Debian-based Distro?
 
-See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os)
+**WSL2 on Windows** is community-supported via the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — covers two install paths (native Docker and Docker Desktop) with all known gotchas documented and empirical performance numbers comparing WSL2 to bare-metal.
+
+**macOS and other non-Debian Linux distros** aren't officially supported. See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) for details.
 
 ## Why does NOMAD require a Debian-based OS?
 
@@ -28,7 +30,7 @@ Project N.O.M.A.D. is currently designed to run on Debian-based Linux distributi
 
 Support for other operating systems will come in the future, but because our development resources are limited as a free and open-source project, we needed to prioritize our efforts and focus on a narrower set of supported platforms for the initial release. We chose Debian-based Linux as our starting point because it's widely used, easy to spin up, and provides a stable environment for running Docker containers.
 
-Community members have provided guides for running N.O.M.A.D. on other platforms (e.g. WSL2, Mac, etc.) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise.
+For Windows users, the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) provides a community-supported path. Community members have also published guides for other platforms (e.g. macOS) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise.
 
 ## Can I run NOMAD on a Raspberry Pi or other ARM-based device?
 Project N.O.M.A.D. is currently designed to run on x86-64 architecture, and we have not yet tested or optimized it for ARM-based devices like the Raspberry Pi (and have not published any official images for ARM architecture).

README.md

@@ -32,7 +32,7 @@ sudo bash install_nomad.sh
 
 Project N.O.M.A.D. is now installed on your device! Open a browser and navigate to `http://localhost:8080` (or `http://DEVICE_IP:8080`) to start exploring!
 
-For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install).
+For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). For Windows users, see the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — community-supported path covering native Docker and Docker Desktop install routes.
 
 ### Advanced Installation
 For more control over the installation process, copy and paste the [Docker Compose template](https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/management_compose.yaml) into a `docker-compose.yml` file and customize it to your liking (be sure to replace any placeholders with your actual values). Then, run `docker compose up -d` to start the Command Center and its dependencies. Note: this method is recommended for advanced users only, as it requires familiarity with Docker and manual configuration before starting.

admin/adonisrc.ts

@@ -55,6 +55,9 @@ export default defineConfig({
     () => import('@adonisjs/transmit/transmit_provider'),
     () => import('#providers/map_static_provider'),
     () => import('#providers/kiwix_migration_provider'),
+    () => import('#providers/qdrant_restart_policy_provider'),
+    () => import('#providers/version_check_provider'),
+    () => import('#providers/gpu_passthrough_remediation_provider'),
   ],
 
   /*
@@ -106,6 +109,10 @@ export default defineConfig({
       pattern: 'resources/views/**/*.edge',
       reloadServer: false,
     },
+    {
+      pattern: 'resources/geodata/**/*.geojson',
+      reloadServer: false,
+    },
     {
       pattern: 'public/**',
       reloadServer: false,

admin/app/controllers/maps_controller.ts

@@ -4,6 +4,8 @@ import {
   assertNotPrivateUrl,
   downloadCollectionValidator,
   filenameParamValidator,
+  mapExtractPreflightValidator,
+  mapExtractValidator,
   remoteDownloadValidator,
   remoteDownloadValidatorOptional,
 } from '#validators/common'
@@ -87,6 +89,28 @@ export default class MapsController {
     }
   }
 
+  async listCountries({}: HttpContext) {
+    return { countries: await this.mapService.listCountries() }
+  }
+
+  async listCountryGroups({}: HttpContext) {
+    return { groups: await this.mapService.listCountryGroups() }
+  }
+
+  async extractPreflight({ request }: HttpContext) {
+    const payload = await request.validateUsing(mapExtractPreflightValidator)
+    return await this.mapService.extractPreflight(payload)
+  }
+
+  async extractRegion({ request }: HttpContext) {
+    const payload = await request.validateUsing(mapExtractValidator)
+    const result = await this.mapService.extractRegion(payload)
+    return {
+      message: 'Extract started successfully',
+      ...result,
+    }
+  }
+
   async styles({ request, response }: HttpContext) {
     // Automatically ensure base assets are present before generating styles
     const baseAssetsExist = await this.mapService.ensureBaseAssets()
@@ -137,9 +161,11 @@ export default class MapsController {
       vine.compile(
         vine.object({
           name: vine.string().trim().minLength(1).maxLength(255),
-          longitude: vine.number(),
-          latitude: vine.number(),
+          longitude: vine.number().min(-180).max(180),
+          latitude: vine.number().min(-90).max(90),
           color: vine.string().trim().maxLength(20).optional(),
+          notes: vine.string().trim().nullable().optional(),
+          marker_type: vine.string().trim().maxLength(20).optional(),
         })
       )
     )
@@ -148,6 +174,8 @@ export default class MapsController {
       longitude: payload.longitude,
       latitude: payload.latitude,
       color: payload.color ?? 'orange',
+      notes: payload.notes ?? null,
+      marker_type: payload.marker_type ?? 'pin',
     })
     return marker
   }
@@ -163,11 +191,19 @@ export default class MapsController {
         vine.object({
           name: vine.string().trim().minLength(1).maxLength(255).optional(),
           color: vine.string().trim().maxLength(20).optional(),
+          longitude: vine.number().min(-180).max(180).optional(),
+          latitude: vine.number().min(-90).max(90).optional(),
+          notes: vine.string().trim().nullable().optional(),
+          marker_type: vine.string().trim().maxLength(20).optional(),
         })
       )
     )
     if (payload.name !== undefined) marker.name = payload.name
     if (payload.color !== undefined) marker.color = payload.color
+    if (payload.longitude !== undefined) marker.longitude = payload.longitude
+    if (payload.latitude !== undefined) marker.latitude = payload.latitude
+    if (payload.notes !== undefined) marker.notes = payload.notes
+    if (payload.marker_type !== undefined) marker.marker_type = payload.marker_type
     await marker.save()
     return marker
   }

admin/app/controllers/ollama_controller.ts

@@ -5,7 +5,8 @@ import { RagService } from '#services/rag_service'
 import Service from '#models/service'
 import KVStore from '#models/kv_store'
 import { modelNameSchema } from '#validators/download'
-import { chatSchema, getAvailableModelsSchema } from '#validators/ollama'
+import { chatSchema, getAvailableModelsSchema, unloadChatModelsSchema } from '#validators/ollama'
+import { assertNotCloudMetadataUrl } from '#validators/common'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import { RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js'
@@ -33,6 +34,19 @@ export default class OllamaController {
     })
   }
 
+  /**
+   * Send Ollama `keep_alive: 0` hints to every currently-loaded chat model
+   * except the embedding model and (optionally) a target model to preserve.
+   * Used by the chat UI to enforce the "one chat model at a time" invariant
+   * on model-switch, session-switch, and page-load. Best-effort: a failure
+   * here should not block the calling flow.
+   */
+  async unloadChatModels({ request, response }: HttpContext) {
+    const { targetModel } = await request.validateUsing(unloadChatModelsSchema)
+    const unloaded = await this.ollamaService.unloadAllChatModelsExcept(targetModel ?? null)
+    return response.status(200).json({ unloaded })
+  }
+
   async chat({ request, response }: HttpContext) {
     const reqData = await request.validateUsing(chatSchema)
 
@@ -229,11 +243,12 @@ export default class OllamaController {
       }
     }
 
-    // Validate URL format
-    if (!remoteUrl.startsWith('http')) {
+    try {
+      assertNotCloudMetadataUrl(remoteUrl)
+    } catch (err) {
       return response.status(400).send({
         success: false,
-        message: 'Invalid URL. Must start with http:// or https://',
+        message: err instanceof Error ? err.message : 'Invalid URL.',
       })
     }
 
@@ -383,10 +398,15 @@ export default class OllamaController {
       // Get recent conversation history (last 6 messages for 3 turns)
       const recentMessages = messages.slice(-6)
 
-      // Skip rewriting for short conversations. Rewriting adds latency with
-      // little RAG benefit until there is enough context to matter.
+      // Skip rewriting on the very first turn — with only one user message
+      // there is no prior context to fold in, so the rewrite would just echo
+      // the message back at the cost of an extra LLM round-trip. From the
+      // first follow-up onward we need the rewrite so the RAG query carries
+      // entities and topics from earlier turns ("the bars" → "Hershey's bars
+      // chocolate poisoning dog"); without it, embeddings match nothing and
+      // the assistant loses the thread.
       const userMessages = recentMessages.filter(msg => msg.role === 'user')
-      if (userMessages.length <= 2) {
+      if (userMessages.length < 2) {
         return lastUserMessage?.content || null
       }
 

admin/app/controllers/rag_controller.ts

@@ -1,11 +1,13 @@
 import { RagService } from '#services/rag_service'
 import { EmbedFileJob } from '#jobs/embed_file_job'
+import KbRatioRegistry from '#models/kb_ratio_registry'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import app from '@adonisjs/core/services/app'
 import { randomBytes } from 'node:crypto'
 import { sanitizeFilename } from '../utils/fs.js'
-import { deleteFileSchema, getJobStatusSchema } from '#validators/rag'
+import { basename } from 'node:path'
+import { deleteFileSchema, embedFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag'
 import logger from '@adonisjs/core/services/logger'
 
 @inject()
@@ -66,6 +68,11 @@ export default class RagController {
     return response.status(200).json({ files })
   }
 
+  public async getFileWarnings({ response }: HttpContext) {
+    const result = await this.ragService.computeFileWarnings()
+    return response.status(200).json(result)
+  }
+
   public async deleteFile({ request, response }: HttpContext) {
     const { source } = await request.validateUsing(deleteFileSchema)
     const result = await this.ragService.deleteFileBySource(source)
@@ -75,6 +82,21 @@ export default class RagController {
     return response.status(200).json({ message: result.message })
   }
 
+  public async embedFile({ request, response }: HttpContext) {
+    const { source, force } = await request.validateUsing(embedFileSchema)
+    const result = await this.ragService.embedSingleFile(source, force ?? false)
+    if (!result.success) {
+      const status = {
+        not_found: 404,
+        inflight: 409,
+        delete_failed: 500,
+        dispatch_failed: 500,
+      }[result.code]
+      return response.status(status).json({ error: result.message, code: result.code })
+    }
+    return response.status(202).json({ message: result.message })
+  }
+
   public async getFailedJobs({ response }: HttpContext) {
     const jobs = await EmbedFileJob.listFailedJobs()
     return response.status(200).json(jobs)
@@ -88,6 +110,11 @@ export default class RagController {
     })
   }
 
+  public async policyPromptState({ response }: HttpContext) {
+    const result = await this.ragService.getPolicyPromptState()
+    return response.status(200).json(result)
+  }
+
   public async scanAndSync({ response }: HttpContext) {
     try {
       const syncResult = await this.ragService.scanAndSyncStorage()
@@ -97,4 +124,42 @@ export default class RagController {
       return response.status(500).json({ error: 'Error scanning and syncing storage' })
     }
   }
+
+  public async reembedAll({ response }: HttpContext) {
+    try {
+      const result = await this.ragService.reembedAll()
+      return response.status(200).json(result)
+    } catch (error) {
+      logger.error({ err: error }, '[RagController] Error during re-embed all')
+      return response.status(500).json({ error: 'Error during re-embed all' })
+    }
+  }
+
+  public async resetAndRebuild({ response }: HttpContext) {
+    try {
+      const result = await this.ragService.resetAndRebuild()
+      return response.status(200).json(result)
+    } catch (error) {
+      logger.error({ err: error }, '[RagController] Error during reset and rebuild')
+      return response.status(500).json({ error: 'Error during reset and rebuild' })
+    }
+  }
+
+  public async health({ response }: HttpContext) {
+    const result = await this.ragService.checkQdrantHealth()
+    return response.status(200).json(result)
+  }
+
+  public async estimateBatch({ request, response }: HttpContext) {
+    const { files } = await request.validateUsing(estimateBatchSchema)
+    // The registry matches on basename prefixes; if a caller passes a full path
+    // (e.g. /app/storage/zim/wikipedia_en_simple_…), strip directories first so
+    // patterns like `wikipedia_en_simple_` still match.
+    const normalized = files.map((f) => ({
+      filename: basename(f.filename),
+      sizeBytes: f.sizeBytes,
+    }))
+    const result = await KbRatioRegistry.estimateBatch(normalized)
+    return response.status(200).json(result)
+  }
 }

admin/app/controllers/zim_controller.ts

@@ -6,7 +6,7 @@ import {
   remoteDownloadWithMetadataValidator,
   selectWikipediaValidator,
 } from '#validators/common'
-import { listRemoteZimValidator } from '#validators/zim'
+import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 
@@ -85,4 +85,51 @@ export default class ZimController {
     const payload = await request.validateUsing(selectWikipediaValidator)
     return this.zimService.selectWikipedia(payload.optionId)
   }
+
+  // Custom library endpoints
+
+  async listCustomLibraries({}: HttpContext) {
+    return this.zimService.listCustomLibraries()
+  }
+
+  async addCustomLibrary({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(addCustomLibraryValidator)
+    assertNotPrivateUrl(payload.base_url)
+    try {
+      const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url)
+      return { message: 'Custom library added', library: source }
+    } catch (error) {
+      if (error.message === 'Maximum of 10 custom libraries allowed') {
+        return response.status(400).send({ message: error.message })
+      }
+      throw error
+    }
+  }
+
+  async removeCustomLibrary({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(idParamValidator)
+    try {
+      await this.zimService.removeCustomLibrary(payload.params.id)
+      return { message: 'Custom library removed' }
+    } catch (error) {
+      if (error.message === 'Custom library not found') {
+        return response.status(404).send({ message: error.message })
+      }
+      throw error
+    }
+  }
+
+  async browseLibrary({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(browseLibraryValidator)
+    try {
+      return await this.zimService.browseLibraryUrl(payload.url)
+    } catch (error) {
+      if (error.message?.includes('loopback or link-local')) {
+        return response.status(400).send({ message: error.message })
+      }
+      return response.status(502).send({
+        message: 'Could not fetch directory listing from the provided URL',
+      })
+    }
+  }
 }

admin/app/jobs/check_service_updates_job.ts

@@ -95,7 +95,7 @@ export class CheckServiceUpdatesJob {
   }
 
   static async scheduleNightly() {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
 
     await queue.upsertJobScheduler(
@@ -114,7 +114,7 @@ export class CheckServiceUpdatesJob {
   }
 
   static async dispatch() {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
 
     const job = await queue.add(

admin/app/jobs/check_update_job.ts

@@ -42,7 +42,7 @@ export class CheckUpdateJob {
   }
 
   static async scheduleNightly() {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
 
     await queue.upsertJobScheduler(
@@ -61,7 +61,7 @@ export class CheckUpdateJob {
   }
 
   static async dispatch() {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
 
     const job = await queue.add(this.key, {}, {

admin/app/jobs/download_model_job.ts

@@ -34,7 +34,7 @@ export class DownloadModelJob {
 
   /** Signal cancellation via Redis so the worker process can pick it up on its next poll tick */
   static async signalCancel(jobId: string): Promise<void> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const client = await queue.client
     await client.set(this.cancelKey(jobId), '1', 'EX', 300) // 5 min TTL
@@ -66,7 +66,7 @@ export class DownloadModelJob {
     DownloadModelJob.abortControllers.set(job.id!, abortController)
 
     // Get Redis client for checking cancel signals from the API process
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const cancelRedis = await queueService.getQueue(DownloadModelJob.queue).client
 
     // Track whether cancellation was explicitly requested by the user. Only user-initiated
@@ -154,14 +154,14 @@ export class DownloadModelJob {
   }
 
   static async getByModelName(modelName: string): Promise<Job | undefined> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(modelName)
     return await queue.getJob(jobId)
   }
 
   static async dispatch(params: DownloadModelJobParams) {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(params.modelName)
 

admin/app/jobs/embed_file_job.ts

@@ -4,9 +4,11 @@ import { EmbedJobWithProgress } from '../../types/rag.js'
 import { RagService } from '#services/rag_service'
 import { DockerService } from '#services/docker_service'
 import { OllamaService } from '#services/ollama_service'
+import KbIngestState from '#models/kb_ingest_state'
 import { createHash } from 'crypto'
 import logger from '@adonisjs/core/services/logger'
 import fs from 'node:fs/promises'
+import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
 
 export interface EmbedFileJobParams {
   filePath: string
@@ -27,6 +29,12 @@ export class EmbedFileJob {
     return 'embed-file'
   }
 
+  // Delay between continuation batches when embedding runs CPU-only. Gives the OS
+  // scheduler a brief idle window so sshd / disk-collector / other services don't
+  // starve during long multi-batch ZIM ingestions. Skipped entirely when the
+  // embedding model is GPU-offloaded — see OllamaService.isEmbeddingGpuAccelerated().
+  static readonly CPU_BATCH_DELAY_MS = 1000
+
   static getJobId(filePath: string): string {
     return createHash('sha256').update(filePath).digest('hex').slice(0, 16)
   }
@@ -77,8 +85,16 @@ export class EmbedFileJob {
 
       logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`)
 
-      // Update progress starting
-      await this.safeUpdateProgress(job, 5)
+      // Anchor initial progress to where we are in the overall file. For a
+      // continuation batch midway through a multi-batch ZIM (e.g. offset 100k of
+      // 600k), the hardcoded 5 used to make the gauge briefly flash 0→5→real,
+      // which read as a backward jump. Fall back to 5 for single-batch files
+      // where totalArticles isn't set.
+      const initialPercent =
+        totalArticles && totalArticles > 0
+          ? Math.min(99, Math.round(((batchOffset || 0) / totalArticles) * 100))
+          : 5
+      await this.safeUpdateProgress(job, initialPercent)
       await job.updateData({
         ...job.data,
         status: 'processing',
@@ -87,9 +103,25 @@ export class EmbedFileJob {
 
       logger.info(`[EmbedFileJob] Processing file: ${filePath}`)
 
-      // Progress callback: maps service-reported 0-100% into the 5-95% job range
+      // Progress callback. For multi-batch ZIM ingestions, scale the service-reported
+      // 0-100% (which is % through the current batch's chunks) into the overall-file
+      // frame so the UI gauge climbs monotonically across the many continuation jobs
+      // BullMQ creates per file. Without this, every new continuation jobId resets the
+      // gauge to ~5% and the user sees ingestion progress "jumping around" between
+      // each batch's local frame and the end-of-batch overall-file overwrite below.
+      //
+      // For single-batch files (uploaded PDFs, txts) totalArticles is undefined and
+      // we fall back to the original 5-95% per-job range, which is what the UI expects
+      // for a one-shot file with no continuations.
       const onProgress = async (percent: number) => {
-        await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9)))
+        const useOverallFrame = totalArticles && totalArticles > 0
+        if (useOverallFrame) {
+          const articlesDone = (batchOffset || 0) + (percent / 100) * ZIM_BATCH_SIZE
+          const overallPercent = Math.min(99, Math.round((articlesDone / totalArticles) * 100))
+          await this.safeUpdateProgress(job, overallPercent)
+        } else {
+          await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9)))
+        }
       }
 
       // Process and embed the file
@@ -114,6 +146,19 @@ export class EmbedFileJob {
           `[EmbedFileJob] Batch complete. Dispatching next batch at offset ${nextOffset}`
         )
 
+        // Pace continuation batches when embedding is CPU-bound. Sustained 100% CPU
+        // saturation across all cores during multi-batch ZIM ingestion can starve
+        // other services (sshd has been seen to lose responsiveness hard enough to
+        // require a power-cycle). When GPU-accelerated, embeddings stream through
+        // the GPU and CPUs stay free — no pacing needed.
+        const isGpuAccelerated = await ollamaService.isEmbeddingGpuAccelerated()
+        if (!isGpuAccelerated) {
+          logger.info(
+            `[EmbedFileJob] Embedding is CPU-only — pacing ${EmbedFileJob.CPU_BATCH_DELAY_MS}ms before dispatching next batch`
+          )
+          await new Promise((resolve) => setTimeout(resolve, EmbedFileJob.CPU_BATCH_DELAY_MS))
+        }
+
         // Dispatch next batch (not final yet)
         await EmbedFileJob.dispatch({
           filePath,
@@ -157,6 +202,18 @@ export class EmbedFileJob {
         chunks: totalChunks,
       })
 
+      // Persist the post-job state so scanAndSyncStorage knows this file is done.
+      // BullMQ's :completed retention (50 jobs) ages out, so the state row is
+      // the only durable record of "this file finished embedding".
+      try {
+        await KbIngestState.markIndexed(filePath, totalChunks)
+      } catch (stateErr) {
+        logger.warn(
+          `[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`,
+          stateErr instanceof Error ? stateErr.message : String(stateErr)
+        )
+      }
+
       const batchMsg = isZimBatch ? ` (final batch, total chunks: ${totalChunks})` : ''
       logger.info(
         `[EmbedFileJob] Successfully embedded ${result.chunks} chunks from file: ${fileName}${batchMsg}`
@@ -179,64 +236,125 @@ export class EmbedFileJob {
         error: error instanceof Error ? error.message : 'Unknown error',
       })
 
+      // Only persist `failed` for unrecoverable errors. Retryable errors get
+      // automatic BullMQ retries (30 attempts); marking state failed on every
+      // transient blip would suppress the retry-driven recovery path.
+      if (error instanceof UnrecoverableError) {
+        try {
+          await KbIngestState.markFailed(
+            filePath,
+            error instanceof Error ? error.message : 'Unknown error'
+          )
+        } catch (stateErr) {
+          logger.warn(
+            `[EmbedFileJob] Failed to persist failed state for ${fileName}: %s`,
+            stateErr instanceof Error ? stateErr.message : String(stateErr)
+          )
+        }
+      }
+
       throw error
     }
   }
 
   static async listActiveJobs(): Promise<EmbedJobWithProgress[]> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
 
-    return jobs.map((job) => ({
-      jobId: job.id!.toString(),
-      fileName: (job.data as EmbedFileJobParams).fileName,
-      filePath: (job.data as EmbedFileJobParams).filePath,
-      progress: typeof job.progress === 'number' ? job.progress : 0,
-      status: ((job.data as any).status as string) ?? 'waiting',
-    }))
+    return jobs.map((job) => {
+      const data = job.data as EmbedFileJobParams & {
+        status?: string
+        lastBatchAt?: number
+        startedAt?: number
+        chunks?: number
+      }
+      return {
+        jobId: job.id!.toString(),
+        fileName: data.fileName,
+        filePath: data.filePath,
+        progress: typeof job.progress === 'number' ? job.progress : 0,
+        status: data.status ?? 'waiting',
+        lastBatchAt: data.lastBatchAt,
+        startedAt: data.startedAt,
+        chunks: data.chunks,
+      }
+    })
   }
 
   static async getByFilePath(filePath: string): Promise<Job | undefined> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(filePath)
     return await queue.getJob(jobId)
   }
 
-  static async dispatch(params: EmbedFileJobParams) {
-    const queueService = new QueueService()
+  static async dispatch(params: EmbedFileJobParams, options?: { force?: boolean }) {
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
-    const jobId = this.getJobId(params.filePath)
+
+    // Continuation batches (batchOffset > 0) must NOT reuse the deterministic
+    // per-file jobId. Two BullMQ dedupe paths would otherwise silently swallow them:
+    //   1) The parent batch's handle() calls dispatch() before returning, so the
+    //      parent job is still `active` and locked — queue.add() with the same
+    //      jobId returns the locked parent rather than enqueueing the new batch.
+    //   2) After the parent completes, its entry stays in `completed` (held by
+    //      `removeOnComplete: { count: 50 }`), still tripping jobId dedupe.
+    // Letting BullMQ auto-generate a unique jobId for continuation batches stacks
+    // them as independent queue entries that each process via handle().
+    // Initial dispatches keep the deterministic jobId so re-triggering an install
+    // (UI re-click, sync rescan, etc.) is still idempotent.
+    // `force` skips the deterministic jobId for bulk callers (reembedAll /
+    // resetAndRebuild) where historical entries in :completed would otherwise
+    // silently swallow the new dispatch.
+    const isContinuation = !!(params.batchOffset && params.batchOffset > 0)
+    const force = !!options?.force
+    const initialJobId = this.getJobId(params.filePath)
+
+    const jobOptions: Parameters<typeof queue.add>[2] = {
+      attempts: 30,
+      backoff: {
+        type: 'fixed',
+        delay: 60000, // Check every 60 seconds for service readiness
+      },
+      removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history
+      removeOnFail: { count: 20 }, // Keep last 20 failed jobs for debugging
+    }
+    if (!isContinuation && !force) {
+      jobOptions.jobId = initialJobId
+    }
 
     try {
-      const job = await queue.add(this.key, params, {
-        jobId,
-        attempts: 30,
-        backoff: {
-          type: 'fixed',
-          delay: 60000, // Check every 60 seconds for service readiness
-        },
-        removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history
-        removeOnFail: { count: 20 } // Keep last 20 failed jobs for debugging
-      })
+      const job = await queue.add(this.key, params, jobOptions)
 
-      logger.info(`[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}`)
+      const label = isContinuation
+        ? ` (continuation @ offset ${params.batchOffset})`
+        : force
+          ? ' (forced re-dispatch)'
+          : ''
+      logger.info(
+        `[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}${label}`
+      )
 
       return {
         job,
         created: true,
-        jobId,
+        jobId: job.id ?? initialJobId,
         message: `File queued for embedding: ${params.fileName}`,
       }
     } catch (error) {
-      if (error.message && error.message.includes('job already exists')) {
-        const existing = await queue.getJob(jobId)
+      if (
+        !isContinuation &&
+        !force &&
+        error.message &&
+        error.message.includes('job already exists')
+      ) {
+        const existing = await queue.getJob(initialJobId)
         logger.info(`[EmbedFileJob] Job already exists for file: ${params.fileName}`)
         return {
           job: existing,
           created: false,
-          jobId,
+          jobId: initialJobId,
           message: `Embedding job already exists for: ${params.fileName}`,
         }
       }
@@ -245,7 +363,7 @@ export class EmbedFileJob {
   }
 
   static async listFailedJobs(): Promise<EmbedJobWithProgress[]> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     // Jobs that have failed at least once are in 'delayed' (retrying) or terminal 'failed' state.
     // We identify them by job.data.status === 'failed' set in the catch block of handle().
@@ -264,7 +382,7 @@ export class EmbedFileJob {
   }
 
   static async cleanupFailedJobs(): Promise<{ cleaned: number; filesDeleted: number }> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const allJobs = await queue.getJobs(['waiting', 'delayed', 'failed'])
     const failedJobs = allJobs.filter((job) => (job.data as any).status === 'failed')

admin/app/jobs/run_benchmark_job.ts

@@ -53,7 +53,7 @@ export class RunBenchmarkJob {
   }
 
   static async dispatch(params: RunBenchmarkJobParams) {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
 
     try {
@@ -89,7 +89,7 @@ export class RunBenchmarkJob {
   }
 
   static async getJob(benchmarkId: string): Promise<Job | undefined> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     return await queue.getJob(benchmarkId)
   }

admin/app/jobs/run_download_job.ts

@@ -31,7 +31,7 @@ export class RunDownloadJob {
 
   /** Signal cancellation via Redis so the worker process can pick it up */
   static async signalCancel(jobId: string): Promise<void> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const client = await queue.client
     await client.set(this.cancelKey(jobId), '1', 'EX', 300) // 5 min TTL
@@ -46,7 +46,7 @@ export class RunDownloadJob {
     RunDownloadJob.abortControllers.set(job.id!, abortController)
 
     // Get Redis client for checking cancel signals from the API process
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const cancelRedis = await queueService.getQueue(RunDownloadJob.queue).client
 
     let lastKnownProgress: Pick<DownloadProgressData, 'downloadedBytes' | 'totalBytes'> = {
@@ -147,13 +147,40 @@ export class RunDownloadJob {
               // Only dispatch embedding job if AI Assistant (Ollama) is installed
               const ollamaUrl = await dockerService.getServiceURL('nomad_ollama')
               if (ollamaUrl) {
-                try {
-                  await EmbedFileJob.dispatch({
-                    fileName: url.split('/').pop() || '',
-                    filePath: filepath,
-                  })
-                } catch (error) {
-                  console.error(`[RunDownloadJob] Error dispatching EmbedFileJob for URL ${url}:`, error)
+                // Respect the global ingest policy. Under Manual, record the file
+                // as pending_decision so the KB panel surfaces the per-file Index
+                // affordance (PR #909) instead of silently auto-embedding behind
+                // the user's back. Unset is treated as Always to preserve legacy
+                // behavior — mirrors rag_service.ts:1587-1588.
+                const { default: KVStore } = await import('#models/kv_store')
+                const { default: KbIngestState } = await import('#models/kb_ingest_state')
+                const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy')
+                const policy: 'Always' | 'Manual' = policyRaw === 'Manual' ? 'Manual' : 'Always'
+
+                if (policy === 'Manual') {
+                  try {
+                    // firstOrCreate so a re-download doesn't demote an existing
+                    // indexed/failed row — user keeps prior state and can re-index
+                    // explicitly from the KB panel if they want fresh content.
+                    await KbIngestState.firstOrCreate(
+                      { file_path: filepath },
+                      { file_path: filepath, state: 'pending_decision', chunks_embedded: 0 }
+                    )
+                  } catch (error) {
+                    console.error(
+                      `[RunDownloadJob] Error recording pending_decision state for ${filepath}:`,
+                      error
+                    )
+                  }
+                } else {
+                  try {
+                    await EmbedFileJob.dispatch({
+                      fileName: url.split('/').pop() || '',
+                      filePath: filepath,
+                    })
+                  } catch (error) {
+                    console.error(`[RunDownloadJob] Error dispatching EmbedFileJob for URL ${url}:`, error)
+                  }
                 }
               }
             } else if (filetype === 'map') {
@@ -199,7 +226,7 @@ export class RunDownloadJob {
   }
 
   static async getByUrl(url: string): Promise<Job | undefined> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(url)
     return await queue.getJob(jobId)
@@ -229,7 +256,7 @@ export class RunDownloadJob {
   }
 
   static async dispatch(params: RunDownloadJobParams) {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(params.url)
 

admin/app/jobs/run_extract_pmtiles_job.ts

@@ -0,0 +1,294 @@
+import { Job, UnrecoverableError } from 'bullmq'
+import { spawn, ChildProcess } from 'child_process'
+import { createHash } from 'crypto'
+import { readdir, stat } from 'fs/promises'
+import { basename, dirname, join } from 'path'
+import { QueueService } from '#services/queue_service'
+import logger from '@adonisjs/core/services/logger'
+import { DownloadProgressData } from '../../types/downloads.js'
+import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js'
+import { deleteFileIfExists } from '../utils/fs.js'
+
+export interface RunExtractPmtilesJobParams {
+  sourceUrl: string
+  outputFilepath: string
+  /** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */
+  regionFilepath: string
+  maxzoom?: number
+  /** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */
+  estimatedBytes?: number
+  filetype: 'map'
+  title?: string
+  resourceMetadata?: {
+    resource_id: string
+    version: string
+    collection_ref: string | null
+  }
+}
+
+export class RunExtractPmtilesJob {
+  static get queue() {
+    return 'pmtiles-extract'
+  }
+
+  static get key() {
+    return 'run-pmtiles-extract'
+  }
+
+  /** In-memory registry of active child processes so in-process cancels can SIGTERM them */
+  static childProcesses: Map<string, ChildProcess> = new Map()
+
+  static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string {
+    const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null })
+    return createHash('sha256').update(payload).digest('hex').slice(0, 16)
+  }
+
+  /** Redis key used to signal cancellation across processes */
+  static cancelKey(jobId: string): string {
+    return `nomad:download:pmtiles-cancel:${jobId}`
+  }
+
+  static async signalCancel(jobId: string): Promise<void> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const client = await queue.client
+    await client.set(this.cancelKey(jobId), '1', 'EX', 300)
+  }
+
+  /** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1),
+   *  which occur when the job was removed from Redis (e.g. cancelled) between
+   *  the await being issued and the Redis write completing. Anything else
+   *  re-throws so it's caught by the surrounding try rather than becoming an
+   *  unhandled rejection. */
+  private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise<void> {
+    try {
+      await job.updateProgress(progress)
+    } catch (err: any) {
+      if (err?.code !== -1) throw err
+    }
+  }
+
+  async handle(job: Job) {
+    const params = job.data as RunExtractPmtilesJobParams
+    const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params
+
+    logger.info(
+      `[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` +
+        `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}`
+    )
+
+    const queueService = QueueService.getInstance()
+    const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client
+
+    let userCancelled = false
+    let proc: ChildProcess | null = null
+    let lastReportedBytes = -1
+
+    // One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles
+    // writes incrementally but rewrites directories near the end so progress isn't strictly
+    // monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter.
+    const tick = setInterval(async () => {
+      try {
+        const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!))
+        if (val) {
+          await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!))
+          userCancelled = true
+          proc?.kill('SIGTERM')
+        }
+      } catch {
+        // Redis errors non-fatal — in-memory handle also covers same-process cancels
+      }
+
+      try {
+        const fileStat = await stat(outputFilepath)
+        const downloadedBytes = Number(fileStat.size)
+        if (downloadedBytes === lastReportedBytes) return
+        lastReportedBytes = downloadedBytes
+
+        const totalBytes = estimatedBytes ?? 0
+        const percent =
+          totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0
+
+        await this.safeUpdateProgress(job, {
+          percent,
+          downloadedBytes,
+          totalBytes,
+          lastProgressTime: Date.now(),
+        } as DownloadProgressData)
+      } catch {
+        // File doesn't exist yet (subprocess still setting up)
+      }
+    }, 2000)
+
+    try {
+      const args = buildPmtilesExtractArgs({
+        sourceUrl,
+        outputFilepath,
+        regionFilepath,
+        maxzoom,
+        downloadThreads: 8,
+        overfetch: 0.2,
+      })
+      proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] })
+      RunExtractPmtilesJob.childProcesses.set(job.id!, proc)
+
+      proc.stdout?.on('data', (chunk) => {
+        logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
+      })
+      proc.stderr?.on('data', (chunk) => {
+        logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
+      })
+
+      const exitCode: number = await new Promise((resolve, reject) => {
+        proc!.on('close', (code) => resolve(code ?? -1))
+        proc!.on('error', (err) => reject(err))
+      })
+
+      if (exitCode !== 0) {
+        await deleteFileIfExists(outputFilepath)
+        if (userCancelled) {
+          throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`)
+        }
+        throw new Error(`pmtiles extract exited with code ${exitCode}`)
+      }
+
+      // Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract
+      const finalStat = await stat(outputFilepath)
+      await this.safeUpdateProgress(job, {
+        percent: 100,
+        downloadedBytes: Number(finalStat.size),
+        totalBytes: estimatedBytes ?? Number(finalStat.size),
+        lastProgressTime: Date.now(),
+      } as DownloadProgressData)
+
+      // Reuse the HTTP download path's post-download hook so the file is registered and
+      // the previous version (if any) is deleted
+      await this.onComplete(params)
+
+      logger.info(
+        `[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes`
+      )
+
+      return { sourceUrl, outputFilepath }
+    } catch (error: any) {
+      if (userCancelled && !(error instanceof UnrecoverableError)) {
+        throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`)
+      }
+      throw error
+    } finally {
+      clearInterval(tick)
+      RunExtractPmtilesJob.childProcesses.delete(job.id!)
+    }
+  }
+
+  private async onComplete(params: RunExtractPmtilesJobParams) {
+    if (!params.resourceMetadata) return
+
+    const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([
+      import('#models/installed_resource'),
+      import('luxon'),
+      import('../utils/fs.js'),
+    ])
+
+    const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath)
+
+    const existing = await InstalledResource.query()
+      .where('resource_id', params.resourceMetadata.resource_id)
+      .where('resource_type', 'map')
+      .first()
+    const oldFilePath = existing?.file_path ?? null
+
+    await InstalledResource.updateOrCreate(
+      {
+        resource_id: params.resourceMetadata.resource_id,
+        resource_type: 'map',
+      },
+      {
+        version: params.resourceMetadata.version,
+        collection_ref: params.resourceMetadata.collection_ref,
+        url: params.sourceUrl,
+        file_path: params.outputFilepath,
+        file_size_bytes: fileStat ? Number(fileStat.size) : null,
+        installed_at: DateTime.now(),
+      }
+    )
+
+    if (oldFilePath && oldFilePath !== params.outputFilepath) {
+      try {
+        await fsUtils.deleteFileIfExists(oldFilePath)
+      } catch (err) {
+        logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`)
+      }
+    }
+
+    // Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB
+    // lookup above didn't catch — e.g. a prior extract crashed before writing its
+    // InstalledResource row, or an earlier bug wrote a file without registering it.
+    // Matches both curated (`<id>_YYYY-MM.pmtiles`) and regional (`<id>_YYYYMMDD_zN.pmtiles`)
+    // naming — prefix-only so new filename formats don't silently miss.
+    const dir = dirname(params.outputFilepath)
+    const keepName = basename(params.outputFilepath)
+    const prefix = `${params.resourceMetadata.resource_id}_`
+    try {
+      const entries = await readdir(dir)
+      for (const entry of entries) {
+        if (entry === keepName || !entry.endsWith('.pmtiles')) continue
+        if (!entry.startsWith(prefix)) continue
+        const orphanPath = join(dir, entry)
+        if (orphanPath === oldFilePath) continue
+        try {
+          await fsUtils.deleteFileIfExists(orphanPath)
+          logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`)
+        } catch (err) {
+          logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`)
+        }
+      }
+    } catch (err) {
+      logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`)
+    }
+  }
+
+  static async getById(jobId: string): Promise<Job | undefined> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    return await queue.getJob(jobId)
+  }
+
+  static async dispatch(params: RunExtractPmtilesJobParams) {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom)
+
+    const existing = await queue.getJob(jobId)
+    if (existing) {
+      const state = await existing.getState()
+      if (state === 'active' || state === 'waiting' || state === 'delayed') {
+        return {
+          job: existing,
+          created: false,
+          message: `Extract job already exists for these params`,
+        }
+      }
+      // Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id
+      try {
+        await existing.remove()
+      } catch {
+        // Already gone or locked — add() below will still report a meaningful error
+      }
+    }
+
+    // Fewer attempts than HTTP downloads — a failed extract usually means the source URL
+    // rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI
+    const job = await queue.add(this.key, params, {
+      jobId,
+      attempts: 3,
+      backoff: { type: 'exponential', delay: 60000 },
+      removeOnComplete: true,
+    })
+    return {
+      job,
+      created: true,
+      message: `Dispatched pmtiles extract job`,
+    }
+  }
+}

admin/app/middleware/compression_middleware.ts

@@ -3,7 +3,21 @@ import type { HttpContext } from '@adonisjs/core/http'
 import type { NextFn } from '@adonisjs/core/types/http'
 import compression from 'compression'
 
-const compress = env.get('DISABLE_COMPRESSION') ? null : compression()
+// Skip compression for Server-Sent Events. The compression library buffers
+// response writes to determine encoding, which collapses per-token streaming
+// into a single block delivered after generation completes (regression in
+// v1.31.0-rc.2, reported in #781 by @toasterking).
+const compress = env.get('DISABLE_COMPRESSION')
+  ? null
+  : compression({
+      filter: (req: any, res: any) => {
+        const contentType = res.getHeader('Content-Type')
+        if (typeof contentType === 'string' && contentType.includes('text/event-stream')) {
+          return false
+        }
+        return compression.filter(req, res)
+      },
+    })
 
 export default class CompressionMiddleware {
   async handle({ request, response }: HttpContext, next: NextFn) {

admin/app/models/chat_message.ts

@@ -18,7 +18,7 @@ export default class ChatMessage extends BaseModel {
   @column()
   declare content: string
 
-  @belongsTo(() => ChatSession, { foreignKey: 'id', localKey: 'session_id' })
+  @belongsTo(() => ChatSession, { foreignKey: 'session_id', localKey: 'id' })
   declare session: BelongsTo<typeof ChatSession>
 
   @column.dateTime({ autoCreate: true })

admin/app/models/custom_library_source.ts

@@ -0,0 +1,24 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+
+export default class CustomLibrarySource extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare name: string
+
+  @column()
+  declare base_url: string
+
+  @column()
+  declare is_default: boolean
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/models/kb_ingest_state.ts

@@ -0,0 +1,77 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
+
+const LAST_ERROR_MAX_LEN = 1024
+
+/**
+ * Tracks the per-file decision and outcome of AI knowledge-base ingestion.
+ *
+ * The row exists for any embeddable file the scanner has seen and is independent
+ * of `installed_resources` (which only covers curated downloads). Replaces the
+ * earlier "any chunks in qdrant ⇒ embedded" binary check, which conflated
+ * partially-stalled ingestions with fully-indexed files. See RFC #883.
+ */
+export default class KbIngestState extends BaseModel {
+  static table = 'kb_ingest_state'
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare file_path: string
+
+  @column()
+  declare state: KbIngestStateValue
+
+  @column()
+  declare chunks_embedded: number
+
+  @column()
+  declare last_error: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+
+  static async getOrCreate(filePath: string): Promise<KbIngestState> {
+    return this.firstOrCreate(
+      { file_path: filePath },
+      { file_path: filePath, state: 'pending_decision', chunks_embedded: 0 }
+    )
+  }
+
+  static async markIndexed(filePath: string, chunksEmbedded: number): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'indexed'
+    row.chunks_embedded = chunksEmbedded
+    row.last_error = null
+    await row.save()
+  }
+
+  static async markFailed(filePath: string, errorMessage: string): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'failed'
+    row.last_error = errorMessage.slice(0, LAST_ERROR_MAX_LEN)
+    await row.save()
+  }
+
+  static async markBrowseOnly(filePath: string): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'browse_only'
+    await row.save()
+  }
+
+  static async markStalled(filePath: string): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'stalled'
+    await row.save()
+  }
+
+  static async remove(filePath: string): Promise<void> {
+    await this.query().where('file_path', filePath).delete()
+  }
+}

admin/app/models/kb_ratio_registry.ts

@@ -0,0 +1,67 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import {
+  findChunksPerMb,
+  estimateChunkCount,
+  estimateBatch,
+  type BatchEstimate,
+  type BatchEstimateInput,
+} from '../utils/kb_ratio_lookup.js'
+
+/**
+ * Self-calibrating registry of `{filename-prefix → chunks_per_mb}` ratios used
+ * for disk-footprint and time-to-embed estimates surfaced in the KB panel.
+ *
+ * Migration seeds the registry with heuristic defaults from the RFC #883
+ * appendix; Phase 4 self-calibration will update rows in place as ZIMs finish
+ * ingesting and the real ratio becomes known. Lookup is longest-prefix-match
+ * (see `kb_ratio_lookup.ts`) so a specific entry (`wikipedia_en_simple_`)
+ * overrides a broader one (`wikipedia_en_`).
+ */
+export default class KbRatioRegistry extends BaseModel {
+  static table = 'kb_ratio_registry'
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare pattern: string
+
+  @column()
+  declare chunks_per_mb: number
+
+  @column()
+  declare sample_count: number
+
+  @column()
+  declare notes: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+
+  /** Look up chunks_per_mb for a filename by longest-prefix match. */
+  static async lookup(filename: string): Promise<number | null> {
+    const rows = await this.all()
+    return findChunksPerMb(filename, rows)
+  }
+
+  /** Estimate total chunks for a file of the given size on disk. */
+  static async estimateChunks(filename: string, fileSizeBytes: number): Promise<number | null> {
+    const rows = await this.all()
+    return estimateChunkCount(filename, fileSizeBytes, rows)
+  }
+
+  /**
+   * Aggregate an embedding-disk-cost estimate across a batch of files. Used by
+   * the curated-tier-change UI to show "you're about to add ~X GB of
+   * embeddings on top of the ZIM downloads" before the user commits.
+   */
+  static async estimateBatch(files: BatchEstimateInput[]): Promise<BatchEstimate> {
+    const rows = await this.all()
+    return estimateBatch(files, rows)
+  }
+}

admin/app/services/benchmark_service.ts

@@ -317,6 +317,23 @@ export class BenchmarkService {
         }
       }
 
+      // Fallback: AMD discrete cards. si.graphics() returns empty inside Docker for AMD,
+      // the nvidia-smi path doesn't apply, and the APU regex only catches integrated parts.
+      // SystemService.getSystemInfo() already handles AMD via the marker file + Ollama log
+      // probe added in PR #804, so reuse that plumbing rather than duplicating it here.
+      if (!gpuModel) {
+        try {
+          const systemService = new (await import('./system_service.js')).SystemService(this.dockerService)
+          const sysInfo = await systemService.getSystemInfo()
+          const sysGpuModel = sysInfo?.graphics?.controllers?.[0]?.model
+          if (sysGpuModel) {
+            gpuModel = sysGpuModel
+          }
+        } catch (sysError: any) {
+          logger.warn(`[BenchmarkService] system_service AMD fallback failed: ${sysError.message}`)
+        }
+      }
+
       return {
         cpu_model: `${cpu.manufacturer} ${cpu.brand}`,
         cpu_cores: cpu.physicalCores,

admin/app/services/collection_manifest_service.ts

@@ -5,6 +5,8 @@ import { DateTime } from 'luxon'
 import { join } from 'path'
 import CollectionManifest from '#models/collection_manifest'
 import InstalledResource from '#models/installed_resource'
+import { QueueService } from './queue_service.js'
+import { RunDownloadJob } from '#jobs/run_download_job'
 import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema } from '#validators/curated_collections'
 import {
   ensureDirectoryExists,
@@ -98,10 +100,74 @@ export class CollectionManifestService {
     const installedResources = await InstalledResource.query().where('resource_type', 'zim')
     const installedMap = new Map(installedResources.map((r) => [r.resource_id, r]))
 
-    return spec.categories.map((category) => ({
-      ...category,
-      installedTierSlug: this.getInstalledTierForCategory(category.tiers, installedMap),
-    }))
+    // In-flight ZIM download resource IDs from the BullMQ queue. Used to
+    // surface the user's tier intent immediately on submit, before any single
+    // file has finished downloading. Failed jobs are excluded so a stuck
+    // queue entry doesn't keep claiming the user's pick forever.
+    const inFlightIds = await this.getInFlightZimResourceIds()
+
+    return spec.categories.map((category) => {
+      const installedTierSlug = this.getInstalledTierForCategory(category.tiers, installedMap)
+      const downloadingTierSlug = this.getDownloadingTierForCategory(
+        category.tiers,
+        installedMap,
+        inFlightIds,
+        installedTierSlug
+      )
+      return { ...category, installedTierSlug, downloadingTierSlug }
+    })
+  }
+
+  private async getInFlightZimResourceIds(): Promise<Set<string>> {
+    const ids = new Set<string>()
+    try {
+      const queue = QueueService.getInstance().getQueue(RunDownloadJob.queue)
+      const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
+      for (const job of jobs) {
+        if (job.data?.filetype !== 'zim') continue
+        const resourceId = job.data?.resourceMetadata?.resource_id
+        if (typeof resourceId === 'string') ids.add(resourceId)
+      }
+    } catch (error) {
+      // Don't fail the whole categories endpoint if the queue is briefly
+      // unreachable — just report no in-flight downloads.
+      logger.warn('[CollectionManifestService] Could not read download queue:', error?.message || error)
+    }
+    return ids
+  }
+
+  /**
+   * Highest tier whose every resource is installed OR has an in-flight
+   * download. Returns undefined when there are no in-flight downloads for this
+   * category, or when the result would just duplicate installedTierSlug (i.e.
+   * everything that's downloading is already installed — nothing new to show).
+   */
+  getDownloadingTierForCategory(
+    tiers: SpecTier[],
+    installedMap: Map<string, InstalledResource>,
+    inFlightIds: Set<string>,
+    installedTierSlug: string | undefined
+  ): string | undefined {
+    if (inFlightIds.size === 0) return undefined
+
+    // Cheap pre-check: any of this category's resources actually in flight?
+    const anyInFlight = tiers.some((tier) =>
+      CollectionManifestService.resolveTierResources(tier, tiers).some((r) => inFlightIds.has(r.id))
+    )
+    if (!anyInFlight) return undefined
+
+    const reversedTiers = [...tiers].reverse()
+    for (const tier of reversedTiers) {
+      const resolved = CollectionManifestService.resolveTierResources(tier, tiers)
+      if (resolved.length === 0) continue
+      const allAccountedFor = resolved.every(
+        (r) => installedMap.has(r.id) || inFlightIds.has(r.id)
+      )
+      if (allAccountedFor) {
+        return tier.slug === installedTierSlug ? undefined : tier.slug
+      }
+    }
+    return undefined
   }
 
   async getMapCollectionsWithStatus(): Promise<CollectionWithStatus[]> {

admin/app/services/collection_update_service.ts

@@ -53,8 +53,10 @@ export class CollectionUpdateService {
         `[CollectionUpdateService] Update check complete: ${response.data.length} update(s) available`
       )
 
+      const updates = await this.enrichWithSizes(response.data)
+
       return {
-        updates: response.data,
+        updates,
         checked_at: new Date().toISOString(),
       }
     } catch (error) {
@@ -105,6 +107,8 @@ export class CollectionUpdateService {
         update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES,
       forceNew: true,
       filetype: update.resource_type,
+      title: update.resource_id,
+      totalBytes: update.size_bytes,
       resourceMetadata: {
         resource_id: update.resource_id,
         version: update.latest_version,
@@ -126,21 +130,42 @@ export class CollectionUpdateService {
   async applyAllUpdates(
     updates: ResourceUpdateInfo[]
   ): Promise<{ results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }> }> {
-    const results: Array<{
-      resource_id: string
-      success: boolean
-      jobId?: string
-      error?: string
-    }> = []
-
-    for (const update of updates) {
-      const result = await this.applyUpdate(update)
-      results.push({ resource_id: update.resource_id, ...result })
-    }
+    const results = await Promise.all(
+      updates.map(async (update) => {
+        const result = await this.applyUpdate(update)
+        return { resource_id: update.resource_id, ...result }
+      })
+    )
 
     return { results }
   }
 
+  /**
+   * Fetch Content-Length for each update URL in parallel. HEAD failures are non-fatal —
+   * the update row just renders without a size. Bounded to HEAD_TIMEOUT_MS so a slow
+   * mirror doesn't block the whole check.
+   */
+  private async enrichWithSizes(updates: ResourceUpdateInfo[]): Promise<ResourceUpdateInfo[]> {
+    const HEAD_TIMEOUT_MS = 5000
+
+    return await Promise.all(
+      updates.map(async (update) => {
+        if (update.size_bytes) return update // Trust upstream if it already gave us one
+        try {
+          const head = await axios.head(update.download_url, {
+            timeout: HEAD_TIMEOUT_MS,
+            maxRedirects: 5,
+            validateStatus: (s) => s >= 200 && s < 400,
+          })
+          const len = Number(head.headers['content-length'])
+          return Number.isFinite(len) && len > 0 ? { ...update, size_bytes: len } : update
+        } catch {
+          return update
+        }
+      })
+    )
+  }
+
   private buildFilename(update: ResourceUpdateInfo): string {
     if (update.resource_type === 'zim') {
       return `${update.resource_id}_${update.latest_version}.zim`

admin/app/services/countries_service.ts

@@ -0,0 +1,308 @@
+import { access, readFile, writeFile, mkdir } from 'fs/promises'
+import { join, resolve } from 'path'
+import { createHash } from 'crypto'
+import { tmpdir } from 'os'
+import logger from '@adonisjs/core/services/logger'
+import type { Country, CountryCode, CountryGroup } from '../../types/maps.js'
+
+interface NEFeature {
+  type: 'Feature'
+  properties: Record<string, any>
+  geometry: unknown
+}
+
+interface NEFeatureCollection {
+  type: 'FeatureCollection'
+  features: NEFeature[]
+}
+
+const COUNTRY_GEOJSON_PATH = join(
+  process.cwd(),
+  'resources',
+  'geodata',
+  'ne_50m_admin_0_countries.geojson'
+)
+
+// Natural Earth country polygons are land-only (no territorial waters), so a
+// strict intersect leaves tiles fully over the ocean out of the extract —
+// coastal cities render as grey off their coast. Inflate each polygon outward
+// by ~11 km to pull in adjacent tiles without ballooning the extract size.
+const REGION_BUFFER_DEGREES = 0.1
+
+const GROUP_ORDER = [
+  'north-america',
+  'south-america',
+  'europe',
+  'africa',
+  'asia',
+  'oceania',
+]
+
+const GROUP_META: Record<string, { id: string; name: string; description: string }> = {
+  'North America': {
+    id: 'north-america',
+    name: 'North America',
+    description: 'All countries in North America and the Caribbean.',
+  },
+  'South America': {
+    id: 'south-america',
+    name: 'South America',
+    description: 'All countries in South America.',
+  },
+  Europe: {
+    id: 'europe',
+    name: 'Europe',
+    description: 'All countries in Europe.',
+  },
+  Africa: {
+    id: 'africa',
+    name: 'Africa',
+    description: 'All countries in Africa.',
+  },
+  Asia: {
+    id: 'asia',
+    name: 'Asia',
+    description: 'All countries in Asia.',
+  },
+  Oceania: {
+    id: 'oceania',
+    name: 'Oceania',
+    description: 'Australia, New Zealand, and Pacific island nations.',
+  },
+}
+
+export class CountriesService {
+  private static instance: CountriesService | null = null
+  private loadPromise: Promise<void> | null = null
+  private countries: Country[] = []
+  private byCode: Map<CountryCode, { country: Country; feature: NEFeature }> = new Map()
+  private groups: CountryGroup[] = []
+
+  static getInstance(): CountriesService {
+    if (!this.instance) {
+      this.instance = new CountriesService()
+    }
+    return this.instance
+  }
+
+  private async ensureLoaded(): Promise<void> {
+    if (this.byCode.size > 0) return
+    if (!this.loadPromise) {
+      this.loadPromise = this.load()
+    }
+    await this.loadPromise
+  }
+
+  private async load(): Promise<void> {
+    const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8')
+    const fc = JSON.parse(raw) as NEFeatureCollection
+
+    // Natural Earth reuses a sovereign state's ISO_A2 for its dependencies
+    // (e.g. AU covers both Australia and Australian territories). Sort so the
+    // sovereign mainland wins the ISO-code slot, and skip any subsequent
+    // same-code dependency — otherwise the "AU" entry ends up being some tiny
+    // island territory.
+    const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b))
+
+    const countries: Country[] = []
+    const byCode = new Map<CountryCode, { country: Country; feature: NEFeature }>()
+    const groupCodes: Record<string, CountryCode[]> = {}
+
+    for (const feature of sortedFeatures) {
+      const p = feature.properties
+      const code = resolveIso2(p)
+      if (!code) continue
+      if (byCode.has(code)) continue
+
+      const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other'
+      if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue
+
+      const country: Country = {
+        code,
+        code3: resolveIso3(p) ?? code,
+        name: typeof p.NAME === 'string' ? p.NAME : code,
+        continent,
+        subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent,
+        population: typeof p.POP_EST === 'number' ? p.POP_EST : 0,
+      }
+
+      countries.push(country)
+      byCode.set(code, { country, feature })
+
+      if (GROUP_META[continent]) {
+        const groupId = GROUP_META[continent].id
+        if (!groupCodes[groupId]) groupCodes[groupId] = []
+        groupCodes[groupId].push(code)
+      }
+    }
+
+    countries.sort((a, b) => a.name.localeCompare(b.name))
+
+    const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => {
+      const meta = Object.values(GROUP_META).find((m) => m.id === groupId)
+      if (!meta) return []
+      const codes = (groupCodes[groupId] ?? []).slice().sort()
+      if (codes.length === 0) return []
+      return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }]
+    })
+
+    this.countries = countries
+    this.byCode = byCode
+    this.groups = groups
+
+    logger.info(
+      `[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups`
+    )
+  }
+
+  async list(): Promise<Country[]> {
+    await this.ensureLoaded()
+    return this.countries
+  }
+
+  async listGroups(): Promise<CountryGroup[]> {
+    await this.ensureLoaded()
+    return this.groups
+  }
+
+  /** Throws when a supplied code does not map to a known country. */
+  async resolveCodes(codes: CountryCode[]): Promise<CountryCode[]> {
+    await this.ensureLoaded()
+    const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort()
+    const unknown = normalized.filter((c) => !this.byCode.has(c))
+    if (unknown.length > 0) {
+      throw new Error(`Unknown country code(s): ${unknown.join(', ')}`)
+    }
+    return normalized
+  }
+
+  /**
+   * Filename is keyed on a hash of the sorted ISO codes + buffer size so
+   * repeated calls with the same selection reuse the same path, and bumping
+   * the buffer auto-invalidates stale files.
+   */
+  async writeRegionFile(codes: CountryCode[]): Promise<string> {
+    await this.ensureLoaded()
+    const resolved = await this.resolveCodes(codes)
+    const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}`
+    const hash = createHash('sha1').update(key).digest('hex').slice(0, 12)
+
+    const dir = resolve(tmpdir(), 'nomad-pmtiles-regions')
+    await mkdir(dir, { recursive: true })
+    const filepath = join(dir, `region-${hash}.geojson`)
+
+    try {
+      await access(filepath)
+      return filepath
+    } catch {}
+
+    const fc = {
+      type: 'FeatureCollection',
+      features: resolved.map((code) => {
+        const entry = this.byCode.get(code)!
+        return {
+          type: 'Feature',
+          properties: { iso: code, name: entry.country.name },
+          geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES),
+        }
+      }),
+    }
+
+    await writeFile(filepath, JSON.stringify(fc))
+    return filepath
+  }
+}
+
+function typeRank(f: NEFeature): number {
+  const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : ''
+  if (t === 'Sovereign country') return 0
+  if (t === 'Country') return 1
+  if (t === 'Sovereignty') return 2
+  if (t === 'Disputed') return 3
+  if (t === 'Dependency') return 4
+  return 5
+}
+
+function resolveIso2(p: Record<string, any>): CountryCode | null {
+  // Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for
+  // Taiwan or "-99" for countries involved in disputes. Only accept clean
+  // 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code).
+  const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null
+  if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase()
+  const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null
+  if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase()
+  return null
+}
+
+/**
+ * Inflate each polygon ring outward by `buffer` degrees via per-vertex
+ * averaged-normal offset. Not geodesically accurate — but at small buffers
+ * (<= 0.2°) it's within a few percent of a proper geodesic buffer at
+ * country scale, which is plenty for tile-inclusion purposes.
+ */
+function bufferGeometry(geometry: unknown, buffer: number): unknown {
+  const geom = geometry as { type: string; coordinates: any }
+  if (geom?.type === 'Polygon') {
+    return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) }
+  }
+  if (geom?.type === 'MultiPolygon') {
+    return {
+      type: 'MultiPolygon',
+      coordinates: geom.coordinates.map((poly: number[][][]) =>
+        bufferPolygonRings(poly, buffer)
+      ),
+    }
+  }
+  return geometry
+}
+
+function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] {
+  return rings.map((ring) => bufferRing(ring, buffer))
+}
+
+function bufferRing(ring: number[][], buffer: number): number[][] {
+  if (ring.length < 4) return ring
+  const sign = signedArea(ring) > 0 ? 1 : -1
+  const n = ring.length - 1
+  const out: number[][] = []
+  for (let i = 0; i < n; i++) {
+    const prev = ring[(i - 1 + n) % n]
+    const curr = ring[i]
+    const next = ring[(i + 1) % n]
+    const e1x = curr[0] - prev[0]
+    const e1y = curr[1] - prev[1]
+    const e2x = next[0] - curr[0]
+    const e2y = next[1] - curr[1]
+    const l1 = Math.hypot(e1x, e1y) || 1
+    const l2 = Math.hypot(e2x, e2y) || 1
+    const n1x = (e1y / l1) * sign
+    const n1y = (-e1x / l1) * sign
+    const n2x = (e2y / l2) * sign
+    const n2y = (-e2x / l2) * sign
+    const sumX = n1x + n2x
+    const sumY = n1y + n2y
+    const sl = Math.hypot(sumX, sumY) || 1
+    out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer])
+  }
+  out.push(out[0])
+  return out
+}
+
+function signedArea(ring: number[][]): number {
+  let a = 0
+  for (let i = 0; i < ring.length - 1; i++) {
+    a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1]
+  }
+  return a / 2
+}
+
+function resolveIso3(p: Record<string, any>): string | null {
+  const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null
+  if (primary && primary !== '-99') return primary.toUpperCase()
+  const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null
+  if (fallback && fallback !== '-99') return fallback.toUpperCase()
+  const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null
+  if (adm && adm !== '-99') return adm.toUpperCase()
+  return null
+}
+

admin/app/services/docker_service.ts

@@ -10,7 +10,7 @@ import { KiwixLibraryService } from './kiwix_library_service.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { exec } from 'child_process'
 import { promisify } from 'util'
-// import { readdir } from 'fs/promises'
+import { readFile } from 'node:fs/promises'
 import KVStore from '#models/kv_store'
 import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
 import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
@@ -291,8 +291,12 @@ export class DockerService {
 
   /**
    * Force reinstall a service by stopping, removing, and recreating its container.
-   * This method will also clear any associated volumes/data.
-   * Handles edge cases gracefully (e.g., container not running, container not found).
+   *
+   * Volume handling: removes Docker-managed named volumes whose name equals
+   * `serviceName`, starts with `${serviceName}_`, or carries a `service=${serviceName}`
+   * label. Host bind mounts are NOT touched — any data living on a bind-mounted
+   * host path (ZIM stores, model caches, MySQL data dir, etc.) survives the reinstall.
+   * Anonymous volumes (random hash names) are also not matched.
    */
   async forceReinstall(serviceName: string): Promise<{ success: boolean; message: string }> {
     try {
@@ -365,7 +369,10 @@ export class DockerService {
         const volumes = await this.docker.listVolumes()
         const serviceVolumes =
           volumes.Volumes?.filter(
-            (v) => v.Name.includes(serviceName) || v.Labels?.service === serviceName
+            (v) =>
+              v.Name === serviceName ||
+              v.Name.startsWith(`${serviceName}_`) ||
+              v.Labels?.service === serviceName
           ) || []
 
         for (const vol of serviceVolumes) {
@@ -500,6 +507,7 @@ export class DockerService {
       // GPU-aware configuration for Ollama
       let finalImage = service.container_image
       let gpuHostConfig = containerConfig?.HostConfig || {}
+      let amdGpuConfigured = false
 
       if (service.service_name === SERVICE_NAMES.OLLAMA) {
         const gpuResult = await this._detectGPUType()
@@ -523,16 +531,51 @@ export class DockerService {
             ],
           }
         } else if (gpuResult.type === 'amd') {
-          this._broadcast(
-            service.service_name,
-            'gpu-config',
-            `AMD GPU detected. ROCm GPU acceleration is not yet supported in this version — proceeding with CPU-only configuration. GPU support for AMD will be available in a future update.`
-          )
-          logger.warn('[DockerService] AMD GPU detected but ROCm support is not yet enabled. Using CPU-only configuration.')
-          // TODO: Re-enable AMD GPU support once ROCm image and device discovery are validated.
-          // When re-enabling:
-          //   1. Switch image to 'ollama/ollama:rocm'
-          //   2. Restore _discoverAMDDevices() to map /dev/kfd and /dev/dri/* into the container
+          // AMD acceleration is opt-out via the 'ai.amdGpuAcceleration' KV key (default-on).
+          // Per memory feedback: KV values can be string or boolean — coerce explicitly.
+          const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
+          const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
+
+          if (amdAccelerationEnabled) {
+            this._broadcast(
+              service.service_name,
+              'gpu-config',
+              `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...`
+            )
+
+            finalImage = 'ollama/ollama:rocm'
+
+            // The pull-if-missing earlier in this function used service.container_image
+            // (the DB-pinned tag, e.g. ollama/ollama:0.18.2). The AMD branch overrides
+            // to a different tag — so we need to pull :rocm separately if it's not local.
+            const rocmImageExists = await this._checkImageExists(finalImage)
+            if (!rocmImageExists) {
+              this._broadcast(
+                service.service_name,
+                'pulling',
+                `Pulling Docker image ${finalImage}...`
+              )
+              const rocmPullStream = await this.docker.pull(finalImage)
+              await new Promise((res) => this.docker.modem.followProgress(rocmPullStream, res))
+            }
+
+            const amdDevices = await this._discoverAMDDevices()
+            gpuHostConfig = {
+              ...gpuHostConfig,
+              Devices: amdDevices,
+            }
+            amdGpuConfigured = true
+            logger.info(
+              `[DockerService] Configured ROCm image and ${amdDevices.length} AMD device entries for Ollama`
+            )
+          } else {
+            this._broadcast(
+              service.service_name,
+              'gpu-config',
+              `AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.`
+            )
+            logger.info('[DockerService] AMD GPU acceleration disabled by KV opt-out; using CPU-only configuration.')
+          }
         } else if (gpuResult.toolkitMissing) {
           this._broadcast(
             service.service_name,
@@ -555,6 +598,14 @@ export class DockerService {
         if (flashAttentionEnabled !== false) {
           ollamaEnv.push('OLLAMA_FLASH_ATTENTION=1')
         }
+        if (amdGpuConfigured) {
+          // gfx-aware HSA override — only set for cards that actually need it. See
+          // _resolveAmdHsaOverride() for the resolution order and gfx → version mapping.
+          const hsaOverride = await this._resolveAmdHsaOverride()
+          if (hsaOverride) {
+            ollamaEnv.push(`HSA_OVERRIDE_GFX_VERSION=${hsaOverride}`)
+          }
+        }
       }
 
       this._broadcast(
@@ -857,7 +908,10 @@ export class DockerService {
   /**
    * Detect GPU type and toolkit availability.
    * Primary: Check Docker runtimes via docker.info() (works from inside containers).
-   * Fallback: lspci for host-based installs and AMD detection.
+   * Secondary: Read /app/storage/.nomad-gpu-type written by install_nomad.sh — needed
+   *   for AMD detection because lspci isn't available inside the admin container and
+   *   AMD has no Docker runtime registration to query.
+   * Fallback: lspci for host-based installs.
    */
   private async _detectGPUType(): Promise<{ type: 'nvidia' | 'amd' | 'none'; toolkitMissing?: boolean }> {
     try {
@@ -874,6 +928,24 @@ export class DockerService {
         logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`)
       }
 
+      // Secondary: install_nomad.sh writes the host-detected GPU type to a marker file in
+      // the storage volume so the admin container (which lacks lspci) can read it.
+      try {
+        const marker = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
+        if (marker === 'nvidia') {
+          // Hardware present but Docker doesn't have nvidia runtime → toolkit missing
+          logger.warn('[DockerService] NVIDIA GPU recorded in marker file but NVIDIA Container Toolkit is not installed')
+          return { type: 'none', toolkitMissing: true }
+        }
+        if (marker === 'amd') {
+          logger.info('[DockerService] AMD GPU detected via install-time marker file')
+          await this._persistGPUType('amd')
+          return { type: 'amd' }
+        }
+      } catch {
+        // No marker file — fall through to lspci attempt for host-based installs
+      }
+
       // Fallback: lspci for host-based installs (not available inside Docker)
       const execAsync = promisify(exec)
 
@@ -937,60 +1009,84 @@ export class DockerService {
   }
 
   /**
-   * Discover AMD GPU DRI devices dynamically.
-   * Returns an array of device configurations for Docker.
+   * Resolve the HSA_OVERRIDE_GFX_VERSION value for the host's AMD GPU.
+   *
+   * gfx1030 (RX 6800/6700/etc.), gfx1100/1101/1102 (RX 7900/7800/7600) are on AMD's
+   * official ROCm allowlist — forcing an override on these breaks GPU discovery.
+   * gfx1035 / gfx1036 (RDNA 2 iGPUs like 680M) need 10.3.0 to coerce to gfx1030.
+   * gfx1103 / gfx1150 / gfx1151 (RDNA 3/3.5 iGPUs like 780M / 890M / Strix Halo) need 11.0.0.
+   *
+   * Resolution order:
+   *   1. KV `ai.amdHsaOverride` — manual user override; accepts 'none' (disable) or a semver-style value.
+   *   2. Marker file `/app/storage/.nomad-amd-gfx` written by install_nomad.sh.
+   *   3. Default: '11.0.0' — preserves prior behavior so existing iGPU users don't regress on
+   *      upgrade. Discrete-card users on existing installs can opt out via the KV.
+   *
+   * Returns null when no override should be applied.
    */
-  // private async _discoverAMDDevices(): Promise<
-  //   Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
-  // > {
-  //   try {
-  //     const devices: Array<{
-  //       PathOnHost: string
-  //       PathInContainer: string
-  //       CgroupPermissions: string
-  //     }> = []
+  private async _resolveAmdHsaOverride(): Promise<string | null> {
+    const manualRaw = await KVStore.getValue('ai.amdHsaOverride')
+    if (manualRaw !== null && manualRaw !== undefined && String(manualRaw).trim() !== '') {
+      const manual = String(manualRaw).trim().toLowerCase()
+      if (manual === 'none' || manual === 'off' || manual === 'false') {
+        logger.info('[DockerService] HSA override disabled via ai.amdHsaOverride')
+        return null
+      }
+      if (/^\d+\.\d+\.\d+$/.test(manual)) {
+        logger.info(`[DockerService] HSA override forced to ${manual} via ai.amdHsaOverride`)
+        return manual
+      }
+      logger.warn(`[DockerService] Ignoring invalid ai.amdHsaOverride value: ${manualRaw}`)
+    }
 
-  //     // Always add /dev/kfd (Kernel Fusion Driver)
-  //     devices.push({
-  //       PathOnHost: '/dev/kfd',
-  //       PathInContainer: '/dev/kfd',
-  //       CgroupPermissions: 'rwm',
-  //     })
+    try {
+      const gfx = (await readFile('/app/storage/.nomad-amd-gfx', 'utf8')).trim()
+      const mapped = this._mapGfxToHsaOverride(gfx)
+      logger.info(`[DockerService] AMD gfx marker '${gfx}' → HSA override ${mapped ?? 'none'}`)
+      return mapped
+    } catch {
+      // Marker absent — most likely an existing install upgraded without re-running
+      // install_nomad.sh. Fall through to the default.
+    }
 
-  //     // Discover DRI devices in /dev/dri/
-  //     try {
-  //       const driDevices = await readdir('/dev/dri')
-  //       for (const device of driDevices) {
-  //         const devicePath = `/dev/dri/${device}`
-  //         devices.push({
-  //           PathOnHost: devicePath,
-  //           PathInContainer: devicePath,
-  //           CgroupPermissions: 'rwm',
-  //         })
-  //       }
-  //       logger.info(
-  //         `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}`
-  //       )
-  //     } catch (error) {
-  //       logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`)
-  //       // Fallback to common device names if directory read fails
-  //       const fallbackDevices = ['card0', 'renderD128']
-  //       for (const device of fallbackDevices) {
-  //         devices.push({
-  //           PathOnHost: `/dev/dri/${device}`,
-  //           PathInContainer: `/dev/dri/${device}`,
-  //           CgroupPermissions: 'rwm',
-  //         })
-  //       }
-  //       logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`)
-  //     }
+    logger.info('[DockerService] No AMD gfx marker; defaulting HSA override to 11.0.0 for backward compatibility')
+    return '11.0.0'
+  }
 
-  //     return devices
-  //   } catch (error) {
-  //     logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`)
-  //     return []
-  //   }
-  // }
+  private _mapGfxToHsaOverride(gfx: string): string | null {
+    // Officially supported by ROCm — no override needed
+    if (gfx === 'gfx1030' || gfx === 'gfx1100' || gfx === 'gfx1101' || gfx === 'gfx1102') {
+      return null
+    }
+    // RDNA 2 variants + iGPUs (gfx1031..gfx1036, e.g. Rembrandt 680M)
+    if (/^gfx103[1-6]$/.test(gfx)) {
+      return '10.3.0'
+    }
+    // RDNA 3 / 3.5 mobile parts (Phoenix 780M = gfx1103, Strix 890M = gfx1150, Strix Halo = gfx1151)
+    if (gfx === 'gfx1103' || gfx === 'gfx1150' || gfx === 'gfx1151') {
+      return '11.0.0'
+    }
+    return '11.0.0'
+  }
+
+  /**
+   * Build the Docker Devices array for AMD GPU passthrough.
+   *
+   * Returns /dev/kfd (Kernel Fusion Driver, required by ROCm) and /dev/dri (the DRM
+   * device tree). Passing /dev/dri as a single directory entry mirrors Docker CLI
+   * --device behavior — the daemon expands it to all child devices (card*, renderD*)
+   * regardless of how the host enumerates them. This avoids the brittle hardcoded
+   * fallback (card0/renderD128) the prior implementation used, which was wrong on
+   * systems where the AMD GPU enumerates as card1+ (e.g. UM890 Pro 780M iGPU).
+   */
+  private async _discoverAMDDevices(): Promise<
+    Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
+  > {
+    return [
+      { PathOnHost: '/dev/kfd', PathInContainer: '/dev/kfd', CgroupPermissions: 'rwm' },
+      { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' },
+    ]
+  }
 
   /**
    * Update a service container to a new image version while preserving volumes and data.
@@ -1014,16 +1110,68 @@ export class DockerService {
 
       this.activeInstallations.add(serviceName)
 
-      // Compute new image string
+      // newImage = the semver tag we record in the DB after the update (e.g. ollama/ollama:0.23.2).
+      // runtimeImage = the tag we actually pull and run. For AMD-on-Ollama these diverge: we run
+      // the rolling :rocm tag because per-version ROCm tags aren't always published, but the DB
+      // must keep the semver tag so the Apps page shows the actual version (not literally "rocm")
+      // and the registry update-check parses a valid tag (instead of looping on the same update).
       const currentImage = service.container_image
       const imageBase = currentImage.includes(':')
         ? currentImage.substring(0, currentImage.lastIndexOf(':'))
         : currentImage
       const newImage = `${imageBase}:${targetVersion}`
+      let runtimeImage = newImage
 
-      // Step 1: Pull new image
-      this._broadcast(serviceName, 'update-pulling', `Pulling image ${newImage}...`)
-      const pullStream = await this.docker.pull(newImage)
+      // GPU detection runs before the pull so AMD updates pull ollama/ollama:rocm rather
+      // than the standard tag. Detection result is reused below when building the new
+      // container config (devices, env). Non-Ollama services skip this entirely.
+      let updatedDeviceRequests: any[] | undefined = undefined
+      let updatedAmdDevices: any[] | undefined = undefined
+      let updatedAmdGpuConfigured = false
+      if (serviceName === SERVICE_NAMES.OLLAMA) {
+        const gpuResult = await this._detectGPUType()
+        if (gpuResult.type === 'nvidia') {
+          this._broadcast(
+            serviceName,
+            'update-gpu-config',
+            `NVIDIA container runtime detected. Configuring updated container with GPU support...`
+          )
+          updatedDeviceRequests = [
+            { Driver: 'nvidia', Count: -1, Capabilities: [['gpu']] },
+          ]
+        } else if (gpuResult.type === 'amd') {
+          const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
+          const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
+          if (amdAccelerationEnabled) {
+            this._broadcast(
+              serviceName,
+              'update-gpu-config',
+              `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...`
+            )
+            runtimeImage = 'ollama/ollama:rocm'
+            updatedAmdDevices = await this._discoverAMDDevices()
+            updatedAmdGpuConfigured = true
+          } else {
+            this._broadcast(
+              serviceName,
+              'update-gpu-config',
+              `AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.`
+            )
+          }
+        } else if (gpuResult.toolkitMissing) {
+          this._broadcast(
+            serviceName,
+            'update-gpu-config',
+            `NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html`
+          )
+        } else {
+          this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`)
+        }
+      }
+
+      // Step 1: Pull new image (runtimeImage diverges from newImage for AMD, see above)
+      this._broadcast(serviceName, 'update-pulling', `Pulling image ${runtimeImage}...`)
+      const pullStream = await this.docker.pull(runtimeImage)
       await new Promise((res) => this.docker.modem.followProgress(pullStream, res))
 
       // Step 2: Find and stop existing container
@@ -1054,48 +1202,23 @@ export class DockerService {
 
       const hostConfig = inspectData.HostConfig || {}
 
-      // Re-run GPU detection for Ollama so updates always reflect the current GPU environment.
-      // This handles cases where the NVIDIA Container Toolkit was installed after the initial
-      // Ollama setup, and ensures DeviceRequests are always built fresh rather than relying on
-      // round-tripping the Docker inspect format back into the create API.
-      let updatedDeviceRequests: any[] | undefined = undefined
-      if (serviceName === SERVICE_NAMES.OLLAMA) {
-        const gpuResult = await this._detectGPUType()
-
-        if (gpuResult.type === 'nvidia') {
-          this._broadcast(
-            serviceName,
-            'update-gpu-config',
-            `NVIDIA container runtime detected. Configuring updated container with GPU support...`
-          )
-          updatedDeviceRequests = [
-            {
-              Driver: 'nvidia',
-              Count: -1,
-              Capabilities: [['gpu']],
-            },
-          ]
-        } else if (gpuResult.type === 'amd') {
-          this._broadcast(
-            serviceName,
-            'update-gpu-config',
-            `AMD GPU detected. ROCm GPU acceleration is not yet supported — using CPU-only configuration.`
-          )
-        } else if (gpuResult.toolkitMissing) {
-          this._broadcast(
-            serviceName,
-            'update-gpu-config',
-            `NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html`
-          )
-        } else {
-          this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`)
+      // GPU detection already ran above (before the pull) so we know the right image, devices,
+      // and whether HSA_OVERRIDE needs injection. For AMD, replace any prior HSA_OVERRIDE in
+      // the inspect-captured env so updates from older containers pick up the current value.
+      const baseEnv = inspectData.Config?.Env || []
+      let finalEnv = baseEnv
+      if (updatedAmdGpuConfigured) {
+        const hsaOverride = await this._resolveAmdHsaOverride()
+        finalEnv = baseEnv.filter((e: string) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION='))
+        if (hsaOverride) {
+          finalEnv.push(`HSA_OVERRIDE_GFX_VERSION=${hsaOverride}`)
         }
       }
 
       const newContainerConfig: any = {
-        Image: newImage,
+        Image: runtimeImage,
         name: serviceName,
-        Env: inspectData.Config?.Env || undefined,
+        Env: finalEnv.length > 0 ? finalEnv : undefined,
         Cmd: inspectData.Config?.Cmd || undefined,
         ExposedPorts: inspectData.Config?.ExposedPorts || undefined,
         WorkingDir: inspectData.Config?.WorkingDir || undefined,
@@ -1105,7 +1228,7 @@ export class DockerService {
           PortBindings: hostConfig.PortBindings || undefined,
           RestartPolicy: hostConfig.RestartPolicy || undefined,
           DeviceRequests: serviceName === SERVICE_NAMES.OLLAMA ? updatedDeviceRequests : (hostConfig.DeviceRequests || undefined),
-          Devices: hostConfig.Devices || undefined,
+          Devices: serviceName === SERVICE_NAMES.OLLAMA && updatedAmdDevices ? updatedAmdDevices : (hostConfig.Devices || undefined),
         },
         NetworkingConfig: inspectData.NetworkSettings?.Networks
           ? {

admin/app/services/download_service.ts

@@ -1,13 +1,19 @@
 import { inject } from '@adonisjs/core'
 import { QueueService } from './queue_service.js'
 import { RunDownloadJob } from '#jobs/run_download_job'
+import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
+import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job'
 import { DownloadModelJob } from '#jobs/download_model_job'
 import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js'
+import type { Job, Queue } from 'bullmq'
 import { normalize } from 'path'
 import { deleteFileIfExists } from '../utils/fs.js'
 import transmit from '@adonisjs/transmit/services/main'
 import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
 
+type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
+type TaggedJob = { job: Job; state: FileJobState }
+
 @inject()
 export class DownloadService {
   constructor(private queueService: QueueService) {}
@@ -26,27 +32,32 @@ export class DownloadService {
     return { percent: parseInt(String(progress), 10) || 0 }
   }
 
-  async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
-    // Get regular file download jobs (zim, map, etc.) — query each state separately so we can
-    // tag each job with its actual BullMQ state rather than guessing from progress data.
-    const queue = this.queueService.getQueue(RunDownloadJob.queue)
-    type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
-
-    const [waitingJobs, activeJobs, delayedJobs, failedJobs] = await Promise.all([
+  /** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */
+  private async fetchJobsWithStates(queueName: string): Promise<TaggedJob[]> {
+    const queue = this.queueService.getQueue(queueName)
+    const [waiting, active, delayed, failed] = await Promise.all([
       queue.getJobs(['waiting']),
       queue.getJobs(['active']),
       queue.getJobs(['delayed']),
       queue.getJobs(['failed']),
     ])
-
-    const taggedFileJobs: Array<{ job: (typeof waitingJobs)[0]; state: FileJobState }> = [
-      ...waitingJobs.map((j) => ({ job: j, state: 'waiting' as const })),
-      ...activeJobs.map((j) => ({ job: j, state: 'active' as const })),
-      ...delayedJobs.map((j) => ({ job: j, state: 'delayed' as const })),
-      ...failedJobs.map((j) => ({ job: j, state: 'failed' as const })),
+    return [
+      ...waiting.map((j) => ({ job: j, state: 'waiting' as const })),
+      ...active.map((j) => ({ job: j, state: 'active' as const })),
+      ...delayed.map((j) => ({ job: j, state: 'delayed' as const })),
+      ...failed.map((j) => ({ job: j, state: 'failed' as const })),
     ]
+  }
 
-    const fileDownloads = taggedFileJobs.map(({ job, state }) => {
+  async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
+    const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
+    const [fileTagged, extractTagged, modelJobs] = await Promise.all([
+      this.fetchJobsWithStates(RunDownloadJob.queue),
+      this.fetchJobsWithStates(RunExtractPmtilesJob.queue),
+      modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']),
+    ])
+
+    const fileDownloads = fileTagged.map(({ job, state }) => {
       const parsed = this.parseProgress(job.progress)
       return {
         jobId: job.id!.toString(),
@@ -63,26 +74,36 @@ export class DownloadService {
       }
     })
 
-    // Get Ollama model download jobs
-    const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
-    const modelJobs = await modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed'])
+    const extractDownloads = extractTagged.map(({ job, state }) => {
+      const parsed = this.parseProgress(job.progress)
+      return {
+        jobId: job.id!.toString(),
+        url: job.data.sourceUrl,
+        progress: parsed.percent,
+        filepath: normalize(job.data.outputFilepath),
+        filetype: job.data.filetype || 'map',
+        title: job.data.title || undefined,
+        downloadedBytes: parsed.downloadedBytes,
+        totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined,
+        lastProgressTime: parsed.lastProgressTime,
+        status: state,
+        failedReason: job.failedReason || undefined,
+      }
+    })
 
     const modelDownloads = modelJobs.map((job) => ({
       jobId: job.id!.toString(),
-      url: job.data.modelName || 'Unknown Model', // Use model name as url
+      url: job.data.modelName || 'Unknown Model',
       progress: parseInt(job.progress.toString(), 10),
-      filepath: job.data.modelName || 'Unknown Model', // Use model name as filepath
+      filepath: job.data.modelName || 'Unknown Model',
       filetype: 'model',
       status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed',
       failedReason: job.failedReason || undefined,
     }))
 
-    const allDownloads = [...fileDownloads, ...modelDownloads]
-
-    // Filter by filetype if specified
+    const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads]
     const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype)
 
-    // Sort: active downloads first (by progress desc), then failed at the bottom
     return filtered.sort((a, b) => {
       if (a.status === 'failed' && b.status !== 'failed') return 1
       if (a.status !== 'failed' && b.status === 'failed') return -1
@@ -91,7 +112,11 @@ export class DownloadService {
   }
 
   async removeFailedJob(jobId: string): Promise<void> {
-    for (const queueName of [RunDownloadJob.queue, DownloadModelJob.queue]) {
+    for (const queueName of [
+      RunDownloadJob.queue,
+      RunExtractPmtilesJob.queue,
+      DownloadModelJob.queue,
+    ]) {
       const queue = this.queueService.getQueue(queueName)
       const job = await queue.getJob(jobId)
       if (job) {
@@ -113,7 +138,6 @@ export class DownloadService {
   }
 
   async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> {
-    // Try the file download queue first (the original PR #554 path)
     const queue = this.queueService.getQueue(RunDownloadJob.queue)
     const job = await queue.getJob(jobId)
 
@@ -121,7 +145,13 @@ export class DownloadService {
       return await this._cancelFileDownloadJob(jobId, job, queue)
     }
 
-    // Fall through to the model download queue
+    const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue)
+    const extractJob = await extractQueue.getJob(jobId)
+
+    if (extractJob) {
+      return await this._cancelExtractJob(jobId, extractJob, extractQueue)
+    }
+
     const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
     const modelJob = await modelQueue.getJob(jobId)
 
@@ -129,11 +159,37 @@ export class DownloadService {
       return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue)
     }
 
-    // Not found in either queue
     return { success: true, message: 'Job not found (may have already completed)' }
   }
 
-  /** Cancel a content download (zim, map, pmtiles, etc.) — original PR #554 logic */
+  private async _cancelExtractJob(
+    jobId: string,
+    job: Job<RunExtractPmtilesJobParams>,
+    queue: Queue<RunExtractPmtilesJobParams>
+  ): Promise<{ success: boolean; message: string }> {
+    const outputFilepath = job.data.outputFilepath
+
+    await RunExtractPmtilesJob.signalCancel(jobId)
+
+    // Same-process fallback when worker and API share a process
+    RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM')
+    RunExtractPmtilesJob.childProcesses.delete(jobId)
+
+    await this._pollForTerminalState(job, jobId)
+    await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId)
+
+    if (outputFilepath) {
+      try {
+        await deleteFileIfExists(outputFilepath)
+      } catch {
+        // File may not exist yet (subprocess may not have opened it)
+      }
+    }
+
+    return { success: true, message: 'Extract cancelled and partial file deleted' }
+  }
+
+  /** Cancel a content download (zim, map, pmtiles, etc.) */
   private async _cancelFileDownloadJob(
     jobId: string,
     job: any,

admin/app/services/map_service.ts

@@ -16,11 +16,35 @@ import {
 import { join, resolve, sep } from 'path'
 import urlJoin from 'url-join'
 import { RunDownloadJob } from '#jobs/run_download_job'
+import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
 import logger from '@adonisjs/core/services/logger'
 import { assertNotPrivateUrl } from '#validators/common'
 import InstalledResource from '#models/installed_resource'
 import { CollectionManifestService } from './collection_manifest_service.js'
 import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js'
+import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js'
+import {
+  EXTRACT_DEFAULT_MAX_ZOOM,
+  EXTRACT_MAX_ZOOM,
+  EXTRACT_MIN_ZOOM,
+  PMTILES_BINARY_PATH,
+  WORLD_BASEMAP_FILENAME,
+  WORLD_BASEMAP_MAX_ZOOM,
+  WORLD_BASEMAP_SOURCE_NAME,
+  buildPmtilesExtractArgs,
+} from '../../constants/map_regions.js'
+import { CountriesService } from './countries_service.js'
+import { execFile } from 'child_process'
+import { createHash, randomBytes } from 'crypto'
+import { tmpdir } from 'os'
+import { promisify } from 'util'
+
+const execFileAsync = promisify(execFile)
+const DRY_RUN_TIMEOUT_MS = 60_000
+const DRY_RUN_MAX_BUFFER = 256 * 1024
+// Real extract of z0-5 world tiles; generous to tolerate slow/metered links
+// since a failure leaves the map grey for uncovered regions.
+const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000
 
 const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json'
 const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com'
@@ -53,10 +77,15 @@ export class MapService implements IMapService {
   private readonly baseAssetsTarFile = 'base-assets.tar.gz'
   private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
   private baseAssetsExistCache: boolean | null = null
+  private worldBasemapReady = false
+  private worldBasemapInFlight: Promise<void> | null = null
 
   async listRegions() {
     const files = (await this.listAllMapStorageItems()).filter(
-      (item) => item.type === 'file' && item.name.endsWith('.pmtiles')
+      (item) =>
+        item.type === 'file' &&
+        item.name.endsWith('.pmtiles') &&
+        item.name !== WORLD_BASEMAP_FILENAME
     )
 
     return {
@@ -327,11 +356,76 @@ export class MapService implements IMapService {
 
   async ensureBaseAssets(): Promise<boolean> {
     const exists = await this.checkBaseAssetsExist()
-    if (exists) {
-      return true
+    if (!exists) {
+      const downloaded = await this.downloadBaseAssets()
+      if (!downloaded) return false
     }
 
-    return await this.downloadBaseAssets()
+    try {
+      await this.ensureWorldBasemap()
+    } catch (err) {
+      logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`)
+    }
+
+    return true
+  }
+
+  /**
+   * Extract a low-zoom global basemap once so the map isn't grey outside a
+   * regional extract's polygon. Cheap (~15 MB, a handful of HTTP range
+   * requests) and layered underneath regional sources at render time.
+   *
+   * Memoizes success in-process, and de-duplicates concurrent callers via a
+   * shared in-flight promise so two simultaneous `/maps` requests on a cold
+   * start don't both launch `pmtiles extract` against the same output path.
+   */
+  private async ensureWorldBasemap(): Promise<void> {
+    if (this.worldBasemapReady) return
+    if (this.worldBasemapInFlight) return this.worldBasemapInFlight
+    this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => {
+      this.worldBasemapInFlight = null
+    })
+    return this.worldBasemapInFlight
+  }
+
+  private async _setupWorldBasemap(): Promise<void> {
+    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
+    const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME))
+    if (!filepath.startsWith(basePath + sep)) {
+      throw new Error('Invalid world basemap path')
+    }
+
+    await ensureDirectoryExists(basePath)
+
+    const existing = await getFileStatsIfExists(filepath)
+    if (existing && Number(existing.size) > 0) {
+      this.worldBasemapReady = true
+      return
+    }
+
+    const info = await this.getGlobalMapInfo()
+    const args = buildPmtilesExtractArgs({
+      sourceUrl: info.url,
+      outputFilepath: filepath,
+      maxzoom: WORLD_BASEMAP_MAX_ZOOM,
+      downloadThreads: 4,
+    })
+
+    logger.info(
+      `[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}`
+    )
+    try {
+      await execFileAsync(PMTILES_BINARY_PATH, args, {
+        timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS,
+        maxBuffer: DRY_RUN_MAX_BUFFER,
+      })
+      this.worldBasemapReady = true
+    } catch (err: any) {
+      await deleteFileIfExists(filepath)
+      throw new Error(
+        `pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}`
+      )
+    }
   }
 
   private async checkBaseAssetsExist(useCache: boolean = true): Promise<boolean> {
@@ -367,6 +461,19 @@ export class MapService implements IMapService {
     const sources: BaseStylesFile['sources'][] = []
     const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol)
 
+    // World basemap goes first so its layers render underneath regional extracts.
+    // Only emitted when ensureWorldBasemap() succeeded — otherwise the style would
+    // reference a file that doesn't exist and produce 404s on every tile request.
+    if (this.worldBasemapReady) {
+      const worldSource: BaseStylesFile['sources'] = {}
+      worldSource[WORLD_BASEMAP_SOURCE_NAME] = {
+        type: 'vector',
+        attribution: PMTILES_ATTRIBUTION,
+        url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`,
+      }
+      sources.push(worldSource)
+    }
+
     for (const region of regions) {
       if (region.type === 'file' && region.name.endsWith('.pmtiles')) {
         // Strip .pmtiles and date suffix (e.g. "alaska_2025-12" -> "alaska") for stable source names
@@ -489,12 +596,206 @@ export class MapService implements IMapService {
     }
   }
 
+  async listCountries(): Promise<Country[]> {
+    return CountriesService.getInstance().list()
+  }
+
+  async listCountryGroups(): Promise<CountryGroup[]> {
+    return CountriesService.getInstance().listGroups()
+  }
+
+  async extractPreflight(params: {
+    countries: CountryCode[]
+    maxzoom?: number
+  }): Promise<MapExtractPreflight> {
+    this.validateMaxzoom(params.maxzoom)
+    const countries = await CountriesService.getInstance().resolveCodes(params.countries)
+    const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries)
+    const info = await this.getGlobalMapInfo()
+    return this.runDryRun(info, regionFilepath, params.maxzoom)
+  }
+
+  private async runDryRun(
+    info: { url: string; date: string; key: string },
+    regionFilepath: string,
+    maxzoom?: number
+  ): Promise<MapExtractPreflight> {
+    const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`)
+    const args = buildPmtilesExtractArgs({
+      sourceUrl: info.url,
+      outputFilepath: dryRunOutput,
+      regionFilepath,
+      maxzoom,
+      dryRun: true,
+    })
+
+    let stdout = ''
+    let stderr = ''
+    try {
+      const result = await execFileAsync(PMTILES_BINARY_PATH, args, {
+        timeout: DRY_RUN_TIMEOUT_MS,
+        maxBuffer: DRY_RUN_MAX_BUFFER,
+      })
+      stdout = result.stdout
+      stderr = result.stderr
+    } catch (err: any) {
+      throw new Error(
+        `pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}`
+      )
+    }
+
+    const parsed = this.parseDryRunOutput(stdout + '\n' + stderr)
+
+    return {
+      tiles: parsed.tiles,
+      bytes: parsed.bytes,
+      source: { url: info.url, date: info.date, key: info.key },
+    }
+  }
+
+  async extractRegion(params: {
+    countries: CountryCode[]
+    maxzoom?: number
+    label?: string
+    estimatedBytes?: number
+  }): Promise<{ filename: string; jobId?: string }> {
+    this.validateMaxzoom(params.maxzoom)
+    const countriesService = CountriesService.getInstance()
+    const countries = await countriesService.resolveCodes(params.countries)
+    const regionFilepath = await countriesService.writeRegionFile(countries)
+    const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM
+
+    const [baseAssetsExist, info, groups] = await Promise.all([
+      this.ensureBaseAssets(),
+      this.getGlobalMapInfo(),
+      countriesService.listGroups(),
+    ])
+    if (!baseAssetsExist) {
+      throw new Error(
+        'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
+      )
+    }
+
+    const groupMatch = findExactGroupMatch(countries, groups)
+    const slug = this.buildRegionSlug(countries, groupMatch)
+    const dateSlug = info.key.replace('.pmtiles', '')
+    const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles`
+    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
+    const filepath = resolve(join(basePath, filename))
+
+    if (!filepath.startsWith(basePath + sep)) {
+      throw new Error('Invalid filename')
+    }
+
+    let estimatedBytes = params.estimatedBytes ?? 0
+    if (estimatedBytes === 0) {
+      try {
+        const preflight = await this.runDryRun(info, regionFilepath, maxzoom)
+        estimatedBytes = preflight.bytes
+      } catch (err) {
+        logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`)
+      }
+    }
+
+    const title = params.label ?? this.buildRegionTitle(countries, groupMatch)
+
+    const result = await RunExtractPmtilesJob.dispatch({
+      sourceUrl: info.url,
+      outputFilepath: filepath,
+      regionFilepath,
+      maxzoom,
+      estimatedBytes,
+      filetype: 'map',
+      title,
+      resourceMetadata: {
+        resource_id: slug,
+        version: dateSlug,
+        collection_ref: null,
+      },
+    })
+
+    if (!result.job) {
+      throw new Error('Failed to dispatch extract job')
+    }
+
+    logger.info(
+      `[MapService] Dispatched extract job ${result.job.id} for ${filename} ` +
+        `(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)`
+    )
+
+    return {
+      filename,
+      jobId: result.job.id,
+    }
+  }
+
+  private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string {
+    if (groupMatch) return groupMatch.id
+    if (countries.length === 1) return countries[0].toLowerCase()
+    const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8)
+    return `custom-${hash}`
+  }
+
+  private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string {
+    if (groupMatch) return groupMatch.name
+    if (countries.length === 1) return countries[0]
+    if (countries.length <= 3) return countries.join(', ')
+    return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more`
+  }
+
+  private validateMaxzoom(maxzoom: number | undefined): void {
+    if (typeof maxzoom !== 'number') return
+    if (
+      !Number.isInteger(maxzoom) ||
+      maxzoom < EXTRACT_MIN_ZOOM ||
+      maxzoom > EXTRACT_MAX_ZOOM
+    ) {
+      throw new Error(
+        `maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]`
+      )
+    }
+  }
+
+  // go-pmtiles output format isn't stable across versions — parse loosely and
+  // fall back to zeros. The extract can still proceed without an estimate.
+  private parseDryRunOutput(output: string): { tiles: number; bytes: number } {
+    let bytes = 0
+    let tiles = 0
+
+    const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i)
+    if (byteLine) {
+      const raw = parseFloat(byteLine[1].replace(/,/g, ''))
+      const unit = (byteLine[2] ?? 'B').toUpperCase()
+      const multipliers: Record<string, number> = {
+        B: 1,
+        BYTE: 1,
+        BYTES: 1,
+        KB: 1_000,
+        MB: 1_000_000,
+        GB: 1_000_000_000,
+        TB: 1_000_000_000_000,
+      }
+      bytes = Math.round(raw * (multipliers[unit] ?? 1))
+    }
+
+    const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i)
+    if (tileLine) {
+      tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0
+    }
+
+    return { tiles, bytes }
+  }
+
   async delete(file: string): Promise<void> {
     let fileName = file
     if (!fileName.endsWith('.pmtiles')) {
       fileName += '.pmtiles'
     }
 
+    if (fileName === WORLD_BASEMAP_FILENAME) {
+      throw new Error('The world basemap cannot be deleted')
+    }
+
     const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
     const fullPath = resolve(join(basePath, fileName))
 
@@ -573,3 +874,16 @@ export class MapService implements IMapService {
     return baseUrl
   }
 }
+
+function findExactGroupMatch(
+  countries: CountryCode[],
+  groups: CountryGroup[]
+): CountryGroup | null {
+  return (
+    groups.find(
+      (g) =>
+        g.countries.length === countries.length &&
+        g.countries.every((c, i) => c === countries[i])
+    ) ?? null
+  )
+}

admin/app/services/ollama_service.ts

@@ -3,7 +3,7 @@ import OpenAI from 'openai'
 import type { ChatCompletionChunk, ChatCompletionMessageParam } from 'openai/resources/chat/completions.js'
 import type { Stream } from 'openai/streaming.js'
 import { NomadOllamaModel } from '../../types/ollama.js'
-import { FALLBACK_RECOMMENDED_OLLAMA_MODELS } from '../../constants/ollama.js'
+import { EMBEDDING_MODEL_NAME, FALLBACK_RECOMMENDED_OLLAMA_MODELS } from '../../constants/ollama.js'
 import fs from 'node:fs/promises'
 import path from 'node:path'
 import logger from '@adonisjs/core/services/logger'
@@ -469,6 +469,18 @@ export class OllamaService {
     }
   }
 
+  /**
+   * Hard char cap per embed input, applied as a runtime safety net regardless of
+   * which backend path runs. The chunker in RagService caps at MAX_SAFE_TOKENS=1600
+   * (3200 chars at the conservative 2 chars/token estimate), but dense technical
+   * content has been observed to slip past on multi-batch ZIM ingestion (#881).
+   *
+   * 4000 chars ≈ 1000–2000 tokens depending on density, which keeps us comfortably
+   * under nomic-embed-text:v1.5's default 2048-token context even on the OpenAI-compat
+   * fallback path (which can't pass `truncate:true`/`num_ctx` to the model).
+   */
+  public static readonly EMBED_MAX_INPUT_CHARS = 4000
+
   /**
    * Generate embeddings for the given input strings.
    * Tries the Ollama native /api/embed endpoint first, falls back to /v1/embeddings.
@@ -479,11 +491,44 @@ export class OllamaService {
       throw new Error('AI service is not initialized.')
     }
 
+    // Runtime safety net (#881). The OpenAI-compat fallback has no equivalent of
+    // truncate:true, so a chunk that exceeds the model's loaded context_length
+    // (often 2048 for nomic-embed-text:v1.5) returns 400 and the chunk is silently
+    // dropped from Qdrant. Pre-capping at the input layer protects both paths.
+    const safeInput = input.map((s) =>
+      s.length > OllamaService.EMBED_MAX_INPUT_CHARS
+        ? s.slice(0, OllamaService.EMBED_MAX_INPUT_CHARS)
+        : s
+    )
+    const truncatedCount = input.reduce(
+      (n, s) => (s.length > OllamaService.EMBED_MAX_INPUT_CHARS ? n + 1 : n),
+      0
+    )
+    if (truncatedCount > 0) {
+      logger.debug(
+        '[OllamaService] embed: pre-capped %d/%d inputs at %d chars',
+        truncatedCount,
+        input.length,
+        OllamaService.EMBED_MAX_INPUT_CHARS
+      )
+    }
+
     try {
-      // Prefer Ollama native endpoint (supports batch input natively)
+      // Prefer Ollama native endpoint (supports batch input natively).
+      // Pass num_ctx explicitly so we don't depend on the embedding model's
+      // modelfile defaults. Some installs ship nomic-embed-text:v1.5 with
+      // num_ctx=2048, which our chunker (sized for ~1500 tokens) can exceed
+      // on dense content, causing "input length exceeds context length" errors.
+      // truncate:true is a runtime safety net for any chunk that still overshoots.
+      // 8192 matches nomic-embed-text:v1.5's RoPE-extrapolated max.
       const response = await axios.post(
         `${this.baseUrl}/api/embed`,
-        { model, input },
+        {
+          model,
+          input: safeInput,
+          truncate: true,
+          options: { num_ctx: 8192 },
+        },
         { timeout: 60000 }
       )
       // Some backends (e.g. LM Studio) return HTTP 200 for unknown endpoints with an incompatible
@@ -492,16 +537,130 @@ export class OllamaService {
         throw new Error('Invalid /api/embed response — missing embeddings array')
       }
       return { embeddings: response.data.embeddings }
-    } catch {
-      // Fall back to OpenAI-compatible /v1/embeddings
+    } catch (err) {
+      // Capture the original error so we know *why* we fell back. Earlier bare
+      // catches here masked recurring "input length exceeds context length"
+      // failures for months (#369, #670, #881) — without this log we have no
+      // signal that /api/embed is the broken path vs the fallback.
+      logger.warn(
+        '[OllamaService] /api/embed failed, falling back to /v1/embeddings: %s',
+        err instanceof Error ? err.message : String(err)
+      )
+      // Fall back to OpenAI-compatible /v1/embeddings.
       // Explicitly request float format — some backends (e.g. LM Studio) don't reliably
       // implement the base64 encoding the OpenAI SDK requests by default.
-      logger.info('[OllamaService] /api/embed unavailable, falling back to /v1/embeddings')
-      const results = await this.openai.embeddings.create({ model, input, encoding_format: 'float' })
+      const results = await this.openai.embeddings.create({
+        model,
+        input: safeInput,
+        encoding_format: 'float',
+      })
       return { embeddings: results.data.map((e) => e.embedding as number[]) }
     }
   }
 
+  /**
+   * Returns true if Ollama is currently running an embedding model with non-zero VRAM
+   * (i.e., GPU-offloaded). Returns false if the model is running CPU-only OR if it's
+   * not currently loaded OR if /api/ps is unreachable.
+   *
+   * Used by EmbedFileJob to pace continuation batches when the embedding model is
+   * CPU-bound — sustained 100% CPU on a multi-batch ZIM ingestion can starve other
+   * services (sshd, etc.) hard enough to require a power-cycle. AMD ROCm installs
+   * hit this today because Ollama's ROCm build doesn't accelerate nomic-bert; on
+   * NVIDIA, nomic-embed-text runs at 100% GPU and pacing is unnecessary.
+   *
+   * Only the Ollama-native endpoint is supported — backends that expose
+   * `/v1/embeddings` (LM Studio, llama.cpp) don't surface placement info.
+   */
+  public async isEmbeddingGpuAccelerated(): Promise<boolean> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) return false
+
+    try {
+      const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 })
+      const models: Array<{ name?: string; size_vram?: number }> = response.data?.models ?? []
+      // Match any loaded model whose name signals it's an embedding model.
+      // nomic-embed-text, mxbai-embed-large, snowflake-arctic-embed, etc. all follow this convention.
+      return models.some(
+        (m) => m.name?.toLowerCase().includes('embed') && (m.size_vram ?? 0) > 0
+      )
+    } catch (err: any) {
+      // /api/ps unreachable (Ollama down, non-native backend, etc.) — fail closed: assume CPU,
+      // which means we'll pace. Better to over-pace than risk box-killing CPU saturation.
+      logger.warn(
+        `[OllamaService] Could not check embedding placement via /api/ps: ${err?.message ?? err}`
+      )
+      return false
+    }
+  }
+
+  /**
+   * Enforces the "at most one chat model resident in VRAM" invariant by firing
+   * `keep_alive: 0` against every currently-loaded model except (a) the
+   * embedding model (always exempt) and (b) `targetModel` (the one we want
+   * loaded next — leaving it alone preserves a hot model when the target is
+   * already loaded).
+   *
+   * Best-effort: queries `/api/ps` and POSTs unload hints in parallel. Network
+   * or Ollama errors are swallowed and logged — neither chat nor page-load
+   * should fail just because the unload housekeeping didn't go through.
+   *
+   * Returns the list of model names that were sent the unload hint, so the
+   * caller (and tests) can confirm what actually happened.
+   *
+   * Pass `targetModel: null` to unload every chat model (used for the future
+   * "free up VRAM" path; not exposed yet but the helper supports it).
+   *
+   * Note that `keep_alive: 0` is a post-completion hint, not a force-kill —
+   * Ollama defers eviction until the runner is idle, so in-flight inference
+   * on the same model is never interrupted. See the design doc for the race
+   * analysis behind this.
+   */
+  public async unloadAllChatModelsExcept(targetModel: string | null): Promise<string[]> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) return []
+
+    let loadedModels: string[] = []
+    try {
+      const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 })
+      loadedModels = (response.data?.models ?? [])
+        .map((m: { name?: string }) => m.name)
+        .filter((name: unknown): name is string => typeof name === 'string')
+    } catch (err: any) {
+      logger.warn(
+        `[OllamaService] unloadAllChatModelsExcept: /api/ps unreachable, skipping unload sweep: ${err?.message ?? err}`
+      )
+      return []
+    }
+
+    const toUnload = loadedModels.filter(
+      (name) => name !== EMBEDDING_MODEL_NAME && name !== targetModel
+    )
+
+    await Promise.all(
+      toUnload.map(async (modelName) => {
+        try {
+          await axios.post(
+            `${this.baseUrl}/api/generate`,
+            { model: modelName, prompt: '', keep_alive: 0 },
+            { timeout: 10000 }
+          )
+        } catch (err: any) {
+          logger.warn(
+            `[OllamaService] Failed to send unload hint for ${modelName}: ${err?.message ?? err}`
+          )
+        }
+      })
+    )
+
+    if (toUnload.length > 0) {
+      logger.info(
+        `[OllamaService] Sent unload hint for ${toUnload.length} chat model(s): ${toUnload.join(', ')}`
+      )
+    }
+    return toUnload
+  }
+
   public async getModels(includeEmbeddings = false): Promise<NomadInstalledModel[]> {
     await this._ensureDependencies()
     if (!this.baseUrl) {

admin/app/services/queue_service.ts

@@ -1,9 +1,25 @@
 import { Queue } from 'bullmq'
 import queueConfig from '#config/queue'
 
+// Process-wide singleton. Each `Queue` opens two ioredis connections (one for
+// commands, one blocking). Instantiating a fresh QueueService per dispatch /
+// status lookup leaks both, and under sustained job churn (e.g. multi-batch ZIM
+// ingestion enqueueing a continuation every few seconds) it saturates Redis's
+// maxclients within hours.
 export class QueueService {
   private queues: Map<string, Queue> = new Map()
 
+  private static _instance: QueueService | null = null
+
+  private constructor() {}
+
+  static getInstance(): QueueService {
+    if (!QueueService._instance) {
+      QueueService._instance = new QueueService()
+    }
+    return QueueService._instance
+  }
+
   getQueue(name: string): Queue {
     if (!this.queues.has(name)) {
       const queue = new Queue(name, {
@@ -18,5 +34,6 @@ export class QueueService {
     for (const queue of this.queues.values()) {
       await queue.close()
     }
+    this.queues.clear()
   }
 }

admin/app/services/rag_service.ts

@@ -16,10 +16,27 @@ import { removeStopwords } from 'stopword'
 import { randomUUID } from 'node:crypto'
 import { join, resolve, sep } from 'node:path'
 import KVStore from '#models/kv_store'
+import KbIngestState from '#models/kb_ingest_state'
+import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js'
+import KbRatioRegistry from '#models/kb_ratio_registry'
+import { decideWarnings } from '../utils/kb_warning_decision.js'
+import type { FileWarning, FileWarningsResult, StoredFileInfo } from '../../types/rag.js'
+import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
 import { ZIMExtractionService } from './zim_extraction_service.js'
 import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
+import { EMBEDDING_MODEL_NAME } from '../../constants/ollama.js'
 import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js'
 
+export type EmbedSingleFileFailureCode =
+  | 'not_found'
+  | 'inflight'
+  | 'delete_failed'
+  | 'dispatch_failed'
+
+export type EmbedSingleFileResult =
+  | { success: true; message: string }
+  | { success: false; code: EmbedSingleFileFailureCode; message: string }
+
 @inject()
 export class RagService {
   private qdrant: QdrantClient | null = null
@@ -28,7 +45,6 @@ export class RagService {
   private resolvedEmbeddingModel: string | null = null
   public static UPLOADS_STORAGE_PATH = 'storage/kb_uploads'
   public static CONTENT_COLLECTION_NAME = 'nomad_knowledge_base'
-  public static EMBEDDING_MODEL = 'nomic-embed-text:v1.5'
   public static EMBEDDING_DIMENSION = 768 // Nomic Embed Text v1.5 dimension is 768
   public static MODEL_CONTEXT_LENGTH = 2048 // nomic-embed-text has 2K token context
   public static MAX_SAFE_TOKENS = 1600 // Leave buffer for prefix and tokenization variance
@@ -52,14 +68,33 @@ export class RagService {
       this.qdrantInitPromise = (async () => {
         const qdrantUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.QDRANT)
         if (!qdrantUrl) {
-          throw new Error('Qdrant service is not installed or running.')
+          throw new Error('Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.')
         }
         this.qdrant = new QdrantClient({ url: qdrantUrl })
-      })()
+      })().catch((err) => {
+        this.qdrantInitPromise = null
+        this.qdrant = null
+        throw err
+      })
     }
     return this.qdrantInitPromise
   }
 
+  public async checkQdrantHealth(): Promise<{ online: boolean; message?: string }> {
+    try {
+      await this._ensureDependencies()
+      await this.qdrant!.getCollections()
+      return { online: true }
+    } catch {
+      this.qdrant = null
+      this.qdrantInitPromise = null
+      return {
+        online: false,
+        message: 'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.',
+      }
+    }
+  }
+
   private async _ensureDependencies() {
     if (!this.qdrant) {
       await this._initializeQdrantClient()
@@ -251,25 +286,25 @@ export class RagService {
       if (!this.embeddingModelVerified) {
         const allModels = await this.ollamaService.getModels(true)
         const embeddingModel =
-          allModels.find((model) => model.name === RagService.EMBEDDING_MODEL) ??
+          allModels.find((model) => model.name === EMBEDDING_MODEL_NAME) ??
           allModels.find((model) => model.name.toLowerCase().includes('nomic-embed-text'))
 
         if (!embeddingModel) {
           try {
-            const downloadResult = await this.ollamaService.downloadModel(RagService.EMBEDDING_MODEL)
+            const downloadResult = await this.ollamaService.downloadModel(EMBEDDING_MODEL_NAME)
             if (!downloadResult.success) {
               throw new Error(downloadResult.message || 'Unknown error during model download')
             }
           } catch (modelError) {
             logger.error(
-              `[RAG] Embedding model ${RagService.EMBEDDING_MODEL} not found locally and failed to download:`,
+              `[RAG] Embedding model ${EMBEDDING_MODEL_NAME} not found locally and failed to download:`,
               modelError
             )
             this.embeddingModelVerified = false
             return null
           }
         }
-        this.resolvedEmbeddingModel = embeddingModel?.name ?? RagService.EMBEDDING_MODEL
+        this.resolvedEmbeddingModel = embeddingModel?.name ?? EMBEDDING_MODEL_NAME
         this.embeddingModelVerified = true
       }
 
@@ -326,7 +361,7 @@ export class RagService {
 
         logger.debug(`[RAG] Embedding batch ${batchIdx + 1}/${totalBatches} (${batch.length} chunks)`)
 
-        const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? RagService.EMBEDDING_MODEL, batch)
+        const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? EMBEDDING_MODEL_NAME, batch)
 
         embeddings.push(...response.embeddings)
 
@@ -481,13 +516,13 @@ export class RagService {
       `[RAG] Extracting ZIM content (batch: offset=${startOffset}, size=${ZIM_BATCH_SIZE})`
     )
 
-    const zimChunks = await zimExtractionService.extractZIMContent(filepath, {
-      startOffset,
-      batchSize: ZIM_BATCH_SIZE,
-    })
+    const { chunks: zimChunks, totalArticles } = await zimExtractionService.extractZIMContent(
+      filepath,
+      { startOffset, batchSize: ZIM_BATCH_SIZE }
+    )
 
     logger.info(
-      `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata`
+      `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata (file totalArticles=${totalArticles})`
     )
 
     // Process each chunk individually with its metadata
@@ -563,6 +598,7 @@ export class RagService {
       chunks: totalChunks,
       hasMoreBatches,
       articlesProcessed: articlesInBatch,
+      totalArticles,
     }
   }
 
@@ -782,12 +818,12 @@ export class RagService {
       if (!this.embeddingModelVerified) {
         const allModels = await this.ollamaService.getModels(true)
         const embeddingModel =
-          allModels.find((model) => model.name === RagService.EMBEDDING_MODEL) ??
+          allModels.find((model) => model.name === EMBEDDING_MODEL_NAME) ??
           allModels.find((model) => model.name.toLowerCase().includes('nomic-embed-text'))
 
         if (!embeddingModel) {
           logger.warn(
-            `[RAG] ${RagService.EMBEDDING_MODEL} not found. Cannot perform similarity search.`
+            `[RAG] ${EMBEDDING_MODEL_NAME} not found. Cannot perform similarity search.`
           )
           this.embeddingModelVerified = false
           return []
@@ -819,7 +855,7 @@ export class RagService {
         return []
       }
 
-      const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? RagService.EMBEDDING_MODEL, [prefixedQuery])
+      const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? EMBEDDING_MODEL_NAME, [prefixedQuery])
 
       // Perform semantic search with a higher limit to enable reranking
       const searchLimit = limit * 3 // Get more results for reranking
@@ -1026,7 +1062,7 @@ export class RagService {
     }
   }
 
-  public async getStoredFiles(): Promise<string[]> {
+  public async getStoredFiles(): Promise<StoredFileInfo[]> {
     try {
       await this._ensureCollection(
         RagService.CONTENT_COLLECTION_NAME,
@@ -1057,13 +1093,159 @@ export class RagService {
         offset = scrollResult.next_page_offset || null
       } while (offset !== null)
 
-      return Array.from(sources)
+      // Union the Qdrant-derived list with the disk-backed file paths the
+      // state machine has tracked. Without this, files known to the scanner
+      // but with zero embedded chunks (video-only ZIMs, failed-before-first-
+      // chunk ingestions, browse_only opt-outs) never get a row in Stored
+      // Files — which means warnings keyed off those files (#895 zero_chunks
+      // in particular) have no row to attach to. The state machine is the
+      // authoritative "what's on disk?" view; Qdrant is "what made it into
+      // the vector store?". Both are needed to render the KB UI honestly.
+      const stateByPath = new Map<string, { state: KbIngestStateValue; chunks_embedded: number }>()
+      try {
+        const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded')
+        for (const row of stateRows) {
+          sources.add(row.file_path)
+          stateByPath.set(row.file_path, {
+            state: row.state,
+            chunks_embedded: row.chunks_embedded,
+          })
+        }
+      } catch (error) {
+        // Non-fatal: if the state machine query fails for any reason we'd
+        // rather return the Qdrant-derived list than 500 the whole panel.
+        logger.warn(
+          { err: error },
+          '[RagService.getStoredFiles] state-machine union skipped; returning Qdrant-only list'
+        )
+      }
+
+      return Array.from(sources).map((source) => {
+        const row = stateByPath.get(source)
+        return {
+          source,
+          state: row?.state ?? null,
+          chunksEmbedded: row?.chunks_embedded ?? 0,
+        }
+      })
     } catch (error) {
       logger.error('Error retrieving stored files:', error)
       return []
     }
   }
 
+  /**
+   * Compute whether the first-chat JIT prompt should fire and surface the file
+   * count the banner uses in its copy ("Index your N existing files?"). The
+   * banner appears when the user hasn't yet picked a global ingest policy
+   * (`rag.defaultIngestPolicy` unset) and the scanner has actually seen at
+   * least one embeddable file — i.e., the prompt is actionable, not theoretical
+   * on a freshly-installed empty NOMAD.
+   *
+   * Once the user picks a policy (Always or Manual) via the banner buttons or
+   * the KB modal toggle, `shouldPrompt` flips to false for good.
+   */
+  public async getPolicyPromptState(): Promise<{
+    shouldPrompt: boolean
+    hasContent: boolean
+    totalFiles: number
+  }> {
+    const policy = await KVStore.getValue('rag.defaultIngestPolicy')
+    const countRow = await KbIngestState.query().count('* as total').first()
+    const totalFiles = Number((countRow as any)?.$extras?.total ?? 0)
+    return {
+      shouldPrompt: policy === null && totalFiles > 0,
+      hasContent: totalFiles > 0,
+      totalFiles,
+    }
+  }
+
+  /**
+   * Compute conditional warnings (RFC #883 §6) for every source the scanner
+   * sees on disk. Returns `{ ok, warnings }` — `ok: false` distinguishes a
+   * computation failure (Qdrant unreachable, DB outage, FS error) from the
+   * healthy-but-empty case, which is critical because the whole point of this
+   * surface is to expose silent failures; reporting "everything healthy" when
+   * we couldn't actually check would reintroduce the bug we set out to fix.
+   *
+   * Per-source chunk counts come from a single Qdrant scroll over the
+   * collection's points; expected-chunk estimates come from the ratio
+   * registry. Files in the scanner's directories that have no qdrant points
+   * at all show up with `chunksInQdrant: 0` so Warning A can fire.
+   */
+  public async computeFileWarnings(): Promise<FileWarningsResult> {
+    try {
+      await this._ensureCollection(
+        RagService.CONTENT_COLLECTION_NAME,
+        RagService.EMBEDDING_DIMENSION
+      )
+
+      // Per-source chunk count from a single scroll. We deliberately don't
+      // assume `kb_ingest_state.chunks_embedded` here so this PR stays
+      // independent of the state-machine PR (#888) — but a future cleanup can
+      // read from there for efficiency once both have landed.
+      const chunksBySource = new Map<string, number>()
+      let offset: string | number | null | Record<string, unknown> = null
+      const batchSize = 100
+      do {
+        const scrollResult = await this.qdrant!.scroll(RagService.CONTENT_COLLECTION_NAME, {
+          limit: batchSize,
+          offset,
+          with_payload: ['source'],
+          with_vector: false,
+        })
+        for (const point of scrollResult.points) {
+          const source = point.payload?.source
+          if (source && typeof source === 'string') {
+            chunksBySource.set(source, (chunksBySource.get(source) ?? 0) + 1)
+          }
+        }
+        offset = scrollResult.next_page_offset || null
+      } while (offset !== null)
+
+      // Scan the filesystem the same way scanAndSyncStorage does so Warning A
+      // can fire on files with zero qdrant points (the headline "video-only
+      // ZIM" case).
+      const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH)
+      const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH)
+      const allSources = new Set<string>(chunksBySource.keys())
+      const sizeByPath = new Map<string, number>()
+
+      for (const dir of [KB_UPLOADS_PATH, ZIM_PATH]) {
+        try {
+          const entries = await listDirectoryContentsRecursive(dir)
+          for (const entry of entries) {
+            if (entry.type !== 'file') continue
+            allSources.add(entry.key)
+            const stat = await getFileStatsIfExists(entry.key)
+            if (stat) sizeByPath.set(entry.key, Number(stat.size))
+          }
+        } catch (error: any) {
+          if (error?.code !== 'ENOENT') throw error
+        }
+      }
+
+      const out: Record<string, FileWarning[]> = {}
+      for (const source of allSources) {
+        const fileSizeBytes = sizeByPath.get(source) ?? 0
+        const chunksInQdrant = chunksBySource.get(source) ?? 0
+        const fileName = source.split(/[/\\]/).pop() ?? source
+        const expectedChunks =
+          fileSizeBytes > 0
+            ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes)
+            : null
+
+        const warnings = decideWarnings({ fileSizeBytes, chunksInQdrant, expectedChunks })
+        if (warnings.length > 0) out[source] = warnings
+      }
+
+      return { ok: true, warnings: out }
+    } catch (error) {
+      logger.error('[RAG] Error computing file warnings:', error)
+      return { ok: false, warnings: {} }
+    }
+  }
+
   /**
    * Delete all Qdrant points associated with a given source path and remove
    * the corresponding file from disk if it lives under the uploads directory.
@@ -1098,6 +1280,11 @@ export class RagService {
         logger.warn(`[RAG] File was removed from knowledge base but doesn't live in Nomad's uploads directory, so it can't be safely removed. Skipping deletion of physical file...`)
       }
 
+      // Drop the ingest state row last so the file disappears entirely. Without
+      // this, the next scanAndSyncStorage would see `indexed + no chunks` for a
+      // path that no longer exists in storage and try to re-embed nothing.
+      await KbIngestState.remove(source)
+
       return { success: true, message: 'File removed from knowledge base.' }
     } catch (error) {
       logger.error('[RAG] Error deleting file from knowledge base:', error)
@@ -1162,12 +1349,182 @@ export class RagService {
     }
   }
 
+  /**
+   * Walk kb_uploads and zim storage directories, returning the full path of
+   * every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are
+   * filtered out so they aren't dispatched only to fail with "Unsupported file
+   * type" and retry on every sync.
+   */
+  private async _discoverKbFiles(): Promise<string[]> {
+    const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH)
+    const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH)
+    const filesInStorage: string[] = []
+
+    for (const [label, dirPath] of [
+      [RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const,
+      [ZIM_STORAGE_PATH, ZIM_PATH] as const,
+    ]) {
+      try {
+        const contents = await listDirectoryContentsRecursive(dirPath)
+        contents.forEach((entry) => {
+          if (entry.type === 'file') filesInStorage.push(entry.key)
+        })
+        logger.debug(`[RAG] Found ${contents.length} files in ${label}`)
+      } catch (error) {
+        if (error.code === 'ENOENT') {
+          logger.debug(`[RAG] ${label} directory does not exist, skipping`)
+        } else {
+          throw error
+        }
+      }
+    }
+
+    return filesInStorage.filter((f) => determineFileType(f) !== 'unknown')
+  }
+
+  /**
+   * Dispatch one EmbedFileJob per file path. Returns honest counts: `queuedCount`
+   * is jobs newly enqueued, `dedupedCount` is jobs that hit BullMQ's per-file
+   * jobId dedupe (an existing :completed/:waiting/etc. entry was returned
+   * instead of a new enqueue), and `failedPaths` lists files whose dispatch
+   * threw. Pass `force: true` for bulk callers that need to bypass dedupe
+   * entirely. Per-file errors are logged but don't abort the batch — callers
+   * must inspect `failedPaths` to surface partial failure to the operator.
+   */
+  private async _dispatchEmbedJobsFor(
+    filePaths: string[],
+    options?: { force?: boolean }
+  ): Promise<{ queuedCount: number; dedupedCount: number; failedPaths: string[] }> {
+    const { EmbedFileJob } = await import('#jobs/embed_file_job')
+    let queuedCount = 0
+    let dedupedCount = 0
+    const failedPaths: string[] = []
+    for (const filePath of filePaths) {
+      try {
+        const fileName = filePath.split(/[/\\]/).pop() || filePath
+        const stats = await getFileStatsIfExists(filePath)
+        const result = await EmbedFileJob.dispatch(
+          {
+            filePath,
+            fileName,
+            fileSize: stats?.size,
+          },
+          { force: options?.force }
+        )
+        if (result.created) {
+          queuedCount++
+        } else {
+          dedupedCount++
+        }
+      } catch (fileError) {
+        failedPaths.push(filePath)
+        logger.error(`[RAG] Error dispatching job for file ${filePath}:`, fileError)
+      }
+    }
+    return { queuedCount, dedupedCount, failedPaths }
+  }
+
+  /**
+   * Dispatch an embed job for a single stored file. Wraps `_dispatchEmbedJobsFor`
+   * with the safety checks needed for a user-triggered per-row action:
+   *   1. The source must be known to the scanner OR have a state row — prevents
+   *      arbitrary path dispatch from the public API.
+   *   2. We refuse if any inflight job (waiting/active/delayed/paused) already
+   *      targets this filePath. Otherwise a double-click or a rapid retry could
+   *      enqueue duplicate jobs, producing duplicate chunks.
+   *   3. When `force` is true (Re-embed of an already-indexed file), we
+   *      pre-delete the prior Qdrant points so the new run doesn't stack on
+   *      top of the old ones. For force=false (Index of a never-embedded file),
+   *      there's nothing to clear.
+   */
+  public async embedSingleFile(
+    source: string,
+    force: boolean = false
+  ): Promise<EmbedSingleFileResult> {
+    const stateRow = await KbIngestState.query().where('file_path', source).first()
+    if (!stateRow) {
+      const knownFiles = await this._discoverKbFiles()
+      if (!knownFiles.includes(source)) {
+        return {
+          success: false,
+          code: 'not_found',
+          message: 'File is not a tracked knowledge-base source.',
+        }
+      }
+    }
+
+    const { EmbedFileJob } = await import('#jobs/embed_file_job')
+    const { QueueService } = await import('#services/queue_service')
+    const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue)
+    const inflight = await queue.getJobs(['waiting', 'active', 'delayed', 'paused'])
+    if (inflight.some((j) => j.data?.filePath === source)) {
+      return {
+        success: false,
+        code: 'inflight',
+        message: 'A job for this file is already in progress. Wait for it to finish before re-queuing.',
+      }
+    }
+
+    if (force) {
+      try {
+        await this._deletePointsBySource(source)
+      } catch (err) {
+        logger.error(`[RAG] Failed to delete prior points for ${source}; aborting re-embed:`, err)
+        return {
+          success: false,
+          code: 'delete_failed',
+          message: 'Failed to clear prior embeddings before re-embed.',
+        }
+      }
+    }
+
+    const result = await this._dispatchEmbedJobsFor([source], { force })
+    if (result.failedPaths.length > 0) {
+      return {
+        success: false,
+        code: 'dispatch_failed',
+        message: 'Failed to dispatch embed job for this file.',
+      }
+    }
+    return {
+      success: true,
+      message: force ? 'Re-embed queued for this file.' : 'Indexing queued for this file.',
+    }
+  }
+
+  /**
+   * Delete all Qdrant points whose `source` payload matches the given path.
+   * Unlike deleteFileBySource(), this does NOT touch the file on disk — used
+   * by reembedAll() where the file must remain so it can be re-ingested.
+   */
+  private async _deletePointsBySource(source: string): Promise<void> {
+    await this._ensureCollection(
+      RagService.CONTENT_COLLECTION_NAME,
+      RagService.EMBEDDING_DIMENSION
+    )
+    await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, {
+      filter: { must: [{ key: 'source', match: { value: source } }] },
+    })
+  }
+
+  /**
+   * Returns true if the file-embeddings queue has any in-flight work
+   * (waiting, active, delayed, or paused). Bulk re-embed actions use this
+   * to refuse mid-flight to avoid racing with deletes/dispatches already
+   * in progress.
+   */
+  private async _hasInflightEmbedJobs(): Promise<boolean> {
+    const { EmbedFileJob } = await import('#jobs/embed_file_job')
+    const { QueueService } = await import('#services/queue_service')
+    const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue)
+    const counts = await queue.getJobCounts('waiting', 'active', 'delayed', 'paused')
+    return (counts.waiting || 0) + (counts.active || 0) + (counts.delayed || 0) + (counts.paused || 0) > 0
+  }
+
   /**
    * Scans the knowledge base storage directories and syncs with Qdrant.
    * Identifies files that exist in storage but haven't been embedded yet,
    * and dispatches EmbedFileJob for each missing file.
-   *
-   * @returns Object containing success status, message, and counts of scanned/queued files
    */
   public async scanAndSyncStorage(): Promise<{
     success: boolean
@@ -1178,91 +1535,111 @@ export class RagService {
     try {
       logger.info('[RAG] Starting knowledge base sync scan')
 
-      const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH)
-      const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH)
-
-      const filesInStorage: string[] = []
-
-      // Force resync of Nomad docs
       await this.discoverNomadDocs(true).catch((error) => {
         logger.error('[RAG] Error during Nomad docs discovery in sync process:', error)
       })
 
-      // Scan kb_uploads directory
-      try {
-        const kbContents = await listDirectoryContentsRecursive(KB_UPLOADS_PATH)
-        kbContents.forEach((entry) => {
-          if (entry.type === 'file') {
-            filesInStorage.push(entry.key)
-          }
-        })
-        logger.debug(`[RAG] Found ${kbContents.length} files in ${RagService.UPLOADS_STORAGE_PATH}`)
-      } catch (error) {
-        if (error.code === 'ENOENT') {
-          logger.debug(`[RAG] ${RagService.UPLOADS_STORAGE_PATH} directory does not exist, skipping`)
-        } else {
-          throw error
-        }
-      }
+      const filesInStorage = await this._discoverKbFiles()
+      logger.info(`[RAG] Found ${filesInStorage.length} embeddable files in storage`)
 
-      // Scan zim directory
-      try {
-        const zimContents = await listDirectoryContentsRecursive(ZIM_PATH)
-        zimContents.forEach((entry) => {
-          if (entry.type === 'file') {
-            filesInStorage.push(entry.key)
-          }
-        })
-        logger.debug(`[RAG] Found ${zimContents.length} files in ${ZIM_STORAGE_PATH}`)
-      } catch (error) {
-        if (error.code === 'ENOENT') {
-          logger.debug(`[RAG] ${ZIM_STORAGE_PATH} directory does not exist, skipping`)
-        } else {
-          throw error
-        }
-      }
-
-      logger.info(`[RAG] Found ${filesInStorage.length} total files in storage directories`)
-
-      // Get all stored sources from Qdrant
       await this._ensureCollection(
         RagService.CONTENT_COLLECTION_NAME,
         RagService.EMBEDDING_DIMENSION
       )
 
+      // Collect every unique `source` already in Qdrant so we can skip files
+      // that have already been embedded.
       const sourcesInQdrant = new Set<string>()
       let offset: string | number | null | Record<string, unknown> = null
-      const batchSize = 100
-
-      // Scroll through all points to get sources
       do {
         const scrollResult = await this.qdrant!.scroll(RagService.CONTENT_COLLECTION_NAME, {
-          limit: batchSize,
-          offset: offset,
-          with_payload: ['source'], // Only fetch source field for efficiency
+          limit: 100,
+          offset,
+          with_payload: ['source'],
           with_vector: false,
         })
-
         scrollResult.points.forEach((point) => {
           const source = point.payload?.source
-          if (source && typeof source === 'string') {
-            sourcesInQdrant.add(source)
-          }
+          if (source && typeof source === 'string') sourcesInQdrant.add(source)
         })
-
         offset = scrollResult.next_page_offset || null
       } while (offset !== null)
 
       logger.info(`[RAG] Found ${sourcesInQdrant.size} unique sources in Qdrant`)
 
-      // Find files that are in storage, not already in Qdrant, and have an embeddable type.
-      // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise be
-      // dispatched to EmbedFileJob, fail with "Unsupported file type", and retry on every sync.
-      const filesToEmbed = filesInStorage.filter(
-        (filePath) => !sourcesInQdrant.has(filePath) && determineFileType(filePath) !== 'unknown'
+      // Load all known per-file ingest states. The state row is authoritative
+      // over the "any chunks in Qdrant" heuristic — it captures user choices
+      // (browse_only) and terminal outcomes (failed, stalled) that aren't visible
+      // from Qdrant alone. See RFC #883 for the full state machine.
+      const stateRows = await KbIngestState.all()
+      const stateByPath = new Map(stateRows.map((row) => [row.file_path, row]))
+
+      // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise
+      // be dispatched to EmbedFileJob, fail with "Unsupported file type", and retry
+      // on every sync — filter them out before state decisions.
+      const embeddableFiles = filesInStorage.filter(
+        (filePath) => determineFileType(filePath) !== 'unknown'
       )
 
-      logger.info(`[RAG] Found ${filesToEmbed.length} files that need embedding`)
+      // Read the global ingest policy. Unset is treated as 'Always' so legacy
+      // installs keep their current behavior until the user explicitly opts
+      // into Manual mode from the KB panel.
+      const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy')
+      const policy: IngestPolicy = policyRaw === 'Manual' ? 'Manual' : 'Always'
+
+      const filesToEmbed: string[] = []
+      let backfilled = 0
+      let createdRows = 0
+      let createdPending = 0
+      let skipped = 0
+
+      for (const filePath of embeddableFiles) {
+        const stateRow = stateByPath.get(filePath) ?? null
+        const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath), policy)
+
+        switch (action.kind) {
+          case 'skip':
+            skipped++
+            break
+          case 'backfill_indexed':
+            // Pre-RFC install (or a fresh admin pointed at an existing Qdrant volume):
+            // chunks already exist with no state row, so trust Qdrant and record
+            // `indexed` without re-embedding. chunks_embedded is left 0 because
+            // we don't count points-per-source during the scroll above.
+            await KbIngestState.create({
+              file_path: filePath,
+              state: 'indexed',
+              chunks_embedded: 0,
+            })
+            backfilled++
+            break
+          case 'create_pending':
+            // Manual mode: record that we've seen the file but don't dispatch.
+            // The KB panel surfaces a per-card "Index" affordance for these.
+            await KbIngestState.create({
+              file_path: filePath,
+              state: 'pending_decision',
+              chunks_embedded: 0,
+            })
+            createdPending++
+            break
+          case 'dispatch':
+            if (action.createStateRow) {
+              await KbIngestState.create({
+                file_path: filePath,
+                state: 'pending_decision',
+                chunks_embedded: 0,
+              })
+              createdRows++
+            }
+            filesToEmbed.push(filePath)
+            break
+        }
+      }
+
+      logger.info(
+        `[RAG] Scan results (policy=${policy}): ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${createdPending} waiting on user, ${skipped} skipped`
+      )
 
       if (filesToEmbed.length === 0) {
         return {
@@ -1273,41 +1650,193 @@ export class RagService {
         }
       }
 
-      // Import EmbedFileJob dynamically to avoid circular dependencies
-      const { EmbedFileJob } = await import('#jobs/embed_file_job')
-
-      // Dispatch jobs for files that need embedding
-      let queuedCount = 0
-      for (const filePath of filesToEmbed) {
-        try {
-          const fileName = filePath.split(/[/\\]/).pop() || filePath
-          const stats = await getFileStatsIfExists(filePath)
-
-          logger.info(`[RAG] Dispatching embed job for: ${fileName}`)
-          await EmbedFileJob.dispatch({
-            filePath: filePath,
-            fileName: fileName,
-            fileSize: stats?.size,
-          })
-          queuedCount++
-          logger.debug(`[RAG] Successfully dispatched job for ${fileName}`)
-        } catch (fileError) {
-          logger.error(`[RAG] Error dispatching job for file ${filePath}:`, fileError)
-        }
-      }
-
+      const { queuedCount, dedupedCount } = await this._dispatchEmbedJobsFor(filesToEmbed)
+      const dedupeNote = dedupedCount > 0 ? ` (${dedupedCount} already queued)` : ''
       return {
         success: true,
-        message: `Scanned ${filesInStorage.length} files, queued ${queuedCount} for embedding`,
+        message: `Scanned ${filesInStorage.length} files, queued ${queuedCount} for embedding${dedupeNote}`,
         filesScanned: filesInStorage.length,
         filesQueued: queuedCount,
       }
     } catch (error) {
       logger.error('[RAG] Error scanning and syncing knowledge base:', error)
-      return {
-        success: false,
-        message: 'Error scanning and syncing knowledge base',
+      return { success: false, message: 'Error scanning and syncing knowledge base' }
+    }
+  }
+
+  /**
+   * Re-embed every file on disk (per-file replace). For each discovered file:
+   * delete its existing Qdrant points by `source` match, then dispatch a fresh
+   * EmbedFileJob. Files are NOT removed from disk. Any orphan points (points
+   * whose source file no longer exists) are intentionally preserved — use
+   * resetAndRebuild() if a clean slate is required.
+   *
+   * Refuses to run if the embeddings queue already has in-flight work.
+   */
+  public async reembedAll(): Promise<{
+    success: boolean
+    message: string
+    filesScanned?: number
+    filesQueued?: number
+    failedPaths?: string[]
+  }> {
+    try {
+      if (await this._hasInflightEmbedJobs()) {
+        return {
+          success: false,
+          message: 'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a bulk re-embed.',
+        }
       }
+
+      logger.info('[RAG] Starting full re-embed (per-file replace)')
+
+      await this.discoverNomadDocs(true).catch((error) => {
+        logger.error('[RAG] Error re-running Nomad docs discovery during re-embed:', error)
+      })
+
+      const filesInStorage = await this._discoverKbFiles()
+
+      await this._ensureCollection(
+        RagService.CONTENT_COLLECTION_NAME,
+        RagService.EMBEDDING_DIMENSION
+      )
+
+      // Per-file: delete-then-dispatch. We tried dispatch-then-delete but that
+      // opens a race where a fast worker can write new points before our
+      // delete-by-source runs, wiping both. Instead we delete first, then
+      // dispatch — and if dispatch fails, we surface the failed paths in the
+      // response so the operator knows which files dropped out (rather than
+      // silently leaving them unindexed). A subsequent sync rescan picks them
+      // back up. Note: a delete-failure aborts the per-file pair (we don't
+      // dispatch a job whose old points are still present, since they'd live
+      // alongside the new vectors forever).
+      const { EmbedFileJob } = await import('#jobs/embed_file_job')
+      let queuedCount = 0
+      const failedPaths: string[] = []
+      for (const filePath of filesInStorage) {
+        try {
+          await this._deletePointsBySource(filePath)
+        } catch (err) {
+          logger.error(`[RAG] Failed to delete prior points for ${filePath}; skipping dispatch:`, err)
+          failedPaths.push(filePath)
+          continue
+        }
+        try {
+          const fileName = filePath.split(/[/\\]/).pop() || filePath
+          const stats = await getFileStatsIfExists(filePath)
+          const result = await EmbedFileJob.dispatch(
+            { filePath, fileName, fileSize: stats?.size },
+            { force: true }
+          )
+          if (result.created) queuedCount++
+        } catch (fileError) {
+          // Old points already deleted but the new job never made it onto the
+          // queue. Logged + surfaced so an operator can rerun a sync.
+          logger.error(`[RAG] Re-embed dispatch failed for ${filePath} after delete; file is now unindexed until next sync:`, fileError)
+          failedPaths.push(filePath)
+        }
+      }
+
+      logger.info(
+        `[RAG] Re-embed dispatched ${queuedCount}/${filesInStorage.length} files` +
+          (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '')
+      )
+
+      const failureSuffix =
+        failedPaths.length > 0
+          ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.`
+          : ''
+
+      return {
+        success: failedPaths.length === 0,
+        message:
+          `Re-embedding ${queuedCount} file${queuedCount === 1 ? '' : 's'}. Existing points were replaced.` +
+          failureSuffix,
+        filesScanned: filesInStorage.length,
+        filesQueued: queuedCount,
+        ...(failedPaths.length > 0 ? { failedPaths } : {}),
+      }
+    } catch (error) {
+      logger.error('[RAG] Error during re-embed:', error)
+      return { success: false, message: 'Error during re-embed' }
+    }
+  }
+
+  /**
+   * Destructive rebuild. Drops the entire Qdrant collection (wiping every
+   * point including orphans), recreates it with the correct dimension, clears
+   * the Nomad-docs discovery flag, then dispatches an EmbedFileJob for every
+   * file currently on disk.
+   *
+   * Refuses to run if the embeddings queue already has in-flight work.
+   */
+  public async resetAndRebuild(): Promise<{
+    success: boolean
+    message: string
+    filesScanned?: number
+    filesQueued?: number
+    failedPaths?: string[]
+  }> {
+    try {
+      if (await this._hasInflightEmbedJobs()) {
+        return {
+          success: false,
+          message: 'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a reset.',
+        }
+      }
+
+      logger.info('[RAG] Starting destructive reset & rebuild')
+
+      await this._initializeQdrantClient()
+      try {
+        await this.qdrant!.deleteCollection(RagService.CONTENT_COLLECTION_NAME)
+        logger.info(`[RAG] Dropped collection ${RagService.CONTENT_COLLECTION_NAME}`)
+      } catch (err) {
+        // Collection may not exist yet on a fresh install — log and continue.
+        logger.warn(`[RAG] deleteCollection failed (may not exist): ${(err as Error).message}`)
+      }
+
+      await this._ensureCollection(
+        RagService.CONTENT_COLLECTION_NAME,
+        RagService.EMBEDDING_DIMENSION
+      )
+
+      // Force Nomad docs to be re-dispatched.
+      await KVStore.setValue('rag.docsEmbedded', false)
+      await this.discoverNomadDocs(true).catch((error) => {
+        logger.error('[RAG] Error re-running Nomad docs discovery after reset:', error)
+      })
+
+      const filesInStorage = await this._discoverKbFiles()
+      const { queuedCount, failedPaths } = await this._dispatchEmbedJobsFor(filesInStorage, {
+        force: true,
+      })
+
+      logger.info(
+        `[RAG] Reset complete — dispatched ${queuedCount}/${filesInStorage.length} files` +
+          (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '')
+      )
+
+      // Collection was already dropped, so dispatch failures here mean the
+      // file is gone from Qdrant with no pending job to repopulate it. Surface
+      // the count + paths so the operator can rerun a sync rescan to recover.
+      const failureSuffix =
+        failedPaths.length > 0
+          ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.`
+          : ''
+
+      return {
+        success: failedPaths.length === 0,
+        message:
+          `Collection wiped. Queued ${queuedCount} file${queuedCount === 1 ? '' : 's'} for a full rebuild.` +
+          failureSuffix,
+        filesScanned: filesInStorage.length,
+        filesQueued: queuedCount,
+        ...(failedPaths.length > 0 ? { failedPaths } : {}),
+      }
+    } catch (error) {
+      logger.error('[RAG] Error during reset & rebuild:', error)
+      return { success: false, message: 'Error during reset & rebuild' }
     }
   }
 }

admin/app/services/system_service.ts

@@ -12,6 +12,7 @@ import {
 } from '../../types/system.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { readFileSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
 import path, { join } from 'node:path'
 import { getAllFilesystems, getFile } from '../utils/fs.js'
 import axios from 'axios'
@@ -72,6 +73,89 @@ export class SystemService {
     return false
   }
 
+  /**
+   * Probe Ollama startup logs for the canonical "inference compute" line that records
+   * which compute backend was selected. This catches silent CPU fallback (e.g. when
+   * /dev/kfd is mounted but ROCm initialization fails, or NVML dies after an update)
+   * which the older nvidia-smi exec probe could not detect.
+   *
+   * Returns the parsed library, GPU model name, and VRAM in MiB, or null when:
+   *   - the Ollama container is not running
+   *   - the line has not been emitted (Ollama still starting up)
+   *   - logs show CPU-only operation (no GPU detected)
+   */
+  async getOllamaInferenceComputeFromLogs(): Promise<{
+    library: 'CUDA' | 'ROCm'
+    name: string
+    vramMiB: number
+  } | null> {
+    try {
+      const containers = await this.dockerService.docker.listContainers({ all: false })
+      const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
+      if (!ollamaContainer) return null
+
+      const container = this.dockerService.docker.getContainer(ollamaContainer.Id)
+
+      // Read logs only from the first 5 minutes after container start. The
+      // "inference compute" line is written once during Ollama's GPU discovery
+      // phase, within seconds of startup. Using tail:N here is fragile: under
+      // active embedding workloads we've seen >1000 lines/min, which pushes the
+      // line past any reasonable tail in minutes. Pinning to the startup window
+      // is bounded (~5 min of logs regardless of container uptime) and never
+      // ages out.
+      //
+      // Fall back to the previous tail:500 strategy if StartedAt is missing or
+      // unparseable — we can't construct a since/until window without it, but
+      // tail:500 is still useful when the container just started and the line
+      // is still recent.
+      const inspect = await container.inspect()
+      const startedAtRaw = inspect?.State?.StartedAt
+      const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN
+      const hasValidStartedAt = Number.isFinite(startedAtMs) && startedAtMs > 0
+
+      const logsOpts: { stdout: true; stderr: true; follow: false; since?: number; until?: number; tail?: number } = {
+        stdout: true,
+        stderr: true,
+        follow: false,
+      }
+      if (hasValidStartedAt) {
+        const startedAtSec = Math.floor(startedAtMs / 1000)
+        logsOpts.since = startedAtSec
+        logsOpts.until = startedAtSec + 300 // 5-minute window
+      } else {
+        logger.warn(
+          `[SystemService] nomad_ollama State.StartedAt missing or invalid (${startedAtRaw ?? 'undefined'}); falling back to tail:500 for inference-compute probe`
+        )
+        logsOpts.tail = 500
+      }
+      const buf = (await container.logs(logsOpts)) as unknown as Buffer
+      const logs = buf.toString('utf8')
+
+      const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"'))
+      if (lines.length === 0) return null
+
+      const lastLine = lines[lines.length - 1]
+      const libraryMatch = lastLine.match(/library=(CUDA|ROCm)/)
+      if (!libraryMatch) return null
+
+      const descMatch = lastLine.match(/description="([^"]+)"/)
+      const totalMatch = lastLine.match(/total="([0-9.]+)\s*GiB"/)
+
+      return {
+        library: libraryMatch[1] as 'CUDA' | 'ROCm',
+        name:
+          descMatch?.[1] ||
+          (libraryMatch[1] === 'CUDA' ? 'NVIDIA GPU' : 'AMD GPU'),
+        vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0,
+      }
+    } catch (error) {
+      logger.warn(
+        `[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}`
+      )
+      return null
+    }
+  }
+
   async getNvidiaSmiInfo(): Promise<
     | Array<{ vendor: string; model: string; vram: number }>
     | { error: string }
@@ -317,10 +401,14 @@ export class SystemService {
         logger.error('Error reading disk info file:', error)
       }
 
-      // GPU health tracking — detect when host has NVIDIA GPU but Ollama can't access it
+      // GPU health tracking — detect when host has a GPU runtime but Ollama can't access it.
+      // Primary probe: parse Ollama's "inference compute" startup log line for both NVIDIA
+      // and AMD. Secondary probe (NVIDIA only): nvidia-smi exec, retained as a fallback for
+      // hardware enrichment when log parsing has not yet captured a startup line.
       let gpuHealth: GpuHealthStatus = {
         status: 'no_gpu',
         hasNvidiaRuntime: false,
+        hasRocmRuntime: false,
         ollamaGpuAccessible: false,
       }
 
@@ -339,28 +427,85 @@ export class SystemService {
           os.kernel = dockerInfo.KernelVersion
         }
 
-        // If si.graphics() returned no controllers (common inside Docker),
-        // fall back to nvidia runtime + nvidia-smi detection
-        if (!graphics.controllers || graphics.controllers.length === 0) {
+        // si.graphics() in the admin container uses lspci (pciutils ships in
+        // the image for AMD detection). lspci has no real VRAM info for
+        // discrete GPUs, so systeminformation parses the first PCI memory
+        // Region (BAR0, typically 1-32 MiB) as `vram`. nvidia-smi / ROCm
+        // tooling enrichment also can't run since neither is in the admin
+        // image. No real dGPU has under 256 MiB, so any discrete-GPU controller
+        // below that threshold needs the probes below to give us real data.
+        // Applies to both NVIDIA and AMD; Intel iGPUs are exempt because their
+        // shared-system-memory VRAM reading via lspci can legitimately be small.
+        const DGPU_BOGUS_VRAM_THRESHOLD_MIB = 256
+        const isDiscreteGpuVendor = (vendor: string) =>
+          /nvidia|advanced micro devices|amd|ati/i.test(vendor)
+        const isBogusDgpuVram = (c: { vendor?: string; vram?: number | null }) =>
+          isDiscreteGpuVendor(c.vendor || '') &&
+          typeof c.vram === 'number' &&
+          c.vram < DGPU_BOGUS_VRAM_THRESHOLD_MIB
+
+        // Clear the bogus value up front. If a probe replaces the entry below
+        // we get the real VRAM; if no probe succeeds (Ollama not installed,
+        // passthrough_failed) the UI falls back to "N/A" instead of showing
+        // "1 MB" / "32 MB". The lspci model/vendor strings stay since they're
+        // still useful for identifying the card.
+        const hasLspciBogusDgpuVram = (graphics.controllers || []).some(isBogusDgpuVram)
+        if (hasLspciBogusDgpuVram) {
+          for (const c of graphics.controllers) {
+            if (isBogusDgpuVram(c)) c.vram = null
+          }
+        }
+
+        // Run the probes when controllers are empty (common inside Docker) or
+        // when lspci gave us bogus discrete-GPU BAR0 values that need replacing.
+        if (
+          !graphics.controllers ||
+          graphics.controllers.length === 0 ||
+          hasLspciBogusDgpuVram
+        ) {
           const runtimes = dockerInfo.Runtimes || {}
-          if ('nvidia' in runtimes) {
-            gpuHealth.hasNvidiaRuntime = true
-            const nvidiaInfo = await this.getNvidiaSmiInfo()
-            if (Array.isArray(nvidiaInfo)) {
-              graphics.controllers = nvidiaInfo.map((gpu) => ({
-                model: gpu.model,
-                vendor: gpu.vendor,
-                bus: '',
-                vram: gpu.vram,
-                vramDynamic: false, // assume false here, we don't actually use this field for our purposes.
-              }))
+          gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes
+
+          // AMD doesn't register a Docker runtime. Detection sources, in priority order:
+          //   1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install)
+          //   2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh)
+          // The marker file matters because the System page should reflect AMD presence
+          // even before AI Assistant has been installed for the first time.
+          let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined
+          if (!savedGpuType) {
+            try {
+              savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
+            } catch {}
+          }
+          const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
+          const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
+          gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled
+
+          if (gpuHealth.hasNvidiaRuntime || gpuHealth.hasRocmRuntime) {
+            gpuHealth.gpuVendor = gpuHealth.hasNvidiaRuntime ? 'nvidia' : 'amd'
+
+            // Primary probe: Ollama log parsing — works for both vendors and catches silent fallback
+            const logInfo = await this.getOllamaInferenceComputeFromLogs()
+            if (logInfo) {
+              graphics.controllers = [
+                {
+                  model: logInfo.name,
+                  vendor: logInfo.library === 'CUDA' ? 'NVIDIA' : 'AMD',
+                  bus: '',
+                  vram: logInfo.vramMiB,
+                  vramDynamic: false,
+                },
+              ]
               gpuHealth.status = 'ok'
               gpuHealth.ollamaGpuAccessible = true
-            } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
-              // No local Ollama container — check if a remote Ollama URL is configured
-              const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
-              if (externalOllamaGpu) {
-                graphics.controllers = externalOllamaGpu.map((gpu) => ({
+            } else if (gpuHealth.hasNvidiaRuntime) {
+              // NVIDIA secondary path: nvidia-smi exec preserves prior behavior when
+              // the log parser hasn't seen a startup line yet (e.g. log rotation,
+              // very fresh container). Distinguishes "no Ollama container" from
+              // "container exists but GPU broken".
+              const nvidiaInfo = await this.getNvidiaSmiInfo()
+              if (Array.isArray(nvidiaInfo)) {
+                graphics.controllers = nvidiaInfo.map((gpu) => ({
                   model: gpu.model,
                   vendor: gpu.vendor,
                   bus: '',
@@ -369,25 +514,66 @@ export class SystemService {
                 }))
                 gpuHealth.status = 'ok'
                 gpuHealth.ollamaGpuAccessible = true
+              } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
+                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
+                if (externalOllamaGpu) {
+                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
+                    model: gpu.model,
+                    vendor: gpu.vendor,
+                    bus: '',
+                    vram: gpu.vram,
+                    vramDynamic: false,
+                  }))
+                  gpuHealth.status = 'ok'
+                  gpuHealth.ollamaGpuAccessible = true
+                } else {
+                  gpuHealth.status = 'ollama_not_installed'
+                }
               } else {
-                gpuHealth.status = 'ollama_not_installed'
+                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
+                if (externalOllamaGpu) {
+                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
+                    model: gpu.model,
+                    vendor: gpu.vendor,
+                    bus: '',
+                    vram: gpu.vram,
+                    vramDynamic: false,
+                  }))
+                  gpuHealth.status = 'ok'
+                  gpuHealth.ollamaGpuAccessible = true
+                } else {
+                  gpuHealth.status = 'passthrough_failed'
+                  logger.warn(
+                    `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`
+                  )
+                }
               }
             } else {
-              const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
-              if (externalOllamaGpu) {
-                graphics.controllers = externalOllamaGpu.map((gpu) => ({
-                  model: gpu.model,
-                  vendor: gpu.vendor,
-                  bus: '',
-                  vram: gpu.vram,
-                  vramDynamic: false,
-                }))
-                gpuHealth.status = 'ok'
-                gpuHealth.ollamaGpuAccessible = true
+              // AMD path: no nvidia-smi equivalent worth running — log parser is authoritative.
+              // Distinguish "Ollama not running" from "Ollama running but no GPU log line".
+              const containers = await this.dockerService.docker.listContainers({ all: false })
+              const ollamaRunning = containers.some((c) =>
+                c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
+              )
+              if (!ollamaRunning) {
+                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
+                if (externalOllamaGpu) {
+                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
+                    model: gpu.model,
+                    vendor: gpu.vendor,
+                    bus: '',
+                    vram: gpu.vram,
+                    vramDynamic: false,
+                  }))
+                  gpuHealth.status = 'ok'
+                  gpuHealth.ollamaGpuAccessible = true
+                } else {
+                  gpuHealth.status = 'ollama_not_installed'
+                }
               } else {
                 gpuHealth.status = 'passthrough_failed'
                 logger.warn(
-                  `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`
+                  'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed'
                 )
               }
             }

admin/app/services/zim_extraction_service.ts

@@ -40,7 +40,10 @@ export class ZIMExtractionService {
      * @param filePath - Path to the ZIM file
      * @param opts - Options including maxArticles, strategy, onProgress, startOffset, and batchSize
      */
-    async extractZIMContent(filePath: string, opts: ExtractZIMContentOptions = {}): Promise<ZIMContentChunk[]> {
+    async extractZIMContent(
+        filePath: string,
+        opts: ExtractZIMContentOptions = {}
+    ): Promise<{ chunks: ZIMContentChunk[]; totalArticles: number }> {
         try {
             logger.info(`[ZIMExtractionService]: Processing ZIM file at path: ${filePath}`)
             
@@ -161,7 +164,7 @@ export class ZIMExtractionService {
                 textPreview: c.text.substring(0, 100)
             })))
             logger.debug("Total structured sections extracted:", toReturn.length)
-            return toReturn
+            return { chunks: toReturn, totalArticles: archive.articleCount }
         } catch (error) {
             logger.error('Error processing ZIM file:', error)
             throw error

admin/app/services/zim_service.ts

@@ -4,8 +4,10 @@ import {
   RemoteZimFileEntry,
 } from '../../types/zim.js'
 import axios from 'axios'
+import * as cheerio from 'cheerio'
 import { XMLParser } from 'fast-xml-parser'
 import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js'
+import { findReplacedWikipediaFiles } from '../utils/zim_filename.js'
 import logger from '@adonisjs/core/services/logger'
 import { DockerService } from './docker_service.js'
 import { inject } from '@adonisjs/core'
@@ -27,6 +29,8 @@ import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { CollectionManifestService } from './collection_manifest_service.js'
 import { KiwixLibraryService } from './kiwix_library_service.js'
 import type { CategoryWithStatus } from '../../types/collections.js'
+import CustomLibrarySource from '#models/custom_library_source'
+import { assertNotPrivateUrl } from '#validators/common'
 
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
 const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
@@ -40,7 +44,21 @@ export class ZimService {
     await ensureDirectoryExists(dirPath)
 
     const all = await listDirectoryContents(dirPath)
-    const files = all.filter((item) => item.name.endsWith('.zim'))
+    const zimEntries = all.filter((item) => item.name.endsWith('.zim'))
+
+    const files = await Promise.all(
+      zimEntries.map(async (entry) => {
+        const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name)
+        const stats = await getFileStatsIfExists(filePath)
+        return {
+          ...entry,
+          title: null,
+          summary: null,
+          author: null,
+          size_bytes: stats ? Number(stats.size) : null,
+        }
+      })
+    )
 
     return {
       files,
@@ -297,7 +315,7 @@ export class ZimService {
     if (restart) {
       // Check if there are any remaining ZIM download jobs before restarting
       const { QueueService } = await import('./queue_service.js')
-      const queueService = new QueueService()
+      const queueService = QueueService.getInstance()
       const queue = queueService.getQueue('downloads')
 
       // Get all active and waiting jobs
@@ -573,40 +591,192 @@ export class ZimService {
   }
 
   async onWikipediaDownloadComplete(url: string, success: boolean): Promise<void> {
+    const filename = url.split('/').pop() || ''
     const selection = await this.getWikipediaSelection()
 
-    if (!selection || selection.url !== url) {
-      logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`)
-      return
+    // Determine which Wikipedia option this file belongs to by matching filename
+    let matchedOptionId: string | null = null
+    try {
+      const options = await this.getWikipediaOptions()
+      for (const opt of options) {
+        if (opt.url && opt.url.split('/').pop() === filename) {
+          matchedOptionId = opt.id
+          break
+        }
+      }
+    } catch {
+      // If we can't fetch options, try to continue with existing selection
     }
 
     if (success) {
-      // Update status to installed
-      selection.status = 'installed'
-      await selection.save()
+      // Update or create the selection record
+      // Match by filename (not URL) so mirror downloads are recognized
+      if (selection) {
+        selection.option_id = matchedOptionId || selection.option_id
+        selection.url = url
+        selection.filename = filename
+        selection.status = 'installed'
+        await selection.save()
+      } else {
+        await WikipediaSelection.create({
+          option_id: matchedOptionId || 'unknown',
+          url: url,
+          filename: filename,
+          status: 'installed',
+        })
+      }
 
-      logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`)
+      logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`)
 
-      // Delete the old Wikipedia file if it exists and is different
-      // We need to find what was previously installed
+      // Delete prior versions of THIS specific Wikipedia variant only.
+      // Earlier logic deleted anything starting with `wikipedia_en_`, which silently
+      // wiped distinct corpora the user had installed independently (issue #884).
       const existingFiles = await this.list()
-      const wikipediaFiles = existingFiles.files.filter((f) =>
-        f.name.startsWith('wikipedia_en_') && f.name !== selection.filename
+      const wikipediaFiles = findReplacedWikipediaFiles(
+        filename,
+        existingFiles.files.map((f) => f.name)
       )
 
       for (const oldFile of wikipediaFiles) {
         try {
-          await this.delete(oldFile.name)
-          logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile.name}`)
+          await this.delete(oldFile)
+          logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile}`)
         } catch (error) {
-          logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile.name}`, error)
+          logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile}`, error)
         }
       }
     } else {
-      // Download failed - keep the selection record but mark as failed
-      selection.status = 'failed'
-      await selection.save()
-      logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`)
+      // Download failed - update selection if it matches this file
+      if (selection && (!selection.filename || selection.filename === filename)) {
+        selection.status = 'failed'
+        await selection.save()
+        logger.error(`[ZimService] Wikipedia download failed for: ${filename}`)
+      } else {
+        logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`)
+      }
     }
   }
+
+  // Custom library source management
+
+  async listCustomLibraries(): Promise<CustomLibrarySource[]> {
+    return CustomLibrarySource.all()
+  }
+
+  async addCustomLibrary(name: string, baseUrl: string): Promise<CustomLibrarySource> {
+    const count = await CustomLibrarySource.query().count('* as total')
+    const total = Number(count[0].$extras.total)
+    if (total >= 10) {
+      throw new Error('Maximum of 10 custom libraries allowed')
+    }
+
+    // Ensure URL ends with /
+    const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/'
+
+    return CustomLibrarySource.create({
+      name,
+      base_url: normalizedUrl,
+    })
+  }
+
+  async removeCustomLibrary(id: number): Promise<void> {
+    const source = await CustomLibrarySource.find(id)
+    if (!source) {
+      throw new Error('Custom library not found')
+    }
+    if (source.is_default) {
+      throw new Error('Cannot remove a built-in mirror')
+    }
+    await source.delete()
+  }
+
+  async browseLibraryUrl(url: string): Promise<{
+    directories: { name: string; url: string }[]
+    files: { name: string; url: string; size_bytes: number | null }[]
+  }> {
+    assertNotPrivateUrl(url)
+
+    const normalizedUrl = url.endsWith('/') ? url : url + '/'
+
+    const res = await axios.get(normalizedUrl, {
+      responseType: 'text',
+      timeout: 15000,
+      headers: {
+        'Accept': 'text/html',
+      },
+    })
+
+    const html: string = res.data
+    const directories: { name: string; url: string }[] = []
+    const files: { name: string; url: string; size_bytes: number | null }[] = []
+
+    const $ = cheerio.load(html)
+
+    $('a').each((_, el) => {
+      const href = el.attribs?.href
+      if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) {
+        return
+      }
+      if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) {
+        return
+      }
+
+      if (href.endsWith('/')) {
+        const dirName = decodeURIComponent(href.replace(/\/$/, ''))
+        directories.push({
+          name: dirName,
+          url: new URL(href, normalizedUrl).toString(),
+        })
+        return
+      }
+
+      if (href.endsWith('.zim')) {
+        const fileName = decodeURIComponent(href)
+
+        // Apache/Nginx autoindex put the date + size in the text node directly
+        // following </a> within a <pre>. Walk forward across text siblings until
+        // we find a parseable size token.
+        let trailingText = ''
+        let sibling = el.next
+        while (sibling && sibling.type === 'text') {
+          trailingText += sibling.data
+          if (/\n/.test(sibling.data)) break
+          sibling = sibling.next
+        }
+
+        files.push({
+          name: fileName,
+          url: new URL(href, normalizedUrl).toString(),
+          size_bytes: this._parseListingSize(trailingText),
+        })
+      }
+    })
+
+    directories.sort((a, b) => a.name.localeCompare(b.name))
+    files.sort((a, b) => a.name.localeCompare(b.name))
+
+    return { directories, files }
+  }
+
+  /**
+   * Parse a directory-listing size token out of the text that follows an anchor.
+   * Apache renders e.g. `   2024-01-15 10:30  5.1G`; Nginx renders raw bytes.
+   * Returns bytes or null if no size token is found.
+   */
+  private _parseListingSize(text: string): number | null {
+    // Skip the date/time columns; grab the last numeric token (with optional suffix)
+    // before a newline. Matches `5.1G`, `5368709120`, `1.2T`, etc.
+    const sizeMatch = /([\d.]+\s*[KMGT]?B?|\d+)\s*$/i.exec(text.split('\n')[0].trim())
+    if (!sizeMatch) return null
+
+    const sizeStr = sizeMatch[1].replace(/\s|B$/gi, '')
+    const num = parseFloat(sizeStr)
+    if (isNaN(num)) return null
+
+    if (/^\d+$/.test(sizeStr)) return num
+
+    const suffix = sizeStr.slice(-1).toUpperCase()
+    const multipliers: Record<string, number> = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }
+    return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null
+  }
 }

admin/app/utils/downloads.ts

@@ -47,7 +47,14 @@ export async function doResumableDownload({
     timeout,
   })
 
-  const contentType = headResponse.headers['content-type'] || ''
+  // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a
+  // Content-Type header at all. Per RFC 7231 §3.1.1.5, "if no Content-Type is
+  // provided" the recipient may treat it as application/octet-stream — which is
+  // already in every binary-content allowlist we use (ZIM, PMTILES, base assets).
+  // Without this default, the validator below throws `MIME type  is not allowed`
+  // and breaks all downloads from kiwix's primary host (#848).
+  const contentType =
+    headResponse.headers['content-type'] || 'application/octet-stream'
   const totalBytes = parseInt(headResponse.headers['content-length'] || '0')
   const supportsRangeRequests = headResponse.headers['accept-ranges'] === 'bytes'
 

admin/app/utils/kb_ingest_decision.ts

@@ -0,0 +1,70 @@
+import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
+
+/**
+ * Decision returned by `decideScanAction` describing what scanAndSyncStorage
+ * should do for one file given its current state row (if any), whether Qdrant
+ * already has chunks for it, and the global ingest policy.
+ *
+ * - `skip` — file is in a settled state (already indexed, deliberately not
+ *   indexed, or in a manual-recovery state); no auto-dispatch.
+ * - `dispatch` — file needs to be (re-)embedded; an EmbedFileJob should be
+ *   dispatched. `createStateRow` indicates whether a new state row needs to
+ *   be created before dispatch (i.e. first time the scanner has seen it).
+ * - `backfill_indexed` — Qdrant has chunks but no state row exists yet
+ *   (pre-RFC install, or new admin instance pointed at an existing Qdrant
+ *   volume). Create a row in `indexed` state without re-embedding.
+ * - `create_pending` — Manual mode: record that we've seen the file but
+ *   don't dispatch. Frontend surfaces a per-card "Index" affordance.
+ */
+export type ScanAction =
+  | { kind: 'skip' }
+  | { kind: 'dispatch'; createStateRow: boolean }
+  | { kind: 'backfill_indexed' }
+  | { kind: 'create_pending' }
+
+export interface KbIngestStateRow {
+  state: KbIngestStateValue
+}
+
+/**
+ * Global auto-index policy stored at KV `rag.defaultIngestPolicy`. Unset is
+ * treated as `Always` so existing installs keep their current behavior until
+ * the user opts into Manual mode through the KB panel.
+ */
+export type IngestPolicy = 'Always' | 'Manual'
+
+/**
+ * Decide what scanAndSyncStorage should do for a single embeddable file.
+ *
+ * Replaces the earlier `!sourcesInQdrant.has(filePath)` binary check, which
+ * couldn't tell a fully-indexed file from a stalled mid-batch ingestion, and
+ * couldn't honor a user's "browse only" choice. The state row is now the
+ * authoritative answer; Qdrant chunk presence is corroborating evidence.
+ */
+export function decideScanAction(
+  stateRow: KbIngestStateRow | null,
+  hasChunksInQdrant: boolean,
+  policy: IngestPolicy = 'Always'
+): ScanAction {
+  if (!stateRow) {
+    if (hasChunksInQdrant) return { kind: 'backfill_indexed' }
+    return policy === 'Always'
+      ? { kind: 'dispatch', createStateRow: true }
+      : { kind: 'create_pending' }
+  }
+
+  switch (stateRow.state) {
+    case 'indexed':
+      return hasChunksInQdrant ? { kind: 'skip' } : { kind: 'dispatch', createStateRow: false }
+    case 'pending_decision':
+      // Manual mode: file is waiting for the user to opt in via per-card Index.
+      // Always mode: treat as "user-equivalent of auto-index" and dispatch.
+      return policy === 'Always'
+        ? { kind: 'dispatch', createStateRow: false }
+        : { kind: 'skip' }
+    case 'browse_only':
+    case 'failed':
+    case 'stalled':
+      return { kind: 'skip' }
+  }
+}

admin/app/utils/kb_job_health.ts

@@ -0,0 +1,50 @@
+/**
+ * Visual status assigned to an in-flight (or stuck) embedding job, used to
+ * pick the colored status pill in the KB Processing Queue. See RFC #883 §5.
+ *
+ * - `waiting`  — queued, no batch has started yet
+ * - `healthy`  — last batch < 2 minutes ago
+ * - `slow`     — last batch 2-5 minutes ago (CPU-paced multi-batch ingestion
+ *                falls into this band; not necessarily a problem)
+ * - `stalled`  — last batch > 5 minutes ago (likely a real problem)
+ * - `failed`   — job recorded a failed status
+ */
+export type JobHealthStatus = 'waiting' | 'healthy' | 'slow' | 'stalled' | 'failed'
+
+export interface JobHealthInput {
+  /** BullMQ job.data.status — set by EmbedFileJob.handle on transitions. */
+  status: string
+  /** 0-100. 0 means no work observed yet on this job-row. */
+  progress: number
+  /** ms epoch of the last completed batch. Multi-batch ZIMs update this on
+   * every continuation; single-batch jobs leave it unset until completion. */
+  lastBatchAt?: number
+  /** ms epoch of the first batch start. Used as a fallback "last activity"
+   * signal for jobs that haven't yet completed their first batch. */
+  startedAt?: number
+  /** Current ms epoch. Injected for testability. */
+  now: number
+}
+
+const SLOW_THRESHOLD_MS = 2 * 60 * 1000
+const STALLED_THRESHOLD_MS = 5 * 60 * 1000
+
+export function computeJobHealth(input: JobHealthInput): JobHealthStatus {
+  if (input.status === 'failed') return 'failed'
+
+  // No progress recorded and no activity timestamps — job is still queued.
+  if (
+    input.progress === 0 &&
+    input.lastBatchAt === undefined &&
+    input.startedAt === undefined
+  ) {
+    return 'waiting'
+  }
+
+  const lastActivity = input.lastBatchAt ?? input.startedAt ?? input.now
+  const stalenessMs = input.now - lastActivity
+
+  if (stalenessMs > STALLED_THRESHOLD_MS) return 'stalled'
+  if (stalenessMs > SLOW_THRESHOLD_MS) return 'slow'
+  return 'healthy'
+}

admin/app/utils/kb_ratio_lookup.ts

@@ -0,0 +1,97 @@
+export interface RatioRow {
+  pattern: string
+  chunks_per_mb: number
+}
+
+/**
+ * Bytes of on-disk storage one embedded chunk consumes inside Qdrant.
+ *
+ * Rough composition for our pipeline:
+ *   - vector: 768 dims × float32 = 3,072 B
+ *   - chunk text payload: ~3,000 B (target 1,500 tokens × 2 chars/token)
+ *   - source/metadata payload + Qdrant indexes: ~2,000 B
+ *
+ * Used for surfacing pre-ingest disk-cost estimates; the actual figure
+ * varies with collection params and will be replaced by self-calibration
+ * (RFC #883 Phase 4) once we have real measurements.
+ */
+export const BYTES_PER_CHUNK_ON_DISK = 8_000
+
+export interface BatchEstimateInput {
+  filename: string
+  sizeBytes: number
+}
+
+export interface BatchEstimate {
+  totalChunks: number
+  totalBytes: number
+  hasUnknown: boolean
+}
+
+/**
+ * Aggregate an embedding-disk-cost estimate across a batch of files (curated
+ * tier add, multi-upload, sync preview, etc). `hasUnknown` is true when at
+ * least one file did not match any registry row — the totals only include
+ * matched files, so callers should annotate "estimate excludes unknown files"
+ * when surfacing the figure.
+ */
+export function estimateBatch(
+  files: BatchEstimateInput[],
+  rows: RatioRow[]
+): BatchEstimate {
+  let totalChunks = 0
+  let hasUnknown = false
+  for (const f of files) {
+    const chunks = estimateChunkCount(f.filename, f.sizeBytes, rows)
+    if (chunks === null) {
+      hasUnknown = true
+    } else {
+      totalChunks += chunks
+    }
+  }
+  return {
+    totalChunks,
+    totalBytes: totalChunks * BYTES_PER_CHUNK_ON_DISK,
+    hasUnknown,
+  }
+}
+
+/**
+ * Pick the chunks_per_mb estimate for a filename by longest-prefix match.
+ *
+ * Patterns are filename prefixes (`devdocs_`, `wikipedia_en_simple_`, ...).
+ * The longest matching prefix wins, so a specific entry (`wikipedia_en_simple_`)
+ * overrides the broader fallback (`wikipedia_en_`). An empty-string pattern in
+ * the registry serves as a catch-all that matches every input.
+ *
+ * Returns `null` if no row matches and no empty-string fallback is present —
+ * caller decides whether to surface "unknown" or use its own default.
+ */
+export function findChunksPerMb(filename: string, rows: RatioRow[]): number | null {
+  let best: RatioRow | null = null
+  for (const row of rows) {
+    if (!filename.startsWith(row.pattern)) continue
+    if (best === null || row.pattern.length > best.pattern.length) {
+      best = row
+    }
+  }
+  return best === null ? null : best.chunks_per_mb
+}
+
+/**
+ * Estimate the number of embedding chunks a ZIM-style file will produce given
+ * its size on disk in bytes. Returns `null` when the registry has nothing to
+ * match against. Caller is responsible for converting the estimate into either
+ * a disk-footprint estimate (chunks × bytes-per-chunk in Qdrant) or a time
+ * estimate (chunks ÷ chunks-per-minute-on-this-hardware).
+ */
+export function estimateChunkCount(
+  filename: string,
+  fileSizeBytes: number,
+  rows: RatioRow[]
+): number | null {
+  const ratio = findChunksPerMb(filename, rows)
+  if (ratio === null) return null
+  const megabytes = fileSizeBytes / (1024 * 1024)
+  return Math.round(ratio * megabytes)
+}

admin/app/utils/kb_warning_decision.ts

@@ -0,0 +1,70 @@
+/**
+ * Conditional warnings surfaced on Stored Files rows in the KB panel.
+ * See RFC #883 §6 — these warnings appear ONLY when their triggering condition
+ * is met, never on healthy files, to keep the panel silent in the common case.
+ *
+ * - `zero_chunks`   — a non-trivial file produced 0 embedding chunks. Common
+ *                     cause: video-only or image-only ZIMs that the pipeline
+ *                     completes "successfully" with no extractable text.
+ *                     AI Assistant cannot reference this content.
+ * - `partial_stall` — the file has embedded chunks but well below the count
+ *                     expected from the ratio registry. Likely a mid-batch
+ *                     stall (which the binary "any chunks ⇒ embedded" check
+ *                     used to mask). Surfaces a Retry affordance.
+ */
+import type { FileWarning } from '../../types/rag.js'
+
+export type { FileWarning }
+
+/** Files smaller than this are too small to flag as suspicious zero-chunk
+ *  cases — a 5 KB upload that produces 0 chunks is much more likely to be a
+ *  legitimate edge case (placeholder file) than the gigabyte-scale video ZIM
+ *  problem this warning targets. */
+export const ZERO_CHUNKS_MIN_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB
+
+/** Fraction of expected chunks below which we consider a file partially
+ *  stalled. 0.5 (50%) matches the threshold described in RFC #883 §6 Warning B. */
+export const PARTIAL_STALL_RATIO_THRESHOLD = 0.5
+
+export interface WarningInputs {
+  /** Source file size on disk in bytes. */
+  fileSizeBytes: number
+  /** Distinct chunks present in Qdrant for this source. */
+  chunksInQdrant: number
+  /** Best estimate of chunks the file should produce, from the ratio
+   *  registry. `null` when no registry pattern matches and no fallback is
+   *  configured — Warning B is suppressed in that case (we'd rather be silent
+   *  than wrong). */
+  expectedChunks: number | null
+}
+
+export function decideWarnings(inputs: WarningInputs): FileWarning[] {
+  const warnings: FileWarning[] = []
+
+  // Warning A: file is large but produced nothing. Almost always a video-only
+  // or image-only ZIM; AI Assistant literally cannot reference this content.
+  if (
+    inputs.chunksInQdrant === 0 &&
+    inputs.fileSizeBytes > ZERO_CHUNKS_MIN_SIZE_BYTES
+  ) {
+    warnings.push({ kind: 'zero_chunks', fileSizeBytes: inputs.fileSizeBytes })
+  }
+
+  // Warning B: chunks present but far below expectation. Suppresses when we
+  // have no expectation (registry miss) since the comparison would be
+  // meaningless and we'd rather under-warn than mislead.
+  if (
+    inputs.expectedChunks !== null &&
+    inputs.expectedChunks > 0 &&
+    inputs.chunksInQdrant > 0 &&
+    inputs.chunksInQdrant < inputs.expectedChunks * PARTIAL_STALL_RATIO_THRESHOLD
+  ) {
+    warnings.push({
+      kind: 'partial_stall',
+      chunksEmbedded: inputs.chunksInQdrant,
+      chunksExpected: inputs.expectedChunks,
+    })
+  }
+
+  return warnings
+}

admin/app/utils/zim_filename.ts

@@ -0,0 +1,26 @@
+/**
+ * Strip the trailing `_YYYY-MM(-DD).zim` date suffix from a Kiwix-style ZIM
+ * filename so different release dates of the same variant share a stem
+ * (e.g., `wikipedia_en_all_nopic`) while distinct corpora keep distinct stems
+ * (`wikipedia_en_simple_all_nopic`, `wikipedia_en_medicine_nopic`, etc.).
+ */
+export function zimFilenameStem(name: string): string {
+  return name.replace(/_\d{4}-\d{2}(?:-\d{2})?\.zim$/i, '')
+}
+
+/**
+ * Of the existing files, return only those that are prior-version replacements
+ * of `currentFilename` — same Wikipedia variant stem, different release. Used
+ * by the post-download cleanup to avoid deleting unrelated Wikipedia corpora
+ * the user has installed independently (issue #884).
+ */
+export function findReplacedWikipediaFiles(
+  currentFilename: string,
+  existingNames: string[]
+): string[] {
+  const currentStem = zimFilenameStem(currentFilename)
+  return existingNames.filter(
+    (n) =>
+      n.startsWith('wikipedia_en_') && n !== currentFilename && zimFilenameStem(n) === currentStem
+  )
+}

admin/app/validators/common.ts

@@ -1,4 +1,5 @@
 import vine from '@vinejs/vine'
+import ipaddr from 'ipaddr.js'
 
 /**
  * Checks whether a URL points to a loopback or link-local address.
@@ -15,15 +16,18 @@ export function assertNotPrivateUrl(urlString: string): void {
   const parsed = new URL(urlString)
   const hostname = parsed.hostname.toLowerCase()
 
+  // `URL.hostname` strips the surrounding brackets from IPv6 literals
+  // (e.g. `http://[::1]/` → hostname `::1`), so IPv6 patterns must match
+  // the unbracketed form.
   const blockedPatterns = [
     /^localhost$/,
     /^127\.\d+\.\d+\.\d+$/,
     /^0\.0\.0\.0$/,
     /^169\.254\.\d+\.\d+$/, // Link-local / cloud metadata
-    /^\[::1\]$/,
-    /^\[?fe80:/i, // IPv6 link-local
-    /^\[::ffff:/i, // IPv4-mapped IPv6 (e.g. [::ffff:7f00:1] = 127.0.0.1)
-    /^\[::\]$/, // IPv6 all-zeros (equivalent to 0.0.0.0)
+    /^::1$/, // IPv6 loopback
+    /^fe80:/i, // IPv6 link-local
+    /^::ffff:/i, // IPv4-mapped IPv6 (e.g. ::ffff:7f00:1 = 127.0.0.1)
+    /^::$/, // IPv6 all-zeros (equivalent to 0.0.0.0)
   ]
 
   if (blockedPatterns.some((re) => re.test(hostname))) {
@@ -31,6 +35,63 @@ export function assertNotPrivateUrl(urlString: string): void {
   }
 }
 
+/**
+ * Narrower SSRF guard for "remote service" URLs the user points NOMAD at
+ * (e.g. an OpenAI-compatible endpoint like LM Studio, llama.cpp, vLLM, or a
+ * sibling Ollama container). Unlike `assertNotPrivateUrl`, this intentionally
+ * ALLOWS loopback, link-local-ish, and RFC1918 hosts because the legitimate
+ * target is frequently on the same host or LAN (host.docker.internal,
+ * the docker bridge gateway, or a LAN IP).
+ *
+ * It blocks only:
+ *   - the cloud instance-metadata IP (169.254.169.254), to avoid leaking
+ *     IAM creds on a misconfigured cloud VM
+ *   - non-HTTP schemes (file:, gopher:, etc.)
+ */
+// Canonical cloud instance-metadata addresses. AWS, GCP, Azure, DigitalOcean,
+// Oracle Cloud, and Alibaba all expose IMDS at 169.254.169.254 over IPv4;
+// AWS additionally exposes it at fd00:ec2::254 over IPv6.
+// Compared after `ipaddr.toNormalizedString()`, which expands IPv6 to its
+// fully-zero-padded form (e.g. `fd00:ec2::254` → `fd00:ec2:0:0:0:0:0:254`).
+const BLOCKED_METADATA_IPV4 = new Set(['169.254.169.254'])
+const BLOCKED_METADATA_IPV6 = new Set([
+  ipaddr.parse('fd00:ec2::254').toNormalizedString(),
+])
+
+export function assertNotCloudMetadataUrl(urlString: string): void {
+  const parsed = new URL(urlString)
+
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new Error(`URL must use http or https scheme: ${parsed.protocol}`)
+  }
+
+  // Node's WHATWG URL parser keeps the brackets on IPv6 literals
+  // (`http://[::1]/` → hostname `[::1]`), so strip them before parsing.
+  const hostname = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, '')
+
+  // If the hostname isn't an IP literal it's a DNS name; allow it. (DNS
+  // rebinding is out of scope here — would require resolving and re-checking
+  // at fetch time.)
+  if (!ipaddr.isValid(hostname)) return
+
+  let addr = ipaddr.parse(hostname)
+
+  // Unwrap IPv4-mapped IPv6 (e.g. ::ffff:169.254.169.254, ::ffff:a9fe:a9fe,
+  // and the fully-expanded 0:0:0:0:0:ffff:a9fe:a9fe) so the IPv4 check below
+  // sees the embedded address.
+  if (addr.kind() === 'ipv6' && (addr as ipaddr.IPv6).isIPv4MappedAddress()) {
+    addr = (addr as ipaddr.IPv6).toIPv4Address()
+  }
+
+  const canonical = addr.toNormalizedString()
+
+  const blocked =
+    addr.kind() === 'ipv4' ? BLOCKED_METADATA_IPV4 : BLOCKED_METADATA_IPV6
+  if (blocked.has(canonical)) {
+    throw new Error(`URL must not point to the cloud instance metadata endpoint: ${canonical}`)
+  }
+}
+
 export const remoteDownloadValidator = vine.compile(
   vine.object({
     url: vine
@@ -100,6 +161,7 @@ const resourceUpdateInfoBase = vine.object({
   installed_version: vine.string().trim(),
   latest_version: vine.string().trim().minLength(1),
   download_url: vine.string().url({ require_tld: false }).trim(),
+  size_bytes: vine.number().positive().optional(),
 })
 
 export const applyContentUpdateValidator = vine.compile(resourceUpdateInfoBase)
@@ -111,3 +173,31 @@ export const applyAllContentUpdatesValidator = vine.compile(
       .minLength(1),
   })
 )
+
+// --- Map extract (regional pmtiles download) ---
+
+// ISO 3166-1 alpha-2, 2 letters. Loose regex; CountriesService.resolveCodes
+// does the authoritative check against the polygon dataset.
+const countryCodeSchema = vine
+  .string()
+  .trim()
+  .toUpperCase()
+  .regex(/^[A-Z]{2}$/)
+
+const countriesArraySchema = vine.array(countryCodeSchema).minLength(1).maxLength(300)
+
+export const mapExtractPreflightValidator = vine.compile(
+  vine.object({
+    countries: countriesArraySchema.clone(),
+    maxzoom: vine.number().min(0).max(15).optional(),
+  })
+)
+
+export const mapExtractValidator = vine.compile(
+  vine.object({
+    countries: countriesArraySchema.clone(),
+    maxzoom: vine.number().min(0).max(15).optional(),
+    label: vine.string().trim().minLength(1).maxLength(64).optional(),
+    estimatedBytes: vine.number().min(0).optional(),
+  })
+)

admin/app/validators/ollama.ts

@@ -14,6 +14,12 @@ export const chatSchema = vine.compile(
   })
 )
 
+export const unloadChatModelsSchema = vine.compile(
+  vine.object({
+    targetModel: vine.string().trim().minLength(1).nullable().optional(),
+  })
+)
+
 export const getAvailableModelsSchema = vine.compile(
   vine.object({
     sort: vine.enum(['pulls', 'name'] as const).optional(),

admin/app/validators/rag.ts

@@ -11,3 +11,24 @@ export const deleteFileSchema = vine.compile(
     source: vine.string(),
   })
 )
+
+export const embedFileSchema = vine.compile(
+  vine.object({
+    source: vine.string().minLength(1),
+    force: vine.boolean().optional(),
+  })
+)
+
+export const estimateBatchSchema = vine.compile(
+  vine.object({
+    files: vine
+      .array(
+        vine.object({
+          filename: vine.string().minLength(1).maxLength(255),
+          sizeBytes: vine.number().min(0),
+        })
+      )
+      .minLength(1)
+      .maxLength(500),
+  })
+)

admin/app/validators/zim.ts

@@ -7,3 +7,30 @@ export const listRemoteZimValidator = vine.compile(
     query: vine.string().optional(),
   })
 )
+
+export const addCustomLibraryValidator = vine.compile(
+  vine.object({
+    name: vine.string().trim().minLength(1).maxLength(100),
+    base_url: vine
+      .string()
+      .url({ require_tld: false })
+      .trim(),
+  })
+)
+
+export const browseLibraryValidator = vine.compile(
+  vine.object({
+    url: vine
+      .string()
+      .url({ require_tld: false })
+      .trim(),
+  })
+)
+
+export const idParamValidator = vine.compile(
+  vine.object({
+    params: vine.object({
+      id: vine.number(),
+    }),
+  })
+)

admin/commands/queue/work.ts

@@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import queueConfig from '#config/queue'
 import { RunDownloadJob } from '#jobs/run_download_job'
+import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
 import { DownloadModelJob } from '#jobs/download_model_job'
 import { RunBenchmarkJob } from '#jobs/run_benchmark_job'
 import { EmbedFileJob } from '#jobs/embed_file_job'
@@ -126,6 +127,7 @@ export default class QueueWork extends BaseCommand {
     const queues = new Map<string, string>()
 
     handlers.set(RunDownloadJob.key, new RunDownloadJob())
+    handlers.set(RunExtractPmtilesJob.key, new RunExtractPmtilesJob())
     handlers.set(DownloadModelJob.key, new DownloadModelJob())
     handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob())
     handlers.set(EmbedFileJob.key, new EmbedFileJob())
@@ -133,6 +135,7 @@ export default class QueueWork extends BaseCommand {
     handlers.set(CheckServiceUpdatesJob.key, new CheckServiceUpdatesJob())
 
     queues.set(RunDownloadJob.key, RunDownloadJob.queue)
+    queues.set(RunExtractPmtilesJob.key, RunExtractPmtilesJob.queue)
     queues.set(DownloadModelJob.key, DownloadModelJob.queue)
     queues.set(RunBenchmarkJob.key, RunBenchmarkJob.queue)
     queues.set(EmbedFileJob.key, EmbedFileJob.queue)
@@ -149,6 +152,9 @@ export default class QueueWork extends BaseCommand {
   private getConcurrencyForQueue(queueName: string): number {
     const concurrencyMap: Record<string, number> = {
       [RunDownloadJob.queue]: 3,
+      // pmtiles extract hits the Protomaps CDN with many parallel range reads per job;
+      // cap concurrency at 2 so a second extract doesn't starve the first.
+      [RunExtractPmtilesJob.queue]: 2,
       [DownloadModelJob.queue]: 2, // Lower concurrency for resource-intensive model downloads
       [RunBenchmarkJob.queue]: 1, // Run benchmarks one at a time for accurate results
       [EmbedFileJob.queue]: 2, // Lower concurrency for embedding jobs, can be resource intensive

admin/constants/kv_store.ts

@@ -1,3 +1,3 @@
 import { KVStoreKey } from "../types/kv_store.js";
 
-export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention'];
+export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy'];

admin/constants/map_regions.ts

@@ -0,0 +1,32 @@
+export const PMTILES_BINARY_PATH = '/usr/local/bin/pmtiles'
+
+// Clamp these so a user can't ask for nonsense that never extracts
+export const EXTRACT_MIN_ZOOM = 0
+export const EXTRACT_MAX_ZOOM = 15
+export const EXTRACT_DEFAULT_MAX_ZOOM = 15
+
+// Low-zoom global fallback extracted once during base-asset setup (~15 MB). Layered
+// underneath regional extracts so the map isn't grey outside a region's polygon.
+export const WORLD_BASEMAP_FILENAME = 'world.pmtiles'
+export const WORLD_BASEMAP_MAX_ZOOM = 5
+export const WORLD_BASEMAP_SOURCE_NAME = 'world'
+
+export interface PmtilesExtractArgOptions {
+  sourceUrl: string
+  outputFilepath: string
+  regionFilepath?: string
+  maxzoom?: number
+  dryRun?: boolean
+  downloadThreads?: number
+  overfetch?: number
+}
+
+export function buildPmtilesExtractArgs(opts: PmtilesExtractArgOptions): string[] {
+  const args = ['extract', opts.sourceUrl, opts.outputFilepath]
+  if (opts.regionFilepath) args.push(`--region=${opts.regionFilepath}`)
+  if (typeof opts.maxzoom === 'number') args.push(`--maxzoom=${opts.maxzoom}`)
+  if (opts.dryRun) args.push('--dry-run')
+  if (typeof opts.downloadThreads === 'number') args.push(`--download-threads=${opts.downloadThreads}`)
+  if (typeof opts.overfetch === 'number') args.push(`--overfetch=${opts.overfetch}`)
+  return args
+}

admin/constants/ollama.ts

@@ -64,6 +64,8 @@ export const FALLBACK_RECOMMENDED_OLLAMA_MODELS: NomadOllamaModel[] = [
 
 export const DEFAULT_QUERY_REWRITE_MODEL = 'qwen2.5:3b' // default to qwen2.5 for query rewriting with good balance of text task performance and resource usage
 
+export const EMBEDDING_MODEL_NAME = 'nomic-embed-text:v1.5'
+
 /**
  * Adaptive RAG context limits based on model size.
  * Smaller models get overwhelmed with too much context, so we cap it.

admin/database/migrations/1775100000001_create_custom_library_sources_table.ts

@@ -0,0 +1,42 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'custom_library_sources'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.increments('id').primary()
+      table.string('name', 100).notNullable()
+      table.string('base_url', 2048).notNullable()
+      table.boolean('is_default').notNullable().defaultTo(false)
+      table.timestamp('created_at').notNullable()
+      table.timestamp('updated_at').notNullable()
+    })
+
+    // Seed default Kiwix mirrors
+    const now = new Date().toISOString().slice(0, 19).replace('T', ' ')
+    const defaults = [
+      { name: 'Debian CDN (Global)', base_url: 'https://cdimage.debian.org/mirror/kiwix.org/zim/' },
+      { name: 'Your.org (US)', base_url: 'https://ftpmirror.your.org/pub/kiwix/zim/' },
+      { name: 'FAU Erlangen (DE)', base_url: 'https://ftp.fau.de/kiwix/zim/' },
+      { name: 'Dotsrc (DK)', base_url: 'https://mirrors.dotsrc.org/kiwix/zim/' },
+      { name: 'MirrorService (UK)', base_url: 'https://www.mirrorservice.org/sites/download.kiwix.org/zim/' },
+    ]
+
+    for (const d of defaults) {
+      await this.defer(async (db) => {
+        await db.table(this.tableName).insert({
+          name: d.name,
+          base_url: d.base_url,
+          is_default: true,
+          created_at: now,
+          updated_at: now,
+        })
+      })
+    }
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}

admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts

@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'kb_ingest_state'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.increments('id').primary()
+      // utf8mb4 caps an indexed varchar at 768 chars (3072 byte InnoDB key limit);
+      // 512 leaves headroom and is plenty for any NOMAD-managed file path.
+      table.string('file_path', 512).notNullable().unique()
+      table
+        .enum('state', ['pending_decision', 'indexed', 'browse_only', 'failed', 'stalled'])
+        .notNullable()
+        .defaultTo('pending_decision')
+      table.integer('chunks_embedded').notNullable().defaultTo(0)
+      table.text('last_error').nullable()
+      table.timestamp('created_at').notNullable()
+      table.timestamp('updated_at').notNullable()
+    })
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}

admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts

@@ -0,0 +1,64 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import { DateTime } from 'luxon'
+
+const SEED_ROWS: Array<{ pattern: string; chunks_per_mb: number; notes: string }> = [
+  // Dense technical reference — every paragraph carries content
+  { pattern: 'devdocs_', chunks_per_mb: 1100, notes: 'Heuristic seed: dense API references' },
+  // Encyclopedia prose — Simple English & general Wikipedia variants
+  {
+    pattern: 'wikipedia_en_simple_',
+    chunks_per_mb: 270,
+    notes: 'Heuristic seed: Simple English Wikipedia',
+  },
+  {
+    pattern: 'wikipedia_en_',
+    chunks_per_mb: 270,
+    notes: 'Heuristic seed: general Wikipedia variants',
+  },
+  // Sparse text, image-heavy
+  { pattern: 'ifixit_', chunks_per_mb: 50, notes: 'Heuristic seed: image-heavy repair guides' },
+  // Q&A pages — moderate density, mostly short answers
+  {
+    pattern: 'cooking.stackexchange.com_',
+    chunks_per_mb: 200,
+    notes: 'Heuristic seed: Stack Exchange Q&A',
+  },
+  // Video-only ZIMs produce zero text chunks. Listing these explicitly keeps
+  // the cost estimator from spinning up "indexing in progress" UI for content
+  // that has no embeddable text whatsoever.
+  { pattern: 'lrnselfreliance_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' },
+  { pattern: 'ted_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' },
+  { pattern: 'freedom-of-religion_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' },
+  // Empty-pattern fallback — every filename startsWith('') is true. The lookup
+  // picks the longest matching pattern, so this only fires for ZIMs that match
+  // none of the above (medium prose density).
+  { pattern: '', chunks_per_mb: 100, notes: 'Heuristic fallback' },
+]
+
+export default class extends BaseSchema {
+  protected tableName = 'kb_ratio_registry'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.increments('id').primary()
+      table.string('pattern', 255).notNullable().unique()
+      table.integer('chunks_per_mb').unsigned().notNullable()
+      // 0 = heuristic seed, >0 = number of observed ZIMs that have updated this entry.
+      // Phase 4 self-calibration increments this on each successful ingestion.
+      table.integer('sample_count').notNullable().defaultTo(0)
+      table.text('notes').nullable()
+      table.timestamp('created_at').notNullable()
+      table.timestamp('updated_at').notNullable()
+    })
+
+    const now = DateTime.utc().toSQL({ includeOffset: false }) as string
+    const rows = SEED_ROWS.map((row) => ({ ...row, created_at: now, updated_at: now }))
+    this.defer(async (db) => {
+      await db.table(this.tableName).multiInsert(rows)
+    })
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}

admin/docs/api-reference.md

@@ -148,6 +148,15 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix.
 | POST | `/api/maps/download-collection` | Download an entire collection by slug (async) |
 | DELETE | `/api/maps/:filename` | Delete a local map file |
 
+### Map Markers
+
+| Method | Path | Description |
+|--------|------|-------------|
+| GET | `/api/maps/markers` | List map markers |
+| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) |
+| PATCH | `/api/maps/markers/{id}` | Update a  map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted|
+| DELETE | `/api/maps/markers/{id}` | Delete a map marker | 
+
 ---
 
 ## Downloads

admin/docs/faq.md

@@ -114,6 +114,18 @@ The Maps feature requires downloaded map data. If you see a blank area:
 3. Wait for downloads to complete
 4. Return to Maps and refresh
 
+### ERROR: Failed to load the XML library file '/data/kiwix-library.xml'
+
+This usually means the Information Library service started before its Kiwix library index was fully initialized.
+
+Try this recovery flow:
+1. Go to **[Apps](/settings/apps)**
+2. Stop **Information Library (Kiwix)**
+3. Wait 10-15 seconds, then start it again
+4. If the error persists, run **Force Reinstall** for Information Library from the same page
+
+After restart/reinstall completes, refresh the Information Library page.
+
 ### AI responses are slow
 
 Local AI requires significant computing power. To improve speed:

admin/docs/release-notes.md

@@ -1,6 +1,94 @@
 # Release Notes
 
-## Unreleased
+## Version 1.31.1 - April 21, 2026
+
+### Features
+- feat(content): custom ZIM library sources with pre-seeded mirrors (#593). Thanks @chriscrosstalk!
+- feat(content-manager): add sortable file size column (#698). Thanks @chriscrosstalk!
+- feat(ai-chat): allow cancelling in-progress model downloads (#701). Thanks @chriscrosstalk!
+- feat(content-updates): show size, surface downloads in Active Downloads (#773). Thanks @chriscrosstalk!
+- feat(maps): regional map downloads via go-pmtiles extract (#780). Thanks @bgauger!
+- feat(maps): show map coordinates on mouse move (#786). Thanks @kennethbrewer3!
+- feat(AI): re-enable AMD GPU acceleration for Ollama via ROCm + HSA override (#804). Thanks @chriscrosstalk!
+- feat(GPU): auto-remediate nomad_ollama passthrough loss on admin boot (#878). Thanks @chriscrosstalk!
+- feat(KB): per-file ingest state machine (Phase 1 of RFC #883) (#888). Thanks @chriscrosstalk!
+- feat(KB): ratio registry for disk + time estimates (Phase 1B of RFC #883) (#891). Thanks @chriscrosstalk!
+- feat(KB): group admin docs into single row in Stored Files (§9) (#892). Thanks @chriscrosstalk!
+- feat(KB): status pill + last-activity on Processing Queue (§5/§10) (#893). Thanks @chriscrosstalk!
+- feat(KB): Always/Manual ingest policy toggle (§1/§4) (#894). Thanks @chriscrosstalk!
+- feat(KB): conditional warnings A + B on Stored Files (§6) (#895). Thanks @chriscrosstalk!
+- feat(KB): surface embedding-disk estimate in curated tier-change modal (§1) (#897). Thanks @chriscrosstalk!
+- feat(KB): first-chat JIT prompt for ingest policy (Phase 3 task 12) (#899). Thanks @chriscrosstalk!
+- feat(KB): wizard AI policy step (Phase 3 task 13) (#900). Thanks @chriscrosstalk!
+- feat(KB): guardrail modal at 50GB / 10%-free thresholds (§7) (#901). Thanks @chriscrosstalk!
+- feat(easy-setup): split AI into its own conditional step (#908). Thanks @chriscrosstalk!
+- feat(KB): per-file ingest action + state indicator on Stored Files (§5) (#909). Thanks @chriscrosstalk!
+- feat(chat): confirm-on-switch + one-chat-model-at-a-time enforcement (#916). Thanks @chriscrosstalk!
+
+### Bug Fixes
+- fix(downloads): stage downloads to .tmp to prevent Kiwix loading partial files (#448). Thanks @artbird309!
+- fix(security): close remaining security audit items 3 & 4 (CWE-918, CWE-209) (#552). Thanks @LuisMIguelFurlanettoSousa!
+- fix(ai-chat): add null check to model name (#645). Thanks @hestela!
+- fix(ai-chat): qwen2.5 loading on every chat message (#649). Thanks @hestela!
+- fix(disk-collector): fix storage reporting for NFS mounts (#686). Thanks @bgauger!
+- fix(rag): add start button in kb modal and ensure restart policy exists (#700). Thanks @hestela!
+- fix(admin): only hide global map banner after download (#702). Thanks @Gujiassh!
+- fix(maps): wire delete confirmation to API (#732). Thanks @cuyua9!
+- fix: prevent ZIM corrupt file crash and deduplicate Ollama download logs (#741). Thanks @jakeaturner!
+- fix(ai): stop local nomad_ollama when remote Ollama is configured (#744). Thanks @chriscrosstalk!
+- fix(rag): repair ZIM embedding pipeline (sync filter, batch gate, DOM walk) (#745). Thanks @chriscrosstalk!
+- fix(zim): accumulate across Kiwix pages to prevent empty Content Explorer (#746). Thanks @chriscrosstalk!
+- fix(qdrant): disable anonymous telemetry by default (#747). Thanks @chriscrosstalk!
+- fix(disk-display): gate NAS Storage label on network filesystem type (#749). Thanks @bgauger!
+- fix(docker): write /app/version.json from VERSION build-arg (#754). Thanks @chriscrosstalk!
+- fix(rag): pass num_ctx and truncate to Ollama embed call (#763). Thanks @chriscrosstalk!
+- fix(api): accept notes, marker_type, and position on markers endpoints (#770). Thanks @jrsphoto!
+- fix(install): warn loudly on non-x86_64 architectures before pulling images (#797). Thanks @chriscrosstalk!
+- fix(stream): skip compression for Server-Sent Events (#798). Thanks @chriscrosstalk!
+- fix(maps): Country Picker UX polish + auto-refresh stored files (#817). Thanks @chriscrosstalk!
+- fix(System): self-heal stale updateAvailable flag after sidecar-driven update (#825). Thanks @jakeaturner!
+- fix(settings/update): four UI/UX fixes for the System Update page (#827). Thanks @chriscrosstalk!
+- fix(Maps): send filename instead of full path to delete endpoint (#829). Thanks @bgauger!
+- fix(Maps): render notes in marker popup when populated (#830). Thanks @chriscrosstalk!
+- fix(AI): vendor-aware AMD HSA override + benchmark discrete-GPU detection (#832). Thanks @chriscrosstalk!
+- fix(System): correct NVIDIA VRAM in Graphics card (#850). Thanks @bgauger!
+- fix(Downloads): treat missing Content-Type as octet-stream (#859). Thanks @bgauger!
+- fix(AI): preserve semver tag in DB on AMD Ollama updates (#868). Thanks @chriscrosstalk!
+- fix(AI): rewrite RAG query on first chat follow-up (#869). Thanks @chriscrosstalk!
+- fix(RAG): unbreak multi-batch ZIM ingestion (jobId dedupe) (#872). Thanks @chriscrosstalk!
+- fix(RAG): pace continuation batches when embedding is CPU-only (#873). Thanks @chriscrosstalk!
+- fix(queue): singleton QueueService to stop ioredis connection leak (#877). Thanks @chriscrosstalk!
+- fix(System): correct AMD VRAM in Graphics card + harden log probe (#879). Thanks @chriscrosstalk!
+- fix(RAG): report ZIM ingestion progress in overall-file frame (#880). Thanks @chriscrosstalk!
+- fix(KB): add re-embed and reset & rebuild options to fix broken embeddings (#886). Thanks @jakeaturner!
+- fix(ZIM): preserve co-existing Wikipedia corpora on cleanup (#887). Thanks @chriscrosstalk!
+- fix(RAG): anchor continuation-batch initial progress to overall-file frame (#889). Thanks @chriscrosstalk!
+- fix(AI): pre-cap embed input + log fallback reason (#890). Thanks @chriscrosstalk!
+- fix(KB): remove redundant Refresh button from Processing Queue (#896). Thanks @chriscrosstalk!
+- fix(KB): union Stored Files list with state-machine file paths (#898). Thanks @chriscrosstalk!
+- fix(KB): blank-screen on panel open + tooltips on bulk-action buttons (#907). Thanks @chriscrosstalk!
+- fix(KB): TierSelectionModal hook order + register IconLibrary (#917). Thanks @chriscrosstalk!
+- fix(content): show selected tier on cards while downloads are in flight (#918). Thanks @chriscrosstalk!
+- fix(KB): respect Manual ingest policy on post-download dispatch (#919). Thanks @chriscrosstalk!
+- fix(AI): improve remote Ollama url validation to prevent SSRF vuln (#920). Thanks @jakeaturner!
+- fix(models): correct inverted belongsTo keys on ChatMessage.session (#921). Thanks @jakeaturner!
+
+### Improvements
+- docs: add Community Add-Ons page with field manuals + W3Schools packs (#753). Thanks @chriscrosstalk!
+- docs: add map marker API reference (#783). Thanks @kennethbrewer3!
+- docs: require linked issue for non-trivial PRs (#799). Thanks @chriscrosstalk!
+- docs(map): updated notes on the map pin api (#803). Thanks @kennethbrewer3!
+- docs: link to new WSL2 install guide from README and FAQ (#811). Thanks @chriscrosstalk!
+- build(deps): bump picomatch in /admin (#544). Thanks @dependabot[bot]!
+- build(deps): bump lodash from 4.17.23 to 4.18.1 in /admin (#643). Thanks @dependabot[bot]!
+- build(deps-dev): bump vite from 6.4.1 to 6.4.2 in /admin (#677). Thanks @dependabot[bot]!
+- build(deps): bump axios from 1.13.5 to 1.15.0 in /admin (#708). Thanks @dependabot[bot]!
+- build(deps): bump @adonisjs/http-server from 7.8.0 to 7.8.1 in /admin (#724). Thanks @dependabot[bot]!
+- build(deps): bump follow-redirects from 1.15.11 to 1.16.0 in /admin (#729). Thanks @dependabot[bot]!
+- build(deps): bump protocol-buffers-schema from 3.6.0 to 3.6.1 in /admin (#736). Thanks @dependabot[bot]!
+- build(deps): bump protobufjs from 7.5.4 to 7.5.5 in /admin (#737). Thanks @dependabot[bot]!
+
+## Version 1.31.1 - April 21, 2026
 
 ### Features
 

admin/inertia/components/ActiveEmbedJobs.tsx

@@ -1,6 +1,12 @@
+import { useEffect, useState } from 'react'
 import useEmbedJobs from '~/hooks/useEmbedJobs'
 import HorizontalBarChart from './HorizontalBarChart'
 import StyledSectionHeader from './StyledSectionHeader'
+import {
+  JOB_HEALTH_DISPLAY,
+  computeJobHealth,
+  formatTimeAgo,
+} from '~/lib/kb_job_health_display'
 
 interface ActiveEmbedJobsProps {
   withHeader?: boolean
@@ -9,31 +15,70 @@ interface ActiveEmbedJobsProps {
 const ActiveEmbedJobs = ({ withHeader = false }: ActiveEmbedJobsProps) => {
   const { data: jobs } = useEmbedJobs()
 
+  // Re-render every 5s to keep per-job "last activity Xs ago" timestamps fresh.
+  const [tick, setTick] = useState(() => Date.now())
+  useEffect(() => {
+    const id = setInterval(() => setTick(Date.now()), 5000)
+    return () => clearInterval(id)
+  }, [])
+
   return (
     <>
       {withHeader && (
         <StyledSectionHeader title="Processing Queue" className="mt-12 mb-4" />
       )}
+
       <div className="space-y-4">
         {jobs && jobs.length > 0 ? (
-          jobs.map((job) => (
-            <div
-              key={job.jobId}
-              className="bg-desert-white rounded-lg p-4 border border-desert-stone-light shadow-sm hover:shadow-lg transition-shadow"
-            >
-              <HorizontalBarChart
-                items={[
-                  {
-                    label: job.fileName,
-                    value: job.progress,
-                    total: '100%',
-                    used: `${job.progress}%`,
-                    type: job.status,
-                  },
-                ]}
-              />
-            </div>
-          ))
+          jobs.map((job) => {
+            const health = computeJobHealth({
+              status: job.status,
+              progress: job.progress,
+              lastBatchAt: job.lastBatchAt,
+              startedAt: job.startedAt,
+              now: tick,
+            })
+            const display = JOB_HEALTH_DISPLAY[health]
+            const lastActivityMs = job.lastBatchAt ?? job.startedAt
+            return (
+              <div
+                key={job.jobId}
+                className="bg-desert-white rounded-lg p-4 border border-desert-stone-light shadow-sm hover:shadow-lg transition-shadow"
+              >
+                <div className="flex items-center gap-3 mb-2">
+                  <span
+                    className={`inline-block w-2.5 h-2.5 rounded-full ${display.dot}`}
+                    aria-label={display.ariaLabel}
+                    title={display.ariaLabel}
+                  />
+                  <span className="text-sm font-medium text-text-primary">
+                    {display.label}
+                  </span>
+                  {lastActivityMs !== undefined && (
+                    <span className="text-xs text-text-muted">
+                      · last activity {formatTimeAgo(lastActivityMs, tick)}
+                    </span>
+                  )}
+                  {typeof job.chunks === 'number' && job.chunks > 0 && (
+                    <span className="text-xs text-text-muted">
+                      · {job.chunks.toLocaleString()} chunks
+                    </span>
+                  )}
+                </div>
+                <HorizontalBarChart
+                  items={[
+                    {
+                      label: job.fileName,
+                      value: job.progress,
+                      total: '100%',
+                      used: `${job.progress}%`,
+                      type: job.status,
+                    },
+                  ]}
+                />
+              </div>
+            )
+          })
         ) : (
           <p className="text-text-muted">No files are currently being processed</p>
         )}

admin/inertia/components/CategoryCard.tsx

@@ -2,7 +2,7 @@ import { formatBytes } from '~/lib/util'
 import DynamicIcon, { DynamicIconName } from './DynamicIcon'
 import type { CategoryWithStatus, SpecTier } from '../../types/collections'
 import classNames from 'classnames'
-import { IconChevronRight, IconCircleCheck } from '@tabler/icons-react'
+import { IconChevronRight, IconCircleCheck, IconLoader2 } from '@tabler/icons-react'
 
 export interface CategoryCardProps {
   category: CategoryWithStatus
@@ -29,14 +29,34 @@ const CategoryCard: React.FC<CategoryCardProps> = ({ category, selectedTier, onC
   const minSize = getTierTotalSize(category.tiers[0], category.tiers)
   const maxSize = getTierTotalSize(category.tiers[category.tiers.length - 1], category.tiers)
 
-  // Determine which tier to highlight: selectedTier (wizard) > installedTierSlug (persisted)
-  const highlightedTierSlug = selectedTier?.slug || category.installedTierSlug
+  // Priority order for the prominent corner badge + lime border:
+  //   1. selectedTier — in-session wizard pick (highest priority, reflects
+  //      what the user is editing right now)
+  //   2. downloadingTierSlug — backend-derived from in-flight downloads, so
+  //      the card shows the user's intent immediately after Submit, before
+  //      any single file has finished downloading
+  //   3. installedTierSlug — fully on disk
+  const downloadingTier = !selectedTier && category.downloadingTierSlug
+    ? category.tiers.find((t) => t.slug === category.downloadingTierSlug)
+    : null
+  const installedTier = !selectedTier && !downloadingTier && category.installedTierSlug
+    ? category.tiers.find((t) => t.slug === category.installedTierSlug)
+    : null
+  const badgeTier = selectedTier || downloadingTier || installedTier
+  const badgeStatus: 'selected' | 'downloading' | 'installed' | null = selectedTier
+    ? 'selected'
+    : downloadingTier
+      ? 'downloading'
+      : installedTier
+        ? 'installed'
+        : null
+  const highlightedTierSlug = badgeTier?.slug
 
   return (
     <div
       className={classNames(
         'flex flex-col bg-desert-green rounded-lg p-6 text-white border shadow-sm hover:shadow-lg transition-shadow cursor-pointer h-80',
-        selectedTier ? 'border-lime-400 border-2' : 'border-desert-green'
+        badgeTier ? 'border-lime-400 border-2' : 'border-desert-green'
       )}
       onClick={() => onClick?.(category)}
     >
@@ -46,10 +66,17 @@ const CategoryCard: React.FC<CategoryCardProps> = ({ category, selectedTier, onC
             <DynamicIcon icon={category.icon as DynamicIconName} className="w-6 h-6 mr-2" />
             <h3 className="text-lg font-semibold">{category.name}</h3>
           </div>
-          {selectedTier ? (
+          {badgeTier ? (
             <div className="flex items-center">
-              <IconCircleCheck className="w-5 h-5 text-lime-400" />
-              <span className="text-lime-400 text-sm ml-1">{selectedTier.name}</span>
+              {badgeStatus === 'downloading' ? (
+                <IconLoader2 className="w-5 h-5 text-lime-400 animate-spin" />
+              ) : (
+                <IconCircleCheck className="w-5 h-5 text-lime-400" />
+              )}
+              <span className="text-lime-400 text-sm ml-1">
+                {badgeTier.name}
+                {badgeStatus === 'downloading' && ' (downloading)'}
+              </span>
             </div>
           ) : (
             <IconChevronRight className="w-5 h-5 text-white opacity-70" />

admin/inertia/components/CountryPickerModal.tsx

@@ -0,0 +1,414 @@
+import { useEffect, useMemo, useRef, useState } from 'react'
+import { useQuery } from '@tanstack/react-query'
+import { IconCheck, IconSearch, IconX } from '@tabler/icons-react'
+import StyledModal, { StyledModalProps } from './StyledModal'
+import LoadingSpinner from './LoadingSpinner'
+import api from '~/lib/api'
+import { formatBytes } from '~/lib/util'
+import classNames from '~/lib/classNames'
+import {
+  EXTRACT_DEFAULT_MAX_ZOOM,
+  EXTRACT_MAX_ZOOM,
+  EXTRACT_MIN_ZOOM,
+} from '../../constants/map_regions'
+import type {
+  Country,
+  CountryCode,
+  CountryGroup,
+  MapExtractPreflight,
+} from '../../types/maps'
+
+export type CountryPickerModalProps = Omit<
+  StyledModalProps,
+  | 'onConfirm'
+  | 'open'
+  | 'confirmText'
+  | 'cancelText'
+  | 'confirmVariant'
+  | 'children'
+  | 'title'
+  | 'large'
+> & {
+  onDownloadStart?: () => void
+  /** Filenames of pmtiles already on disk; used to badge already-installed countries. */
+  installedFilenames?: string[]
+}
+
+// Single-country extracts use the slug `{iso2 lowercase}_{dateSlug}_z{maxzoom}.pmtiles`,
+// matching MapService.buildRegionSlug (which lowercases the alpha-2 country code).
+// dateSlug comes from the upstream pmtiles key with `.pmtiles` stripped — currently
+// YYYYMMDD but we accept any digits/dashes. Group / custom filenames don't reverse-map
+// to country codes, so we skip them here.
+const SINGLE_COUNTRY_FILENAME_RE = /^([a-z]{2})_[\w-]+_z\d+\.pmtiles$/
+
+const CountryPickerModal: React.FC<CountryPickerModalProps> = ({
+  onDownloadStart,
+  installedFilenames = [],
+  ...modalProps
+}) => {
+  const [selected, setSelected] = useState<Set<CountryCode>>(new Set())
+  const [search, setSearch] = useState('')
+  const [maxzoom, setMaxzoom] = useState<number>(EXTRACT_DEFAULT_MAX_ZOOM)
+  const [preflight, setPreflight] = useState<MapExtractPreflight | null>(null)
+  const [loading, setLoading] = useState(false)
+  const [downloading, setDownloading] = useState(false)
+  const [errorMessage, setErrorMessage] = useState<string | null>(null)
+  const preflightRequestIdRef = useRef(0)
+
+  const { data: countries = [], isLoading: countriesLoading } = useQuery({
+    queryKey: ['maps-countries'],
+    queryFn: () => api.listCountries(),
+    staleTime: Infinity,
+  })
+
+  const { data: groups = [] } = useQuery({
+    queryKey: ['maps-country-groups'],
+    queryFn: () => api.listCountryGroups(),
+    staleTime: Infinity,
+  })
+
+  const grouped = useMemo(() => {
+    const q = search.trim().toLowerCase()
+    const filtered = q
+      ? countries.filter(
+          (c) => c.name.toLowerCase().includes(q) || c.code.toLowerCase().includes(q)
+        )
+      : countries
+
+    const buckets: Record<string, Country[]> = {}
+    for (const country of filtered) {
+      if (!buckets[country.continent]) buckets[country.continent] = []
+      buckets[country.continent].push(country)
+    }
+    return Object.entries(buckets).sort(([a], [b]) => a.localeCompare(b))
+  }, [countries, search])
+
+  const selectedCountries = useMemo(
+    () => countries.filter((c) => selected.has(c.code)),
+    [countries, selected]
+  )
+
+  const installedCountrySet = useMemo(() => {
+    const set = new Set<CountryCode>()
+    for (const filename of installedFilenames) {
+      const match = SINGLE_COUNTRY_FILENAME_RE.exec(filename)
+      if (match) set.add(match[1].toUpperCase() as CountryCode)
+    }
+    return set
+  }, [installedFilenames])
+
+  function toggleCountry(code: CountryCode) {
+    setSelected((prev) => {
+      const next = new Set(prev)
+      if (next.has(code)) next.delete(code)
+      else next.add(code)
+      return next
+    })
+  }
+
+  function toggleGroup(group: CountryGroup) {
+    setSelected((prev) => {
+      const next = new Set(prev)
+      const allIn = group.countries.every((c) => next.has(c))
+      if (allIn) {
+        group.countries.forEach((c) => next.delete(c))
+      } else {
+        group.countries.forEach((c) => next.add(c))
+      }
+      return next
+    })
+  }
+
+  function clearAll() {
+    setSelected(new Set())
+  }
+
+  // Auto-refresh the preflight whenever selection or maxzoom changes. Debounced
+  // so rapid multi-select clicks and slider drags collapse into a single CDN
+  // round-trip. Loading state only flips after the debounce expires so the UI
+  // stays interactive during the wait. Stale-safe via requestId so an earlier
+  // slow response can't clobber a later one.
+  useEffect(() => {
+    if (selected.size === 0) {
+      setPreflight(null)
+      setErrorMessage(null)
+      setLoading(false)
+      preflightRequestIdRef.current++
+      return
+    }
+
+    setErrorMessage(null)
+    const timer = setTimeout(async () => {
+      const requestId = ++preflightRequestIdRef.current
+      setLoading(true)
+      try {
+        const res = await api.extractMapPreflight({
+          countries: [...selected],
+          maxzoom,
+        })
+        if (requestId !== preflightRequestIdRef.current) return
+        if (!res) throw new Error('Preflight returned no data')
+        setPreflight(res)
+      } catch (err: any) {
+        if (requestId !== preflightRequestIdRef.current) return
+        console.error('Preflight failed:', err)
+        setErrorMessage(err?.message ?? 'Estimate failed')
+      } finally {
+        if (requestId === preflightRequestIdRef.current) setLoading(false)
+      }
+    }, 1500)
+
+    return () => clearTimeout(timer)
+  }, [selected, maxzoom])
+
+  async function startDownload() {
+    if (selected.size === 0) {
+      setErrorMessage('Pick at least one country before downloading.')
+      return
+    }
+    if (loading || !preflight) {
+      setErrorMessage('Still estimating size — hold on a moment.')
+      return
+    }
+    try {
+      setDownloading(true)
+      setErrorMessage(null)
+      await api.extractMapRegion({
+        countries: [...selected],
+        maxzoom,
+        estimatedBytes: preflight?.bytes,
+      })
+      onDownloadStart?.()
+    } catch (err: any) {
+      console.error('Extract dispatch failed:', err)
+      setErrorMessage(err?.message ?? 'Download failed')
+    } finally {
+      setDownloading(false)
+    }
+  }
+
+  return (
+    <StyledModal
+      {...modalProps}
+      title="Download map by country or region"
+      open={true}
+      confirmText="Start Download"
+      confirmIcon="IconDownload"
+      cancelText="Cancel"
+      confirmVariant="primary"
+      confirmLoading={loading || downloading}
+      cancelLoading={loading || downloading}
+      onConfirm={startDownload}
+      large
+    >
+      <div className="flex flex-col text-left gap-4 min-h-[60vh]">
+        <div className="flex gap-3 items-stretch">
+          <div className="relative flex-1">
+            <IconSearch className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted" />
+            <input
+              type="text"
+              value={search}
+              onChange={(e) => setSearch(e.target.value)}
+              placeholder={`Search ${countries.length} countries...`}
+              className="w-full pl-9 pr-3 py-2 rounded-md border border-border-default bg-surface-primary text-text-primary text-sm focus:outline-none focus:ring-2 focus:ring-desert-green"
+            />
+          </div>
+          {selected.size > 0 && (
+            <button
+              type="button"
+              onClick={clearAll}
+              className="text-sm text-text-muted hover:text-text-primary px-3 cursor-pointer"
+            >
+              Clear all
+            </button>
+          )}
+        </div>
+
+        {groups.length > 0 && (
+          <div>
+            <p className="text-xs uppercase tracking-wide text-text-muted mb-2">
+              Quick picks
+            </p>
+            <div className="flex flex-wrap gap-2">
+              {groups.map((group) => {
+                const allIn =
+                  group.countries.length > 0 &&
+                  group.countries.every((c) => selected.has(c))
+                return (
+                  <button
+                    key={group.id}
+                    type="button"
+                    onClick={() => toggleGroup(group)}
+                    className={classNames(
+                      'px-3 py-1.5 rounded-full text-xs font-medium border transition-colors cursor-pointer',
+                      allIn
+                        ? 'bg-desert-green text-white border-desert-green'
+                        : 'bg-surface-primary text-text-primary border-border-default hover:border-desert-green'
+                    )}
+                  >
+                    {allIn && <IconCheck className="inline w-3 h-3 mr-1" />}
+                    {group.name}{' '}
+                    <span className="opacity-60">({group.countries.length})</span>
+                  </button>
+                )
+              })}
+            </div>
+          </div>
+        )}
+
+        <div className="flex-1 overflow-y-auto max-h-96 border border-border-default rounded-md bg-surface-secondary">
+          {countriesLoading ? (
+            <div className="flex items-center justify-center h-40">
+              <LoadingSpinner />
+            </div>
+          ) : grouped.length === 0 ? (
+            <p className="text-text-muted text-sm p-6 text-center">
+              No countries match "{search}".
+            </p>
+          ) : (
+            grouped.map(([continent, list]) => (
+              <div key={continent}>
+                <div className="sticky top-0 bg-surface-secondary border-b border-border-default px-4 py-2 text-xs uppercase tracking-wide text-text-muted font-semibold z-10">
+                  {continent}
+                </div>
+                <ul>
+                  {list.map((country) => {
+                    const isSelected = selected.has(country.code)
+                    const isInstalled = installedCountrySet.has(country.code)
+                    return (
+                      <li key={country.code}>
+                        <button
+                          type="button"
+                          onClick={() => toggleCountry(country.code)}
+                          className={classNames(
+                            'w-full flex items-center gap-3 px-4 py-2 text-left text-sm transition-colors cursor-pointer',
+                            isSelected
+                              ? 'bg-desert-green/10 hover:bg-desert-green/15'
+                              : 'hover:bg-surface-primary'
+                          )}
+                        >
+                          <span
+                            className={classNames(
+                              'w-4 h-4 rounded border flex items-center justify-center shrink-0',
+                              isSelected
+                                ? 'bg-desert-green border-desert-green'
+                                : 'border-border-default'
+                            )}
+                          >
+                            {isSelected && <IconCheck className="w-3 h-3 text-white" />}
+                          </span>
+                          <span className="flex-1 text-text-primary">{country.name}</span>
+                          {isInstalled && (
+                            <span
+                              className="text-[10px] uppercase tracking-wide font-semibold px-1.5 py-0.5 rounded bg-desert-green/15 text-desert-green border border-desert-green/30"
+                              title="Already downloaded — re-select to update with a different zoom"
+                            >
+                              Installed
+                            </span>
+                          )}
+                          <span className="text-xs font-mono text-text-muted">
+                            {country.code}
+                          </span>
+                        </button>
+                      </li>
+                    )
+                  })}
+                </ul>
+              </div>
+            ))
+          )}
+        </div>
+
+        {selectedCountries.length > 0 && (
+          <div>
+            <p className="text-xs uppercase tracking-wide text-text-muted mb-2">
+              {selectedCountries.length} selected
+            </p>
+            <div className="flex flex-wrap gap-2 max-h-24 overflow-y-auto">
+              {selectedCountries.map((country) => (
+                <span
+                  key={country.code}
+                  className="inline-flex items-center gap-1 px-2 py-1 rounded bg-desert-green text-white text-xs"
+                >
+                  {country.name}
+                  <button
+                    type="button"
+                    onClick={() => toggleCountry(country.code)}
+                    className="hover:bg-white/20 rounded cursor-pointer"
+                    aria-label={`Remove ${country.name}`}
+                  >
+                    <IconX className="w-3 h-3" />
+                  </button>
+                </span>
+              ))}
+            </div>
+          </div>
+        )}
+
+        <div>
+          <label className="block text-sm text-text-primary font-medium mb-2">
+            Max zoom level: <span className="font-mono">{maxzoom}</span>
+          </label>
+          <input
+            type="range"
+            min={EXTRACT_MIN_ZOOM}
+            max={EXTRACT_MAX_ZOOM}
+            step={1}
+            value={maxzoom}
+            onChange={(e) => setMaxzoom(parseInt(e.target.value, 10))}
+            className="w-full accent-desert-green"
+            disabled={downloading}
+          />
+          <div className="flex justify-between text-xs text-text-muted mt-1 font-mono">
+            <span>z{EXTRACT_MIN_ZOOM} (world)</span>
+            <span>z{EXTRACT_MAX_ZOOM} (street)</span>
+          </div>
+          <p className="text-xs text-text-muted mt-2">
+            Lower zoom = smaller file, less detail. Zoom 15 shows individual streets;
+            zoom 10 shows city-level detail.
+          </p>
+        </div>
+
+        <div className="bg-surface-secondary border border-border-default rounded-md p-3 min-h-14 text-sm font-mono">
+          <PreflightStatus
+            errorMessage={errorMessage}
+            loading={loading}
+            preflight={preflight}
+            hasSelection={selected.size > 0}
+          />
+        </div>
+
+      </div>
+    </StyledModal>
+  )
+}
+
+type PreflightStatusProps = {
+  errorMessage: string | null
+  loading: boolean
+  preflight: MapExtractPreflight | null
+  hasSelection: boolean
+}
+
+function PreflightStatus({ errorMessage, loading, preflight, hasSelection }: PreflightStatusProps) {
+  if (errorMessage) {
+    return <p className="text-desert-red">{errorMessage}</p>
+  }
+  if (loading) {
+    return <p className="text-text-muted">Estimating size…</p>
+  }
+  if (preflight) {
+    return (
+      <p className="text-text-primary">
+        {preflight.tiles.toLocaleString()} tiles, ~{formatBytes(preflight.bytes, 1)}{' '}
+        <span className="text-text-muted">(source build {preflight.source.date})</span>
+      </p>
+    )
+  }
+  if (!hasSelection) {
+    return <p className="text-text-muted">Pick at least one country to estimate size.</p>
+  }
+  return <p className="text-text-muted">Estimating size…</p>
+}
+
+export default CountryPickerModal

admin/inertia/components/KbGuardrailModal.tsx

@@ -0,0 +1,109 @@
+import { Fragment } from 'react'
+import { Dialog, Transition } from '@headlessui/react'
+import { IconAlertTriangle, IconX } from '@tabler/icons-react'
+import { formatBytes } from '~/lib/util'
+import StyledButton from './StyledButton'
+import type { GuardrailVerdict } from '~/lib/kb_guardrail'
+
+/**
+ * One-time confirmation modal for bulk indexing actions that trip the
+ * disk-usage thresholds in `lib/kb_guardrail.ts`. The caller (e.g.
+ * TierSelectionModal) decides whether to show the modal by evaluating the
+ * guardrail BEFORE submit; this component just presents the verdict and
+ * passes the user's choice back via `onConfirm` / `onCancel`.
+ */
+interface KbGuardrailModalProps {
+  isOpen: boolean
+  verdict: GuardrailVerdict
+  onConfirm: () => void
+  onCancel: () => void
+}
+
+export default function KbGuardrailModal({
+  isOpen,
+  verdict,
+  onConfirm,
+  onCancel,
+}: KbGuardrailModalProps) {
+  // The primary number to surface — every triggered reason carries the same
+  // estimateBytes, so just grab the first one. `0` is a defensive fallback
+  // for the (impossible-by-construction) "open with empty verdict" case.
+  const estimateBytes = verdict.reasons[0]?.estimateBytes ?? 0
+  const freeReason = verdict.reasons.find((r) => r.kind === 'over_free_disk')
+
+  return (
+    <Transition appear show={isOpen} as={Fragment}>
+      <Dialog as="div" className="relative z-[60]" onClose={onCancel}>
+        <Transition.Child
+          as={Fragment}
+          enter="ease-out duration-200"
+          enterFrom="opacity-0"
+          enterTo="opacity-100"
+          leave="ease-in duration-150"
+          leaveFrom="opacity-100"
+          leaveTo="opacity-0"
+        >
+          <div className="fixed inset-0 bg-black/50" />
+        </Transition.Child>
+
+        <div className="fixed inset-0 overflow-y-auto">
+          <div className="flex min-h-full items-center justify-center p-4">
+            <Transition.Child
+              as={Fragment}
+              enter="ease-out duration-200"
+              enterFrom="opacity-0 scale-95"
+              enterTo="opacity-100 scale-100"
+              leave="ease-in duration-150"
+              leaveFrom="opacity-100 scale-100"
+              leaveTo="opacity-0 scale-95"
+            >
+              <Dialog.Panel className="w-full max-w-lg transform overflow-hidden rounded-lg bg-surface-primary shadow-xl transition-all">
+                <div className="bg-amber-50 dark:bg-amber-950/30 px-6 py-4 border-b border-amber-200 dark:border-amber-800 flex items-start justify-between gap-3">
+                  <div className="flex items-start gap-3">
+                    <IconAlertTriangle className="h-6 w-6 text-amber-600 dark:text-amber-300 flex-shrink-0 mt-0.5" />
+                    <Dialog.Title className="text-lg font-semibold text-text-primary">
+                      Confirm large AI indexing operation
+                    </Dialog.Title>
+                  </div>
+                  <button
+                    onClick={onCancel}
+                    className="text-text-muted hover:text-text-primary transition-colors flex-shrink-0"
+                    aria-label="Cancel"
+                  >
+                    <IconX size={20} />
+                  </button>
+                </div>
+
+                <div className="px-6 py-5 space-y-3">
+                  <p className="text-text-primary text-sm">
+                    Indexing this batch for the AI Assistant will use approximately{' '}
+                    <strong>{formatBytes(estimateBytes, 1)}</strong> of disk space for embeddings, on top of the raw downloads.
+                  </p>
+
+                  {freeReason && (
+                    <p className="text-text-secondary text-sm">
+                      That's more than 10% of your remaining free disk space ({formatBytes(freeReason.freeBytes, 1)} free). Embedding can take several hours and is hard to interrupt cleanly once started.
+                    </p>
+                  )}
+
+                  <p className="text-text-secondary text-sm">
+                    If you'd rather review per-item before indexing, cancel here and switch your Auto-index setting to <strong>Manual</strong> from the Knowledge Base panel.
+                  </p>
+                </div>
+
+                <div className="bg-surface-secondary px-6 py-4 flex justify-end gap-3">
+                  <StyledButton variant="outline" size="md" onClick={onCancel}>
+                    Cancel
+                  </StyledButton>
+                  <StyledButton variant="primary" size="md" onClick={onConfirm}>
+                    Proceed anyway
+                  </StyledButton>
+                </div>
+              </Dialog.Panel>
+            </Transition.Child>
+          </div>
+        </div>
+      </Dialog>
+    </Transition>
+  )
+}

admin/inertia/components/TierSelectionModal.tsx

@@ -1,12 +1,28 @@
-import { Fragment, useState, useEffect } from 'react'
+import { Fragment, useState, useEffect, useMemo } from 'react'
 import { Dialog, Transition } from '@headlessui/react'
 import { IconX, IconCheck, IconInfoCircle } from '@tabler/icons-react'
+import { useQuery } from '@tanstack/react-query'
 import type { CategoryWithStatus, SpecTier, SpecResource } from '../../types/collections'
 import { resolveTierResources } from '~/lib/collections'
 import { formatBytes } from '~/lib/util'
+import api from '~/lib/api'
 import classNames from 'classnames'
 import DynamicIcon, { DynamicIconName } from './DynamicIcon'
 import StyledButton from './StyledButton'
+import KbGuardrailModal from './KbGuardrailModal'
+import { evaluateGuardrail, type GuardrailVerdict } from '~/lib/kb_guardrail'
+import { useSystemInfo } from '~/hooks/useSystemInfo'
+import { getPrimaryDiskInfo } from '~/hooks/useDiskDisplayData'
+
+/**
+ * Filename for the embed-estimate registry lookup. Strips the URL path so
+ * patterns like `wikipedia_en_simple_` continue to match upstream filenames
+ * regardless of mirror domain.
+ */
+function resourceFilename(resource: SpecResource): string {
+  const last = resource.url.split('/').pop()
+  return last && last.length > 0 ? last : resource.id
+}
 
 interface TierSelectionModalProps {
   isOpen: boolean
@@ -33,13 +49,70 @@ const TierSelectionModal: React.FC<TierSelectionModalProps> = ({
     }
   }, [isOpen, category, selectedTierSlug])
 
-  if (!category) return null
-
-  // Get all resources for a tier (including inherited resources)
+  // Get all resources for a tier (including inherited resources). Defined as a
+  // hook-safe closure (always callable, returns [] when no category) so the
+  // memo below can depend on `category` without breaking hook order.
   const getAllResourcesForTier = (tier: SpecTier): SpecResource[] => {
+    if (!category) return []
     return resolveTierResources(tier, category.tiers)
   }
 
+  // Pre-compute the selected tier's resources outside the JSX so hooks below
+  // don't re-run on every render. Empty array when no selection.
+  const selectedTierResources = useMemo<SpecResource[]>(() => {
+    if (!category || !localSelectedSlug) return []
+    const tier = category.tiers.find((t) => t.slug === localSelectedSlug)
+    return tier ? resolveTierResources(tier, category.tiers) : []
+  }, [category, localSelectedSlug])
+
+  const embedEstimateRequest = useMemo(
+    () =>
+      selectedTierResources.map((r) => ({
+        filename: resourceFilename(r),
+        sizeBytes: Math.round(r.size_mb * 1024 * 1024),
+      })),
+    [selectedTierResources]
+  )
+
+  const { data: embedEstimate, isLoading: isEstimating } = useQuery({
+    queryKey: ['embedEstimateBatch', embedEstimateRequest],
+    queryFn: () => api.estimateEmbeddingBatch(embedEstimateRequest),
+    enabled: embedEstimateRequest.length > 0,
+    staleTime: 5 * 60_000,
+  })
+
+  const { data: ingestPolicySetting } = useQuery({
+    queryKey: ['ingestPolicy'],
+    queryFn: () => api.getSetting('rag.defaultIngestPolicy'),
+  })
+
+  // System info for the disk-free side of the guardrail. Shared queryKey with
+  // the home / easy-setup pages so we don't refetch when the user already has
+  // a fresh copy in cache from a sibling component.
+  const { data: systemInfo } = useSystemInfo({ enabled: true })
+
+  // Open state for the guardrail modal — separate from the tier modal so the
+  // user sees the warning as an overlay without losing their tier selection
+  // underneath. Cancel returns to the tier modal as-is; Proceed closes both
+  // and runs the original onSelectTier path.
+  const [guardrailVerdict, setGuardrailVerdict] = useState<GuardrailVerdict | null>(null)
+
+  // Compute disk-free bytes from system info; 0 means "unknown", which the
+  // guardrail helper treats as "skip the relative-disk check".
+  // Must be declared before the `!category` early return so the hook count
+  // stays constant across renders (category transitions null → non-null when
+  // the user opens the modal).
+  const freeBytes = useMemo<number>(() => {
+    const primary = getPrimaryDiskInfo(systemInfo?.disk, systemInfo?.fsSize)
+    if (!primary) return 0
+    return Math.max(0, primary.totalSize - primary.totalUsed)
+  }, [systemInfo])
+
+  const ingestPolicy: 'Always' | 'Manual' =
+    ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always'
+
+  if (!category) return null
+
   const getTierTotalSize = (tier: SpecTier): number => {
     return getAllResourcesForTier(tier).reduce((acc, r) => acc + r.size_mb * 1024 * 1024, 0)
   }
@@ -53,17 +126,43 @@ const TierSelectionModal: React.FC<TierSelectionModalProps> = ({
     }
   }
 
-  const handleSubmit = () => {
-    if (!localSelectedSlug) return
-
-    const selectedTier = category.tiers.find(t => t.slug === localSelectedSlug)
+  /**
+   * Runs the original onSelectTier-then-onClose flow. Pulled out of
+   * handleSubmit so the guardrail modal's confirm path can call it after
+   * the user has consented to the large operation.
+   */
+  const finalizeSubmit = () => {
+    if (!localSelectedSlug || !category) return
+    const selectedTier = category.tiers.find((t) => t.slug === localSelectedSlug)
     if (selectedTier) {
       onSelectTier(category, selectedTier)
     }
     onClose()
   }
 
+  const handleSubmit = () => {
+    if (!localSelectedSlug || !category) return
+
+    // Guardrail only runs when we have an estimate AND the global policy
+    // would auto-index this batch. Under Manual the user has already opted
+    // out of automatic ingestion, so the bulk-disk warning would be a false
+    // alarm — the files would just queue as pending_decision.
+    if (ingestPolicy === 'Always' && embedEstimate) {
+      const verdict = evaluateGuardrail({
+        estimateBytes: embedEstimate.totalBytes,
+        freeBytes,
+      })
+      if (verdict.trips) {
+        setGuardrailVerdict(verdict)
+        return
+      }
+    }
+
+    finalizeSubmit()
+  }
+
   return (
+    <>
     <Transition appear show={isOpen} as={Fragment}>
       <Dialog as="div" className="relative z-50" onClose={onClose}>
         <Transition.Child
@@ -203,8 +302,41 @@ const TierSelectionModal: React.FC<TierSelectionModalProps> = ({
                     })}
                   </div>
 
+                  {/* Embedding-cost preview — visible whenever a tier is
+                      selected. The estimate uses #891's ratio registry to
+                      project how much extra disk space the AI Assistant will
+                      need for these files on top of the raw downloads. */}
+                  {localSelectedSlug && embedEstimate && embedEstimate.totalBytes > 0 && (
+                    <div className="mt-4 bg-surface-secondary border border-border-subtle rounded p-3 text-sm">
+                      <div className="flex items-start gap-2">
+                        <DynamicIcon icon="IconBrain" className="w-5 h-5 text-desert-green flex-shrink-0 mt-0.5" />
+                        <div className="flex-1">
+                          <p className="text-text-primary">
+                            <span className="font-medium">+~{formatBytes(embedEstimate.totalBytes, 1)}</span>
+                            {' '}of additional storage if these are indexed for the AI Assistant
+                            {embedEstimate.hasUnknown && (
+                              <span className="text-text-muted"> (estimate excludes some files we have no prior data for)</span>
+                            )}
+                            .
+                          </p>
+                          <p className="text-text-muted text-xs mt-1">
+                            {ingestPolicy === 'Always' ? (
+                              <>
+                                Your <strong>Auto-index</strong> setting is <strong>Always</strong>, so these files will be indexed automatically once downloaded. You can change this in the Knowledge Base settings.
+                              </>
+                            ) : (
+                              <>
+                                Your <strong>Auto-index</strong> setting is <strong>Manual</strong>, so these files will sit unindexed until you opt in from the Knowledge Base settings.
+                              </>
+                            )}
+                          </p>
+                        </div>
+                      </div>
+                    </div>
+                  )}
+
                   {/* Info note */}
-                  <div className="mt-6 flex items-start gap-2 text-sm text-text-muted bg-blue-50 p-3 rounded">
+                  <div className="mt-4 flex items-start gap-2 text-sm text-text-muted bg-blue-50 p-3 rounded">
                     <IconInfoCircle size={18} className="text-blue-500 flex-shrink-0 mt-0.5" />
                     <p>
                       You can change your selection at any time. Click Submit to confirm your choice.
@@ -218,7 +350,7 @@ const TierSelectionModal: React.FC<TierSelectionModalProps> = ({
                     variant='primary'
                     size='lg'
                     onClick={handleSubmit}
-                    disabled={!localSelectedSlug}
+                    disabled={!localSelectedSlug || (embedEstimateRequest.length > 0 && isEstimating)}
                   >
                     Submit
                   </StyledButton>
@@ -229,6 +361,18 @@ const TierSelectionModal: React.FC<TierSelectionModalProps> = ({
         </div>
       </Dialog>
     </Transition>
+    {guardrailVerdict && (
+      <KbGuardrailModal
+        isOpen={true}
+        verdict={guardrailVerdict}
+        onConfirm={() => {
+          setGuardrailVerdict(null)
+          finalizeSubmit()
+        }}
+        onCancel={() => setGuardrailVerdict(null)}
+      />
+    )}
+    </>
   )
 }
 

admin/inertia/components/chat/KbPolicyPromptBanner.tsx

@@ -0,0 +1,126 @@
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
+import { usePage } from '@inertiajs/react'
+import { IconBrain } from '@tabler/icons-react'
+import api from '~/lib/api'
+import StyledButton from '~/components/StyledButton'
+import { useNotifications } from '~/context/NotificationContext'
+
+/**
+ * First-chat onboarding banner (RFC #883 Phase 3 task 12).
+ *
+ * Renders above the chat header when the scanner has seen at least one
+ * embeddable file AND the user has not yet picked a global ingest policy
+ * (`rag.defaultIngestPolicy` unset). Two buttons let the user decide once,
+ * after which the prompt never returns:
+ *
+ *   - "Index existing content" → sets policy=Always and dispatches a sync so
+ *     anything already on disk + in `pending_decision` gets queued for embed.
+ *   - "Maybe later"            → sets policy=Manual. New content waits in
+ *     `pending_decision` until the user opts in from the KB modal.
+ *
+ * The "dismiss without deciding" X is intentionally NOT here. Dismissing
+ * without setting policy would make the banner reappear on every visit until
+ * a choice is recorded — annoying. The two action buttons each set policy,
+ * and the user can change their mind any time via the Always/Manual radio in
+ * the KB modal.
+ */
+export default function KbPolicyPromptBanner() {
+  const queryClient = useQueryClient()
+  const { addNotification } = useNotifications()
+  // Inertia injects `aiAssistantName` as a shared page prop on chat-mounted
+  // pages so the banner pulls the user-set name when surfaced. Default to
+  // "AI Assistant" when accessed outside that context (no-op for chat pages,
+  // but keeps the component safe for future reuse elsewhere).
+  const aiAssistantName =
+    usePage<{ aiAssistantName?: string }>().props?.aiAssistantName || 'AI Assistant'
+
+  const { data: promptState } = useQuery({
+    queryKey: ['kbPolicyPromptState'],
+    queryFn: () => api.getKbPolicyPromptState(),
+    staleTime: Infinity,
+  })
+
+  const indexNowMutation = useMutation({
+    mutationFn: async () => {
+      await api.updateSetting('rag.defaultIngestPolicy', 'Always')
+      await api.syncRAGStorage()
+    },
+    onSuccess: () => {
+      addNotification({
+        type: 'success',
+        message: `${aiAssistantName} will index your existing content. You can track progress in the Knowledge Base panel.`,
+      })
+      queryClient.invalidateQueries({ queryKey: ['kbPolicyPromptState'] })
+      queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] })
+      queryClient.invalidateQueries({ queryKey: ['embed-jobs'] })
+      queryClient.invalidateQueries({ queryKey: ['storedFiles'] })
+    },
+    onError: (error: any) => {
+      addNotification({
+        type: 'error',
+        message: error?.message || 'Could not start indexing. Try again from the Knowledge Base panel.',
+      })
+    },
+  })
+
+  const maybeLaterMutation = useMutation({
+    mutationFn: () => api.updateSetting('rag.defaultIngestPolicy', 'Manual'),
+    onSuccess: () => {
+      addNotification({
+        type: 'success',
+        message: 'Your content stays unindexed for now. You can opt in any time from the Knowledge Base panel.',
+      })
+      queryClient.invalidateQueries({ queryKey: ['kbPolicyPromptState'] })
+      queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] })
+    },
+    onError: (error: any) => {
+      addNotification({
+        type: 'error',
+        message: error?.message || 'Could not save your choice. Try again.',
+      })
+    },
+  })
+
+  if (!promptState?.shouldPrompt) return null
+
+  const fileCount = promptState.totalFiles
+  const isBusy = indexNowMutation.isPending || maybeLaterMutation.isPending
+
+  return (
+    <div className="px-6 py-3 bg-blue-50 dark:bg-blue-950/30 border-b border-blue-200 dark:border-blue-800 flex-shrink-0">
+      <div className="flex items-center gap-3">
+        <IconBrain className="h-6 w-6 text-blue-600 dark:text-blue-300 flex-shrink-0" />
+        <div className="flex-1 min-w-0">
+          <p className="text-sm text-text-primary">
+            <strong>
+              {fileCount === 1
+                ? `Index your existing file for ${aiAssistantName}?`
+                : `Index your ${fileCount.toLocaleString()} existing files for ${aiAssistantName}?`}
+            </strong>
+            {' '}When indexed, {aiAssistantName} can reference them while answering your questions.
+          </p>
+        </div>
+        <div className="flex gap-2 flex-shrink-0">
+          <StyledButton
+            onClick={() => indexNowMutation.mutate()}
+            variant="primary"
+            size="sm"
+            disabled={isBusy}
+            loading={indexNowMutation.isPending}
+          >
+            Index existing content
+          </StyledButton>
+          <StyledButton
+            onClick={() => maybeLaterMutation.mutate()}
+            variant="ghost"
+            size="sm"
+            disabled={isBusy}
+            loading={maybeLaterMutation.isPending}
+          >
+            Maybe later
+          </StyledButton>
+        </div>
+      </div>
+    </div>
+  )
+}

admin/inertia/components/chat/KnowledgeBaseModal.tsx

@@ -1,24 +1,96 @@
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
-import { useRef, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import FileUploader from '~/components/file-uploader'
 import StyledButton from '~/components/StyledButton'
+import type { DynamicIconName } from '~/lib/icons'
 import StyledSectionHeader from '~/components/StyledSectionHeader'
 import StyledTable from '~/components/StyledTable'
 import { useNotifications } from '~/context/NotificationContext'
 import api from '~/lib/api'
+import {
+  groupAndSortKbFiles,
+  type KbFileGroup,
+} from '~/lib/kb_file_grouping'
+import type { KbIngestStateValue } from '../../../types/kb_ingest_state'
 import { IconX } from '@tabler/icons-react'
 import { useModals } from '~/context/ModalContext'
 import StyledModal from '../StyledModal'
 import ActiveEmbedJobs from '~/components/ActiveEmbedJobs'
+import { SERVICE_NAMES } from '../../../constants/service_names'
 
 interface KnowledgeBaseModalProps {
   aiAssistantName?: string
   onClose: () => void
 }
 
-function sourceToDisplayName(source: string): string {
-  const parts = source.split(/[/\\]/)
-  return parts[parts.length - 1]
+/**
+ * Compact label for the per-row ingestion state. Files that exist in Qdrant
+ * with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883
+ * installs whose chunks are real, so we display them as "Indexed" rather than
+ * surfacing the absent-row detail. Admin-docs group has no pill (the "Managed
+ * by NOMAD" message in the action column carries the same signal).
+ */
+function renderStatePill(record: KbFileGroup): React.ReactNode {
+  if (record.bucket === 'admin_docs') return null
+  const effective: KbIngestStateValue = record.state ?? 'indexed'
+
+  const base = 'inline-flex items-center text-xs font-medium rounded px-2 py-0.5 border'
+  switch (effective) {
+    case 'indexed':
+      return (
+        <span className={`${base} text-green-700 bg-green-50 border-green-200 dark:text-green-300 dark:bg-green-950/40 dark:border-green-800`}>
+          Indexed
+        </span>
+      )
+    case 'pending_decision':
+    case 'browse_only':
+      return (
+        <span className={`${base} text-text-secondary bg-surface-secondary border-border-subtle`}>
+          Not Indexed
+        </span>
+      )
+    case 'failed':
+      return (
+        <span className={`${base} text-red-700 bg-red-50 border-red-200 dark:text-red-300 dark:bg-red-950/40 dark:border-red-800`}>
+          Failed
+        </span>
+      )
+    case 'stalled':
+      return (
+        <span className={`${base} text-amber-700 bg-amber-50 border-amber-200 dark:text-amber-300 dark:bg-amber-950/40 dark:border-amber-800`}>
+          Stalled
+        </span>
+      )
+  }
+}
+
+type RowAction =
+  | { kind: 'index'; label: string; force: boolean; variant: 'primary'; icon: DynamicIconName }
+  | { kind: 'reembed'; label: string; force: true; variant: 'secondary'; icon: DynamicIconName }
+
+/**
+ * Pick the single adaptive per-row action button. Returns null when no action
+ * makes sense for the current state (e.g. healthy indexed file with no
+ * warnings — bulk Re-embed All covers that case). `hasWarnings` lets us
+ * surface a Re-embed affordance specifically when a file *looks* indexed but
+ * has zero chunks or a stalled-mid-ingestion warning attached.
+ */
+function pickRowAction(record: KbFileGroup, hasWarnings: boolean): RowAction | null {
+  if (record.bucket === 'admin_docs') return null
+  const effective: KbIngestStateValue = record.state ?? 'indexed'
+  switch (effective) {
+    case 'indexed':
+      return hasWarnings
+        ? { kind: 'reembed', label: 'Re-embed', force: true, variant: 'secondary', icon: 'IconRefreshAlert' }
+        : null
+    case 'pending_decision':
+      return { kind: 'index', label: 'Index', force: false, variant: 'primary', icon: 'IconDownload' }
+    case 'browse_only':
+      return { kind: 'index', label: 'Index', force: true, variant: 'primary', icon: 'IconDownload' }
+    case 'failed':
+    case 'stalled':
+      return { kind: 'index', label: 'Retry', force: true, variant: 'primary', icon: 'IconRefresh' }
+  }
 }
 
 export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", onClose }: KnowledgeBaseModalProps) {
@@ -26,16 +98,75 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
   const [files, setFiles] = useState<File[]>([])
   const [isUploading, setIsUploading] = useState(false)
   const [confirmDeleteSource, setConfirmDeleteSource] = useState<string | null>(null)
+  const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null)
+  const [bulkMode, setBulkMode] = useState<null | 'reembed' | 'reset'>(null)
+  const [resetTyped, setResetTyped] = useState('')
   const fileUploaderRef = useRef<React.ComponentRef<typeof FileUploader>>(null)
   const { openModal, closeModal } = useModals()
   const queryClient = useQueryClient()
 
+  const [isStartingQdrant, setIsStartingQdrant] = useState(false)
+
+  const { data: healthStatus } = useQuery({
+    queryKey: ['qdrantHealth'],
+    queryFn: () => api.checkRAGHealth(),
+    refetchInterval: isStartingQdrant ? 3_000 : 30_000,
+  })
+  const qdrantOffline = healthStatus?.online === false
+
+  useEffect(() => {
+    if (!qdrantOffline) setIsStartingQdrant(false)
+  }, [qdrantOffline])
+
   const { data: storedFiles = [], isLoading: isLoadingFiles } = useQuery({
     queryKey: ['storedFiles'],
     queryFn: () => api.getStoredRAGFiles(),
     select: (data) => data || [],
   })
 
+  // Per-file conditional warnings (RFC #883 §6). `ok: false` means the
+  // computation itself failed (Qdrant/DB/FS) — distinct from `ok: true` with
+  // an empty map, which means everything is healthy. We surface the failure
+  // explicitly so a silent backend failure doesn't masquerade as health.
+  const { data: warningsResult } = useQuery({
+    queryKey: ['kbFileWarnings'],
+    queryFn: () => api.getKbFileWarnings(),
+    refetchInterval: 30_000,
+  })
+  const fileWarnings = warningsResult?.warnings ?? {}
+  const warningsUnavailable = warningsResult !== undefined && warningsResult.ok === false
+
+  // Global auto-index policy. KVStore returns `null` for an unset key, which
+  // we treat as 'Always' for backward compatibility with installs that predate
+  // this UI. The user can opt into Manual mode from the toggle below.
+  const { data: ingestPolicySetting } = useQuery({
+    queryKey: ['ingestPolicy'],
+    queryFn: () => api.getSetting('rag.defaultIngestPolicy'),
+  })
+  const ingestPolicy: 'Always' | 'Manual' =
+    ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always'
+
+  const updateIngestPolicyMutation = useMutation({
+    mutationFn: (policy: 'Always' | 'Manual') =>
+      api.updateSetting('rag.defaultIngestPolicy', policy),
+    onSuccess: (_data, policy) => {
+      queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] })
+      addNotification({
+        type: 'success',
+        message:
+          policy === 'Always'
+            ? 'New content will be auto-indexed for AI.'
+            : 'New content will wait for you to opt in.',
+      })
+    },
+    onError: (error: any) => {
+      addNotification({
+        type: 'error',
+        message: error?.message || 'Failed to update indexing policy.',
+      })
+    },
+  })
+
   const uploadMutation = useMutation({
     mutationFn: (file: File) => api.uploadDocument(file),
   })
@@ -53,6 +184,25 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
     },
   })
 
+  const embedMutation = useMutation({
+    mutationFn: ({ source, force }: { source: string; force: boolean }) =>
+      api.embedSingleRAGFile(source, force),
+    onSuccess: (data) => {
+      addNotification({
+        type: 'success',
+        message: data?.message || 'File queued for embedding.',
+      })
+      setConfirmReembed(null)
+      queryClient.invalidateQueries({ queryKey: ['storedFiles'] })
+      queryClient.invalidateQueries({ queryKey: ['embed-jobs'] })
+      queryClient.invalidateQueries({ queryKey: ['kbFileWarnings'] })
+    },
+    onError: (error: any) => {
+      addNotification({ type: 'error', message: error?.message || 'Failed to queue file.' })
+      setConfirmReembed(null)
+    },
+  })
+
   const cleanupFailedMutation = useMutation({
     mutationFn: () => api.cleanupFailedEmbedJobs(),
     onSuccess: (data) => {
@@ -64,6 +214,17 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
     },
   })
 
+  const startQdrantMutation = useMutation({
+    mutationFn: () => api.affectService(SERVICE_NAMES.QDRANT, 'start'),
+    onSuccess: () => {
+      setIsStartingQdrant(true)
+      queryClient.invalidateQueries({ queryKey: ['qdrantHealth'] })
+    },
+    onError: (error: any) => {
+      addNotification({ type: 'error', message: error?.message || 'Failed to start Qdrant.' })
+    },
+  })
+
   const syncMutation = useMutation({
     mutationFn: () => api.syncRAGStorage(),
     onSuccess: (data) => {
@@ -80,6 +241,44 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
     },
   })
 
+  const reembedMutation = useMutation({
+    mutationFn: () => api.reembedAllRAG(),
+    onSuccess: (data) => {
+      addNotification({
+        type: data?.success ? 'success' : 'error',
+        message: data?.message || 'Re-embed completed.',
+      })
+      queryClient.invalidateQueries({ queryKey: ['storedFiles'] })
+      queryClient.invalidateQueries({ queryKey: ['embed-jobs'] })
+      setBulkMode(null)
+      setResetTyped('')
+    },
+    onError: () => {
+      addNotification({ type: 'error', message: 'Failed to re-embed knowledge base.' })
+      setBulkMode(null)
+    },
+  })
+
+  const resetMutation = useMutation({
+    mutationFn: () => api.resetAndRebuildRAG(),
+    onSuccess: (data) => {
+      addNotification({
+        type: data?.success ? 'success' : 'error',
+        message: data?.message || 'Reset complete.',
+      })
+      queryClient.invalidateQueries({ queryKey: ['storedFiles'] })
+      queryClient.invalidateQueries({ queryKey: ['embed-jobs'] })
+      setBulkMode(null)
+      setResetTyped('')
+    },
+    onError: () => {
+      addNotification({ type: 'error', message: 'Failed to reset knowledge base.' })
+      setBulkMode(null)
+    },
+  })
+
+  const bulkBusy = reembedMutation.isPending || resetMutation.isPending
+
   const handleUpload = async () => {
     if (files.length === 0) return
     setIsUploading(true)
@@ -149,6 +348,22 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
           </button>
         </div>
         <div className="overflow-y-auto flex-1 p-6">
+          {qdrantOffline && (
+            <div className="mb-4 p-4 bg-red-50 border border-red-200 rounded-lg text-red-700 text-sm dark:bg-red-950 dark:border-red-800 dark:text-red-300 flex items-center justify-between gap-4">
+              <span>
+                <strong>Knowledge Base unavailable:</strong> The Qdrant vector database is offline.
+              </span>
+              <StyledButton
+                variant="danger"
+                size="sm"
+                onClick={() => startQdrantMutation.mutate()}
+                loading={startQdrantMutation.isPending || isStartingQdrant}
+                disabled={startQdrantMutation.isPending || isStartingQdrant}
+              >
+                {isStartingQdrant ? 'Starting…' : 'Start Qdrant'}
+              </StyledButton>
+            </div>
+          )}
           <div className="bg-surface-primary rounded-lg border shadow-md overflow-hidden">
             <div className="p-6">
               <FileUploader
@@ -165,7 +380,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
                   size="lg"
                   icon="IconUpload"
                   onClick={handleUpload}
-                  disabled={files.length === 0 || isUploading}
+                  disabled={files.length === 0 || isUploading || qdrantOffline}
                   loading={isUploading}
                 >
                   Upload
@@ -227,6 +442,48 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
               </div>
             </div>
           </div>
+          <div className="my-8 p-4 rounded-lg border border-border-subtle bg-surface-secondary">
+            <div className="flex flex-wrap items-center justify-between gap-3">
+              <div className="flex-1 min-w-[14rem]">
+                <p className="text-sm font-medium text-text-primary">
+                  Auto-index new content for AI?
+                </p>
+                <p className="text-xs text-text-muted mt-1">
+                  Indexed content typically uses 5–10× the original file size on disk.
+                  Changes apply to new content added after this setting changes.
+                </p>
+              </div>
+              <div
+                role="radiogroup"
+                aria-label="Ingest policy"
+                className="inline-flex rounded-md overflow-hidden border border-border-subtle"
+              >
+                {(['Always', 'Manual'] as const).map((option) => {
+                  const isActive = ingestPolicy === option
+                  return (
+                    <button
+                      key={option}
+                      type="button"
+                      role="radio"
+                      aria-checked={isActive}
+                      onClick={() =>
+                        !isActive && updateIngestPolicyMutation.mutate(option)
+                      }
+                      disabled={updateIngestPolicyMutation.isPending}
+                      className={`px-4 py-2 text-sm font-medium transition-colors ${
+                        isActive
+                          ? 'bg-desert-green text-white'
+                          : 'bg-surface-primary text-text-secondary hover:bg-surface-tertiary'
+                      } ${updateIngestPolicyMutation.isPending ? 'opacity-50 cursor-not-allowed' : ''}`}
+                    >
+                      {option}
+                    </button>
+                  )
+                })}
+              </div>
+            </div>
+          </div>
+
           <div className="my-8">
             <div className="flex items-center justify-between mb-4">
               <StyledSectionHeader title="Processing Queue" className="!mb-0" />
@@ -236,7 +493,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
                 icon="IconTrash"
                 onClick={() => cleanupFailedMutation.mutate()}
                 loading={cleanupFailedMutation.isPending}
-                disabled={cleanupFailedMutation.isPending}
+                disabled={cleanupFailedMutation.isPending || qdrantOffline}
               >
                 Clean Up Failed
               </StyledButton>
@@ -245,20 +502,54 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
           </div>
 
           <div className="my-12">
-            <div className='flex items-center justify-between mb-6'>
+            <div className='flex items-center justify-between mb-6 gap-2 flex-wrap'>
               <StyledSectionHeader title="Stored Knowledge Base Files" className='!mb-0' />
-              <StyledButton
-                variant="secondary"
-                size="md"
-                icon='IconRefresh'
-                onClick={handleConfirmSync}
-                disabled={syncMutation.isPending || isUploading}
-                loading={syncMutation.isPending || isUploading}
-              >
-                Sync Storage
-              </StyledButton>
+              <div className="flex items-center gap-2 flex-wrap">
+                <StyledButton
+                  variant="danger"
+                  size="md"
+                  icon='IconAlertTriangle'
+                  onClick={() => { setResetTyped(''); setBulkMode('reset') }}
+                  disabled={isUploading || qdrantOffline || bulkBusy}
+                  loading={resetMutation.isPending}
+                  title="Drop the entire embeddings collection and re-embed everything from scratch. Permanently removes vectors for files no longer on disk. Destructive: requires typing RESET to confirm."
+                >
+                  Reset & Rebuild
+                </StyledButton>
+                <StyledButton
+                  variant="secondary"
+                  size="md"
+                  icon='IconRefreshAlert'
+                  onClick={() => setBulkMode('reembed')}
+                  disabled={isUploading || qdrantOffline || bulkBusy || storedFiles.length === 0}
+                  loading={reembedMutation.isPending}
+                  title="Re-embed every file on disk, replacing existing vectors file-by-file. Vectors for files no longer on disk are preserved. Use this if the chunker or embedding model has changed."
+                >
+                  Re-embed All
+                </StyledButton>
+                <StyledButton
+                  variant="secondary"
+                  size="md"
+                  icon='IconRefresh'
+                  onClick={handleConfirmSync}
+                  disabled={syncMutation.isPending || isUploading || qdrantOffline || bulkBusy}
+                  loading={syncMutation.isPending || isUploading}
+                  title="Scan storage for new files and queue any that haven't been embedded yet. Safe to run anytime; won't touch already-embedded content."
+                >
+                  Sync Storage
+                </StyledButton>
+
+              </div>
             </div>
-            <StyledTable<{ source: string }>
+            {warningsUnavailable && (
+              <div className="mb-4 inline-flex items-center gap-2 text-xs text-amber-700 dark:text-amber-300 bg-amber-50 dark:bg-amber-950/40 border border-amber-200 dark:border-amber-800 rounded px-3 py-2">
+                <span aria-hidden="true">⚠</span>
+                <span>
+                  File warnings unavailable — couldn't read storage state. Retrying…
+                </span>
+              </div>
+            )}
+            <StyledTable<KbFileGroup>
               className="font-semibold"
               rowLines={true}
               columns={[
@@ -266,13 +557,60 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
                   accessor: 'source',
                   title: 'File Name',
                   render(record) {
-                    return <span className="text-text-primary">{sourceToDisplayName(record.source)}</span>
+                    const warnings = fileWarnings[record.source] ?? []
+                    const pill = renderStatePill(record)
+                    return (
+                      <div className="flex flex-col gap-1.5">
+                        <span className="text-text-primary">
+                          {record.displayName}
+                        </span>
+                        {(pill || warnings.length > 0) && (
+                          <div className="flex flex-wrap items-center gap-1.5">
+                            {pill}
+                            {warnings.map((w, i) => (
+                              <span
+                                key={i}
+                                className="inline-flex items-center gap-1.5 self-start text-xs text-amber-700 dark:text-amber-300 bg-amber-50 dark:bg-amber-950/40 border border-amber-200 dark:border-amber-800 rounded px-2 py-0.5"
+                              >
+                                <span aria-hidden="true">⚠</span>
+                                {w.kind === 'zero_chunks' && (
+                                  <span>
+                                    Embedded 0 chunks — this file has no text content.
+                                    AI Assistant cannot reference it.
+                                  </span>
+                                )}
+                                {w.kind === 'partial_stall' && (
+                                  <span>
+                                    Only {w.chunksEmbedded.toLocaleString()} of est.{' '}
+                                    {w.chunksExpected.toLocaleString()} chunks embedded —
+                                    ingestion may have stalled.
+                                  </span>
+                                )}
+                              </span>
+                            ))}
+                          </div>
+                        )}
+                      </div>
+                    )
                   },
                 },
                 {
                   accessor: 'source',
                   title: '',
                   render(record) {
+                    // Admin docs are auto-discovered and managed by NOMAD itself —
+                    // deleting one would just be re-embedded on the next sync, so
+                    // we surface them as informational only and hide Delete.
+                    if (record.bucket === 'admin_docs') {
+                      return (
+                        <div className="flex justify-end">
+                          <span className="text-sm text-text-muted italic">
+                            Managed by NOMAD
+                          </span>
+                        </div>
+                      )
+                    }
+
                     const isConfirming = confirmDeleteSource === record.source
                     const isDeleting = deleteMutation.isPending && confirmDeleteSource === record.source
                     if (isConfirming) {
@@ -298,14 +636,38 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
                         </div>
                       )
                     }
+
+                    const warnings = fileWarnings[record.source] ?? []
+                    const action = pickRowAction(record, warnings.length > 0)
+                    const actionPendingForThisRow =
+                      embedMutation.isPending && embedMutation.variables?.source === record.source
+
                     return (
-                      <div className="flex justify-end">
+                      <div className="flex justify-end items-center gap-2">
+                        {action && (
+                          <StyledButton
+                            variant={action.variant}
+                            size="sm"
+                            icon={action.icon}
+                            onClick={() => {
+                              if (action.kind === 'reembed') {
+                                setConfirmReembed({ source: record.source, displayName: record.displayName })
+                              } else {
+                                embedMutation.mutate({ source: record.source, force: action.force })
+                              }
+                            }}
+                            disabled={qdrantOffline || deleteMutation.isPending || embedMutation.isPending}
+                            loading={actionPendingForThisRow}
+                          >
+                            {action.label}
+                          </StyledButton>
+                        )}
                         <StyledButton
                           variant="danger"
                           size="sm"
                           icon="IconTrash"
                           onClick={() => setConfirmDeleteSource(record.source)}
-                          disabled={deleteMutation.isPending}
+                          disabled={deleteMutation.isPending || embedMutation.isPending}
                           loading={deleteMutation.isPending && confirmDeleteSource === record.source}
                         >Delete</StyledButton>
                       </div>
@@ -313,12 +675,138 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
                   },
                 },
               ]}
-              data={storedFiles.map((source) => ({ source }))}
+              data={groupAndSortKbFiles(storedFiles)}
               loading={isLoadingFiles}
             />
           </div>
         </div>
       </div>
+
+      {bulkMode === 'reembed' && (
+        <StyledModal
+          title='Re-embed All Documents?'
+          open={true}
+          confirmText={reembedMutation.isPending ? 'Re-embedding…' : 'Re-embed All'}
+          cancelText='Cancel'
+          confirmVariant='primary'
+          confirmLoading={reembedMutation.isPending}
+          onConfirm={() => reembedMutation.mutate()}
+          onCancel={() => setBulkMode(null)}
+        >
+          <div className='text-text-primary text-sm space-y-3 text-left'>
+            <p>
+              This will re-process every document currently in your knowledge base — about
+              <strong> {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'}</strong>.
+              For each file, NOMAD will delete the existing embeddings from Qdrant and queue a fresh
+              embedding job using the current chunking and embedding model.
+            </p>
+            <div className='rounded border border-border-subtle bg-surface-secondary p-3'>
+              <p className='font-semibold mb-1'>What this is for</p>
+              <p className='text-text-secondary'>
+                Use this when the embedding model or chunking logic has changed, or when you suspect
+                stored vectors are stale. Files on disk are <em>not</em> deleted, and any orphan
+                points whose source file is no longer present will be preserved untouched (see
+                <em> Reset &amp; Rebuild </em>if you want a fully clean slate).
+              </p>
+            </div>
+            <div className='rounded border border-amber-300 bg-amber-50 dark:bg-amber-950 dark:border-amber-800 p-3 text-amber-900 dark:text-amber-200'>
+              <p className='font-semibold mb-1'>Heads up</p>
+              <ul className='list-disc pl-5 space-y-1'>
+                <li>Embedding {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'} may take a long time, especially for large PDFs or ZIM archives.</li>
+                <li>On systems without GPU acceleration, expect sustained high CPU usage for the duration.</li>
+                <li>Knowledge Base search results may be incomplete until every file finishes re-embedding.</li>
+                <li>If embed jobs are already in progress, this action will be refused — wait for the queue to drain first.</li>
+              </ul>
+            </div>
+          </div>
+        </StyledModal>
+      )}
+
+      {bulkMode === 'reset' && (
+        <StyledModal
+          title='Reset & Rebuild Knowledge Base?'
+          open={true}
+          confirmText={resetMutation.isPending ? 'Resetting…' : 'Wipe & Rebuild'}
+          cancelText='Cancel'
+          confirmVariant='danger'
+          confirmLoading={resetMutation.isPending}
+          onConfirm={() => {
+            if (resetTyped === 'RESET') resetMutation.mutate()
+          }}
+          onCancel={() => { setBulkMode(null); setResetTyped('') }}
+        >
+          <div className='text-text-primary text-sm space-y-3 text-left'>
+            <p>
+              This will <strong>permanently delete every point</strong> in the
+              <code> nomad_knowledge_base </code>Qdrant collection and rebuild from the
+              <strong> {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'}</strong> currently
+              on disk. The collection is dropped, recreated, and every file is re-queued for embedding.
+            </p>
+            <div className='rounded border border-border-subtle bg-surface-secondary p-3'>
+              <p className='font-semibold mb-1'>How this differs from Re-embed All</p>
+              <ul className='list-disc pl-5 space-y-1 text-text-secondary'>
+                <li><strong>Re-embed All</strong> replaces vectors file-by-file. Any orphan points (vectors whose source file was deleted from disk at some point) are preserved.</li>
+                <li><strong>Reset &amp; Rebuild</strong> drops the entire collection. Orphan points are <strong>gone forever</strong>. Only files currently on disk will exist in Qdrant afterwards.</li>
+              </ul>
+            </div>
+            <div className='rounded border border-red-300 bg-red-50 dark:bg-red-950 dark:border-red-800 p-3 text-red-900 dark:text-red-200'>
+              <p className='font-semibold mb-1'>This action is destructive and cannot be undone</p>
+              <ul className='list-disc pl-5 space-y-1'>
+                <li>Knowledge Base search will be empty until embedding finishes (potentially hours on CPU-only systems).</li>
+                <li>For a few seconds during the reset, the Qdrant collection does not exist — any chat-with-RAG queries in that window may return a "collection not found" error. Avoid using chat until the rebuild has begun.</li>
+                <li>If embed jobs are already in progress, this action will be refused — wait for the queue to drain first.</li>
+              </ul>
+            </div>
+            <div>
+              <label className='block text-sm font-semibold mb-1'>
+                Type <code>RESET</code> to confirm:
+              </label>
+              <input
+                type='text'
+                value={resetTyped}
+                onChange={(e) => setResetTyped(e.target.value)}
+                placeholder='RESET'
+                autoFocus
+                className='w-full rounded border border-border-subtle bg-surface-primary px-3 py-2 text-text-primary focus:outline-none focus:ring-2 focus:ring-red-500'
+              />
+              {resetTyped.length > 0 && resetTyped !== 'RESET' && (
+                <p className='text-xs text-red-600 mt-1'>Type RESET exactly (uppercase, no spaces) to enable the confirm button.</p>
+              )}
+            </div>
+          </div>
+        </StyledModal>
+      )}
+
+      {confirmReembed && (
+        <StyledModal
+          title='Re-embed this file?'
+          open={true}
+          confirmText={embedMutation.isPending ? 'Queuing…' : 'Re-embed'}
+          cancelText='Cancel'
+          confirmVariant='primary'
+          confirmLoading={embedMutation.isPending}
+          onConfirm={() =>
+            embedMutation.mutate({ source: confirmReembed.source, force: true })
+          }
+          onCancel={() => setConfirmReembed(null)}
+        >
+          <div className='text-text-primary text-sm space-y-3 text-left'>
+            <p>
+              This will delete the existing embeddings for{' '}
+              <strong>{confirmReembed.displayName}</strong> and queue
+              a fresh embedding job. The file on disk is not touched.
+            </p>
+            <div className='rounded border border-amber-300 bg-amber-50 dark:bg-amber-950 dark:border-amber-800 p-3 text-amber-900 dark:text-amber-200'>
+              <p className='font-semibold mb-1'>Heads up</p>
+              <ul className='list-disc pl-5 space-y-1'>
+                <li>For large ZIM archives this can take a long time, especially on CPU-only systems.</li>
+                <li>Search results that referenced this file will be incomplete until the new embedding finishes.</li>
+                <li>If a job for this file is already running, the re-embed will be refused — wait for it to finish first.</li>
+              </ul>
+            </div>
+          </div>
+        </StyledModal>
+      )}
     </div>
   )
 }

admin/inertia/components/chat/index.tsx

@@ -2,6 +2,7 @@ import { useState, useCallback, useEffect, useRef, useMemo } from 'react'
 import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
 import ChatSidebar from './ChatSidebar'
 import ChatInterface from './ChatInterface'
+import KbPolicyPromptBanner from './KbPolicyPromptBanner'
 import StyledModal from '../StyledModal'
 import api from '~/lib/api'
 import { formatBytes } from '~/lib/util'
@@ -32,6 +33,8 @@ export default function Chat({
   const [activeSessionId, setActiveSessionId] = useState<string | null>(null)
   const [messages, setMessages] = useState<ChatMessage[]>([])
   const [selectedModel, setSelectedModel] = useState<string>('')
+  const [pendingModelSwitch, setPendingModelSwitch] = useState<string | null>(null)
+  const pageLoadNormalizedRef = useRef(false)
   const [isStreamingResponse, setIsStreamingResponse] = useState(false)
   const streamAbortRef = useRef<AbortController | null>(null)
 
@@ -150,6 +153,62 @@ export default function Chat({
     }
   }, [selectedModel])
 
+  // Page-load normalization: enforce the "one chat model at a time" invariant
+  // when the chat page first mounts. Anything stacked from a prior session
+  // gets `keep_alive: 0` so it can be evicted; the embedding model is exempt
+  // server-side. We wait for `selectedModel` to be populated by the
+  // first-installed / lastModel effect so the request has a target to preserve.
+  useEffect(() => {
+    if (!enabled) return
+    if (!selectedModel) return
+    if (pageLoadNormalizedRef.current) return
+    pageLoadNormalizedRef.current = true
+    api.unloadChatModels(selectedModel).catch((err) => {
+      console.warn('Failed to normalize loaded models on chat-page mount:', err)
+    })
+  }, [enabled, selectedModel])
+
+  const handleUserSelectedModel = useCallback(
+    (newModel: string) => {
+      if (newModel === selectedModel) return
+      // No active chat session yet → no conversation to lose, no popup needed.
+      // Just update the dropdown silently; the next "New Chat" will use it.
+      if (!activeSessionId) {
+        setSelectedModel(newModel)
+        return
+      }
+      // Active session: defer the actual model swap until the user confirms.
+      // Setting `pendingModelSwitch` drives the dropdown's effective value
+      // *and* opens the confirm modal — clearing it on cancel reverts the
+      // visible selection without us having to touch `selectedModel`.
+      setPendingModelSwitch(newModel)
+    },
+    [selectedModel, activeSessionId]
+  )
+
+  const handleConfirmModelSwitch = useCallback(async () => {
+    const newModel = pendingModelSwitch
+    if (!newModel) return
+    // Best-effort unload of the previously-active chat model. Fire-and-forget:
+    // Ollama queues the eviction until the runner is idle, so an in-flight
+    // request on the old model finishes cleanly. We don't await this before
+    // clearing the session — UI responsiveness wins over housekeeping.
+    api.unloadChatModels(newModel).catch((err) => {
+      console.warn('Failed to unload previous chat model:', err)
+    })
+    setSelectedModel(newModel)
+    setPendingModelSwitch(null)
+    // Clear the active session and messages — the next user message will
+    // lazily create a new session via the existing handleSendMessage path,
+    // which already calls api.createChatSession with `selectedModel`.
+    setActiveSessionId(null)
+    setMessages([])
+  }, [pendingModelSwitch])
+
+  const handleCancelModelSwitch = useCallback(() => {
+    setPendingModelSwitch(null)
+  }, [])
+
   const handleNewChat = useCallback(() => {
     // Just clear the active session and messages - don't create a session yet
     setActiveSessionId(null)
@@ -201,8 +260,19 @@ export default function Chat({
       if (sessionData?.model) {
         setSelectedModel(sessionData.model)
       }
+
+      // Enforce the one-chat-model-at-a-time invariant: ask the backend to
+      // unload anything that isn't the target session's model. Fire-and-forget;
+      // this is housekeeping. Note we pass the *session's* model here rather
+      // than reading `selectedModel`, because setSelectedModel above is async
+      // and the effect-driven page-load normalize wouldn't catch a sidebar
+      // click after the first render.
+      const targetModel = sessionData?.model ?? selectedModel ?? null
+      api.unloadChatModels(targetModel).catch((err) => {
+        console.warn('Failed to unload non-target chat models on session switch:', err)
+      })
     },
-    [installedModels, queryClient]
+    [installedModels, queryClient, selectedModel]
   )
 
   const handleSendMessage = useCallback(
@@ -351,6 +421,23 @@ export default function Chat({
   )
 
   return (
+    <>
+    {pendingModelSwitch && (
+      <StyledModal
+        title={`Switch to ${pendingModelSwitch}?`}
+        onConfirm={handleConfirmModelSwitch}
+        onCancel={handleCancelModelSwitch}
+        open={true}
+        confirmText="Switch & New Chat"
+        cancelText="Cancel"
+        confirmVariant="primary"
+      >
+        <p className="text-text-primary">
+          Switching to <strong>{pendingModelSwitch}</strong> will start a new chat. Your current
+          conversation stays available in the sidebar.
+        </p>
+      </StyledModal>
+    )}
     <div
       className={classNames(
         'flex border border-border-subtle overflow-hidden shadow-sm w-full',
@@ -366,6 +453,7 @@ export default function Chat({
         isInModal={isInModal}
       />
       <div className="flex-1 flex flex-col min-h-0">
+        <KbPolicyPromptBanner />
         <div className="px-6 py-3 border-b border-border-subtle bg-surface-secondary flex items-center justify-between h-[75px] flex-shrink-0">
           <h2 className="text-lg font-semibold text-text-primary">
             {activeSession?.title || 'New Chat'}
@@ -394,8 +482,8 @@ export default function Chat({
               ) : (
                 <select
                   id="model-select"
-                  value={selectedModel}
-                  onChange={(e) => setSelectedModel(e.target.value)}
+                  value={pendingModelSwitch ?? selectedModel}
+                  onChange={(e) => handleUserSelectedModel(e.target.value)}
                   className="px-3 py-1.5 border border-border-default rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-desert-green focus:border-transparent bg-surface-primary"
                 >
                   {installedModels.map((model) => (
@@ -431,5 +519,6 @@ export default function Chat({
         />
       </div>
     </div>
+    </>
   )
 }

admin/inertia/components/maps/CoordinateOverlay.tsx

@@ -0,0 +1,25 @@
+type CoordinateOverlayProps = {
+  latitude: number
+  longitude: number
+  x: number
+  y: number
+}
+
+export default function CoordinateOverlay({
+  latitude,
+  longitude,
+  x,
+  y,
+}: CoordinateOverlayProps) {
+  return (
+    <div
+      className="pointer-events-none absolute z-[9999] -translate-x-1/2 whitespace-nowrap rounded bg-black/75 px-2 py-1 font-mono text-[11px] text-white"
+      style={{
+        left: x,
+        top: y - 36,
+      }}
+    >
+      {latitude.toFixed(6)}, {longitude.toFixed(6)}
+    </div>
+  )
+}

admin/inertia/components/maps/MapComponent.tsx

@@ -9,43 +9,111 @@ import Map, {
 import type { MapRef, MapLayerMouseEvent } from 'react-map-gl/maplibre'
 import maplibregl from 'maplibre-gl'
 import 'maplibre-gl/dist/maplibre-gl.css'
+
 import { Protocol } from 'pmtiles'
 import { useEffect, useRef, useState, useCallback } from 'react'
 
-type ScaleUnit = 'imperial' | 'metric'
 import { useMapMarkers, PIN_COLORS } from '~/hooks/useMapMarkers'
 import type { PinColorId } from '~/hooks/useMapMarkers'
+
 import MarkerPin from './MarkerPin'
 import MarkerPanel from './MarkerPanel'
+import CoordinateOverlay from './CoordinateOverlay'
+import ScaleUnitToggle from './ScaleUnitToggle'
 
-export default function MapComponent() {
+type ScaleUnit = 'imperial' | 'metric'
+
+type MapComponentProps = {
+  isHoveringUI: boolean
+  showCoordinatesEnabled: boolean
+}
+
+export default function MapComponent({
+  isHoveringUI,
+  showCoordinatesEnabled,
+}: MapComponentProps) {
   const mapRef = useRef<MapRef>(null)
+  const animationFrameRef = useRef<number | null>(null)
+
   const { markers, addMarker, deleteMarker } = useMapMarkers()
+
+  const [isDraggingMap, setIsDraggingMap] = useState(false)
   const [placingMarker, setPlacingMarker] = useState<{ lng: number; lat: number } | null>(null)
   const [markerName, setMarkerName] = useState('')
   const [markerColor, setMarkerColor] = useState<PinColorId>('orange')
   const [selectedMarkerId, setSelectedMarkerId] = useState<number | null>(null)
+
   const [scaleUnit, setScaleUnit] = useState<ScaleUnit>(
     () => (localStorage.getItem('nomad:map-scale-unit') as ScaleUnit) || 'metric'
   )
 
-  const toggleScaleUnit = useCallback(() => {
-    setScaleUnit((prev) => {
-      const next = prev === 'metric' ? 'imperial' : 'metric'
-      localStorage.setItem('nomad:map-scale-unit', next)
-      return next
-    })
-  }, [])
+  const [cursorLngLat, setCursorLngLat] = useState<{
+    lng: number
+    lat: number
+    x: number
+    y: number
+  } | null>(null)
+
+  const [showCoordinates, setShowCoordinates] = useState(false)
 
-  // Add the PMTiles protocol to maplibre-gl
   useEffect(() => {
-    let protocol = new Protocol()
+    const protocol = new Protocol()
     maplibregl.addProtocol('pmtiles', protocol.tile)
+
     return () => {
       maplibregl.removeProtocol('pmtiles')
     }
   }, [])
 
+  useEffect(() => {
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current)
+      }
+    }
+  }, [])
+
+  const hideCoordinates = useCallback(() => {
+    setShowCoordinates(false)
+    setCursorLngLat(null)
+  }, [])
+
+  const handleScaleUnitChange = useCallback((unit: ScaleUnit) => {
+    setScaleUnit(unit)
+    localStorage.setItem('nomad:map-scale-unit', unit)
+  }, [])
+
+  const handleMouseMove = useCallback(
+    (e: MapLayerMouseEvent) => {
+      const target = e.originalEvent.target as HTMLElement | null
+
+      if (
+        !showCoordinatesEnabled ||
+        isHoveringUI ||
+        isDraggingMap ||
+        target?.closest('.maplibregl-control-container, .maplibregl-ctrl')
+      ) {
+        hideCoordinates()
+        return
+      }
+
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current)
+      }
+
+      animationFrameRef.current = requestAnimationFrame(() => {
+        setShowCoordinates(true)
+        setCursorLngLat({
+          lng: e.lngLat.lng,
+          lat: e.lngLat.lat,
+          x: e.point.x,
+          y: e.point.y,
+        })
+      })
+    },
+    [hideCoordinates, isHoveringUI, isDraggingMap, showCoordinatesEnabled]
+  )
+
   const handleMapClick = useCallback((e: MapLayerMouseEvent) => {
     setPlacingMarker({ lng: e.lngLat.lng, lat: e.lngLat.lat })
     setMarkerName('')
@@ -78,167 +146,185 @@ export default function MapComponent() {
 
   return (
     <MapProvider>
-      <Map
-        ref={mapRef}
-        reuseMaps
-        style={{
-          width: '100%',
-          height: '100vh',
+      <div
+        style={{ position: 'relative', width: '100%', height: '100vh' }}
+        onMouseLeave={() => {
+          setIsDraggingMap(false)
+          hideCoordinates()
         }}
-        mapStyle={`${window.location.protocol}//${window.location.hostname}:${window.location.port}/api/maps/styles`}
-        mapLib={maplibregl}
-        initialViewState={{
-          longitude: -101,
-          latitude: 40,
-          zoom: 3.5,
+        onMouseMoveCapture={(e) => {
+          const target = e.target as HTMLElement | null
+
+          if (
+            target?.closest(
+              '.maplibregl-control-container, .maplibregl-ctrl, .maplibregl-ctrl-group, .maplibregl-ctrl-scale'
+            )
+          ) {
+            hideCoordinates()
+          }
         }}
-        onClick={handleMapClick}
       >
-        <NavigationControl style={{ marginTop: '110px', marginRight: '36px' }} />
-        <FullscreenControl style={{ marginTop: '30px', marginRight: '36px' }} />
-        <ScaleControl position="bottom-left" maxWidth={150} unit={scaleUnit} />
-        <div style={{ position: 'absolute', bottom: '30px', left: '10px', zIndex: 2 }}>
-          <div
-            style={{
-              display: 'inline-flex',
-              borderRadius: '4px',
-              boxShadow: '0 0 0 2px rgba(0,0,0,0.1)',
-              overflow: 'hidden',
-              fontSize: '11px',
-              fontWeight: 600,
-              lineHeight: 1,
-            }}
-          >
-            <button
-              onClick={() => { if (scaleUnit !== 'metric') toggleScaleUnit() }}
-              style={{
-                background: scaleUnit === 'metric' ? '#424420' : 'white',
-                color: scaleUnit === 'metric' ? 'white' : '#666',
-                border: 'none',
-                padding: '4px 8px',
-                cursor: 'pointer',
-              }}
-            >
-              Metric
-            </button>
-            <button
-              onClick={() => { if (scaleUnit !== 'imperial') toggleScaleUnit() }}
-              style={{
-                background: scaleUnit === 'imperial' ? '#424420' : 'white',
-                color: scaleUnit === 'imperial' ? 'white' : '#666',
-                border: 'none',
-                padding: '4px 8px',
-                cursor: 'pointer',
-              }}
-            >
-              Imperial
-            </button>
-          </div>
-        </div>
+        <Map
+          ref={mapRef}
+          reuseMaps
+          style={{ width: '100%', height: '100vh' }}
+          cursor={isDraggingMap ? 'grabbing' : 'crosshair'}
+          mapStyle={`${window.location.protocol}//${window.location.hostname}:${window.location.port}/api/maps/styles`}
+          mapLib={maplibregl}
+          initialViewState={{
+            longitude: -101,
+            latitude: 40,
+            zoom: 3.5,
+          }}
+          onMouseDown={() => {
+            setIsDraggingMap(true)
+            hideCoordinates()
+          }}
+          onMouseUp={() => {
+            setIsDraggingMap(false)
+          }}
+          onDragStart={() => {
+            setIsDraggingMap(true)
+            hideCoordinates()
+          }}
+          onDragEnd={() => {
+            setIsDraggingMap(false)
+            hideCoordinates()
+          }}
+          onClick={handleMapClick}
+          onMouseMove={handleMouseMove}
+          onMouseLeave={hideCoordinates}
+        >
+          <NavigationControl style={{ marginTop: '110px', marginRight: '36px' }} />
+          <FullscreenControl style={{ marginTop: '30px', marginRight: '36px' }} />
+          <ScaleControl position="bottom-left" maxWidth={150} unit={scaleUnit} />
 
-        {/* Existing markers */}
-        {markers.map((marker) => (
-          <Marker
-            key={marker.id}
-            longitude={marker.longitude}
-            latitude={marker.latitude}
-            anchor="bottom"
-            onClick={(e) => {
-              e.originalEvent.stopPropagation()
-              setSelectedMarkerId(marker.id === selectedMarkerId ? null : marker.id)
-              setPlacingMarker(null)
-            }}
-          >
-            <MarkerPin
-              color={PIN_COLORS.find((c) => c.id === marker.color)?.hex}
-              active={marker.id === selectedMarkerId}
+          {showCoordinates && cursorLngLat && (
+            <CoordinateOverlay
+              latitude={cursorLngLat.lat}
+              longitude={cursorLngLat.lng}
+              x={cursorLngLat.x}
+              y={cursorLngLat.y}
             />
-          </Marker>
-        ))}
+          )}
 
-        {/* Popup for selected marker */}
-        {selectedMarker && (
-          <Popup
-            longitude={selectedMarker.longitude}
-            latitude={selectedMarker.latitude}
-            anchor="bottom"
-            offset={[0, -36] as [number, number]}
-            onClose={() => setSelectedMarkerId(null)}
-            closeOnClick={false}
-          >
-            <div className="text-sm font-medium">{selectedMarker.name}</div>
-          </Popup>
-        )}
+          <ScaleUnitToggle
+            scaleUnit={scaleUnit}
+            onChange={handleScaleUnitChange}
+            onMouseEnter={hideCoordinates}
+          />
 
-        {/* Popup for placing a new marker */}
-        {placingMarker && (
-          <Popup
-            longitude={placingMarker.lng}
-            latitude={placingMarker.lat}
-            anchor="bottom"
-            onClose={() => setPlacingMarker(null)}
-            closeOnClick={false}
-          >
-            <div className="p-1">
-              <input
-                autoFocus
-                type="text"
-                placeholder="Name this location"
-                value={markerName}
-                onChange={(e) => setMarkerName(e.target.value)}
-                onKeyDown={(e) => {
-                  if (e.key === 'Enter') handleSaveMarker()
-                  if (e.key === 'Escape') setPlacingMarker(null)
-                }}
-                className="block w-full rounded border border-gray-300 px-2 py-1 text-sm placeholder:text-gray-400 focus:outline-none focus:border-gray-500"
+          {markers.map((marker) => (
+            <Marker
+              key={marker.id}
+              longitude={marker.longitude}
+              latitude={marker.latitude}
+              anchor="bottom"
+              onClick={(e) => {
+                e.originalEvent.stopPropagation()
+                setSelectedMarkerId(marker.id === selectedMarkerId ? null : marker.id)
+                setPlacingMarker(null)
+              }}
+            >
+              <MarkerPin
+                color={PIN_COLORS.find((c) => c.id === marker.color)?.hex}
+                active={marker.id === selectedMarkerId}
               />
-              <div className="mt-1.5 flex gap-1 items-center">
-                {PIN_COLORS.map((c) => (
-                  <button
-                    key={c.id}
-                    onClick={() => setMarkerColor(c.id)}
-                    title={c.label}
-                    className="rounded-full p-0.5 transition-transform"
-                    style={{
-                      outline: markerColor === c.id ? `2px solid ${c.hex}` : '2px solid transparent',
-                      outlineOffset: '1px',
-                    }}
-                  >
-                    <div
-                      className="w-4 h-4 rounded-full"
-                      style={{ backgroundColor: c.hex }}
-                    />
-                  </button>
-                ))}
-              </div>
-              <div className="mt-1.5 flex gap-1.5 justify-end">
-                <button
-                  onClick={() => setPlacingMarker(null)}
-                  className="text-xs text-gray-500 hover:text-gray-700 px-2 py-1 rounded transition-colors"
-                >
-                  Cancel
-                </button>
-                <button
-                  onClick={handleSaveMarker}
-                  disabled={!markerName.trim()}
-                  className="text-xs bg-[#424420] text-white rounded px-2.5 py-1 hover:bg-[#525530] disabled:opacity-40 transition-colors"
-                >
-                  Save
-                </button>
-              </div>
-            </div>
-          </Popup>
-        )}
-      </Map>
+            </Marker>
+          ))}
 
-      {/* Marker panel overlay */}
-      <MarkerPanel
-        markers={markers}
-        onDelete={handleDeleteMarker}
-        onFlyTo={handleFlyTo}
-        onSelect={setSelectedMarkerId}
-        selectedMarkerId={selectedMarkerId}
-      />
+          {selectedMarker && (
+            <Popup
+              longitude={selectedMarker.longitude}
+              latitude={selectedMarker.latitude}
+              anchor="bottom"
+              offset={[0, -36]}
+              onClose={() => setSelectedMarkerId(null)}
+              closeOnClick={false}
+            >
+              <div className="text-sm font-medium">{selectedMarker.name}</div>
+              {selectedMarker.notes && selectedMarker.notes.trim() && (
+                <div className="mt-1 text-xs text-desert-stone-dark whitespace-pre-wrap break-words max-w-[240px]">
+                  {selectedMarker.notes}
+                </div>
+              )}
+            </Popup>
+          )}
+
+          {placingMarker && (
+            <Popup
+              longitude={placingMarker.lng}
+              latitude={placingMarker.lat}
+              anchor="bottom"
+              onClose={() => setPlacingMarker(null)}
+              closeOnClick={false}
+            >
+              <div onMouseEnter={hideCoordinates} className="p-1">
+                <input
+                  autoFocus
+                  type="text"
+                  placeholder="Name this location"
+                  value={markerName}
+                  onChange={(e) => setMarkerName(e.target.value)}
+                  onKeyDown={(e) => {
+                    if (e.key === 'Enter') handleSaveMarker()
+                    if (e.key === 'Escape') setPlacingMarker(null)
+                  }}
+                  className="block w-full rounded border border-gray-300 px-2 py-1 text-sm placeholder:text-gray-400 focus:outline-none focus:border-gray-500"
+                />
+
+                <div className="mt-1.5 flex gap-1 items-center">
+                  {PIN_COLORS.map((c) => (
+                    <button
+                      key={c.id}
+                      type="button"
+                      onClick={() => setMarkerColor(c.id)}
+                      title={c.label}
+                      className="rounded-full p-0.5 transition-transform"
+                      style={{
+                        outline:
+                          markerColor === c.id ? `2px solid ${c.hex}` : '2px solid transparent',
+                        outlineOffset: '1px',
+                      }}
+                    >
+                      <div className="w-4 h-4 rounded-full" style={{ backgroundColor: c.hex }} />
+                    </button>
+                  ))}
+                </div>
+
+                <div className="mt-1.5 flex gap-1.5 justify-end">
+                  <button
+                    type="button"
+                    onClick={() => setPlacingMarker(null)}
+                    className="text-xs text-gray-500 hover:text-gray-700 px-2 py-1 rounded transition-colors"
+                  >
+                    Cancel
+                  </button>
+
+                  <button
+                    type="button"
+                    onClick={handleSaveMarker}
+                    disabled={!markerName.trim()}
+                    className="text-xs bg-[#424420] text-white rounded px-2.5 py-1 hover:bg-[#525530] disabled:opacity-40 transition-colors"
+                  >
+                    Save
+                  </button>
+                </div>
+              </div>
+            </Popup>
+          )}
+        </Map>
+      </div>
+
+      <div onMouseEnter={hideCoordinates}>
+        <MarkerPanel
+          markers={markers}
+          onDelete={handleDeleteMarker}
+          onFlyTo={handleFlyTo}
+          onSelect={setSelectedMarkerId}
+          selectedMarkerId={selectedMarkerId}
+        />
+      </div>
     </MapProvider>
   )
 }

admin/inertia/components/maps/ScaleUnitToggle.tsx

@@ -0,0 +1,46 @@
+type ScaleUnit = 'imperial' | 'metric'
+
+type ScaleUnitToggleProps = {
+  scaleUnit: ScaleUnit
+  onChange: (unit: ScaleUnit) => void
+  onMouseEnter?: () => void
+}
+
+export default function ScaleUnitToggle({
+  scaleUnit,
+  onChange,
+  onMouseEnter,
+}: ScaleUnitToggleProps) {
+  return (
+    <div
+      className="absolute bottom-[30px] left-[10px] z-[2]"
+      onMouseEnter={onMouseEnter}
+    >
+      <div className="inline-flex overflow-hidden rounded text-[11px] font-semibold leading-none shadow-[0_0_0_2px_rgba(0,0,0,0.1)]">
+        <button
+          type="button"
+          onClick={() => onChange('metric')}
+          className="border-0 px-2 py-1"
+          style={{
+            background: scaleUnit === 'metric' ? '#424420' : 'white',
+            color: scaleUnit === 'metric' ? 'white' : '#666',
+          }}
+        >
+          Metric
+        </button>
+
+        <button
+          type="button"
+          onClick={() => onChange('imperial')}
+          className="border-0 px-2 py-1"
+          style={{
+            background: scaleUnit === 'imperial' ? '#424420' : 'white',
+            color: scaleUnit === 'imperial' ? 'white' : '#666',
+          }}
+        >
+          Imperial
+        </button>
+      </div>
+    </div>
+  )
+}

admin/inertia/hooks/useDownloads.ts

@@ -19,8 +19,9 @@ const useDownloads = (props: useDownloadsProps) => {
     queryFn: () => api.listDownloadJobs(props.filetype),
     refetchInterval: (query) => {
       const data = query.state.data
-      // Only poll when there are active downloads; otherwise use a slower interval
-      return data && data.length > 0 ? 2000 : 30000
+      // Idle poll is kept tight so newly-dispatched jobs surface quickly — small ZIM
+      // updates can complete in ~2s, so a 30s idle interval almost always missed them.
+      return data && data.length > 0 ? 2000 : 3000
     },
     enabled: props.enabled ?? true,
   })

admin/inertia/hooks/useMapMarkers.ts

@@ -18,6 +18,7 @@ export interface MapMarker {
   longitude: number
   latitude: number
   color: PinColorId
+  notes: string | null
   createdAt: string
 }
 
@@ -36,6 +37,7 @@ export function useMapMarkers() {
             longitude: m.longitude,
             latitude: m.latitude,
             color: m.color as PinColorId,
+            notes: m.notes ?? null,
             createdAt: m.created_at,
           }))
         )
@@ -54,6 +56,7 @@ export function useMapMarkers() {
           longitude: result.longitude,
           latitude: result.latitude,
           color: result.color as PinColorId,
+          notes: result.notes ?? null,
           createdAt: result.created_at,
         }
         setMarkers((prev) => [...prev, marker])

admin/inertia/lib/api.ts

@@ -4,7 +4,8 @@ import { ServiceSlim } from '../../types/services'
 import { FileEntry } from '../../types/files'
 import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system'
 import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads'
-import { EmbedJobWithProgress } from '../../types/rag'
+import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps'
+import { EmbedJobWithProgress, FileWarningsResult, StoredFileInfo } from '../../types/rag'
 import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections'
 import { catchInternal } from './util'
 import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama'
@@ -130,6 +131,15 @@ class API {
     })()
   }
 
+  async deleteMapRegionFile(filename: string): Promise<{ message: string }> {
+    return catchInternal(async () => {
+      const response = await this.client.delete<{ message: string }>(
+        `/maps/${encodeURIComponent(filename)}`
+      )
+      return response.data
+    })()
+  }
+
   async downloadRemoteZimFile(
     url: string,
     metadata?: { title: string; summary?: string; author?: string; size_bytes?: number }
@@ -262,6 +272,24 @@ class API {
     })()
   }
 
+  /**
+   * Ask the backend to send Ollama `keep_alive: 0` to every currently-loaded
+   * chat model except `targetModel` (and the embedding model, which is always
+   * exempt server-side). Fire-and-forget — the chat UI doesn't await this
+   * before creating a new session, since unload is housekeeping.
+   *
+   * Pass `null` to unload every chat model.
+   */
+  async unloadChatModels(targetModel: string | null) {
+    return catchInternal(async () => {
+      const response = await this.client.post<{ unloaded: string[] }>(
+        '/ollama/unload-chat-models',
+        { targetModel }
+      )
+      return response.data
+    })()
+  }
+
   async getAvailableModels(params: { query?: string; recommendedOnly?: boolean; limit?: number; force?: boolean }) {
     return catchInternal(async () => {
       const response = await this.client.get<{
@@ -451,13 +479,34 @@ class API {
     })()
   }
 
+  async checkRAGHealth() {
+    return catchInternal(async () => {
+      const response = await this.client.get<{ online: boolean; message?: string }>('/rag/health')
+      return response.data
+    })()
+  }
+
   async getStoredRAGFiles() {
     return catchInternal(async () => {
-      const response = await this.client.get<{ files: string[] }>('/rag/files')
+      const response = await this.client.get<{ files: StoredFileInfo[] }>('/rag/files')
       return response.data.files
     })()
   }
 
+  async embedSingleRAGFile(source: string, force: boolean = false) {
+    return catchInternal(async () => {
+      const response = await this.client.post<{ message: string }>('/rag/files/embed', { source, force })
+      return response.data
+    })()
+  }
+
+  async getKbFileWarnings() {
+    return catchInternal(async () => {
+      const response = await this.client.get<FileWarningsResult>('/rag/file-warnings')
+      return response.data
+    })()
+  }
+
   async deleteRAGFile(source: string) {
     return catchInternal(async () => {
       const response = await this.client.delete<{ message: string }>('/rag/files', { data: { source } })
@@ -541,6 +590,46 @@ class API {
     })()
   }
 
+  async listCountries() {
+    return catchInternal(async () => {
+      const response = await this.client.get<{ countries: Country[] }>('/maps/countries')
+      return response.data.countries
+    })()
+  }
+
+  async listCountryGroups() {
+    return catchInternal(async () => {
+      const response = await this.client.get<{ groups: CountryGroup[] }>('/maps/country-groups')
+      return response.data.groups
+    })()
+  }
+
+  async extractMapPreflight(params: { countries: CountryCode[]; maxzoom?: number }) {
+    return catchInternal(async () => {
+      const response = await this.client.post<MapExtractPreflight>(
+        '/maps/extract-preflight',
+        params
+      )
+      return response.data
+    })()
+  }
+
+  async extractMapRegion(params: {
+    countries: CountryCode[]
+    maxzoom?: number
+    label?: string
+    estimatedBytes?: number
+  }) {
+    return catchInternal(async () => {
+      const response = await this.client.post<{
+        message: string
+        filename: string
+        jobId?: string
+      }>('/maps/extract', params)
+      return response.data
+    })()
+  }
+
   async listCuratedMapCollections() {
     return catchInternal(async () => {
       const response = await this.client.get<CollectionWithStatus[]>(
@@ -574,7 +663,7 @@ class API {
   async listMapMarkers() {
     return catchInternal(async () => {
       const response = await this.client.get<
-        Array<{ id: number; name: string; longitude: number; latitude: number; color: string; created_at: string }>
+        Array<{ id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }>
       >('/maps/markers')
       return response.data
     })()
@@ -583,7 +672,7 @@ class API {
   async createMapMarker(data: { name: string; longitude: number; latitude: number; color?: string }) {
     return catchInternal(async () => {
       const response = await this.client.post<
-        { id: number; name: string; longitude: number; latitude: number; color: string; created_at: string }
+        { id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }
       >('/maps/markers', data)
       return response.data
     })()
@@ -624,6 +713,42 @@ class API {
     })()
   }
 
+  async listCustomLibraries() {
+    return catchInternal(async () => {
+      const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>(
+        '/zim/custom-libraries'
+      )
+      return response.data
+    })()
+  }
+
+  async addCustomLibrary(name: string, base_url: string) {
+    return catchInternal(async () => {
+      const response = await this.client.post<{
+        message: string
+        library: { id: number; name: string; base_url: string }
+      }>('/zim/custom-libraries', { name, base_url })
+      return response.data
+    })()
+  }
+
+  async removeCustomLibrary(id: number) {
+    return catchInternal(async () => {
+      const response = await this.client.delete<{ message: string }>(`/zim/custom-libraries/${id}`)
+      return response.data
+    })()
+  }
+
+  async browseLibrary(url: string) {
+    return catchInternal(async () => {
+      const response = await this.client.get<{
+        directories: { name: string; url: string }[]
+        files: { name: string; url: string; size_bytes: number | null }[]
+      }>('/zim/browse-library', { params: { url } })
+      return response.data
+    })()
+  }
+
   async deleteZimFile(filename: string) {
     return catchInternal(async () => {
       const response = await this.client.delete<{ message: string }>(`/zim/${filename}`)
@@ -718,6 +843,52 @@ class API {
     })()
   }
 
+  async reembedAllRAG() {
+    return catchInternal(async () => {
+      const response = await this.client.post<{
+        success: boolean
+        message: string
+        filesScanned?: number
+        filesQueued?: number
+      }>('/rag/re-embed-all')
+      return response.data
+    })()
+  }
+
+  async resetAndRebuildRAG() {
+    return catchInternal(async () => {
+      const response = await this.client.post<{
+        success: boolean
+        message: string
+        filesScanned?: number
+        filesQueued?: number
+      }>('/rag/reset-and-rebuild')
+      return response.data
+    })()
+  }
+
+  async estimateEmbeddingBatch(files: { filename: string; sizeBytes: number }[]) {
+    return catchInternal(async () => {
+      const response = await this.client.post<{
+        totalChunks: number
+        totalBytes: number
+        hasUnknown: boolean
+      }>('/rag/estimate-batch', { files })
+      return response.data
+    })()
+  }
+
+  async getKbPolicyPromptState() {
+    return catchInternal(async () => {
+      const response = await this.client.get<{
+        shouldPrompt: boolean
+        hasContent: boolean
+        totalFiles: number
+      }>('/rag/policy-prompt-state')
+      return response.data
+    })()
+  }
+
   // Wikipedia selector methods
 
   async getWikipediaState(): Promise<WikipediaState | undefined> {

admin/inertia/lib/global_map_banner.ts

@@ -0,0 +1,10 @@
+export function hasDownloadedGlobalMap(
+  globalMapKey: string | null | undefined,
+  storedMapFiles: Array<{ name: string }>
+): boolean {
+  if (!globalMapKey) {
+    return false
+  }
+
+  return storedMapFiles.some((file) => file.name === globalMapKey || /^\d{8}\.pmtiles$/.test(file.name))
+}

admin/inertia/lib/icons.ts

@@ -32,6 +32,7 @@ import {
   IconInfoCircle,
   IconBug,
   IconCopy,
+  IconLibrary,
   IconServer,
   IconMenu2,
   IconArrowLeft,
@@ -75,6 +76,7 @@ export const icons = {
   IconDownload,
   IconHome,
   IconInfoCircle,
+  IconLibrary,
   IconLogs,
   IconMap,
   IconMenu2,

admin/inertia/lib/kb_file_grouping.ts

@@ -0,0 +1,113 @@
+import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
+import type { StoredFileInfo } from '../../types/rag.js'
+
+/**
+ * Knowledge-base files come back as a list of `{source, state, chunksEmbedded}`
+ * objects from `/api/rag/files`. The UI groups them so the user sees the
+ * categories that matter to them — ZIMs, uploaded documents, and a single
+ * rolled-up entry for Project NOMAD's bundled docs (rather than the 12+
+ * individual markdown files those break into).
+ *
+ * Bucket assignment is purely by path prefix; matching is done on `/` so the
+ * server-emitted absolute paths work regardless of which Linux mount the admin
+ * container uses.
+ */
+export type KbFileBucket = 'zim' | 'upload' | 'admin_docs' | 'other'
+
+const ADMIN_DOCS_PREFIXES = ['/app/docs/', '/app/README.md']
+const ZIM_PREFIX = '/app/storage/zim/'
+const UPLOADS_PREFIX = '/app/storage/kb_uploads/'
+
+export function classifyKbFile(source: string): KbFileBucket {
+  if (
+    ADMIN_DOCS_PREFIXES.some((p) =>
+      p.endsWith('/') ? source.startsWith(p) : source === p
+    )
+  ) {
+    return 'admin_docs'
+  }
+  if (source.startsWith(ZIM_PREFIX)) return 'zim'
+  if (source.startsWith(UPLOADS_PREFIX)) return 'upload'
+  return 'other'
+}
+
+export function sourceToDisplayName(source: string): string {
+  const parts = source.split(/[/\\]/)
+  return parts[parts.length - 1] || source
+}
+
+export interface KbFileGroup {
+  bucket: KbFileBucket
+  /** Source path used as the row's stable React key. For collapsed admin docs
+   * this is a synthetic marker; individual file paths live in `members`. */
+  source: string
+  displayName: string
+  /** Number of underlying files this row represents (1 for non-collapsed). */
+  count: number
+  /** All member source paths — populated for collapsed groups, empty otherwise. */
+  members: string[]
+  /** Per-file ingestion state. `null` for the collapsed admin_docs group and
+   * for any source that exists in Qdrant but has no state row yet. */
+  state: KbIngestStateValue | null
+  /** Chunks currently embedded for this source; 0 for state-row-less or
+   * zero-chunk files. Always 0 for the collapsed admin_docs group. */
+  chunksEmbedded: number
+}
+
+const BUCKET_SORT_ORDER: KbFileBucket[] = ['zim', 'upload', 'admin_docs', 'other']
+
+/**
+ * Group stored-file rows into table rows for the Stored Files panel.
+ *
+ * - Admin docs (`/app/docs/*`, README) collapse into a single
+ *   "Project NOMAD documentation · N files" row.
+ * - ZIMs, uploads, and others stay as individual rows, sorted by bucket then
+ *   alphabetically by filename so related items cluster naturally.
+ */
+export function groupAndSortKbFiles(files: StoredFileInfo[]): KbFileGroup[] {
+  const buckets: Record<KbFileBucket, StoredFileInfo[]> = {
+    zim: [],
+    upload: [],
+    admin_docs: [],
+    other: [],
+  }
+  for (const file of files) {
+    buckets[classifyKbFile(file.source)].push(file)
+  }
+
+  const groups: KbFileGroup[] = []
+
+  for (const bucket of BUCKET_SORT_ORDER) {
+    const members = buckets[bucket]
+    if (members.length === 0) continue
+
+    if (bucket === 'admin_docs') {
+      groups.push({
+        bucket,
+        source: '__admin_docs_group__',
+        displayName: `Project NOMAD documentation · ${members.length} file${members.length === 1 ? '' : 's'}`,
+        count: members.length,
+        members: members.map((m) => m.source),
+        state: null,
+        chunksEmbedded: 0,
+      })
+      continue
+    }
+
+    for (const file of members.sort((a, b) =>
+      sourceToDisplayName(a.source).localeCompare(sourceToDisplayName(b.source))
+    )) {
+      groups.push({
+        bucket,
+        source: file.source,
+        displayName: sourceToDisplayName(file.source),
+        count: 1,
+        members: [],
+        state: file.state,
+        chunksEmbedded: file.chunksEmbedded,
+      })
+    }
+  }
+
+  return groups
+}

admin/inertia/lib/kb_guardrail.ts

@@ -0,0 +1,74 @@
+/**
+ * Auto-index guardrail thresholds and pure decision logic (RFC #883 §7).
+ *
+ * The guardrail fires when a user is about to commit to a bulk indexing
+ * action (curated tier change, large multi-file upload, etc.) that would
+ * use a substantial amount of disk for embedding storage. It's a one-time
+ * confirmation step at scary thresholds — it doesn't fire for ordinary
+ * everyday operations. After the user confirms once for a given batch
+ * the action proceeds as it would have without the guardrail.
+ *
+ * Thresholds are intentionally conservative to avoid surprise consumption
+ * of a user's storage. Tweak both constants if the field experience
+ * suggests we're nagging users too aggressively.
+ */
+
+/** Absolute upper bound: estimates at or above this trip the guardrail. */
+export const GUARDRAIL_ABSOLUTE_BYTES = 50 * 1024 * 1024 * 1024 // 50 GB
+
+/** Relative-to-free-disk bound: estimates >= 10% of free disk trip too. */
+export const GUARDRAIL_FREE_DISK_RATIO = 0.1
+
+export type GuardrailReason =
+  | {
+      kind: 'over_absolute'
+      estimateBytes: number
+      thresholdBytes: number
+    }
+  | {
+      kind: 'over_free_disk'
+      estimateBytes: number
+      freeBytes: number
+      thresholdBytes: number
+    }
+
+export type GuardrailVerdict = {
+  trips: boolean
+  reasons: GuardrailReason[]
+}
+
+/**
+ * Decide whether a bulk indexing action should be gated behind the
+ * guardrail modal. Caller passes the precomputed embedding-storage
+ * estimate (from `KbRatioRegistry.estimateBatch` in #891 / #897) and
+ * the free-disk figure from system info. Pass `freeBytes = 0` to skip
+ * the relative-disk check when free space isn't known.
+ */
+export function evaluateGuardrail(input: {
+  estimateBytes: number
+  freeBytes: number
+}): GuardrailVerdict {
+  const reasons: GuardrailReason[] = []
+
+  if (input.estimateBytes >= GUARDRAIL_ABSOLUTE_BYTES) {
+    reasons.push({
+      kind: 'over_absolute',
+      estimateBytes: input.estimateBytes,
+      thresholdBytes: GUARDRAIL_ABSOLUTE_BYTES,
+    })
+  }
+
+  if (input.freeBytes > 0) {
+    const relativeThreshold = input.freeBytes * GUARDRAIL_FREE_DISK_RATIO
+    if (input.estimateBytes >= relativeThreshold) {
+      reasons.push({
+        kind: 'over_free_disk',
+        estimateBytes: input.estimateBytes,
+        freeBytes: input.freeBytes,
+        thresholdBytes: relativeThreshold,
+      })
+    }
+  }
+
+  return { trips: reasons.length > 0, reasons }
+}

admin/inertia/lib/kb_job_health_display.ts

@@ -0,0 +1,63 @@
+import { computeJobHealth, type JobHealthStatus } from '../../app/utils/kb_job_health.js'
+
+export { computeJobHealth, type JobHealthStatus } from '../../app/utils/kb_job_health.js'
+
+/**
+ * Visual presentation for each health status — pill color, dot color, and the
+ * short label rendered alongside the dot. Kept in one place so backend health
+ * decisions (`computeJobHealth`) and frontend rendering stay in sync.
+ */
+export const JOB_HEALTH_DISPLAY: Record<
+  JobHealthStatus,
+  { dot: string; label: string; ariaLabel: string }
+> = {
+  waiting: {
+    dot: 'bg-gray-400 dark:bg-gray-500',
+    label: 'Waiting',
+    ariaLabel: 'Job is queued and waiting to start',
+  },
+  healthy: {
+    dot: 'bg-green-500',
+    label: 'Active',
+    ariaLabel: 'Job is embedding at a normal rate',
+  },
+  slow: {
+    dot: 'bg-yellow-500',
+    label: 'Slow',
+    ariaLabel: 'Job has not made progress for at least 2 minutes',
+  },
+  stalled: {
+    dot: 'bg-red-500',
+    label: 'Stalled',
+    ariaLabel: 'Job has not made progress for at least 5 minutes',
+  },
+  failed: {
+    dot: 'bg-red-700',
+    label: 'Failed',
+    ariaLabel: 'Job failed',
+  },
+}
+
+/**
+ * Format a relative timestamp as "Xs ago", "Xm ago", "Xh ago" with sensible
+ * thresholds for the KB Processing Queue's "Last activity" line.
+ */
+export function formatTimeAgo(timestampMs: number, now: number): string {
+  const seconds = Math.max(0, Math.floor((now - timestampMs) / 1000))
+  if (seconds < 5) return 'just now'
+  if (seconds < 60) return `${seconds}s ago`
+  const minutes = Math.floor(seconds / 60)
+  if (minutes < 60) return `${minutes}m ago`
+  const hours = Math.floor(minutes / 60)
+  return `${hours}h ago`
+}
+
+/**
+ * Convenience wrapper that resolves a job's health status without the caller
+ * having to remember to pass `now`. Mostly for ergonomic frontend use.
+ */
+export function computeJobHealthNow(
+  input: Omit<Parameters<typeof computeJobHealth>[0], 'now'>
+): JobHealthStatus {
+  return computeJobHealth({ ...input, now: Date.now() })
+}

admin/inertia/pages/easy-setup/index.tsx

@@ -106,7 +106,7 @@ const ADDITIONAL_TOOLS: Capability[] = [
   },
 ]
 
-type WizardStep = 1 | 2 | 3 | 4
+type WizardStep = 1 | 2 | 3 | 4 | 5
 
 const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections'
 const CURATED_CATEGORIES_KEY = 'curated-categories'
@@ -122,6 +122,13 @@ export default function EasySetupWizard(props: {
   const [selectedServices, setSelectedServices] = useState<string[]>([])
   const [selectedMapCollections, setSelectedMapCollections] = useState<string[]>([])
   const [selectedAiModels, setSelectedAiModels] = useState<string[]>([])
+  // Auto-index policy for the AI Assistant Knowledge Base. Defaults to
+  // 'Always' so a new user who keeps the default behavior gets the "just
+  // works" experience — downloads become searchable automatically. Persisted
+  // to KVStore['rag.defaultIngestPolicy'] on wizard submit (same key #894's
+  // KB modal toggle reads/writes) so the JIT prompt at first chat sees a
+  // decided policy and doesn't ask again.
+  const [ingestPolicy, setIngestPolicy] = useState<'Always' | 'Manual'>('Always')
   const [isProcessing, setIsProcessing] = useState(false)
   const [showAdditionalTools, setShowAdditionalTools] = useState(false)
   const [remoteOllamaEnabled, setRemoteOllamaEnabled] = useState(
@@ -192,6 +199,19 @@ export default function EasySetupWizard(props: {
   // Services that are already installed
   const installedServices = props.system.services.filter((service) => service.installed)
 
+  // Canonical "is AI part of this user's setup?" predicate (RFC #883 / issue #905).
+  // Single source consumed by step-indicator render, navigation skip logic, the
+  // review summary, and handleFinish. The AI step renders if and only if this
+  // is true; if false, the wizard collapses to 4 steps and the AI step is
+  // skipped on both forward and back nav.
+  const isAiInSetup = useMemo(
+    () =>
+      selectedServices.includes(SERVICE_NAMES.OLLAMA) ||
+      installedServices.some((s) => s.service_name === SERVICE_NAMES.OLLAMA) ||
+      remoteOllamaEnabled,
+    [selectedServices, installedServices, remoteOllamaEnabled]
+  )
+
   const toggleMapCollection = (slug: string) => {
     setSelectedMapCollections((prev) =>
       prev.includes(slug) ? prev.filter((s) => s !== slug) : [...prev, slug]
@@ -306,24 +326,29 @@ export default function EasySetupWizard(props: {
   // Get primary disk/filesystem info for storage projection
   const storageInfo = getPrimaryDiskInfo(systemInfo?.disk, systemInfo?.fsSize)
 
+  // Final step number (4 when AI is off, 5 when AI is on). Centralizing this
+  // here so canProceedToNextStep / handleNext / handleBack / the bottom-bar
+  // Next-vs-Finish switch all read the same value.
+  const finalStep: WizardStep = isAiInSetup ? 5 : 4
+
   const canProceedToNextStep = () => {
     if (!isOnline) return false // Must be online to proceed
-    if (currentStep === 1) return true // Can skip app installation
-    if (currentStep === 2) return true // Can skip map downloads
-    if (currentStep === 3) return true // Can skip ZIM downloads
-    return false
+    // Every step before the review is skippable; the review step shows Finish, not Next.
+    return currentStep < finalStep
   }
 
   const handleNext = () => {
-    if (currentStep < 4) {
-      setCurrentStep((prev) => (prev + 1) as WizardStep)
-    }
+    if (currentStep >= finalStep) return
+    // Skip the AI step (4) on forward nav when isAiInSetup is false.
+    const next = currentStep === 3 && !isAiInSetup ? 5 : currentStep + 1
+    setCurrentStep(next as WizardStep)
   }
 
   const handleBack = () => {
-    if (currentStep > 1) {
-      setCurrentStep((prev) => (prev - 1) as WizardStep)
-    }
+    if (currentStep <= 1) return
+    // Skip the AI step (4) on back nav when isAiInSetup is false.
+    const prev = currentStep === 5 && !isAiInSetup ? 3 : currentStep - 1
+    setCurrentStep(prev as WizardStep)
   }
 
   const handleFinish = async () => {
@@ -338,6 +363,21 @@ export default function EasySetupWizard(props: {
     setIsProcessing(true)
 
     try {
+      // Persist the auto-index policy choice before kicking off downloads so
+      // any content that finishes during this same wizard run sees the right
+      // policy. Skipped when AI is not in the user's setup; the KV stays null
+      // and the first-chat JIT prompt (#899) handles the decision later if/when
+      // the user enables AI. Uses the canonical isAiInSetup predicate so step
+      // 3 / step 4 / step 5 / handleFinish never disagree (issue #905).
+      if (isAiInSetup) {
+        try {
+          await api.updateSetting('rag.defaultIngestPolicy', ingestPolicy)
+        } catch (err) {
+          // Non-fatal: the user can still set the policy from the KB modal.
+          console.warn('Could not persist ingest policy from wizard:', err)
+        }
+      }
+
       // If using remote Ollama, configure it first before other installs
       if (remoteOllamaEnabled && remoteOllamaUrl) {
         const remoteResult = await api.configureRemoteOllama(remoteOllamaUrl)
@@ -430,12 +470,25 @@ export default function EasySetupWizard(props: {
   }, [])
 
   const renderStepIndicator = () => {
-    const steps = [
-      { number: 1, label: 'Apps' },
-      { number: 2, label: 'Maps' },
-      { number: 3, label: 'Content' },
-      { number: 4, label: 'Review' },
-    ]
+    // `step` is the stable WizardStep value (1=Apps, 2=Maps, 3=Content,
+    // 4=AI, 5=Review). `displayNumber` is the sequential position shown in
+    // the dot (always 1..N) so users see "1 2 3 4" when AI is off and
+    // "1 2 3 4 5" when AI is on, with no gap.
+    const baseSteps: Array<{ step: WizardStep; label: string }> = isAiInSetup
+      ? [
+          { step: 1, label: 'Apps' },
+          { step: 2, label: 'Maps' },
+          { step: 3, label: 'Content' },
+          { step: 4, label: 'AI' },
+          { step: 5, label: 'Review' },
+        ]
+      : [
+          { step: 1, label: 'Apps' },
+          { step: 2, label: 'Maps' },
+          { step: 3, label: 'Content' },
+          { step: 5, label: 'Review' },
+        ]
+    const steps = baseSteps.map((s, idx) => ({ ...s, displayNumber: idx + 1 }))
 
     return (
       <nav aria-label="Progress" className="px-6 pt-6">
@@ -444,8 +497,8 @@ export default function EasySetupWizard(props: {
           className="divide-y divide-border-default rounded-md md:flex md:divide-y-0 md:justify-between border border-desert-green"
         >
           {steps.map((step, stepIdx) => (
-            <li key={step.number} className="relative md:flex-1 md:flex md:justify-center">
-              {currentStep > step.number ? (
+            <li key={step.step} className="relative md:flex-1 md:flex md:justify-center">
+              {currentStep > step.step ? (
                 <div className="group flex w-full items-center md:justify-center">
                   <span className="flex items-center px-6 py-2 text-sm font-medium">
                     <span className="flex size-10 shrink-0 items-center justify-center rounded-full bg-desert-green">
@@ -454,13 +507,13 @@ export default function EasySetupWizard(props: {
                     <span className="ml-4 text-lg font-medium text-text-primary">{step.label}</span>
                   </span>
                 </div>
-              ) : currentStep === step.number ? (
+              ) : currentStep === step.step ? (
                 <div
                   aria-current="step"
                   className="flex items-center px-6 py-2 text-sm font-medium md:justify-center"
                 >
                   <span className="flex size-10 shrink-0 items-center justify-center rounded-full bg-desert-green border-2 border-desert-green">
-                    <span className="text-white">{step.number}</span>
+                    <span className="text-white">{step.displayNumber}</span>
                   </span>
                   <span className="ml-4 text-lg font-medium text-desert-green">{step.label}</span>
                 </div>
@@ -468,7 +521,7 @@ export default function EasySetupWizard(props: {
                 <div className="group flex items-center md:justify-center">
                   <span className="flex items-center px-6 py-2 text-sm font-medium">
                     <span className="flex size-10 shrink-0 items-center justify-center rounded-full border-2 border-border-default">
-                      <span className="text-text-muted">{step.number}</span>
+                      <span className="text-text-muted">{step.displayNumber}</span>
                     </span>
                     <span className="ml-4 text-lg font-medium text-text-muted">{step.label}</span>
                   </span>
@@ -486,7 +539,7 @@ export default function EasySetupWizard(props: {
                       fill="none"
                       viewBox="0 0 22 80"
                       preserveAspectRatio="none"
-                      className={`size-full ${currentStep > step.number ? 'text-desert-green' : 'text-text-muted'}`}
+                      className={`size-full ${currentStep > step.step ? 'text-desert-green' : 'text-text-muted'}`}
                     >
                       <path
                         d="M0 -2L20 40L0 82"
@@ -530,6 +583,29 @@ export default function EasySetupWizard(props: {
     if (isCapabilityInstalled(capability)) return
 
     const isSelected = isCapabilitySelected(capability)
+
+    // Toggling AI off needs to clear dependent state that lives in the AI
+    // step (model picks, ingest policy, remote Ollama config). If the user
+    // has any of that filled in, confirm before discarding so a stray click
+    // doesn't quietly wipe their setup.
+    if (capability.id === 'ai' && isSelected) {
+      const hasAiSelections =
+        selectedAiModels.length > 0 ||
+        ingestPolicy !== 'Always' ||
+        remoteOllamaEnabled
+      if (hasAiSelections) {
+        const confirmed = window.confirm(
+          "Turning off AI will discard your AI model picks, indexing policy, and remote Ollama configuration. Continue?"
+        )
+        if (!confirmed) return
+      }
+      setSelectedAiModels([])
+      setIngestPolicy('Always')
+      setRemoteOllamaEnabled(false)
+      setRemoteOllamaUrl('')
+      setRemoteOllamaUrlError(null)
+    }
+
     if (isSelected) {
       // Deselect all services in this capability
       setSelectedServices((prev) => prev.filter((s) => !capability.services.includes(s)))
@@ -811,10 +887,11 @@ export default function EasySetupWizard(props: {
   )
 
   const renderStep3 = () => {
-    // Check if AI or Information capabilities are selected OR already installed
-    const isAiSelected = selectedServices.includes(SERVICE_NAMES.OLLAMA) ||
-      installedServices.some((s) => s.service_name === SERVICE_NAMES.OLLAMA)
-    const isInformationSelected = selectedServices.includes(SERVICE_NAMES.KIWIX) ||
+    // Issue #905: AI moved to its own conditional Step 4. Step 3 is now
+    // content-only (Wikipedia + curated tiers), gated on the Information
+    // capability (Kiwix).
+    const isInformationSelected =
+      selectedServices.includes(SERVICE_NAMES.KIWIX) ||
       installedServices.some((s) => s.service_name === SERVICE_NAMES.KIWIX)
 
     return (
@@ -822,132 +899,31 @@ export default function EasySetupWizard(props: {
         <div className="text-center mb-6">
           <h2 className="text-3xl font-bold text-text-primary mb-2">Choose Content</h2>
           <p className="text-text-secondary">
-            {isAiSelected && isInformationSelected
-              ? 'Select AI models and content categories for offline use.'
-              : isAiSelected
-                ? 'Select AI models to download for offline use.'
-                : isInformationSelected
-                  ? 'Select content categories for offline knowledge.'
-                  : 'Configure content for your selected capabilities.'}
+            {isInformationSelected
+              ? 'Select content categories for offline knowledge.'
+              : 'Configure content for your selected capabilities.'}
           </p>
         </div>
 
-        {/* AI Model Selection - Only show if AI capability is selected */}
-        {isAiSelected && (
+        {/* Wikipedia Selection - Only show if Information capability is selected */}
+        {isInformationSelected && (
           <div className="mb-8">
-            <div className="flex items-center gap-3 mb-4">
-              <div className="w-10 h-10 rounded-full bg-surface-primary border border-border-subtle flex items-center justify-center shadow-sm">
-                <IconCpu className="w-6 h-6 text-text-primary" />
-              </div>
-              <div>
-                <h3 className="text-xl font-semibold text-text-primary">AI Models</h3>
-                <p className="text-sm text-text-muted">Select models to download for offline AI</p>
-              </div>
-            </div>
-            {remoteOllamaEnabled && remoteOllamaUrl ? (
-              <Alert
-                title="Remote Ollama selected"
-                message="Models are managed on the remote machine. You can add models from Settings > AI Assistant after setup, note this is only supported when using Ollama, not LM Studio and other OpenAI API software."
-                type="info"
-                variant="bordered"
-              />
-            ) : isLoadingRecommendedModels ? (
+            {isLoadingWikipedia ? (
               <div className="flex justify-center py-12">
                 <LoadingSpinner />
               </div>
-            ) : recommendedModels && recommendedModels.length > 0 ? (
-              <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
-                {recommendedModels.map((model) => (
-                  <div
-                    key={model.name}
-                    onClick={() => isOnline && toggleAiModel(model.name)}
-                    className={classNames(
-                      'p-4 rounded-lg border-2 transition-all cursor-pointer',
-                      selectedAiModels.includes(model.name)
-                        ? 'border-desert-green bg-desert-green shadow-md'
-                        : 'border-desert-stone-light bg-surface-primary hover:border-desert-green hover:shadow-sm',
-                      !isOnline && 'opacity-50 cursor-not-allowed'
-                    )}
-                  >
-                    <div className="flex items-start justify-between">
-                      <div className="flex-1">
-                        <h4
-                          className={classNames(
-                            'text-lg font-semibold mb-1',
-                            selectedAiModels.includes(model.name) ? 'text-white' : 'text-text-primary'
-                          )}
-                        >
-                          {model.name}
-                        </h4>
-                        <p
-                          className={classNames(
-                            'text-sm mb-2',
-                            selectedAiModels.includes(model.name) ? 'text-white' : 'text-text-secondary'
-                          )}
-                        >
-                          {model.description}
-                        </p>
-                        {model.tags?.[0]?.size && (
-                          <div
-                            className={classNames(
-                              'text-xs',
-                              selectedAiModels.includes(model.name)
-                                ? 'text-green-100'
-                                : 'text-text-muted'
-                            )}
-                          >
-                            Size: {model.tags[0].size}
-                          </div>
-                        )}
-                      </div>
-                      <div
-                        className={classNames(
-                          'ml-4 w-6 h-6 rounded-full border-2 flex items-center justify-center transition-all flex-shrink-0',
-                          selectedAiModels.includes(model.name)
-                            ? 'border-white bg-white'
-                            : 'border-desert-stone'
-                        )}
-                      >
-                        {selectedAiModels.includes(model.name) && (
-                          <IconCheck size={16} className="text-desert-green" />
-                        )}
-                      </div>
-                    </div>
-                  </div>
-                ))}
-              </div>
-            ) : (
-              <div className="text-center py-8 bg-surface-secondary rounded-lg">
-                <p className="text-text-secondary">No recommended AI models available at this time.</p>
-              </div>
-            )}
+            ) : wikipediaState && wikipediaState.options.length > 0 ? (
+              <WikipediaSelector
+                options={wikipediaState.options}
+                currentSelection={wikipediaState.currentSelection}
+                selectedOptionId={selectedWikipedia}
+                onSelect={(optionId) => isOnline && setSelectedWikipedia(optionId)}
+                disabled={!isOnline}
+              />
+            ) : null}
           </div>
         )}
 
-        {/* Wikipedia Selection - Only show if Information capability is selected */}
-        {isInformationSelected && (
-          <>
-            {/* Divider between AI Models and Wikipedia */}
-            {isAiSelected && <hr className="my-8 border-border-subtle" />}
-
-            <div className="mb-8">
-              {isLoadingWikipedia ? (
-                <div className="flex justify-center py-12">
-                  <LoadingSpinner />
-                </div>
-              ) : wikipediaState && wikipediaState.options.length > 0 ? (
-                <WikipediaSelector
-                  options={wikipediaState.options}
-                  currentSelection={wikipediaState.currentSelection}
-                  selectedOptionId={selectedWikipedia}
-                  onSelect={(optionId) => isOnline && setSelectedWikipedia(optionId)}
-                  disabled={!isOnline}
-                />
-              ) : null}
-            </div>
-          </>
-        )}
-
         {/* Curated Categories with Tiers - Only show if Information capability is selected */}
         {isInformationSelected && (
           <>
@@ -999,8 +975,8 @@ export default function EasySetupWizard(props: {
           </>
         )}
 
-        {/* Show message if no capabilities requiring content are selected */}
-        {!isAiSelected && !isInformationSelected && (
+        {/* Show message if no content-bearing capabilities are selected */}
+        {!isInformationSelected && (
           <div className="text-center py-12">
             <p className="text-text-secondary text-lg">
               No content-based capabilities selected. You can skip this step or go back to select
@@ -1013,6 +989,149 @@ export default function EasySetupWizard(props: {
   }
 
   const renderStep4 = () => {
+    // AI step (issue #905). Only rendered when isAiInSetup is true; otherwise
+    // the wizard's step array drops it and forward/back nav jumps Content → Review.
+    return (
+      <div className="space-y-6">
+        <div className="text-center mb-6">
+          <h2 className="text-3xl font-bold text-text-primary mb-2">Configure {aiAssistantName}</h2>
+          <p className="text-text-secondary">
+            Choose models to download and set how {aiAssistantName} handles new content.
+          </p>
+        </div>
+
+        <div className="flex items-center gap-3 mb-4">
+          <div className="w-10 h-10 rounded-full bg-surface-primary border border-border-subtle flex items-center justify-center shadow-sm">
+            <IconCpu className="w-6 h-6 text-text-primary" />
+          </div>
+          <div>
+            <h3 className="text-xl font-semibold text-text-primary">AI Models</h3>
+            <p className="text-sm text-text-muted">Select models to download for offline AI</p>
+          </div>
+        </div>
+        {remoteOllamaEnabled && remoteOllamaUrl ? (
+          <Alert
+            title="Remote Ollama selected"
+            message="Models are managed on the remote machine. You can add models from Settings > AI Assistant after setup, note this is only supported when using Ollama, not LM Studio and other OpenAI API software."
+            type="info"
+            variant="bordered"
+          />
+        ) : isLoadingRecommendedModels ? (
+          <div className="flex justify-center py-12">
+            <LoadingSpinner />
+          </div>
+        ) : recommendedModels && recommendedModels.length > 0 ? (
+          <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
+            {recommendedModels.map((model) => (
+              <div
+                key={model.name}
+                onClick={() => isOnline && toggleAiModel(model.name)}
+                className={classNames(
+                  'p-4 rounded-lg border-2 transition-all cursor-pointer',
+                  selectedAiModels.includes(model.name)
+                    ? 'border-desert-green bg-desert-green shadow-md'
+                    : 'border-desert-stone-light bg-surface-primary hover:border-desert-green hover:shadow-sm',
+                  !isOnline && 'opacity-50 cursor-not-allowed'
+                )}
+              >
+                <div className="flex items-start justify-between">
+                  <div className="flex-1">
+                    <h4
+                      className={classNames(
+                        'text-lg font-semibold mb-1',
+                        selectedAiModels.includes(model.name) ? 'text-white' : 'text-text-primary'
+                      )}
+                    >
+                      {model.name}
+                    </h4>
+                    <p
+                      className={classNames(
+                        'text-sm mb-2',
+                        selectedAiModels.includes(model.name) ? 'text-white' : 'text-text-secondary'
+                      )}
+                    >
+                      {model.description}
+                    </p>
+                    {model.tags?.[0]?.size && (
+                      <div
+                        className={classNames(
+                          'text-xs',
+                          selectedAiModels.includes(model.name)
+                            ? 'text-green-100'
+                            : 'text-text-muted'
+                        )}
+                      >
+                        Size: {model.tags[0].size}
+                      </div>
+                    )}
+                  </div>
+                  <div
+                    className={classNames(
+                      'ml-4 w-6 h-6 rounded-full border-2 flex items-center justify-center transition-all flex-shrink-0',
+                      selectedAiModels.includes(model.name)
+                        ? 'border-white bg-white'
+                        : 'border-desert-stone'
+                    )}
+                  >
+                    {selectedAiModels.includes(model.name) && (
+                      <IconCheck size={16} className="text-desert-green" />
+                    )}
+                  </div>
+                </div>
+              </div>
+            ))}
+          </div>
+        ) : (
+          <div className="text-center py-8 bg-surface-secondary rounded-lg">
+            <p className="text-text-secondary">No recommended AI models available at this time.</p>
+          </div>
+        )}
+
+        {/* Auto-index policy — choose now so the JIT prompt at first chat
+            doesn't ask again (RFC #883 Phase 3 task 13). Persisted to
+            rag.defaultIngestPolicy on wizard submit. */}
+        <div className="mt-8 pt-6 border-t border-border-subtle">
+          <h4 className="text-lg font-semibold text-text-primary mb-1">
+            Auto-index new content for {aiAssistantName}?
+          </h4>
+          <p className="text-sm text-text-muted mb-4">
+            When you add new ZIMs, documents, or curated content, should {aiAssistantName} index them automatically so it can search them while answering your questions?
+          </p>
+          <div className="inline-flex rounded-md border border-border-default overflow-hidden" role="group">
+            <button
+              type="button"
+              onClick={() => setIngestPolicy('Always')}
+              className={classNames(
+                'px-5 py-2 text-sm font-medium transition-colors',
+                ingestPolicy === 'Always'
+                  ? 'bg-desert-green text-white'
+                  : 'bg-surface-primary text-text-secondary hover:bg-surface-secondary'
+              )}
+            >
+              Yes, always
+            </button>
+            <button
+              type="button"
+              onClick={() => setIngestPolicy('Manual')}
+              className={classNames(
+                'px-5 py-2 text-sm font-medium transition-colors border-l border-border-default',
+                ingestPolicy === 'Manual'
+                  ? 'bg-desert-green text-white'
+                  : 'bg-surface-primary text-text-secondary hover:bg-surface-secondary'
+              )}
+            >
+              Ask me first
+            </button>
+          </div>
+          <p className="text-xs text-text-muted mt-3">
+            You can change this any time from the Knowledge Base panel inside AI Chat.
+          </p>
+        </div>
+      </div>
+    )
+  }
+
+  const renderStep5 = () => {
     const hasSelections =
       selectedServices.length > 0 ||
       selectedMapCollections.length > 0 ||
@@ -1157,6 +1276,25 @@ export default function EasySetupWizard(props: {
               </div>
             )}
 
+            {isAiInSetup && (
+              <div className="bg-surface-primary rounded-lg border-2 border-desert-stone-light p-6">
+                <h3 className="text-xl font-semibold text-text-primary mb-2">
+                  Auto-index Setting
+                </h3>
+                <p className="text-text-secondary text-sm">
+                  {ingestPolicy === 'Always' ? (
+                    <>
+                      New content will be <strong>indexed automatically</strong> as it arrives so {aiAssistantName} can search it.
+                    </>
+                  ) : (
+                    <>
+                      New content will <strong>wait for you to opt in</strong> from the Knowledge Base panel before {aiAssistantName} indexes it.
+                    </>
+                  )}
+                </p>
+              </div>
+            )}
+
             <Alert
               title="Ready to Start"
               message="Click 'Complete Setup' to begin installing apps and downloading content. This may take some time depending on your internet connection and the size of the downloads."
@@ -1197,7 +1335,8 @@ export default function EasySetupWizard(props: {
             {currentStep === 1 && renderStep1()}
             {currentStep === 2 && renderStep2()}
             {currentStep === 3 && renderStep3()}
-            {currentStep === 4 && renderStep4()}
+            {currentStep === 4 && isAiInSetup && renderStep4()}
+            {currentStep === 5 && renderStep5()}
 
             <div className="flex justify-between mt-8 pt-4 border-t border-desert-stone-light">
               <div className="flex space-x-4 items-center">
@@ -1235,7 +1374,7 @@ export default function EasySetupWizard(props: {
                   Cancel & Go to Home
                 </StyledButton>
 
-                {currentStep < 4 ? (
+                {currentStep < finalStep ? (
                   <StyledButton
                     onClick={handleNext}
                     disabled={!canProceedToNextStep() || isProcessing}

admin/inertia/pages/maps.tsx

@@ -1,38 +1,66 @@
-import MapsLayout from '~/layouts/MapsLayout'
+import { useState } from 'react'
 import { Head, Link, router } from '@inertiajs/react'
+import { IconArrowLeft } from '@tabler/icons-react'
+
+import MapsLayout from '~/layouts/MapsLayout'
 import MapComponent from '~/components/maps/MapComponent'
 import StyledButton from '~/components/StyledButton'
-import { IconArrowLeft } from '@tabler/icons-react'
-import { FileEntry } from '../../types/files'
 import Alert from '~/components/Alert'
 
+import { FileEntry } from '../../types/files'
+
 export default function Maps(props: {
   maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] }
 }) {
+  const [isHoveringUI, setIsHoveringUI] = useState(false)
+  const [showMapCoordinates, setShowMapCoordinates] = useState(true)
+
   const alertMessage = !props.maps.baseAssetsExist
     ? 'The base map assets have not been installed. Please download them first to enable map functionality.'
     : props.maps.regionFiles.length === 0
-      ? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.'
-      : null
+    ? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.'
+    : null
 
   return (
     <MapsLayout>
       <Head title="Maps" />
+
       <div className="relative w-full h-screen overflow-hidden">
-        {/* Nav and alerts are overlayed */}
-        <div className="absolute top-0 left-0 right-0 z-50 flex justify-between p-4 bg-surface-secondary backdrop-blur-sm shadow-sm">
+        {/* Navbar */}
+        <div
+          className="absolute top-0 left-0 right-0 z-50 flex justify-between p-4 bg-surface-secondary backdrop-blur-sm shadow-sm"
+          onMouseEnter={() => setIsHoveringUI(true)}
+          onMouseLeave={() => setIsHoveringUI(false)}
+        >
           <Link href="/home" className="flex items-center">
             <IconArrowLeft className="mr-2" size={24} />
             <p className="text-lg text-text-secondary">Back to Home</p>
           </Link>
-          <Link href="/settings/maps" className='mr-4'>
-            <StyledButton variant="primary" icon="IconSettings">
-              Manage Map Regions
-            </StyledButton>
-          </Link>
+
+          <div className="flex items-center gap-3 mr-4">
+            <button
+              type="button"
+              onClick={() => setShowMapCoordinates((prev) => !prev)}
+              className="rounded px-3 py-2 text-sm bg-surface-primary text-text-secondary hover:opacity-80 transition"
+            >
+              {showMapCoordinates ? 'Hide Coordinates' : 'Show Coordinates'}
+            </button>
+
+            <Link href="/settings/maps">
+              <StyledButton variant="primary" icon="IconSettings">
+                Manage Map Regions
+              </StyledButton>
+            </Link>
+          </div>
         </div>
+
+        {/* Alert */}
         {alertMessage && (
-          <div className="absolute top-20 left-4 right-4 z-50">
+          <div
+            className="absolute top-20 left-4 right-4 z-50"
+            onMouseEnter={() => setIsHoveringUI(true)}
+            onMouseLeave={() => setIsHoveringUI(false)}
+          >
             <Alert
               title={alertMessage}
               type="warning"
@@ -47,8 +75,13 @@ export default function Maps(props: {
             />
           </div>
         )}
+
+        {/* Map */}
         <div className="absolute inset-0">
-          <MapComponent />
+          <MapComponent
+            isHoveringUI={isHoveringUI}
+            showCoordinatesEnabled={showMapCoordinates}
+          />
         </div>
       </div>
     </MapsLayout>

admin/inertia/pages/settings/maps.tsx

@@ -6,17 +6,19 @@ import { useModals } from '~/context/ModalContext'
 import StyledModal from '~/components/StyledModal'
 import { FileEntry } from '../../../types/files'
 import { useNotifications } from '~/context/NotificationContext'
-import { useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import api from '~/lib/api'
 import DownloadURLModal from '~/components/DownloadURLModal'
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
 import useDownloads from '~/hooks/useDownloads'
 import StyledSectionHeader from '~/components/StyledSectionHeader'
 import CuratedCollectionCard from '~/components/CuratedCollectionCard'
+import CountryPickerModal from '~/components/CountryPickerModal'
 import type { CollectionWithStatus } from '../../../types/collections'
 import ActiveDownloads from '~/components/ActiveDownloads'
 import Alert from '~/components/Alert'
 import { formatBytes } from '~/lib/util'
+import { hasDownloadedGlobalMap } from '~/lib/global_map_banner'
 
 const CURATED_COLLECTIONS_KEY = 'curated-map-collections'
 const GLOBAL_MAP_INFO_KEY = 'global-map-info'
@@ -28,6 +30,7 @@ export default function MapsManager(props: {
   const { openModal, closeAllModals } = useModals()
   const { addNotification } = useNotifications()
   const [downloading, setDownloading] = useState(false)
+  const [deletingFileKey, setDeletingFileKey] = useState<string | null>(null)
 
   const { data: curatedCollections } = useQuery({
     queryKey: [CURATED_COLLECTIONS_KEY],
@@ -35,16 +38,28 @@ export default function MapsManager(props: {
     refetchOnWindowFocus: false,
   })
 
-  const { invalidate: invalidateDownloads } = useDownloads({
+  const { data: activeMapDownloads = [], invalidate: invalidateDownloads } = useDownloads({
     filetype: 'map',
     enabled: true,
   })
 
+  // Refresh the Stored Map Files list when a map download finishes. We pass props.maps.regionFiles
+  // straight through from the server-side render, so without an Inertia partial reload it stays stale
+  // until the user navigates away and back.
+  const prevMapDownloadCountRef = useRef(activeMapDownloads.length)
+  useEffect(() => {
+    if (activeMapDownloads.length < prevMapDownloadCountRef.current) {
+      router.reload({ only: ['maps'] })
+    }
+    prevMapDownloadCountRef.current = activeMapDownloads.length
+  }, [activeMapDownloads.length])
+
   const { data: globalMapInfo } = useQuery({
     queryKey: [GLOBAL_MAP_INFO_KEY],
     queryFn: () => api.getGlobalMapInfo(),
     refetchOnWindowFocus: false,
   })
+  const globalMapAlreadyDownloaded = hasDownloadedGlobalMap(globalMapInfo?.key, props.maps.regionFiles)
 
   const downloadGlobalMap = useMutation({
     mutationFn: () => api.downloadGlobalMap(),
@@ -118,18 +133,40 @@ export default function MapsManager(props: {
     }
   }
 
+  async function deleteFile(file: FileEntry) {
+    if (file.type !== 'file') return
+
+    try {
+      setDeletingFileKey(file.key)
+      await api.deleteMapRegionFile(file.name)
+      addNotification({
+        type: 'success',
+        message: `${file.name} has been deleted.`,
+      })
+      closeAllModals()
+      router.reload({ only: ['maps'] })
+    } catch (error) {
+      console.error('Error deleting map file:', error)
+      addNotification({
+        type: 'error',
+        message: `Failed to delete ${file.name}. Please try again.`,
+      })
+    } finally {
+      setDeletingFileKey(null)
+    }
+  }
+
   async function confirmDeleteFile(file: FileEntry) {
     openModal(
       <StyledModal
         title="Confirm Delete?"
-        onConfirm={() => {
-          closeAllModals()
-        }}
+        onConfirm={() => deleteFile(file)}
         onCancel={closeAllModals}
         open={true}
         confirmText="Delete"
         cancelText="Cancel"
         confirmVariant="danger"
+        confirmLoading={file.type === 'file' && deletingFileKey === file.key}
       >
         <p className="text-text-secondary">
           Are you sure you want to delete {file.name}? This action cannot be undone.
@@ -196,6 +233,24 @@ export default function MapsManager(props: {
     )
   }
 
+  function openCountryPickerModal() {
+    openModal(
+      <CountryPickerModal
+        onCancel={closeAllModals}
+        installedFilenames={(props.maps.regionFiles ?? []).map((f) => f.name)}
+        onDownloadStart={() => {
+          invalidateDownloads()
+          addNotification({
+            type: 'success',
+            message: 'Download queued. Watch progress below.',
+          })
+          closeAllModals()
+        }}
+      />,
+      'country-picker-modal'
+    )
+  }
+
   async function openDownloadModal() {
     openModal(
       <DownloadURLModal
@@ -251,7 +306,23 @@ export default function MapsManager(props: {
               }}
             />
           )}
-          {globalMapInfo && (
+          {globalMapInfo && globalMapAlreadyDownloaded && (
+            <Alert
+              title="Global Map Installed"
+              message={`Your global map build ${globalMapInfo.date} (${formatBytes(globalMapInfo.size, 1)}) is stored locally and ready for offline use.`}
+              type="success"
+              variant="bordered"
+              className="mt-8"
+              icon="IconCircleCheck"
+              buttonProps={{
+                variant: 'secondary',
+                children: 'Download latest build',
+                icon: 'IconRefresh',
+                onClick: () => confirmGlobalMapDownload(),
+              }}
+            />
+          )}
+          {globalMapInfo && !globalMapAlreadyDownloaded && (
             <Alert
               title="Global Map Coverage Available"
               message={`Download a complete worldwide map from Protomaps (${formatBytes(globalMapInfo.size, 1)}, build ${globalMapInfo.date}). This is a large file but covers the entire planet — no individual region downloads needed.`}
@@ -268,6 +339,21 @@ export default function MapsManager(props: {
               }}
             />
           )}
+          <Alert
+            title="Download by country or region"
+            message="Pick the countries you actually need — from a single country to a whole continent — and we'll pull just those tiles from the global Protomaps archive. Much smaller than the full 125 GB global map."
+            type="info-inverted"
+            variant="bordered"
+            className="mt-8"
+            icon="IconMap2"
+            buttonProps={{
+              variant: 'primary',
+              children: 'Choose Countries',
+              icon: 'IconMap2',
+              onClick: openCountryPickerModal,
+            }}
+          />
+
           <div className="mt-8 mb-6 flex items-center justify-between">
             <StyledSectionHeader title="Curated Map Regions" className="!mb-0" />
             <StyledButton

admin/inertia/pages/settings/models.tsx

@@ -283,7 +283,7 @@ export default function ModelsPage(props: {
               type="warning"
               variant="bordered"
               title="GPU Not Accessible"
-              message={`Your system has an NVIDIA GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`}
+              message={`Your system has ${systemInfo?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`}
               className="!mt-6"
               dismissible={true}
               onDismiss={handleDismissGpuBanner}

admin/inertia/pages/settings/system.tsx

@@ -209,7 +209,7 @@ export default function SettingsPage(props: {
                     type="warning"
                     variant="bordered"
                     title="GPU Not Accessible to AI Assistant"
-                    message="Your system has an NVIDIA GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower."
+                    message={`Your system has ${info?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower.`}
                     dismissible={true}
                     onDismiss={handleDismissGpuBanner}
                     buttonProps={{

admin/inertia/pages/settings/update.tsx

@@ -5,16 +5,17 @@ import StyledTable from '~/components/StyledTable'
 import StyledSectionHeader from '~/components/StyledSectionHeader'
 import ActiveDownloads from '~/components/ActiveDownloads'
 import Alert from '~/components/Alert'
-import { useEffect, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import { IconAlertCircle, IconArrowBigUpLines, IconCheck, IconCircleCheck, IconReload } from '@tabler/icons-react'
 import { SystemUpdateStatus } from '../../../types/system'
 import type { ContentUpdateCheckResult, ResourceUpdateInfo } from '../../../types/collections'
 import api from '~/lib/api'
 import Input from '~/components/inputs/Input'
 import Switch from '~/components/inputs/Switch'
-import { useMutation } from '@tanstack/react-query'
+import { useMutation, useQueryClient } from '@tanstack/react-query'
 import { useNotifications } from '~/context/NotificationContext'
 import { useSystemSetting } from '~/hooks/useSystemSetting'
+import { formatBytes } from '~/lib/util'
 
 type Props = {
   updateAvailable: boolean
@@ -23,8 +24,26 @@ type Props = {
   earlyAccess: boolean
 }
 
+const STAGE_LABELS: Record<SystemUpdateStatus['stage'], string> = {
+  idle: 'Preparing Update',
+  starting: 'Starting Update',
+  pulling: 'Pulling Images',
+  pulled: 'Images Pulled',
+  recreating: 'Recreating Containers',
+  complete: 'Update Complete',
+  error: 'Update Failed',
+}
+
+const ADVANCED_STAGES: ReadonlySet<SystemUpdateStatus['stage']> = new Set([
+  'pulling',
+  'pulled',
+  'recreating',
+  'complete',
+])
+
 function ContentUpdatesSection() {
   const { addNotification } = useNotifications()
+  const queryClient = useQueryClient()
   const [checkResult, setCheckResult] = useState<ContentUpdateCheckResult | null>(null)
   const [isChecking, setIsChecking] = useState(false)
   const [applyingIds, setApplyingIds] = useState<Set<string>>(new Set())
@@ -60,6 +79,9 @@ function ContentUpdatesSection() {
             ? { ...prev, updates: prev.updates.filter((u) => u.resource_id !== update.resource_id) }
             : prev
         )
+        // Force Active Downloads to refetch now — small updates finish before the next
+        // idle poll fires, so without this the user wouldn't see them.
+        queryClient.invalidateQueries({ queryKey: ['download-jobs'] })
       } else {
         addNotification({ type: 'error', message: result?.error || 'Failed to start update' })
       }
@@ -95,6 +117,9 @@ function ContentUpdatesSection() {
             ? { ...prev, updates: prev.updates.filter((u) => !successIds.has(u.resource_id)) }
             : prev
         )
+        if (successIds.size > 0) {
+          queryClient.invalidateQueries({ queryKey: ['download-jobs'] })
+        }
       }
     } catch {
       addNotification({ type: 'error', message: 'Failed to apply updates' })
@@ -182,6 +207,15 @@ function ContentUpdatesSection() {
                     </span>
                   ),
                 },
+                {
+                  accessor: 'size_bytes',
+                  title: 'Size',
+                  render: (record) => (
+                    <span className="text-desert-stone-dark">
+                      {record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'}
+                    </span>
+                  ),
+                },
                 {
                   accessor: 'installed_version',
                   title: 'Version',
@@ -234,6 +268,12 @@ export default function SystemUpdatePage(props: { system: Props }) {
   const [email, setEmail] = useState('')
   const [versionInfo, setVersionInfo] = useState<Omit<Props, 'earlyAccess'>>(props.system)
   const [showConnectionLostNotice, setShowConnectionLostNotice] = useState(false)
+  // Tracks whether this update session has progressed past 'idle'/'starting'.
+  // The sidecar sits on 'complete' for ~5s before resetting to 'idle' (see
+  // install/sidecar-updater/update-watcher.sh), and the SPA can miss that
+  // window across the admin container restart. If we resurface to 'idle'
+  // after seeing an advanced stage, treat it as the missed completion.
+  const seenAdvancedStageRef = useRef(false)
 
   const earlyAccessSetting = useSystemSetting({
     key: 'system.earlyAccess', initialData: {
@@ -253,11 +293,22 @@ export default function SystemUpdatePage(props: { system: Props }) {
         }
         setUpdateStatus(response)
 
+        if (ADVANCED_STAGES.has(response.stage)) {
+          seenAdvancedStageRef.current = true
+        }
+
         // If we can connect again, hide the connection lost notice
         setShowConnectionLostNotice(false)
 
-        // Check if update is complete or errored
-        if (response.stage === 'complete') {
+        // Check if update is complete or errored. We also treat a return to
+        // 'idle' as completion if we previously saw an advanced stage — this
+        // catches the race where the sidecar's brief 'complete' window passes
+        // while we're disconnected during the admin container restart.
+        const isComplete =
+          response.stage === 'complete' ||
+          (response.stage === 'idle' && seenAdvancedStageRef.current)
+
+        if (isComplete) {
           // Re-check version so the KV store clears the stale "update available" flag
           // before we reload, otherwise the banner shows "current → current"
           try {
@@ -287,6 +338,7 @@ export default function SystemUpdatePage(props: { system: Props }) {
   const handleStartUpdate = async () => {
     try {
       setError(null)
+      seenAdvancedStageRef.current = false
       setIsUpdating(true)
       const response = await api.startSystemUpdate()
       if (!response || !response.success) {
@@ -351,7 +403,7 @@ export default function SystemUpdatePage(props: { system: Props }) {
     if (updateStatus?.stage === 'error')
       return <IconAlertCircle className="h-12 w-12 text-desert-red" />
     if (isUpdating) return <IconReload className="h-12 w-12 text-desert-green animate-spin" />
-    if (props.system.updateAvailable)
+    if (versionInfo.updateAvailable)
       return <IconArrowBigUpLines className="h-16 w-16 text-desert-green" />
     return <IconCircleCheck className="h-16 w-16 text-desert-olive" />
   }
@@ -363,6 +415,9 @@ export default function SystemUpdatePage(props: { system: Props }) {
     onSuccess: () => {
       addNotification({ message: 'Setting updated successfully.', type: 'success' })
       earlyAccessSetting.refetch()
+      // Toggling Early Access changes which versions are eligible, so re-evaluate
+      // immediately rather than making the user click Check Again.
+      checkVersionMutation.mutate()
     },
     onError: (error) => {
       console.error('Error updating setting:', error)
@@ -444,11 +499,11 @@ export default function SystemUpdatePage(props: { system: Props }) {
               {!isUpdating && (
                 <>
                   <h2 className="text-2xl font-bold text-desert-green mb-2">
-                    {props.system.updateAvailable ? 'Update Available' : 'System Up to Date'}
+                    {versionInfo.updateAvailable ? 'Update Available' : 'System Up to Date'}
                   </h2>
                   <p className="text-desert-stone-dark mb-6">
-                    {props.system.updateAvailable
-                      ? `A new version (${props.system.latestVersion}) is available for your Project N.O.M.A.D. instance.`
+                    {versionInfo.updateAvailable
+                      ? `A new version (${versionInfo.latestVersion}) is available for your Project N.O.M.A.D. instance.`
                       : 'Your system is running the latest version!'}
                   </p>
                 </>
@@ -456,8 +511,8 @@ export default function SystemUpdatePage(props: { system: Props }) {
 
               {isUpdating && updateStatus && (
                 <>
-                  <h2 className="text-2xl font-bold text-desert-green mb-2 capitalize">
-                    {updateStatus.stage === 'idle' ? 'Preparing Update' : updateStatus.stage}
+                  <h2 className="text-2xl font-bold text-desert-green mb-2">
+                    {STAGE_LABELS[updateStatus.stage] ?? updateStatus.stage}
                   </h2>
                   <p className="text-desert-stone-dark mb-6">{updateStatus.message}</p>
                 </>

admin/inertia/pages/settings/zim/index.tsx

@@ -1,5 +1,6 @@
 import { Head } from '@inertiajs/react'
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
+import { useMemo, useState } from 'react'
 import StyledTable from '~/components/StyledTable'
 import SettingsLayout from '~/layouts/SettingsLayout'
 import api from '~/lib/api'
@@ -10,11 +11,18 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
 import Alert from '~/components/Alert'
 import { ZimFileWithMetadata } from '../../../../types/zim'
 import { SERVICE_NAMES } from '../../../../constants/service_names'
+import { formatBytes } from '~/lib/util'
+import { IconArrowDown, IconArrowUp, IconArrowsSort } from '@tabler/icons-react'
+
+type SortKey = 'name' | 'size'
+type SortDirection = 'asc' | 'desc'
 
 export default function ZimPage() {
   const queryClient = useQueryClient()
   const { openModal, closeAllModals } = useModals()
   const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX)
+  const [sortKey, setSortKey] = useState<SortKey>('size')
+  const [sortDirection, setSortDirection] = useState<SortDirection>('desc')
   const { data, isLoading } = useQuery<ZimFileWithMetadata[]>({
     queryKey: ['zim-files'],
     queryFn: getFiles,
@@ -25,6 +33,49 @@ export default function ZimPage() {
     return res.data.files
   }
 
+  const sortedData = useMemo(() => {
+    if (!data) return []
+    const copy = [...data]
+    copy.sort((a, b) => {
+      let cmp = 0
+      if (sortKey === 'size') {
+        const aSize = a.size_bytes ?? 0
+        const bSize = b.size_bytes ?? 0
+        cmp = aSize - bSize
+      } else {
+        const aName = (a.title || a.name).toLowerCase()
+        const bName = (b.title || b.name).toLowerCase()
+        cmp = aName.localeCompare(bName)
+      }
+      return sortDirection === 'asc' ? cmp : -cmp
+    })
+    return copy
+  }, [data, sortKey, sortDirection])
+
+  function toggleSort(key: SortKey) {
+    if (sortKey === key) {
+      setSortDirection((d) => (d === 'asc' ? 'desc' : 'asc'))
+    } else {
+      setSortKey(key)
+      setSortDirection(key === 'size' ? 'desc' : 'asc')
+    }
+  }
+
+  function renderSortHeader(label: string, key: SortKey) {
+    const active = sortKey === key
+    const Icon = !active ? IconArrowsSort : sortDirection === 'asc' ? IconArrowUp : IconArrowDown
+    return (
+      <button
+        type="button"
+        onClick={() => toggleSort(key)}
+        className="flex items-center gap-1 font-semibold text-text-primary hover:text-desert-orange"
+      >
+        {label}
+        <Icon className="size-4" />
+      </button>
+    )
+  }
+
   async function confirmDeleteFile(file: ZimFileWithMetadata) {
     openModal(
       <StyledModal
@@ -83,7 +134,7 @@ export default function ZimPage() {
             columns={[
               {
                 accessor: 'title',
-                title: 'Title',
+                title: renderSortHeader('Title', 'name'),
                 render: (record) => (
                   <span className="font-medium">
                     {record.title || record.name}
@@ -99,6 +150,15 @@ export default function ZimPage() {
                   </span>
                 ),
               },
+              {
+                accessor: 'size_bytes',
+                title: renderSortHeader('Size', 'size'),
+                render: (record) => (
+                  <span className="text-text-secondary tabular-nums">
+                    {record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'}
+                  </span>
+                ),
+              },
               {
                 accessor: 'actions',
                 title: 'Actions',
@@ -117,7 +177,7 @@ export default function ZimPage() {
                 ),
               },
             ]}
-            data={data || []}
+            data={sortedData}
           />
         </main>
       </div>

admin/inertia/pages/settings/zim/remote-explorer.tsx

@@ -21,7 +21,16 @@ import useInternetStatus from '~/hooks/useInternetStatus'
 import Alert from '~/components/Alert'
 import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
 import Input from '~/components/inputs/Input'
-import { IconSearch, IconBooks } from '@tabler/icons-react'
+import {
+  IconSearch,
+  IconBooks,
+  IconFolder,
+  IconFileDownload,
+  IconChevronRight,
+  IconPlus,
+  IconTrash,
+  IconLibrary,
+} from '@tabler/icons-react'
 import useDebounce from '~/hooks/useDebounce'
 import CategoryCard from '~/components/CategoryCard'
 import TierSelectionModal from '~/components/TierSelectionModal'
@@ -34,6 +43,13 @@ import { SERVICE_NAMES } from '../../../../constants/service_names'
 
 const CURATED_CATEGORIES_KEY = 'curated-categories'
 const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
+const CUSTOM_LIBRARIES_KEY = 'custom-libraries'
+
+type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean }
+type BrowseResult = {
+  directories: { name: string; url: string }[]
+  files: { name: string; url: string; size_bytes: number | null }[]
+}
 
 export default function ZimRemoteExplorer() {
   const queryClient = useQueryClient()
@@ -56,6 +72,20 @@ export default function ZimRemoteExplorer() {
   const [selectedWikipedia, setSelectedWikipedia] = useState<string | null>(null)
   const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false)
 
+  // Custom library state - persist selection to localStorage
+  const [selectedSource, setSelectedSource] = useState<'default' | number>(() => {
+    try {
+      const saved = localStorage.getItem('nomad:zim-library-source')
+      if (saved && saved !== 'default') return parseInt(saved, 10)
+    } catch {}
+    return 'default'
+  })
+  const [browseUrl, setBrowseUrl] = useState<string | null>(null)
+  const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([])
+  const [manageModalOpen, setManageModalOpen] = useState(false)
+  const [newLibraryName, setNewLibraryName] = useState('')
+  const [newLibraryUrl, setNewLibraryUrl] = useState('')
+
   const debouncedSetQuery = debounce((val: string) => {
     setQuery(val)
   }, 400)
@@ -79,6 +109,26 @@ export default function ZimRemoteExplorer() {
     enabled: true,
   })
 
+  // Fetch custom libraries
+  const { data: customLibraries } = useQuery({
+    queryKey: [CUSTOM_LIBRARIES_KEY],
+    queryFn: () => api.listCustomLibraries(),
+    refetchOnWindowFocus: false,
+  })
+
+  // Browse custom library directory
+  const {
+    data: browseData,
+    isLoading: isBrowsing,
+    error: browseError,
+  } = useQuery<BrowseResult>({
+    queryKey: ['browse-library', browseUrl],
+    queryFn: () => api.browseLibrary(browseUrl!) as Promise<BrowseResult>,
+    enabled: !!browseUrl && selectedSource !== 'default',
+    refetchOnWindowFocus: false,
+    retry: false,
+  })
+
   const { data, fetchNextPage, isFetching, isLoading } =
     useInfiniteQuery<ListRemoteZimFilesResponse>({
       queryKey: ['remote-zim-files', query],
@@ -97,6 +147,7 @@ export default function ZimRemoteExplorer() {
       getNextPageParam: (lastPage) => (lastPage.has_more ? lastPage.next_start : undefined),
       refetchOnWindowFocus: false,
       placeholderData: keepPreviousData,
+      enabled: selectedSource === 'default',
     })
 
   const flatData = useMemo(() => {
@@ -140,6 +191,50 @@ export default function ZimRemoteExplorer() {
     fetchOnBottomReached(tableParentRef.current)
   }, [fetchOnBottomReached])
 
+  // Restore custom library selection on mount when data loads
+  useEffect(() => {
+    if (selectedSource !== 'default' && customLibraries) {
+      const lib = customLibraries.find((l) => l.id === selectedSource)
+      if (lib && !browseUrl) {
+        setBrowseUrl(lib.base_url)
+        setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
+      } else if (!lib) {
+        // Saved library was deleted
+        setSelectedSource('default')
+        localStorage.setItem('nomad:zim-library-source', 'default')
+      }
+    }
+  }, [customLibraries, selectedSource])
+
+  // When selecting a custom library, navigate to its root
+  const handleSourceChange = (value: string) => {
+    localStorage.setItem('nomad:zim-library-source', value)
+    if (value === 'default') {
+      setSelectedSource('default')
+      setBrowseUrl(null)
+      setBreadcrumbs([])
+    } else {
+      const id = parseInt(value, 10)
+      const lib = customLibraries?.find((l) => l.id === id)
+      if (lib) {
+        setSelectedSource(id)
+        setBrowseUrl(lib.base_url)
+        setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
+      }
+    }
+  }
+
+  const navigateToDirectory = (name: string, url: string) => {
+    setBrowseUrl(url)
+    setBreadcrumbs((prev) => [...prev, { name, url }])
+  }
+
+  const navigateToBreadcrumb = (index: number) => {
+    const crumb = breadcrumbs[index]
+    setBrowseUrl(crumb.url)
+    setBreadcrumbs((prev) => prev.slice(0, index + 1))
+  }
+
   async function confirmDownload(record: RemoteZimFileEntry) {
     openModal(
       <StyledModal
@@ -165,6 +260,31 @@ export default function ZimRemoteExplorer() {
     )
   }
 
+  async function confirmCustomDownload(file: { name: string; url: string; size_bytes: number | null }) {
+    openModal(
+      <StyledModal
+        title="Confirm Download?"
+        onConfirm={() => {
+          downloadCustomFile(file)
+          closeAllModals()
+        }}
+        onCancel={closeAllModals}
+        open={true}
+        confirmText="Download"
+        cancelText="Cancel"
+        confirmVariant="primary"
+      >
+        <p className="text-text-primary">
+          Are you sure you want to download{' '}
+          <strong>{file.name}</strong>
+          {file.size_bytes ? ` (${formatBytes(file.size_bytes)})` : ''}? The Kiwix
+          application will be restarted after the download is complete.
+        </p>
+      </StyledModal>,
+      'confirm-download-custom-modal'
+    )
+  }
+
   async function downloadFile(record: RemoteZimFileEntry) {
     try {
       await api.downloadRemoteZimFile(record.download_url, {
@@ -179,6 +299,26 @@ export default function ZimRemoteExplorer() {
     }
   }
 
+  async function downloadCustomFile(file: { name: string; url: string; size_bytes: number | null }) {
+    try {
+      await api.downloadRemoteZimFile(file.url, {
+        title: file.name.replace(/\.zim$/, ''),
+        size_bytes: file.size_bytes ?? undefined,
+      })
+      addNotification({
+        message: `Started downloading "${file.name}"`,
+        type: 'success',
+      })
+      invalidateDownloads()
+    } catch (error) {
+      console.error('Error downloading file:', error)
+      addNotification({
+        message: 'Failed to start download.',
+        type: 'error',
+      })
+    }
+  }
+
   // Category/tier handlers
   const handleCategoryClick = (category: CategoryWithStatus) => {
     if (!isOnline) return
@@ -264,6 +404,35 @@ export default function ZimRemoteExplorer() {
     },
   })
 
+  // Custom library management
+  const addLibraryMutation = useMutation({
+    mutationFn: () => api.addCustomLibrary(newLibraryName.trim(), newLibraryUrl.trim()),
+    onSuccess: () => {
+      addNotification({ message: 'Custom library added.', type: 'success' })
+      queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
+      setNewLibraryName('')
+      setNewLibraryUrl('')
+    },
+    onError: () => {
+      addNotification({ message: 'Failed to add custom library.', type: 'error' })
+    },
+  })
+
+  const removeLibraryMutation = useMutation({
+    mutationFn: (id: number) => api.removeCustomLibrary(id),
+    onSuccess: (_data, id) => {
+      addNotification({ message: 'Custom library removed.', type: 'success' })
+      queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
+      if (selectedSource === id) {
+        setSelectedSource('default')
+        setBrowseUrl(null)
+        setBreadcrumbs([])
+      }
+    },
+  })
+
+  const hasCustomLibraries = customLibraries && customLibraries.length > 0
+
   return (
     <SettingsLayout>
       <Head title="Content Explorer | Project N.O.M.A.D." />
@@ -302,7 +471,7 @@ export default function ZimRemoteExplorer() {
               Force Refresh Collections
             </StyledButton>
           </div>
-          
+
           {/* Wikipedia Selector */}
           {isLoadingWikipedia ? (
             <div className="mt-8 bg-surface-primary rounded-lg border border-border-subtle p-6">
@@ -360,87 +529,303 @@ export default function ZimRemoteExplorer() {
           ) : (
             <p className="text-text-muted mt-4">No curated content categories available.</p>
           )}
-          <StyledSectionHeader title="Browse the Kiwix Library" className="mt-12 mb-4" />
-          <div className="flex justify-start mt-4">
-            <Input
-              name="search"
-              label=""
-              placeholder="Search available ZIM files..."
-              value={queryUI}
-              onChange={(e) => {
-                setQueryUI(e.target.value)
-                debouncedSetQuery(e.target.value)
-              }}
-              className="w-1/3"
-              leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
-            />
+
+          {/* Kiwix Library / Custom Library Browser */}
+          <div className="mt-12 mb-4 flex items-center justify-between">
+            <StyledSectionHeader title="Browse the Kiwix Library" className="!mb-0" />
+            <StyledButton
+              onClick={() => setManageModalOpen(true)}
+              disabled={!isOnline}
+              icon="IconLibrary"
+            >
+              {hasCustomLibraries ? 'Manage Custom Libraries' : 'Add Custom Library'}
+            </StyledButton>
           </div>
-          <StyledTable<RemoteZimFileEntry & { actions?: any }>
-            data={flatData.map((i, idx) => {
-              const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
-              return {
-                ...i,
-                height: `${row?.size || 48}px`, // Use the size from the virtualizer
-                translateY: row?.start || 0,
-              }
-            })}
-            ref={tableParentRef}
-            loading={isLoading}
-            columns={[
-              {
-                accessor: 'title',
-              },
-              {
-                accessor: 'author',
-              },
-              {
-                accessor: 'summary',
-              },
-              {
-                accessor: 'updated',
-                render(record) {
-                  return new Intl.DateTimeFormat('en-US', {
-                    dateStyle: 'medium',
-                  }).format(new Date(record.updated))
-                },
-              },
-              {
-                accessor: 'size_bytes',
-                title: 'Size',
-                render(record) {
-                  return formatBytes(record.size_bytes)
-                },
-              },
-              {
-                accessor: 'actions',
-                render(record) {
-                  return (
-                    <div className="flex space-x-2">
-                      <StyledButton
-                        icon={'IconDownload'}
-                        onClick={() => {
-                          confirmDownload(record)
-                        }}
+
+          {/* Source selector dropdown */}
+          {hasCustomLibraries && (
+            <div className="flex items-center gap-3 mb-4">
+              <label className="text-sm font-medium text-text-secondary">Source:</label>
+              <select
+                value={selectedSource === 'default' ? 'default' : String(selectedSource)}
+                onChange={(e) => handleSourceChange(e.target.value)}
+                className="rounded-md border border-border-default bg-surface-primary text-text-primary px-3 py-1.5 text-sm focus:outline-none focus:ring-2 focus:ring-desert-green"
+              >
+                <option value="default">Default (Kiwix)</option>
+                {customLibraries.map((lib) => (
+                  <option key={lib.id} value={String(lib.id)}>
+                    {lib.name}
+                  </option>
+                ))}
+              </select>
+            </div>
+          )}
+
+          {/* Default Kiwix library browser */}
+          {selectedSource === 'default' && (
+            <>
+              <div className="flex justify-start mt-4">
+                <Input
+                  name="search"
+                  label=""
+                  placeholder="Search available ZIM files..."
+                  value={queryUI}
+                  onChange={(e) => {
+                    setQueryUI(e.target.value)
+                    debouncedSetQuery(e.target.value)
+                  }}
+                  className="w-1/3"
+                  leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
+                />
+              </div>
+              <StyledTable<RemoteZimFileEntry & { actions?: any }>
+                data={flatData.map((i, idx) => {
+                  const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
+                  return {
+                    ...i,
+                    height: `${row?.size || 48}px`,
+                    translateY: row?.start || 0,
+                  }
+                })}
+                ref={tableParentRef}
+                loading={isLoading}
+                columns={[
+                  {
+                    accessor: 'title',
+                  },
+                  {
+                    accessor: 'author',
+                  },
+                  {
+                    accessor: 'summary',
+                  },
+                  {
+                    accessor: 'updated',
+                    render(record) {
+                      return new Intl.DateTimeFormat('en-US', {
+                        dateStyle: 'medium',
+                      }).format(new Date(record.updated))
+                    },
+                  },
+                  {
+                    accessor: 'size_bytes',
+                    title: 'Size',
+                    render(record) {
+                      return formatBytes(record.size_bytes)
+                    },
+                  },
+                  {
+                    accessor: 'actions',
+                    render(record) {
+                      return (
+                        <div className="flex space-x-2">
+                          <StyledButton
+                            icon={'IconDownload'}
+                            onClick={() => {
+                              confirmDownload(record)
+                            }}
+                          >
+                            Download
+                          </StyledButton>
+                        </div>
+                      )
+                    },
+                  },
+                ]}
+                className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
+                tableBodyStyle={{
+                  position: 'relative',
+                  height: `${virtualizer.getTotalSize()}px`,
+                }}
+                containerProps={{
+                  onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
+                }}
+                compact
+                rowLines
+              />
+            </>
+          )}
+
+          {/* Custom library directory browser */}
+          {selectedSource !== 'default' && (
+            <div className="mt-4">
+              {/* Breadcrumb navigation */}
+              <nav className="flex items-center gap-1 text-sm text-text-muted mb-4 flex-wrap">
+                {breadcrumbs.map((crumb, idx) => (
+                  <span key={idx} className="flex items-center gap-1">
+                    {idx > 0 && <IconChevronRight className="w-4 h-4" />}
+                    {idx < breadcrumbs.length - 1 ? (
+                      <button
+                        onClick={() => navigateToBreadcrumb(idx)}
+                        className="text-desert-green hover:underline"
                       >
-                        Download
-                      </StyledButton>
-                    </div>
-                  )
-                },
-              },
-            ]}
-            className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
-            tableBodyStyle={{
-              position: 'relative',
-              height: `${virtualizer.getTotalSize()}px`,
-            }}
-            containerProps={{
-              onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
-            }}
-            compact
-            rowLines
-          />
+                        {crumb.name}
+                      </button>
+                    ) : (
+                      <span className="text-text-primary font-medium">{crumb.name}</span>
+                    )}
+                  </span>
+                ))}
+              </nav>
+
+              {isBrowsing && (
+                <div className="flex justify-center py-12">
+                  <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-desert-green"></div>
+                </div>
+              )}
+
+              {browseError && (
+                <Alert
+                  title="Could not fetch directory listing from this URL."
+                  message="The server may not support directory browsing, or the URL may be incorrect."
+                  type="error"
+                  variant="solid"
+                />
+              )}
+
+              {!isBrowsing && !browseError && browseData && (
+                <div className="bg-surface-primary rounded-lg border border-border-subtle overflow-hidden relative" style={{ maxHeight: '600px', overflowY: 'auto' }}>
+                  {browseData.directories.length === 0 && browseData.files.length === 0 ? (
+                    <p className="text-text-muted p-6 text-center">
+                      No directories or ZIM files found at this location.
+                    </p>
+                  ) : (
+                    <table className="w-full text-sm">
+                      <thead>
+                        <tr className="border-b border-border-subtle bg-surface-secondary sticky top-0 z-10">
+                          <th className="text-left px-4 py-3 font-medium text-text-secondary">Name</th>
+                          <th className="text-right px-4 py-3 font-medium text-text-secondary w-32">Size</th>
+                          <th className="text-right px-4 py-3 font-medium text-text-secondary w-36"></th>
+                        </tr>
+                      </thead>
+                      <tbody>
+                        {browseData.directories.map((dir) => (
+                          <tr
+                            key={dir.url}
+                            className="border-b border-border-subtle hover:bg-surface-secondary cursor-pointer transition-colors"
+                            onClick={() => navigateToDirectory(dir.name, dir.url)}
+                          >
+                            <td className="px-4 py-3">
+                              <span className="flex items-center gap-2 text-text-primary">
+                                <IconFolder className="w-5 h-5 text-desert-orange" />
+                                {dir.name}
+                              </span>
+                            </td>
+                            <td className="text-right px-4 py-3 text-text-muted">--</td>
+                            <td className="text-right px-4 py-3">
+                              <IconChevronRight className="w-4 h-4 text-text-muted ml-auto" />
+                            </td>
+                          </tr>
+                        ))}
+                        {browseData.files.map((file) => (
+                          <tr
+                            key={file.url}
+                            className="border-b border-border-subtle hover:bg-surface-secondary transition-colors"
+                          >
+                            <td className="px-4 py-3">
+                              <span className="flex items-center gap-2 text-text-primary">
+                                <IconFileDownload className="w-5 h-5 text-desert-green" />
+                                {file.name}
+                              </span>
+                            </td>
+                            <td className="text-right px-4 py-3 text-text-muted">
+                              {file.size_bytes ? formatBytes(file.size_bytes) : '--'}
+                            </td>
+                            <td className="text-right px-4 py-3">
+                              <StyledButton
+                                icon="IconDownload"
+                                onClick={() => confirmCustomDownload(file)}
+                              >
+                                Download
+                              </StyledButton>
+                            </td>
+                          </tr>
+                        ))}
+                      </tbody>
+                    </table>
+                  )}
+                </div>
+              )}
+            </div>
+          )}
+
           <ActiveDownloads filetype="zim" withHeader />
+
+          {/* Manage Custom Libraries Modal */}
+          <StyledModal
+            title="Manage Custom Libraries"
+            open={manageModalOpen}
+            onCancel={() => setManageModalOpen(false)}
+            cancelText="Close"
+          >
+            <div className="space-y-6">
+              <div>
+                <p className="text-sm text-text-muted mb-4">
+                  Add Kiwix mirrors or other ZIM file sources for faster downloads.
+                </p>
+
+                {/* Existing libraries */}
+                {customLibraries && customLibraries.length > 0 && (
+                  <div className="space-y-2 mb-6">
+                    {customLibraries.map((lib) => (
+                      <div
+                        key={lib.id}
+                        className="flex items-center justify-between bg-surface-secondary rounded-lg px-4 py-3 border border-border-subtle"
+                      >
+                        <div className="min-w-0 flex-1">
+                          <p className="font-medium text-text-primary truncate">
+                            {lib.name}
+                            {lib.is_default && (
+                              <span className="ml-2 text-xs text-text-muted font-normal">(built-in)</span>
+                            )}
+                          </p>
+                          <p className="text-xs text-text-muted truncate">{lib.base_url}</p>
+                        </div>
+                        {!lib.is_default && (
+                          <button
+                            onClick={() => removeLibraryMutation.mutate(lib.id)}
+                            className="ml-3 p-1.5 text-text-muted hover:text-red-500 transition-colors rounded"
+                            title="Remove library"
+                          >
+                            <IconTrash className="w-4 h-4" />
+                          </button>
+                        )}
+                      </div>
+                    ))}
+                  </div>
+                )}
+
+                {/* Add new library form */}
+                <div className="space-y-3">
+                  <Input
+                    name="library-name"
+                    label="Library Name"
+                    placeholder="e.g., Debian Mirror"
+                    value={newLibraryName}
+                    onChange={(e) => setNewLibraryName(e.target.value)}
+                  />
+                  <Input
+                    name="library-url"
+                    label="Base URL"
+                    placeholder="e.g., https://cdimage.debian.org/mirror/kiwix.org/zim/"
+                    value={newLibraryUrl}
+                    onChange={(e) => setNewLibraryUrl(e.target.value)}
+                  />
+                  <StyledButton
+                    icon="IconPlus"
+                    onClick={() => addLibraryMutation.mutate()}
+                    disabled={
+                      !newLibraryName.trim() ||
+                      !newLibraryUrl.trim() ||
+                      addLibraryMutation.isPending
+                    }
+                  >
+                    Add Library
+                  </StyledButton>
+                </div>
+              </div>
+            </div>
+          </StyledModal>
         </main>
       </div>
     </SettingsLayout>

admin/package-lock.json

@@ -9,94 +9,95 @@
       "version": "0.0.0",
       "license": "ISC",
       "dependencies": {
-        "@adonisjs/auth": "^9.4.0",
-        "@adonisjs/core": "^6.18.0",
-        "@adonisjs/cors": "^2.2.1",
-        "@adonisjs/inertia": "^3.1.1",
-        "@adonisjs/lucid": "^21.8.2",
-        "@adonisjs/session": "^7.5.1",
-        "@adonisjs/shield": "^8.2.0",
-        "@adonisjs/static": "^1.1.1",
-        "@adonisjs/transmit": "^2.0.2",
-        "@adonisjs/transmit-client": "^1.0.0",
-        "@adonisjs/vite": "^4.0.0",
-        "@chonkiejs/core": "^0.0.7",
-        "@headlessui/react": "^2.2.4",
-        "@inertiajs/react": "^2.0.13",
-        "@markdoc/markdoc": "^0.5.2",
-        "@openzim/libzim": "^4.0.0",
-        "@protomaps/basemaps": "^5.7.0",
-        "@qdrant/js-client-rest": "^1.16.2",
-        "@tabler/icons-react": "^3.34.0",
-        "@tailwindcss/vite": "^4.1.10",
-        "@tanstack/react-query": "^5.81.5",
-        "@tanstack/react-query-devtools": "^5.83.0",
-        "@tanstack/react-virtual": "^3.13.12",
-        "@uppy/core": "^5.2.0",
-        "@uppy/dashboard": "^5.1.0",
-        "@uppy/react": "^5.1.1",
-        "@vinejs/vine": "^3.0.1",
-        "@vitejs/plugin-react": "^4.6.0",
-        "autoprefixer": "^10.4.21",
-        "axios": "^1.15.0",
-        "better-sqlite3": "^12.1.1",
-        "bullmq": "^5.65.1",
-        "cheerio": "^1.2.0",
-        "compression": "^1.8.1",
-        "dockerode": "^4.0.7",
-        "edge.js": "^6.2.1",
-        "fast-xml-parser": "^5.5.7",
-        "fuse.js": "^7.1.0",
-        "jszip": "^3.10.1",
-        "luxon": "^3.6.1",
-        "maplibre-gl": "^4.7.1",
-        "mysql2": "^3.14.1",
-        "ollama": "^0.6.3",
-        "openai": "^6.27.0",
-        "pdf-parse": "^2.4.5",
-        "pdf2pic": "^3.2.0",
-        "pino-pretty": "^13.0.0",
-        "pmtiles": "^4.4.0",
-        "postcss": "^8.5.6",
-        "react": "^19.1.0",
-        "react-adonis-transmit": "^1.0.1",
-        "react-dom": "^19.1.0",
-        "react-map-gl": "^8.1.0",
-        "react-markdown": "^10.1.0",
-        "reflect-metadata": "^0.2.2",
-        "remark-gfm": "^4.0.1",
-        "sharp": "^0.34.5",
-        "stopword": "^3.1.5",
-        "systeminformation": "^5.31.0",
-        "tailwindcss": "^4.2.1",
-        "tar": "^7.5.11",
-        "tesseract.js": "^7.0.0",
-        "url-join": "^5.0.0",
-        "yaml": "^2.8.3"
+        "@adonisjs/auth": "9.6.0",
+        "@adonisjs/core": "6.19.3",
+        "@adonisjs/cors": "2.2.1",
+        "@adonisjs/inertia": "3.1.1",
+        "@adonisjs/lucid": "21.8.2",
+        "@adonisjs/session": "7.7.1",
+        "@adonisjs/shield": "8.2.0",
+        "@adonisjs/static": "1.1.1",
+        "@adonisjs/transmit": "2.0.2",
+        "@adonisjs/transmit-client": "1.1.0",
+        "@adonisjs/vite": "4.0.0",
+        "@chonkiejs/core": "0.0.7",
+        "@headlessui/react": "2.2.9",
+        "@inertiajs/react": "2.3.13",
+        "@markdoc/markdoc": "0.5.4",
+        "@openzim/libzim": "4.0.0",
+        "@protomaps/basemaps": "5.7.0",
+        "@qdrant/js-client-rest": "1.16.2",
+        "@tabler/icons-react": "3.36.1",
+        "@tailwindcss/vite": "4.1.18",
+        "@tanstack/react-query": "5.90.20",
+        "@tanstack/react-query-devtools": "5.91.3",
+        "@tanstack/react-virtual": "3.13.18",
+        "@uppy/core": "5.2.0",
+        "@uppy/dashboard": "5.1.0",
+        "@uppy/react": "5.1.1",
+        "@vinejs/vine": "3.0.1",
+        "@vitejs/plugin-react": "4.7.0",
+        "autoprefixer": "10.4.24",
+        "axios": "1.15.0",
+        "better-sqlite3": "12.6.2",
+        "bullmq": "5.67.2",
+        "cheerio": "1.2.0",
+        "compression": "1.8.1",
+        "dockerode": "4.0.9",
+        "edge.js": "6.4.0",
+        "fast-xml-parser": "5.5.9",
+        "fuse.js": "7.1.0",
+        "ipaddr.js": "^2.4.0",
+        "jszip": "3.10.1",
+        "luxon": "3.7.2",
+        "maplibre-gl": "4.7.1",
+        "mysql2": "3.16.2",
+        "ollama": "0.6.3",
+        "openai": "6.27.0",
+        "pdf-parse": "2.4.5",
+        "pdf2pic": "3.2.0",
+        "pino-pretty": "13.1.3",
+        "pmtiles": "4.4.0",
+        "postcss": "8.5.6",
+        "react": "19.2.4",
+        "react-adonis-transmit": "1.0.1",
+        "react-dom": "19.2.4",
+        "react-map-gl": "8.1.0",
+        "react-markdown": "10.1.0",
+        "reflect-metadata": "0.2.2",
+        "remark-gfm": "4.0.1",
+        "sharp": "0.34.5",
+        "stopword": "3.1.5",
+        "systeminformation": "5.31.0",
+        "tailwindcss": "4.2.2",
+        "tar": "7.5.11",
+        "tesseract.js": "7.0.0",
+        "url-join": "5.0.0",
+        "yaml": "2.8.3"
       },
       "devDependencies": {
-        "@adonisjs/assembler": "^7.8.2",
-        "@adonisjs/eslint-config": "^2.0.0",
-        "@adonisjs/prettier-config": "^1.4.4",
-        "@adonisjs/tsconfig": "^1.4.0",
-        "@japa/assert": "^4.0.1",
-        "@japa/plugin-adonisjs": "^4.0.0",
-        "@japa/runner": "^4.2.0",
+        "@adonisjs/assembler": "7.8.2",
+        "@adonisjs/eslint-config": "2.1.2",
+        "@adonisjs/prettier-config": "1.4.5",
+        "@adonisjs/tsconfig": "1.4.1",
+        "@japa/assert": "4.2.0",
+        "@japa/plugin-adonisjs": "4.0.0",
+        "@japa/runner": "4.5.0",
         "@swc/core": "1.11.24",
-        "@tanstack/eslint-plugin-query": "^5.81.2",
-        "@types/compression": "^1.8.1",
-        "@types/dockerode": "^4.0.1",
-        "@types/luxon": "^3.6.2",
-        "@types/node": "^22.15.18",
-        "@types/react": "^19.1.8",
-        "@types/react-dom": "^19.1.6",
-        "@types/stopword": "^2.0.3",
-        "eslint": "^9.26.0",
-        "hot-hook": "^0.4.0",
-        "prettier": "^3.5.3",
-        "ts-node-maintained": "^10.9.5",
-        "typescript": "~5.8.3",
-        "vite": "^6.4.2"
+        "@tanstack/eslint-plugin-query": "5.91.4",
+        "@types/compression": "1.8.1",
+        "@types/dockerode": "4.0.1",
+        "@types/luxon": "3.7.1",
+        "@types/node": "22.19.7",
+        "@types/react": "19.2.10",
+        "@types/react-dom": "19.2.3",
+        "@types/stopword": "2.0.3",
+        "eslint": "9.39.2",
+        "hot-hook": "0.4.0",
+        "prettier": "3.8.1",
+        "ts-node-maintained": "10.9.6",
+        "typescript": "5.8.3",
+        "vite": "6.4.2"
       }
     },
     "node_modules/@adobe/css-tools": {
@@ -10099,12 +10100,12 @@
       }
     },
     "node_modules/ipaddr.js": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
-      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz",
+      "integrity": "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ==",
       "license": "MIT",
       "engines": {
-        "node": ">= 0.10"
+        "node": ">= 10"
       }
     },
     "node_modules/is-alphabetical": {
@@ -12174,9 +12175,9 @@
       }
     },
     "node_modules/micromatch/node_modules/picomatch": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
-      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
+      "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
       "license": "MIT",
       "engines": {
         "node": ">=8.6"
@@ -13351,9 +13352,9 @@
       "license": "ISC"
     },
     "node_modules/picomatch": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
-      "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz",
+      "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==",
       "license": "MIT",
       "engines": {
         "node": ">=12"
@@ -13790,6 +13791,15 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/proxy-addr/node_modules/ipaddr.js": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
+      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/proxy-from-env": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",

admin/package.json

@@ -38,94 +38,95 @@
     "#jobs/*": "./app/jobs/*.js"
   },
   "devDependencies": {
-    "@adonisjs/assembler": "^7.8.2",
-    "@adonisjs/eslint-config": "^2.0.0",
-    "@adonisjs/prettier-config": "^1.4.4",
-    "@adonisjs/tsconfig": "^1.4.0",
-    "@japa/assert": "^4.0.1",
-    "@japa/plugin-adonisjs": "^4.0.0",
-    "@japa/runner": "^4.2.0",
+    "@adonisjs/assembler": "7.8.2",
+    "@adonisjs/eslint-config": "2.1.2",
+    "@adonisjs/prettier-config": "1.4.5",
+    "@adonisjs/tsconfig": "1.4.1",
+    "@japa/assert": "4.2.0",
+    "@japa/plugin-adonisjs": "4.0.0",
+    "@japa/runner": "4.5.0",
     "@swc/core": "1.11.24",
-    "@tanstack/eslint-plugin-query": "^5.81.2",
-    "@types/compression": "^1.8.1",
-    "@types/dockerode": "^4.0.1",
-    "@types/luxon": "^3.6.2",
-    "@types/node": "^22.15.18",
-    "@types/react": "^19.1.8",
-    "@types/react-dom": "^19.1.6",
-    "@types/stopword": "^2.0.3",
-    "eslint": "^9.26.0",
-    "hot-hook": "^0.4.0",
-    "prettier": "^3.5.3",
-    "ts-node-maintained": "^10.9.5",
-    "typescript": "~5.8.3",
-    "vite": "^6.4.2"
+    "@tanstack/eslint-plugin-query": "5.91.4",
+    "@types/compression": "1.8.1",
+    "@types/dockerode": "4.0.1",
+    "@types/luxon": "3.7.1",
+    "@types/node": "22.19.7",
+    "@types/react": "19.2.10",
+    "@types/react-dom": "19.2.3",
+    "@types/stopword": "2.0.3",
+    "eslint": "9.39.2",
+    "hot-hook": "0.4.0",
+    "prettier": "3.8.1",
+    "ts-node-maintained": "10.9.6",
+    "typescript": "5.8.3",
+    "vite": "6.4.2"
   },
   "dependencies": {
-    "@adonisjs/auth": "^9.4.0",
-    "@adonisjs/core": "^6.18.0",
-    "@adonisjs/cors": "^2.2.1",
-    "@adonisjs/inertia": "^3.1.1",
-    "@adonisjs/lucid": "^21.8.2",
-    "@adonisjs/session": "^7.5.1",
-    "@adonisjs/shield": "^8.2.0",
-    "@adonisjs/static": "^1.1.1",
-    "@adonisjs/transmit": "^2.0.2",
-    "@adonisjs/transmit-client": "^1.0.0",
-    "@adonisjs/vite": "^4.0.0",
-    "@chonkiejs/core": "^0.0.7",
-    "@headlessui/react": "^2.2.4",
-    "@inertiajs/react": "^2.0.13",
-    "@markdoc/markdoc": "^0.5.2",
-    "@openzim/libzim": "^4.0.0",
-    "@protomaps/basemaps": "^5.7.0",
-    "@qdrant/js-client-rest": "^1.16.2",
-    "@tabler/icons-react": "^3.34.0",
-    "@tailwindcss/vite": "^4.1.10",
-    "@tanstack/react-query": "^5.81.5",
-    "@tanstack/react-query-devtools": "^5.83.0",
-    "@tanstack/react-virtual": "^3.13.12",
-    "@uppy/core": "^5.2.0",
-    "@uppy/dashboard": "^5.1.0",
-    "@uppy/react": "^5.1.1",
-    "@vinejs/vine": "^3.0.1",
-    "@vitejs/plugin-react": "^4.6.0",
-    "autoprefixer": "^10.4.21",
-    "axios": "^1.15.0",
-    "better-sqlite3": "^12.1.1",
-    "bullmq": "^5.65.1",
-    "cheerio": "^1.2.0",
-    "compression": "^1.8.1",
-    "dockerode": "^4.0.7",
-    "edge.js": "^6.2.1",
-    "fast-xml-parser": "^5.5.7",
-    "fuse.js": "^7.1.0",
-    "jszip": "^3.10.1",
-    "luxon": "^3.6.1",
-    "maplibre-gl": "^4.7.1",
-    "mysql2": "^3.14.1",
-    "ollama": "^0.6.3",
-    "openai": "^6.27.0",
-    "pdf-parse": "^2.4.5",
-    "pdf2pic": "^3.2.0",
-    "pino-pretty": "^13.0.0",
-    "pmtiles": "^4.4.0",
-    "postcss": "^8.5.6",
-    "react": "^19.1.0",
-    "react-adonis-transmit": "^1.0.1",
-    "react-dom": "^19.1.0",
-    "react-map-gl": "^8.1.0",
-    "react-markdown": "^10.1.0",
-    "reflect-metadata": "^0.2.2",
-    "remark-gfm": "^4.0.1",
-    "sharp": "^0.34.5",
-    "stopword": "^3.1.5",
-    "systeminformation": "^5.31.0",
-    "tailwindcss": "^4.2.1",
-    "tar": "^7.5.11",
-    "tesseract.js": "^7.0.0",
-    "url-join": "^5.0.0",
-    "yaml": "^2.8.3"
+    "@adonisjs/auth": "9.6.0",
+    "@adonisjs/core": "6.19.3",
+    "@adonisjs/cors": "2.2.1",
+    "@adonisjs/inertia": "3.1.1",
+    "@adonisjs/lucid": "21.8.2",
+    "@adonisjs/session": "7.7.1",
+    "@adonisjs/shield": "8.2.0",
+    "@adonisjs/static": "1.1.1",
+    "@adonisjs/transmit": "2.0.2",
+    "@adonisjs/transmit-client": "1.1.0",
+    "@adonisjs/vite": "4.0.0",
+    "@chonkiejs/core": "0.0.7",
+    "@headlessui/react": "2.2.9",
+    "@inertiajs/react": "2.3.13",
+    "@markdoc/markdoc": "0.5.4",
+    "@openzim/libzim": "4.0.0",
+    "@protomaps/basemaps": "5.7.0",
+    "@qdrant/js-client-rest": "1.16.2",
+    "@tabler/icons-react": "3.36.1",
+    "@tailwindcss/vite": "4.1.18",
+    "@tanstack/react-query": "5.90.20",
+    "@tanstack/react-query-devtools": "5.91.3",
+    "@tanstack/react-virtual": "3.13.18",
+    "@uppy/core": "5.2.0",
+    "@uppy/dashboard": "5.1.0",
+    "@uppy/react": "5.1.1",
+    "@vinejs/vine": "3.0.1",
+    "@vitejs/plugin-react": "4.7.0",
+    "autoprefixer": "10.4.24",
+    "axios": "1.15.0",
+    "better-sqlite3": "12.6.2",
+    "bullmq": "5.67.2",
+    "cheerio": "1.2.0",
+    "compression": "1.8.1",
+    "dockerode": "4.0.9",
+    "edge.js": "6.4.0",
+    "fast-xml-parser": "5.5.9",
+    "fuse.js": "7.1.0",
+    "ipaddr.js": "2.4.0",
+    "jszip": "3.10.1",
+    "luxon": "3.7.2",
+    "maplibre-gl": "4.7.1",
+    "mysql2": "3.16.2",
+    "ollama": "0.6.3",
+    "openai": "6.27.0",
+    "pdf-parse": "2.4.5",
+    "pdf2pic": "3.2.0",
+    "pino-pretty": "13.1.3",
+    "pmtiles": "4.4.0",
+    "postcss": "8.5.6",
+    "react": "19.2.4",
+    "react-adonis-transmit": "1.0.1",
+    "react-dom": "19.2.4",
+    "react-map-gl": "8.1.0",
+    "react-markdown": "10.1.0",
+    "reflect-metadata": "0.2.2",
+    "remark-gfm": "4.0.1",
+    "sharp": "0.34.5",
+    "stopword": "3.1.5",
+    "systeminformation": "5.31.0",
+    "tailwindcss": "4.2.2",
+    "tar": "7.5.11",
+    "tesseract.js": "7.0.0",
+    "url-join": "5.0.0",
+    "yaml": "2.8.3"
   },
   "hotHook": {
     "boundaries": [

admin/providers/gpu_passthrough_remediation_provider.ts

@@ -0,0 +1,122 @@
+import logger from '@adonisjs/core/services/logger'
+import type { ApplicationService } from '@adonisjs/core/types'
+
+/**
+ * Auto-remediates NVIDIA GPU passthrough loss after admin / host restart.
+ *
+ * After an update or container recreate, nomad_ollama's HostConfig.DeviceRequests
+ * still lists the nvidia driver, but the NVIDIA Container Toolkit binding inside
+ * the container is torn. `nvidia-smi` inside the container returns
+ * "Failed to initialize NVML: Unknown Error" and Ollama silently falls back to
+ * CPU inference. PR #208 added detection + a one-click "Fix: Reinstall AI Assistant"
+ * banner. This provider does that click automatically on admin boot when the
+ * condition is detected.
+ *
+ * Guards:
+ *   - NVIDIA-only. AMD passthrough_failed has a different fix path (HSA override
+ *     handling in PR #804) and is left to the user.
+ *   - One-shot per admin boot. The provider runs once on startup; if the recreate
+ *     itself fails the banner remains as a fallback.
+ *   - Opt-out via KV `ai.autoFixGpuPassthrough = false`.
+ *   - Skipped entirely when no NVIDIA runtime is registered with Docker.
+ */
+export default class GpuPassthroughRemediationProvider {
+  constructor(protected app: ApplicationService) {}
+
+  async boot() {
+    if (this.app.getEnvironment() !== 'web') return
+
+    setImmediate(async () => {
+      try {
+        const KVStore = (await import('#models/kv_store')).default
+        const { DockerService } = await import('#services/docker_service')
+        const { SERVICE_NAMES } = await import('../constants/service_names.js')
+        const Docker = (await import('dockerode')).default
+
+        const enabledRaw = await KVStore.getValue('ai.autoFixGpuPassthrough')
+        if (String(enabledRaw) === 'false') {
+          logger.info(
+            '[GpuPassthroughRemediationProvider] Auto-fix disabled via KV — skipping.'
+          )
+          return
+        }
+
+        const docker = new Docker({ socketPath: '/var/run/docker.sock' })
+        const dockerInfo = await docker.info()
+        const runtimes = dockerInfo.Runtimes || {}
+        const hasNvidiaRuntime = 'nvidia' in runtimes
+
+        if (!hasNvidiaRuntime) {
+          logger.info(
+            '[GpuPassthroughRemediationProvider] No NVIDIA runtime registered — skipping.'
+          )
+          return
+        }
+
+        const containers = await docker.listContainers({ all: false })
+        const ollama = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
+
+        if (!ollama) {
+          logger.info(
+            '[GpuPassthroughRemediationProvider] nomad_ollama not running — skipping.'
+          )
+          return
+        }
+
+        // Probe: exec nvidia-smi inside the Ollama container. NVML init failure
+        // is the signature of a broken passthrough that DeviceRequests can't see.
+        const container = docker.getContainer(ollama.Id)
+        const exec = await container.exec({
+          Cmd: ['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'],
+          AttachStdout: true,
+          AttachStderr: true,
+        })
+        const stream = await exec.start({ Tty: true })
+        const output = await new Promise<string>((resolve) => {
+          let buf = ''
+          const timer = setTimeout(() => resolve(buf || 'TIMEOUT'), 8000)
+          stream.on('data', (chunk: Buffer) => (buf += chunk.toString('utf8')))
+          stream.on('end', () => {
+            clearTimeout(timer)
+            resolve(buf)
+          })
+        })
+
+        const passthroughBroken =
+          /Failed to initialize NVML|Unknown Error|TIMEOUT/i.test(output) ||
+          !/[A-Za-z]/.test(output)
+
+        if (!passthroughBroken) {
+          logger.info(
+            '[GpuPassthroughRemediationProvider] NVIDIA passthrough healthy — no action needed.'
+          )
+          return
+        }
+
+        logger.warn(
+          '[GpuPassthroughRemediationProvider] NVIDIA passthrough broken (nvidia-smi inside nomad_ollama failed). ' +
+            'Auto-reinstalling nomad_ollama; volumes and installed models are preserved.'
+        )
+
+        const dockerService = new DockerService()
+        const result = await dockerService.forceReinstall(SERVICE_NAMES.OLLAMA)
+
+        if (result.success) {
+          await KVStore.setValue('gpu.autoRemediatedAt', new Date().toISOString())
+          logger.info(
+            '[GpuPassthroughRemediationProvider] nomad_ollama force-reinstall completed successfully.'
+          )
+        } else {
+          logger.error(
+            `[GpuPassthroughRemediationProvider] Force-reinstall failed: ${result.message}. ` +
+              'User can still click the "Fix: Reinstall AI Assistant" banner manually.'
+          )
+        }
+      } catch (err: any) {
+        logger.error(
+          `[GpuPassthroughRemediationProvider] Auto-remediation check failed: ${err?.message ?? err}`
+        )
+      }
+    })
+  }
+}

admin/providers/qdrant_restart_policy_provider.ts

@@ -0,0 +1,62 @@
+import logger from '@adonisjs/core/services/logger'
+import type { ApplicationService } from '@adonisjs/core/types'
+
+/**
+ * Ensures the nomad_qdrant container has the `unless-stopped` restart policy.
+ *
+ * Existing installations may have been created before this policy was enforced
+ * in the service seeder. Docker allows updating a container's restart policy
+ * without recreating it via the container.update() API.
+ *
+ * This provider runs once on every admin startup. If the policy is already
+ * correct, the check is a no-op.
+ */
+export default class QdrantRestartPolicyProvider {
+  constructor(protected app: ApplicationService) {}
+
+  async boot() {
+    if (this.app.getEnvironment() !== 'web') return
+
+    setImmediate(async () => {
+      try {
+        const Service = (await import('#models/service')).default
+        const { SERVICE_NAMES } = await import('../constants/service_names.js')
+        const Docker = (await import('dockerode')).default
+
+        const qdrantService = await Service.query()
+          .where('service_name', SERVICE_NAMES.QDRANT)
+          .first()
+
+        if (!qdrantService?.installed) {
+          logger.info('[QdrantRestartPolicyProvider] Qdrant not installed — skipping restart policy check.')
+          return
+        }
+
+        const docker = new Docker({ socketPath: '/var/run/docker.sock' })
+        const containers = await docker.listContainers({ all: true })
+        const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.QDRANT}`))
+
+        if (!containerInfo) {
+          logger.warn('[QdrantRestartPolicyProvider] Qdrant container not found — skipping restart policy check.')
+          return
+        }
+
+        const container = docker.getContainer(containerInfo.Id)
+        const inspected = await container.inspect()
+        const currentPolicy = inspected.HostConfig?.RestartPolicy?.Name
+
+        if (currentPolicy === 'unless-stopped') {
+          logger.info('[QdrantRestartPolicyProvider] Qdrant already has unless-stopped restart policy — no update needed.')
+          return
+        }
+
+        logger.info(`[QdrantRestartPolicyProvider] Qdrant restart policy is "${currentPolicy ?? 'none'}" — updating to unless-stopped.`)
+        await container.update({ RestartPolicy: { Name: 'unless-stopped', MaximumRetryCount: 0 } })
+        logger.info('[QdrantRestartPolicyProvider] Qdrant restart policy updated successfully.')
+      } catch (err: any) {
+        logger.error(`[QdrantRestartPolicyProvider] Failed to update Qdrant restart policy: ${err.message}`)
+        // Non-fatal: the container will still run, just without auto-restart on crash.
+      }
+    })
+  }
+}

admin/providers/version_check_provider.ts

@@ -0,0 +1,56 @@
+import logger from '@adonisjs/core/services/logger'
+import type { ApplicationService } from '@adonisjs/core/types'
+
+/**
+ * Self-heals stale `system.updateAvailable` after a sidecar-driven update.
+ *
+ * When the admin container is recreated on a new image, the KVStore still
+ * carries pre-update values for `system.updateAvailable` and
+ * `system.latestVersion`. Without intervention the UI keeps showing the
+ * "update available" banner until the next scheduled CheckUpdateJob (could be up to ~12h).
+ *
+ * Synchronous self-heal (no network): if the cached "latest" is not newer
+ * than the version we are now running, clear `updateAvailable`. The next
+ * scheduled CheckUpdateJob refreshes the cache from GitHub — we deliberately
+ * do not hit the network from boot to avoid coupling container startup to
+ * a network request to Github (e.g. container restart loop = flooding GitHub with requests).
+ *
+ * Note: this provider does not set `updateAvailable` to true if the cached
+ * "latest" is newer than the current version. We rely on the next scheduled
+ * CheckUpdateJob to do that, to avoid false positives in case of a stale cache.
+ */
+export default class VersionCheckProvider {
+  constructor(protected app: ApplicationService) { }
+
+  async boot() {
+    if (this.app.getEnvironment() !== 'web') return
+
+    setImmediate(async () => {
+      try {
+        const KVStore = (await import('#models/kv_store')).default
+        const { SystemService } = await import('#services/system_service')
+        const { isNewerVersion } = await import('../app/utils/version.js')
+
+        const current = SystemService.getAppVersion()
+        if (current === 'dev' || current === '0.0.0'){
+          logger.info(`[VersionCheckProvider] Skipping self-heal for version ${current}. Appears to be a dev build without proper version set.`)
+          return
+        }
+
+        logger.info(`[VersionCheckProvider] Checking for stale updateAvailable (current=${current})`)
+
+        const cachedLatest = (await KVStore.getValue('system.latestVersion')) as string | null
+        const earlyAccess = ((await KVStore.getValue('system.earlyAccess')) ?? false) as boolean
+
+        if (cachedLatest && !isNewerVersion(cachedLatest, current, earlyAccess)) {
+          await KVStore.setValue('system.updateAvailable', false)
+          logger.info(
+            `[VersionCheckProvider] Cleared stale updateAvailable (cached=${cachedLatest}, current=${current})`
+          )
+        }
+      } catch (err: any) {
+        logger.warn(`[VersionCheckProvider] Self-heal skipped: ${err?.message ?? err}`)
+      }
+    })
+  }
+}

admin/start/routes.ts

@@ -80,6 +80,10 @@ router
     router.post('/download-collection', [MapsController, 'downloadCollection'])
     router.get('/global-map-info', [MapsController, 'globalMapInfo'])
     router.post('/download-global-map', [MapsController, 'downloadGlobalMap'])
+    router.get('/countries', [MapsController, 'listCountries'])
+    router.get('/country-groups', [MapsController, 'listCountryGroups'])
+    router.post('/extract-preflight', [MapsController, 'extractPreflight'])
+    router.post('/extract', [MapsController, 'extractRegion'])
     router.get('/markers', [MapsController, 'listMarkers'])
     router.post('/markers', [MapsController, 'createMarker'])
     router.patch('/markers/:id', [MapsController, 'updateMarker'])
@@ -114,6 +118,7 @@ router
     router.post('/models', [OllamaController, 'dispatchModelDownload'])
     router.delete('/models', [OllamaController, 'deleteModel'])
     router.get('/installed-models', [OllamaController, 'installedModels'])
+    router.post('/unload-chat-models', [OllamaController, 'unloadChatModels'])
     router.post('/configure-remote', [OllamaController, 'configureRemote'])
     router.get('/remote-status', [OllamaController, 'remoteStatus'])
   })
@@ -137,12 +142,19 @@ router
   .group(() => {
     router.post('/upload', [RagController, 'upload'])
     router.get('/files', [RagController, 'getStoredFiles'])
+    router.get('/file-warnings', [RagController, 'getFileWarnings'])
     router.delete('/files', [RagController, 'deleteFile'])
+    router.post('/files/embed', [RagController, 'embedFile'])
     router.get('/active-jobs', [RagController, 'getActiveJobs'])
     router.get('/failed-jobs', [RagController, 'getFailedJobs'])
     router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs'])
     router.get('/job-status', [RagController, 'getJobStatus'])
     router.post('/sync', [RagController, 'scanAndSync'])
+    router.post('/re-embed-all', [RagController, 'reembedAll'])
+    router.post('/reset-and-rebuild', [RagController, 'resetAndRebuild'])
+    router.post('/estimate-batch', [RagController, 'estimateBatch'])
+    router.get('/policy-prompt-state', [RagController, 'policyPromptState'])
+    router.get('/health', [RagController, 'health'])
   })
   .prefix('/api/rag')
 
@@ -178,6 +190,12 @@ router
 
     router.get('/wikipedia', [ZimController, 'getWikipediaState'])
     router.post('/wikipedia/select', [ZimController, 'selectWikipedia'])
+
+    router.get('/custom-libraries', [ZimController, 'listCustomLibraries'])
+    router.post('/custom-libraries', [ZimController, 'addCustomLibrary'])
+    router.delete('/custom-libraries/:id', [ZimController, 'removeCustomLibrary'])
+    router.get('/browse-library', [ZimController, 'browseLibrary'])
+
     router.delete('/:filename', [ZimController, 'delete'])
   })
   .prefix('/api/zim')

admin/tests/unit/cloud_metadata_url.spec.ts

@@ -0,0 +1,57 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { assertNotCloudMetadataUrl } from '../../app/validators/common.js'
+
+const expectBlocked = (url: string) => {
+  assert.throws(() => assertNotCloudMetadataUrl(url), /instance metadata|http or https/)
+}
+
+const expectAllowed = (url: string) => {
+  assert.doesNotThrow(() => assertNotCloudMetadataUrl(url))
+}
+
+test('blocks plain IPv4 IMDS', () => {
+  expectBlocked('http://169.254.169.254/latest/meta-data/')
+})
+
+test('blocks IPv6 EC2 IMDS', () => {
+  expectBlocked('http://[fd00:ec2::254]/')
+})
+
+test('blocks fully-expanded IPv6 EC2 IMDS', () => {
+  expectBlocked('http://[fd00:ec2:0:0:0:0:0:254]/')
+})
+
+test('blocks IPv4-mapped IPv6 form of IMDS (dotted)', () => {
+  expectBlocked('http://[::ffff:169.254.169.254]/')
+})
+
+test('blocks IPv4-mapped IPv6 form of IMDS (hex)', () => {
+  expectBlocked('http://[::ffff:a9fe:a9fe]/')
+})
+
+test('blocks fully-expanded IPv4-mapped IPv6 form of IMDS', () => {
+  expectBlocked('http://[0:0:0:0:0:ffff:a9fe:a9fe]/')
+})
+
+test('blocks non-http(s) schemes', () => {
+  expectBlocked('file:///etc/passwd')
+  expectBlocked('gopher://169.254.169.254/')
+})
+
+test('allows LAN / loopback / RFC1918 hosts (intentional for remote-ollama use case)', () => {
+  expectAllowed('http://127.0.0.1:11434/')
+  expectAllowed('http://192.168.1.10:11434/')
+  expectAllowed('http://10.0.0.5:11434/')
+  expectAllowed('http://[::1]:11434/')
+})
+
+test('allows DNS hostnames', () => {
+  expectAllowed('http://ollama.lan:11434/')
+  expectAllowed('https://api.example.com/v1')
+})
+
+test('allows other link-local IPv4 addresses (not the metadata IP)', () => {
+  expectAllowed('http://169.254.1.1/')
+})

admin/tests/unit/global_map_banner.spec.ts

@@ -0,0 +1,37 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { hasDownloadedGlobalMap } from '../../inertia/lib/global_map_banner.js'
+
+test('returns true when the global map key already exists on disk', () => {
+  assert.equal(
+    hasDownloadedGlobalMap('20260402.pmtiles', [
+      { name: '20260402.pmtiles' },
+      { name: 'california.pmtiles' },
+    ]),
+    true
+  )
+})
+
+test('returns false when the global map key is missing', () => {
+  assert.equal(
+    hasDownloadedGlobalMap('20260402.pmtiles', [
+      { name: 'california.pmtiles' },
+    ]),
+    false
+  )
+})
+
+test('returns true when an older global map build is already on disk', () => {
+  assert.equal(
+    hasDownloadedGlobalMap('20260402.pmtiles', [
+      { name: '20260315.pmtiles' },
+      { name: 'california.pmtiles' },
+    ]),
+    true
+  )
+})
+
+test('returns false when there is no global map info', () => {
+  assert.equal(hasDownloadedGlobalMap(undefined, [{ name: '20260402.pmtiles' }]), false)
+})

admin/tests/unit/kb_file_grouping.spec.ts

@@ -0,0 +1,108 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import {
+  classifyKbFile,
+  groupAndSortKbFiles,
+  sourceToDisplayName,
+} from '../../inertia/lib/kb_file_grouping.js'
+import type { StoredFileInfo } from '../../types/rag.js'
+
+/** Wrap source paths into the minimal StoredFileInfo shape that
+ * `groupAndSortKbFiles` now expects. State + chunk count are irrelevant to
+ * grouping/sorting behavior; the per-file state-pill rendering is exercised
+ * separately in the modal's component tests (added in the follow-up PR). */
+const asInfos = (sources: string[]): StoredFileInfo[] =>
+  sources.map((source) => ({ source, state: null, chunksEmbedded: 0 }))
+
+test('classifyKbFile distinguishes ZIM, upload, admin_docs, and other', () => {
+  assert.equal(
+    classifyKbFile('/app/storage/zim/devdocs_en_python_2026-02.zim'),
+    'zim'
+  )
+  assert.equal(
+    classifyKbFile('/app/storage/kb_uploads/federalist.txt-8cc4ec95aa8f.txt'),
+    'upload'
+  )
+  assert.equal(classifyKbFile('/app/docs/release-notes.md'), 'admin_docs')
+  assert.equal(classifyKbFile('/app/README.md'), 'admin_docs')
+  assert.equal(classifyKbFile('/unexpected/path/file.txt'), 'other')
+})
+
+test('classifyKbFile does not match /app/READMEs that are not the bundled one', () => {
+  assert.equal(classifyKbFile('/app/README.md.bak'), 'other')
+})
+
+test('sourceToDisplayName returns the basename', () => {
+  assert.equal(
+    sourceToDisplayName('/app/storage/zim/devdocs_en_python_2026-02.zim'),
+    'devdocs_en_python_2026-02.zim'
+  )
+  assert.equal(sourceToDisplayName('/app/docs/release-notes.md'), 'release-notes.md')
+})
+
+test('groupAndSortKbFiles collapses all admin docs into a single row', () => {
+  const groups = groupAndSortKbFiles(asInfos([
+    '/app/docs/release-notes.md',
+    '/app/docs/getting-started.md',
+    '/app/docs/maps.md',
+    '/app/README.md',
+  ]))
+
+  assert.equal(groups.length, 1)
+  assert.equal(groups[0].bucket, 'admin_docs')
+  assert.equal(groups[0].count, 4)
+  assert.equal(groups[0].displayName, 'Project NOMAD documentation · 4 files')
+  assert.deepEqual(groups[0].members.sort(), [
+    '/app/README.md',
+    '/app/docs/getting-started.md',
+    '/app/docs/maps.md',
+    '/app/docs/release-notes.md',
+  ])
+})
+
+test('groupAndSortKbFiles orders buckets ZIM → upload → admin_docs → other', () => {
+  const groups = groupAndSortKbFiles(asInfos([
+    '/app/docs/release-notes.md',
+    '/unexpected/foo.txt',
+    '/app/storage/kb_uploads/upload.pdf',
+    '/app/storage/zim/devdocs.zim',
+  ]))
+
+  assert.deepEqual(
+    groups.map((g) => g.bucket),
+    ['zim', 'upload', 'admin_docs', 'other']
+  )
+})
+
+test('groupAndSortKbFiles alphabetizes within a bucket', () => {
+  const groups = groupAndSortKbFiles(asInfos([
+    '/app/storage/zim/wikipedia.zim',
+    '/app/storage/zim/devdocs.zim',
+    '/app/storage/zim/ifixit.zim',
+  ]))
+
+  assert.deepEqual(
+    groups.map((g) => g.displayName),
+    ['devdocs.zim', 'ifixit.zim', 'wikipedia.zim']
+  )
+})
+
+test('groupAndSortKbFiles uses singular noun when only one admin doc exists', () => {
+  const groups = groupAndSortKbFiles(asInfos(['/app/docs/release-notes.md']))
+  assert.equal(groups[0].displayName, 'Project NOMAD documentation · 1 file')
+})
+
+test('groupAndSortKbFiles handles empty input', () => {
+  assert.deepEqual(groupAndSortKbFiles([]), [])
+})
+
+test('groupAndSortKbFiles preserves a stable synthetic key for the admin docs group', () => {
+  const groups = groupAndSortKbFiles(asInfos([
+    '/app/docs/release-notes.md',
+    '/app/docs/maps.md',
+  ]))
+  // The admin-docs row uses a synthetic source key (not a real path) so it
+  // can be used as a React key without colliding with any real file row.
+  assert.equal(groups[0].source, '__admin_docs_group__')
+})

admin/tests/unit/kb_guardrail.spec.ts

@@ -0,0 +1,106 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import {
+  GUARDRAIL_ABSOLUTE_BYTES,
+  GUARDRAIL_FREE_DISK_RATIO,
+  evaluateGuardrail,
+} from '../../inertia/lib/kb_guardrail.js'
+
+const GB = 1024 * 1024 * 1024
+
+test('small batch does not trip the guardrail', () => {
+  const verdict = evaluateGuardrail({
+    estimateBytes: 1 * GB, // 1 GB
+    freeBytes: 500 * GB,
+  })
+  assert.equal(verdict.trips, false)
+  assert.deepEqual(verdict.reasons, [])
+})
+
+test('batch at exactly the absolute threshold trips', () => {
+  const verdict = evaluateGuardrail({
+    estimateBytes: GUARDRAIL_ABSOLUTE_BYTES,
+    freeBytes: 1000 * GB,
+  })
+  assert.equal(verdict.trips, true)
+  assert.equal(verdict.reasons.length, 1)
+  assert.equal(verdict.reasons[0].kind, 'over_absolute')
+})
+
+test('batch over the absolute threshold trips with over_absolute reason', () => {
+  const verdict = evaluateGuardrail({
+    estimateBytes: 60 * GB,
+    freeBytes: 1000 * GB,
+  })
+  const overAbsolute = verdict.reasons.find((r) => r.kind === 'over_absolute')
+  assert.ok(overAbsolute, 'should include over_absolute reason')
+  assert.equal(verdict.trips, true)
+})
+
+test('batch over 10% of free disk trips with over_free_disk reason', () => {
+  // 5 GB estimate against 40 GB free disk -> 5 > 4 (10% of 40)
+  const verdict = evaluateGuardrail({
+    estimateBytes: 5 * GB,
+    freeBytes: 40 * GB,
+  })
+  const overFree = verdict.reasons.find((r) => r.kind === 'over_free_disk')
+  assert.ok(overFree, 'should include over_free_disk reason')
+  assert.equal(verdict.trips, true)
+})
+
+test('batch can trip BOTH thresholds simultaneously', () => {
+  // 100 GB estimate, 200 GB free
+  // - over absolute (100 > 50)
+  // - over 10% of free (100 > 20)
+  const verdict = evaluateGuardrail({
+    estimateBytes: 100 * GB,
+    freeBytes: 200 * GB,
+  })
+  assert.equal(verdict.trips, true)
+  assert.equal(verdict.reasons.length, 2)
+  assert.ok(verdict.reasons.some((r) => r.kind === 'over_absolute'))
+  assert.ok(verdict.reasons.some((r) => r.kind === 'over_free_disk'))
+})
+
+test('freeBytes = 0 skips the relative-disk check', () => {
+  // 100 MB estimate, no free-disk signal: only the absolute check runs,
+  // and 100 MB is well below the 50 GB absolute threshold
+  const verdict = evaluateGuardrail({
+    estimateBytes: 100 * 1024 * 1024,
+    freeBytes: 0,
+  })
+  assert.equal(verdict.trips, false)
+})
+
+test('freeBytes = 0 still trips the absolute check at 50 GB', () => {
+  const verdict = evaluateGuardrail({
+    estimateBytes: 100 * GB,
+    freeBytes: 0,
+  })
+  assert.equal(verdict.trips, true)
+  assert.equal(verdict.reasons.length, 1)
+  assert.equal(verdict.reasons[0].kind, 'over_absolute')
+})
+
+test('relative-disk threshold computed from GUARDRAIL_FREE_DISK_RATIO constant', () => {
+  // Estimate exactly equal to 10% of free -> trips (>=)
+  const free = 100 * GB
+  const exactlyTenPercent = free * GUARDRAIL_FREE_DISK_RATIO
+  const verdict = evaluateGuardrail({
+    estimateBytes: exactlyTenPercent,
+    freeBytes: free,
+  })
+  const overFree = verdict.reasons.find((r) => r.kind === 'over_free_disk')
+  assert.ok(overFree, 'should trip at exactly the threshold')
+})
+
+test('batch just under both thresholds does not trip', () => {
+  // 4 GB estimate vs 50 GB free -> 10% of 50 = 5 GB, so 4 < 5
+  // Also well below 50 GB absolute
+  const verdict = evaluateGuardrail({
+    estimateBytes: 4 * GB,
+    freeBytes: 50 * GB,
+  })
+  assert.equal(verdict.trips, false)
+})

admin/tests/unit/kb_ingest_decision.spec.ts

@@ -0,0 +1,82 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { decideScanAction } from '../../app/utils/kb_ingest_decision.js'
+
+// ---------- Always-policy cases (default behavior; preserves pre-policy install) ----------
+
+test('Always: no state row, no chunks → dispatch and create row (new file)', () => {
+  assert.deepEqual(decideScanAction(null, false, 'Always'), {
+    kind: 'dispatch',
+    createStateRow: true,
+  })
+})
+
+test('Always: no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => {
+  assert.deepEqual(decideScanAction(null, true, 'Always'), { kind: 'backfill_indexed' })
+})
+
+test('Always: indexed + chunks present → skip', () => {
+  assert.deepEqual(decideScanAction({ state: 'indexed' }, true, 'Always'), { kind: 'skip' })
+})
+
+test('Always: indexed + chunks missing → re-dispatch (Qdrant collection rebuilt or chunks deleted)', () => {
+  assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Always'), {
+    kind: 'dispatch',
+    createStateRow: false,
+  })
+})
+
+test('Always: pending_decision → dispatch', () => {
+  assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Always'), {
+    kind: 'dispatch',
+    createStateRow: false,
+  })
+})
+
+test('Always: browse_only → skip (user opted out of indexing)', () => {
+  assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Always'), { kind: 'skip' })
+})
+
+test('Always: failed → skip (manual retry needed, do not auto-redispatch)', () => {
+  assert.deepEqual(decideScanAction({ state: 'failed' }, false, 'Always'), { kind: 'skip' })
+})
+
+test('Always: stalled → skip (manual retry needed)', () => {
+  assert.deepEqual(decideScanAction({ state: 'stalled' }, false, 'Always'), { kind: 'skip' })
+})
+
+// ---------- Manual-policy cases ----------
+
+test('Manual: no state row, no chunks → create_pending (do not auto-dispatch new content)', () => {
+  assert.deepEqual(decideScanAction(null, false, 'Manual'), { kind: 'create_pending' })
+})
+
+test('Manual: no state row, chunks present → backfill_indexed (same as Always — Qdrant is authoritative)', () => {
+  assert.deepEqual(decideScanAction(null, true, 'Manual'), { kind: 'backfill_indexed' })
+})
+
+test('Manual: pending_decision → skip (waiting for user to opt in via Index button)', () => {
+  assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Manual'), {
+    kind: 'skip',
+  })
+})
+
+test('Manual: indexed + chunks missing → re-dispatch (user has already opted in for this file)', () => {
+  // Policy switch from Always→Manual must not break in-flight or partially-deleted indexes
+  // for files the user previously chose to index.
+  assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Manual'), {
+    kind: 'dispatch',
+    createStateRow: false,
+  })
+})
+
+test('Manual: browse_only → skip (same as Always)', () => {
+  assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Manual'), { kind: 'skip' })
+})
+
+// ---------- Policy default ----------
+
+test('omitted policy defaults to Always (unset KV preserves legacy behavior)', () => {
+  assert.deepEqual(decideScanAction(null, false), { kind: 'dispatch', createStateRow: true })
+})

admin/tests/unit/kb_job_health.spec.ts

@@ -0,0 +1,100 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { computeJobHealth } from '../../app/utils/kb_job_health.js'
+
+const MIN = 60 * 1000
+const NOW = 1_700_000_000_000 // arbitrary fixed epoch for deterministic tests
+
+test('failed status takes precedence over any timing', () => {
+  assert.equal(
+    computeJobHealth({ status: 'failed', progress: 42, lastBatchAt: NOW, now: NOW }),
+    'failed'
+  )
+})
+
+test('no progress + no activity timestamps → waiting', () => {
+  assert.equal(
+    computeJobHealth({ status: 'waiting', progress: 0, now: NOW }),
+    'waiting'
+  )
+})
+
+test('progress > 0 but no lastBatchAt yet → healthy (first batch just started)', () => {
+  assert.equal(
+    computeJobHealth({ status: 'processing', progress: 5, startedAt: NOW, now: NOW }),
+    'healthy'
+  )
+})
+
+test('lastBatchAt 30s ago → healthy', () => {
+  assert.equal(
+    computeJobHealth({
+      status: 'batch_completed',
+      progress: 50,
+      lastBatchAt: NOW - 30 * 1000,
+      now: NOW,
+    }),
+    'healthy'
+  )
+})
+
+test('lastBatchAt 90s ago → still healthy (under 2 min threshold)', () => {
+  assert.equal(
+    computeJobHealth({
+      status: 'batch_completed',
+      progress: 50,
+      lastBatchAt: NOW - 90 * 1000,
+      now: NOW,
+    }),
+    'healthy'
+  )
+})
+
+test('lastBatchAt 3 min ago → slow (CPU-paced ingestion lives here)', () => {
+  assert.equal(
+    computeJobHealth({
+      status: 'batch_completed',
+      progress: 50,
+      lastBatchAt: NOW - 3 * MIN,
+      now: NOW,
+    }),
+    'slow'
+  )
+})
+
+test('lastBatchAt 4:30 ago → still slow (under 5 min stalled threshold)', () => {
+  assert.equal(
+    computeJobHealth({
+      status: 'batch_completed',
+      progress: 50,
+      lastBatchAt: NOW - 4.5 * MIN,
+      now: NOW,
+    }),
+    'slow'
+  )
+})
+
+test('lastBatchAt 5:01 ago → stalled', () => {
+  assert.equal(
+    computeJobHealth({
+      status: 'batch_completed',
+      progress: 50,
+      lastBatchAt: NOW - (5 * MIN + 1000),
+      now: NOW,
+    }),
+    'stalled'
+  )
+})
+
+test('lastBatchAt missing but startedAt 10 min ago → stalled (first-batch-never-finished case)', () => {
+  assert.equal(
+    computeJobHealth({
+      status: 'processing',
+      progress: 5,
+      startedAt: NOW - 10 * MIN,
+      now: NOW,
+    }),
+    'stalled'
+  )
+})

admin/tests/unit/kb_ratio_lookup.spec.ts

@@ -0,0 +1,111 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import {
+  BYTES_PER_CHUNK_ON_DISK,
+  estimateBatch,
+  estimateChunkCount,
+  findChunksPerMb,
+} from '../../app/utils/kb_ratio_lookup.js'
+
+const SEEDED_ROWS = [
+  { pattern: 'devdocs_', chunks_per_mb: 1100 },
+  { pattern: 'wikipedia_en_simple_', chunks_per_mb: 270 },
+  { pattern: 'wikipedia_en_', chunks_per_mb: 250 },
+  { pattern: 'ifixit_', chunks_per_mb: 50 },
+  { pattern: 'lrnselfreliance_', chunks_per_mb: 0 },
+  { pattern: '', chunks_per_mb: 100 },
+]
+
+test('exact prefix match', () => {
+  assert.equal(findChunksPerMb('devdocs_en_python_2026-02.zim', SEEDED_ROWS), 1100)
+})
+
+test('longest-prefix wins over broader sibling', () => {
+  // wikipedia_en_simple_* should pick 270, not the 250 from wikipedia_en_
+  assert.equal(
+    findChunksPerMb('wikipedia_en_simple_all_nopic_2026-02.zim', SEEDED_ROWS),
+    270
+  )
+})
+
+test('broader prefix used when no specific match', () => {
+  // wikipedia_en_medicine_* is not seeded; falls through to wikipedia_en_ at 250
+  assert.equal(findChunksPerMb('wikipedia_en_medicine_nopic_2026-04.zim', SEEDED_ROWS), 250)
+})
+
+test('empty-string fallback catches unmatched filenames', () => {
+  assert.equal(findChunksPerMb('something_unknown_2026-02.zim', SEEDED_ROWS), 100)
+})
+
+test('returns null when no row matches and no fallback is registered', () => {
+  const rowsWithoutFallback = SEEDED_ROWS.filter((r) => r.pattern !== '')
+  assert.equal(findChunksPerMb('something_unknown_2026-02.zim', rowsWithoutFallback), null)
+})
+
+test('zero-ratio entry returns 0, not null (video-only ZIMs)', () => {
+  assert.equal(findChunksPerMb('lrnselfreliance_en_all_2025-12.zim', SEEDED_ROWS), 0)
+})
+
+test('estimateChunkCount scales by file size in MB', () => {
+  // 100 MB * 1100 chunks/MB ≈ 110,000 chunks for devdocs
+  const bytes = 100 * 1024 * 1024
+  assert.equal(estimateChunkCount('devdocs_en_python_2026-02.zim', bytes, SEEDED_ROWS), 110000)
+})
+
+test('estimateChunkCount returns 0 for video-only ZIM regardless of size', () => {
+  const bytes = 5 * 1024 * 1024 * 1024 // 5 GB
+  assert.equal(estimateChunkCount('lrnselfreliance_en_all_2025-12.zim', bytes, SEEDED_ROWS), 0)
+})
+
+test('estimateChunkCount returns null when no match and no fallback', () => {
+  const rowsWithoutFallback = SEEDED_ROWS.filter((r) => r.pattern !== '')
+  assert.equal(
+    estimateChunkCount('something_unknown_2026-02.zim', 50 * 1024 * 1024, rowsWithoutFallback),
+    null
+  )
+})
+
+test('estimateBatch sums chunks and bytes for matched files', () => {
+  const files = [
+    // 100 MB devdocs -> 110,000 chunks
+    { filename: 'devdocs_en_python_2026-02.zim', sizeBytes: 100 * 1024 * 1024 },
+    // 500 MB wikipedia_en_simple -> 135,000 chunks
+    { filename: 'wikipedia_en_simple_all_nopic_2026-02.zim', sizeBytes: 500 * 1024 * 1024 },
+  ]
+  const out = estimateBatch(files, SEEDED_ROWS)
+  assert.equal(out.totalChunks, 110_000 + 135_000)
+  assert.equal(out.totalBytes, (110_000 + 135_000) * BYTES_PER_CHUNK_ON_DISK)
+  assert.equal(out.hasUnknown, false)
+})
+
+test('estimateBatch sets hasUnknown when a file has no registry match', () => {
+  // Drop the empty-string fallback so the unknown file truly has no match
+  const rows = SEEDED_ROWS.filter((r) => r.pattern !== '')
+  const files = [
+    { filename: 'devdocs_en_python_2026-02.zim', sizeBytes: 100 * 1024 * 1024 },
+    { filename: 'something_unknown_2026-02.zim', sizeBytes: 50 * 1024 * 1024 },
+  ]
+  const out = estimateBatch(files, rows)
+  assert.equal(out.totalChunks, 110_000) // only the matched file
+  assert.equal(out.hasUnknown, true)
+})
+
+test('estimateBatch handles video-only ZIMs (0 chunks/MB) without flagging hasUnknown', () => {
+  // A 5 GB video ZIM matches the registry with 0 chunks/MB; that is a
+  // *known* value, not an unknown — totals should be 0 and hasUnknown false.
+  const files = [
+    { filename: 'lrnselfreliance_en_all_2025-12.zim', sizeBytes: 5 * 1024 * 1024 * 1024 },
+  ]
+  const out = estimateBatch(files, SEEDED_ROWS)
+  assert.equal(out.totalChunks, 0)
+  assert.equal(out.totalBytes, 0)
+  assert.equal(out.hasUnknown, false)
+})
+
+test('estimateBatch on empty input returns zeros', () => {
+  const out = estimateBatch([], SEEDED_ROWS)
+  assert.equal(out.totalChunks, 0)
+  assert.equal(out.totalBytes, 0)
+  assert.equal(out.hasUnknown, false)
+})

admin/tests/unit/kb_warning_decision.spec.ts

@@ -0,0 +1,125 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { decideWarnings } from '../../app/utils/kb_warning_decision.js'
+
+const MB = 1024 * 1024
+
+test('healthy file: chunks present and on-target → no warnings', () => {
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 100 * MB,
+      chunksInQdrant: 11_000,
+      expectedChunks: 11_000,
+    }),
+    []
+  )
+})
+
+test('healthy file: chunks slightly above expectation → no warnings', () => {
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 100 * MB,
+      chunksInQdrant: 12_000,
+      expectedChunks: 11_000,
+    }),
+    []
+  )
+})
+
+test('Warning A: large file with 0 chunks (video-only ZIM)', () => {
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 5 * 1024 * MB,
+      chunksInQdrant: 0,
+      expectedChunks: 0,
+    }),
+    [{ kind: 'zero_chunks', fileSizeBytes: 5 * 1024 * MB }]
+  )
+})
+
+test('Warning A: small empty file is silently ignored (under 100 MB threshold)', () => {
+  // A user uploads a 5 KB placeholder.txt that produces nothing → not worth a banner
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 5 * 1024, // 5 KB
+      chunksInQdrant: 0,
+      expectedChunks: null,
+    }),
+    []
+  )
+})
+
+test('Warning B: partial stall — chunks well below expectation', () => {
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 1000 * MB,
+      chunksInQdrant: 266,
+      expectedChunks: 600_000,
+    }),
+    [{ kind: 'partial_stall', chunksEmbedded: 266, chunksExpected: 600_000 }]
+  )
+})
+
+test('Warning B: chunks just under 50% of expected → triggers', () => {
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 100 * MB,
+      chunksInQdrant: 4_999,
+      expectedChunks: 10_000,
+    }),
+    [{ kind: 'partial_stall', chunksEmbedded: 4_999, chunksExpected: 10_000 }]
+  )
+})
+
+test('Warning B: chunks at exactly 50% of expected → does NOT trigger', () => {
+  // Strict less-than threshold leaves room for the boundary
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 100 * MB,
+      chunksInQdrant: 5_000,
+      expectedChunks: 10_000,
+    }),
+    []
+  )
+})
+
+test('Warning B suppressed when expectedChunks is null (registry miss)', () => {
+  // Better to be silent than show a meaningless "266 of unknown" comparison
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 100 * MB,
+      chunksInQdrant: 266,
+      expectedChunks: null,
+    }),
+    []
+  )
+})
+
+test('Warning B suppressed when expectedChunks is 0 (video-only registry entry)', () => {
+  // A `lrnselfreliance_` row in the registry says "expect 0 chunks". A real
+  // file matching it correctly producing 0 chunks must not trigger Warning B.
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 500 * MB,
+      chunksInQdrant: 0,
+      expectedChunks: 0,
+    }),
+    // Note: Warning A triggers here because file > 100 MB and chunks = 0
+    [{ kind: 'zero_chunks', fileSizeBytes: 500 * MB }]
+  )
+})
+
+test('Both warnings can fire on the same file in principle', () => {
+  // Edge case: huge file, 0 chunks, but ratio registry expected 100k.
+  // Warning A fires (large + zero), Warning B suppressed (chunksInQdrant must be > 0).
+  // This documents the chunksInQdrant > 0 guard on Warning B.
+  assert.deepEqual(
+    decideWarnings({
+      fileSizeBytes: 1000 * MB,
+      chunksInQdrant: 0,
+      expectedChunks: 100_000,
+    }),
+    [{ kind: 'zero_chunks', fileSizeBytes: 1000 * MB }]
+  )
+})

admin/tests/unit/zim_filename.spec.ts

@@ -0,0 +1,73 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { findReplacedWikipediaFiles, zimFilenameStem } from '../../app/utils/zim_filename.js'
+
+test('zimFilenameStem strips YYYY-MM date suffix', () => {
+  assert.equal(zimFilenameStem('wikipedia_en_all_nopic_2026-02.zim'), 'wikipedia_en_all_nopic')
+})
+
+test('zimFilenameStem strips YYYY-MM-DD date suffix', () => {
+  assert.equal(zimFilenameStem('wikipedia_en_all_nopic_2026-02-15.zim'), 'wikipedia_en_all_nopic')
+})
+
+test('zimFilenameStem returns input unchanged when no date suffix present', () => {
+  assert.equal(
+    zimFilenameStem('wikipedia_en_my_custom_extract.zim'),
+    'wikipedia_en_my_custom_extract.zim'
+  )
+})
+
+test('findReplacedWikipediaFiles cleans up older version of same variant', () => {
+  assert.deepEqual(
+    findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [
+      'wikipedia_en_all_nopic_2026-02.zim',
+      'wikipedia_en_all_nopic_2026-04.zim',
+    ]),
+    ['wikipedia_en_all_nopic_2026-02.zim']
+  )
+})
+
+test('findReplacedWikipediaFiles preserves co-existing distinct corpora — the #884 regression case', () => {
+  assert.deepEqual(
+    findReplacedWikipediaFiles('wikipedia_en_medicine_nopic_2026-04.zim', [
+      'wikipedia_en_simple_all_nopic_2026-02.zim',
+      'wikipedia_en_medicine_nopic_2026-04.zim',
+    ]),
+    []
+  )
+})
+
+test('findReplacedWikipediaFiles preserves all unrelated variants when a new variant lands', () => {
+  assert.deepEqual(
+    findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [
+      'wikipedia_en_simple_all_nopic_2026-02.zim',
+      'wikipedia_en_medicine_nopic_2026-04.zim',
+      'wikipedia_en_wikivoyage_2026-02.zim',
+      'wikipedia_en_climate_change_2025-08.zim',
+      'wikipedia_en_all_nopic_2026-04.zim',
+    ]),
+    []
+  )
+})
+
+test('findReplacedWikipediaFiles ignores files without wikipedia_en_ prefix', () => {
+  assert.deepEqual(
+    findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [
+      'wiktionary_en_all_2026-02.zim',
+      'gutenberg_en_all_2026-01.zim',
+      'wikipedia_en_all_nopic_2026-04.zim',
+    ]),
+    []
+  )
+})
+
+test('findReplacedWikipediaFiles preserves manually-named files without a date suffix', () => {
+  assert.deepEqual(
+    findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [
+      'wikipedia_en_my_custom_extract.zim',
+      'wikipedia_en_all_nopic_2026-04.zim',
+    ]),
+    []
+  )
+})

admin/types/collections.ts

@@ -64,6 +64,11 @@ export type ResourceStatus = 'installed' | 'not_installed' | 'update_available'
 
 export type CategoryWithStatus = SpecCategory & {
   installedTierSlug?: string
+  // Highest tier whose every resource is either installed OR has an in-flight
+  // download. Set only when it differs from installedTierSlug — i.e. the user
+  // picked something larger and downloads are still running. Lets the UI show
+  // the user's actual intent during the (often long) download window.
+  downloadingTierSlug?: string
 }
 
 export type CollectionWithStatus = SpecCollection & {
@@ -86,6 +91,7 @@ export type ResourceUpdateInfo = {
   installed_version: string
   latest_version: string
   download_url: string
+  size_bytes?: number
 }
 
 export type ContentUpdateCheckResult = {

admin/types/kb_ingest_state.ts

@@ -0,0 +1,9 @@
+export const KB_INGEST_STATES = [
+  'pending_decision',
+  'indexed',
+  'browse_only',
+  'failed',
+  'stalled',
+] as const
+
+export type KbIngestStateValue = (typeof KB_INGEST_STATES)[number]

admin/types/kv_store.ts

@@ -3,6 +3,7 @@ export const KV_STORE_SCHEMA = {
   'chat.suggestionsEnabled':    'boolean',
   'chat.lastModel':             'string',
   'rag.docsEmbedded':           'boolean',
+  'rag.defaultIngestPolicy':    'string',
   'system.updateAvailable':     'boolean',
   'system.latestVersion':       'string',
   'system.earlyAccess':         'boolean',
@@ -12,6 +13,10 @@ export const KV_STORE_SCHEMA = {
   'gpu.type':                   'string',
   'ai.remoteOllamaUrl':         'string',
   'ai.ollamaFlashAttention':    'boolean',
+  'ai.amdGpuAcceleration':      'boolean',
+  'ai.amdHsaOverride':          'string',
+  'ai.autoFixGpuPassthrough':   'boolean',
+  'gpu.autoRemediatedAt':       'string',
 } as const
 
 type KVTagToType<T extends string> = T extends 'boolean' ? boolean : string