feat: auto add NOMAD docs to KB on AI install

Dockerfile

@@ -31,5 +31,6 @@ COPY --from=build /app/build /app
 # Copy root package.json for version info
 COPY package.json /app/version.json
 COPY admin/docs /app/docs
+COPY README.md /app/README.md
 EXPOSE 8080
 CMD ["node", "./bin/server.js"]

admin/README.md

@@ -1,5 +0,0 @@
-
-## Docker container
-```
-docker run --rm -it -p 8080:8080 jturnercosmistack/projectnomad:admin-latest -e PORT=8080 -e HOST=0.0.0.0 -e APP_KEY=secretlongpasswordsecret -e LOG_LEVEL=debug -e DRIVE_DISK=fs
-```

admin/app/jobs/embed_file_job.ts

@@ -35,6 +35,21 @@ export class EmbedFileJob {
     const ragService = new RagService(dockerService, ollamaService)
 
     try {
+      // Check if Ollama and Qdrant services are ready
+      const existingModels = await ollamaService.getModels()
+      if (!existingModels) {
+        logger.warn('[EmbedFileJob] Ollama service not ready yet. Will retry...')
+        throw new Error('Ollama service not ready yet')
+      }
+
+      const qdrantUrl = await dockerService.getServiceURL('nomad_qdrant')
+      if (!qdrantUrl) {
+        logger.warn('[EmbedFileJob] Qdrant service not ready yet. Will retry...')
+        throw new Error('Qdrant service not ready yet')
+      }
+
+      logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`)
+
       // Update progress starting
       await job.updateProgress(0)
       await job.updateData({
@@ -102,10 +117,10 @@ export class EmbedFileJob {
     try {
       const job = await queue.add(this.key, params, {
         jobId,
-        attempts: 3,
+        attempts: 30,
         backoff: {
-          type: 'exponential',
+          type: 'fixed',
-          delay: 5000, // Delay 5 seconds before retrying
+          delay: 60000, // Check every 60 seconds for service readiness
         },
         removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history
         removeOnFail: { count: 20 } // Keep last 20 failed jobs for debugging

admin/app/services/docker_service.ts

@@ -9,7 +9,8 @@ import { ZIM_STORAGE_PATH } from '../utils/fs.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { exec } from 'child_process'
 import { promisify } from 'util'
-import { readdir } from 'fs/promises'
+// import { readdir } from 'fs/promises'
+import KVStore from '#models/kv_store'
 
 @inject()
 export class DockerService {
@@ -473,34 +474,34 @@ export class DockerService {
             ],
           }
         } else if (gpuType === 'amd') {
-          this._broadcast(
+          // this._broadcast(
-            service.service_name,
+          //   service.service_name,
-            'gpu-config',
+          //   'gpu-config',
-            `AMD GPU detected. Using ROCm image and configuring container with GPU support...`
+          //   `AMD GPU detected. Using ROCm image and configuring container with GPU support...`
-          )
+          // )
 
-          // Use ROCm image for AMD
+          // // Use ROCm image for AMD
-          finalImage = 'ollama/ollama:rocm'
+          // finalImage = 'ollama/ollama:rocm'
 
-          // Dynamically discover and add AMD GPU devices
+          // // Dynamically discover and add AMD GPU devices
-          const amdDevices = await this._discoverAMDDevices()
+          // const amdDevices = await this._discoverAMDDevices()
-          if (!amdDevices || amdDevices.length === 0) {
+          // if (!amdDevices || amdDevices.length === 0) {
-            this._broadcast(
+          //   this._broadcast(
-              service.service_name,
+          //     service.service_name,
-              'gpu-config-error',
+          //     'gpu-config-error',
-              `Failed to discover AMD GPU devices. Proceeding with CPU-only configuration...`
+          //     `Failed to discover AMD GPU devices. Proceeding with CPU-only configuration...`
-            )
+          //   )
-            gpuHostConfig = { ...gpuHostConfig } // No GPU devices added
+          //   gpuHostConfig = { ...gpuHostConfig } // No GPU devices added
-            logger.warn(`[DockerService] No AMD GPU devices discovered for Ollama`)
+          //   logger.warn(`[DockerService] No AMD GPU devices discovered for Ollama`)
-          } else {
+          // } else {
-            gpuHostConfig = {
+          //   gpuHostConfig = {
-              ...gpuHostConfig,
+          //     ...gpuHostConfig,
-              Devices: amdDevices,
+          //     Devices: amdDevices,
-            }
+          //   }
-            logger.info(
+          //   logger.info(
-              `[DockerService] Configured ${amdDevices.length} AMD GPU devices for Ollama`
+          //     `[DockerService] Configured ${amdDevices.length} AMD GPU devices for Ollama`
-            )
+          //   )
-          }
+          // }
         } else {
           this._broadcast(
             service.service_name,
@@ -553,6 +554,22 @@ export class DockerService {
       // Remove from active installs tracking
       this.activeInstallations.delete(service.service_name)
 
+      // If Ollama was just installed, trigger Nomad docs discovery and embedding
+      if (service.service_name === SERVICE_NAMES.OLLAMA) {
+        logger.info('[DockerService] Ollama installation complete. Enabling chat suggestions by default.')
+        await KVStore.setValue('chat.suggestionsEnabled', "true")
+
+        logger.info('[DockerService] Ollama installation complete. Triggering Nomad docs discovery...')
+        
+        // Need to use dynamic imports here to avoid circular dependency
+        const ollamaService = new (await import('./ollama_service.js')).OllamaService()
+        const ragService = new (await import('./rag_service.js')).RagService(this, ollamaService)
+
+        ragService.discoverNomadDocs().catch((error) => {
+          logger.error('[DockerService] Failed to discover Nomad docs:', error)
+        })
+      }
+
       this._broadcast(
         service.service_name,
         'completed',
@@ -715,57 +732,57 @@ export class DockerService {
    * Discover AMD GPU DRI devices dynamically.
    * Returns an array of device configurations for Docker.
    */
-  private async _discoverAMDDevices(): Promise<
+  // private async _discoverAMDDevices(): Promise<
-    Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
+  //   Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
-  > {
+  // > {
-    try {
+  //   try {
-      const devices: Array<{
+  //     const devices: Array<{
-        PathOnHost: string
+  //       PathOnHost: string
-        PathInContainer: string
+  //       PathInContainer: string
-        CgroupPermissions: string
+  //       CgroupPermissions: string
-      }> = []
+  //     }> = []
 
-      // Always add /dev/kfd (Kernel Fusion Driver)
+  //     // Always add /dev/kfd (Kernel Fusion Driver)
-      devices.push({
+  //     devices.push({
-        PathOnHost: '/dev/kfd',
+  //       PathOnHost: '/dev/kfd',
-        PathInContainer: '/dev/kfd',
+  //       PathInContainer: '/dev/kfd',
-        CgroupPermissions: 'rwm',
+  //       CgroupPermissions: 'rwm',
-      })
+  //     })
 
-      // Discover DRI devices in /dev/dri/
+  //     // Discover DRI devices in /dev/dri/
-      try {
+  //     try {
-        const driDevices = await readdir('/dev/dri')
+  //       const driDevices = await readdir('/dev/dri')
-        for (const device of driDevices) {
+  //       for (const device of driDevices) {
-          const devicePath = `/dev/dri/${device}`
+  //         const devicePath = `/dev/dri/${device}`
-          devices.push({
+  //         devices.push({
-            PathOnHost: devicePath,
+  //           PathOnHost: devicePath,
-            PathInContainer: devicePath,
+  //           PathInContainer: devicePath,
-            CgroupPermissions: 'rwm',
+  //           CgroupPermissions: 'rwm',
-          })
+  //         })
-        }
+  //       }
-        logger.info(
+  //       logger.info(
-          `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}`
+  //         `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}`
-        )
+  //       )
-      } catch (error) {
+  //     } catch (error) {
-        logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`)
+  //       logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`)
-        // Fallback to common device names if directory read fails
+  //       // Fallback to common device names if directory read fails
-        const fallbackDevices = ['card0', 'renderD128']
+  //       const fallbackDevices = ['card0', 'renderD128']
-        for (const device of fallbackDevices) {
+  //       for (const device of fallbackDevices) {
-          devices.push({
+  //         devices.push({
-            PathOnHost: `/dev/dri/${device}`,
+  //           PathOnHost: `/dev/dri/${device}`,
-            PathInContainer: `/dev/dri/${device}`,
+  //           PathInContainer: `/dev/dri/${device}`,
-            CgroupPermissions: 'rwm',
+  //           CgroupPermissions: 'rwm',
-          })
+  //         })
-        }
+  //       }
-        logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`)
+  //       logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`)
-      }
+  //     }
 
-      return devices
+  //     return devices
-    } catch (error) {
+  //   } catch (error) {
-      logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`)
+  //     logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`)
-      return []
+  //     return []
-    }
+  //   }
-  }
+  // }
 
   private _broadcast(service: string, status: string, message: string) {
     transmit.broadcast('service-installation', {

admin/app/services/ollama_service.ts

@@ -42,8 +42,8 @@ export class OllamaService {
   }
 
   /**
-   * Synchronous version of model download (waits for completion). Should only be used for
+   * Downloads a model from the Ollama service with progress tracking. Where possible,
-   * small models or in contexts where a background job is incompatible.
+   * one should dispatch a background job instead of calling this method directly to avoid long blocking.
    * @param model Model name to download
    * @returns Success status and message
    */

admin/app/services/rag_service.ts

@@ -4,7 +4,7 @@ import { inject } from '@adonisjs/core'
 import logger from '@adonisjs/core/services/logger'
 import { TokenChunker } from '@chonkiejs/core'
 import sharp from 'sharp'
-import { deleteFileIfExists, determineFileType, getFile } from '../utils/fs.js'
+import { deleteFileIfExists, determineFileType, getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js'
 import { PDFParse } from 'pdf-parse'
 import { createWorker } from 'tesseract.js'
 import { fromBuffer } from 'pdf2pic'
@@ -12,6 +12,9 @@ import { OllamaService } from './ollama_service.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { removeStopwords } from 'stopword'
 import { randomUUID } from 'node:crypto'
+import { join } from 'node:path'
+import KVStore from '#models/kv_store'
+import { parseBoolean } from '../utils/misc.js'
 
 @inject()
 export class RagService {
@@ -33,7 +36,7 @@ export class RagService {
   constructor(
     private dockerService: DockerService,
     private ollamaService: OllamaService
-  ) {}
+  ) { }
 
   /**
    * Estimates token count for text. This is a conservative approximation:
@@ -166,9 +169,19 @@ export class RagService {
       const allModels = await this.ollamaService.getModels(true)
       const embeddingModel = allModels.find((model) => model.name === RagService.EMBEDDING_MODEL)
 
-      // TODO: Attempt to download the embedding model if not found
       if (!embeddingModel) {
-        throw new Error(`${RagService.EMBEDDING_MODEL} does not exist and could not be downloaded.`)
+        try {
+          const downloadResult = await this.ollamaService.downloadModel(RagService.EMBEDDING_MODEL)
+          if (!downloadResult.success) {
+            throw new Error(downloadResult.message || 'Unknown error during model download')
+          }
+        } catch (modelError) {
+          logger.error(
+            `[RAG] Embedding model ${RagService.EMBEDDING_MODEL} not found locally and failed to download:`,
+            modelError
+          )
+          return null
+        }
       }
 
       // TokenChunker uses character-based tokenization (1 char = 1 token)
@@ -326,7 +339,8 @@ export class RagService {
    * This includes text extraction, chunking, embedding, and storing in Qdrant.
    */
   public async processAndEmbedFile(
-    filepath: string // Should already be the full path to the uploaded file
+    filepath: string, // Should already be the full path to the uploaded file
+    deleteAfterEmbedding: boolean = false
   ): Promise<{ success: boolean; message: string; chunks?: number }> {
     try {
       const fileType = determineFileType(filepath)
@@ -368,9 +382,15 @@ export class RagService {
         source: filepath
       })
 
-      // Cleanup the file from disk
+      if (!embedResult) {
-      logger.info(`[RAG] Embedding complete, deleting uploaded file: ${filepath}`)
+        return { success: false, message: 'Failed to embed and store the extracted text.' }
-      await deleteFileIfExists(filepath)
+      }
+
+      if (deleteAfterEmbedding) {
+        // Cleanup the file from disk
+        logger.info(`[RAG] Embedding complete, deleting uploaded file: ${filepath}`)
+        await deleteFileIfExists(filepath)
+      }
 
       return {
         success: true,
@@ -656,4 +676,63 @@ export class RagService {
       return []
     }
   }
+
+  public async discoverNomadDocs(force?: boolean): Promise<{ success: boolean; message: string }> {
+    try {
+      const README_PATH = join(process.cwd(), 'README.md')
+      const DOCS_DIR = join(process.cwd(), 'docs')
+
+      const alreadyEmbeddedRaw = await KVStore.getValue('rag.docsEmbedded')
+      if (parseBoolean(alreadyEmbeddedRaw) && !force) {
+        logger.info('[RAG] Nomad docs have already been discovered and queued. Skipping.')
+        return { success: true, message: 'Nomad docs have already been discovered and queued. Skipping.' }
+      }
+
+      const filesToEmbed: Array<{ path: string; source: string }> = []
+
+      const readmeExists = await getFileStatsIfExists(README_PATH)
+      if (readmeExists) {
+        filesToEmbed.push({ path: README_PATH, source: 'README.md' })
+      }
+
+      const dirContents = await listDirectoryContentsRecursive(DOCS_DIR)
+      for (const entry of dirContents) {
+        if (entry.type === 'file') {
+          filesToEmbed.push({ path: entry.key, source: join('docs', entry.name) })
+        }
+      }
+
+      logger.info(`[RAG] Discovered ${filesToEmbed.length} Nomad doc files to embed`)
+
+      // Import EmbedFileJob dynamically to avoid circular dependencies
+      const { EmbedFileJob } = await import('#jobs/embed_file_job')
+
+      // Dispatch an EmbedFileJob for each discovered file
+      for (const fileInfo of filesToEmbed) {
+        try {
+          logger.info(`[RAG] Dispatching embed job for: ${fileInfo.source}`)
+          const stats = await getFileStatsIfExists(fileInfo.path)
+          await EmbedFileJob.dispatch({
+            filePath: fileInfo.path,
+            fileName: fileInfo.source,
+            fileSize: stats?.size,
+          })
+          logger.info(`[RAG] Successfully dispatched job for ${fileInfo.source}`)
+        } catch (fileError) {
+          logger.error(
+            `[RAG] Error dispatching job for file ${fileInfo.source}:`,
+            fileError
+          )
+        }
+      }
+
+      // Update KV store to mark docs as discovered so we don't redo this unnecessarily
+      await KVStore.setValue('rag.docsEmbedded', 'true')
+
+      return { success: true, message: `Nomad docs discovery completed. Dispatched ${filesToEmbed.length} embedding jobs.` }
+    } catch (error) {
+      logger.error('Error discovering Nomad docs:', error)
+      return { success: false, message: 'Error discovering Nomad docs.' }
+    }
+  }
 }

admin/types/kv_store.ts

@@ -1,3 +1,3 @@
 
-export type KVStoreKey = 'chat.suggestionsEnabled'
+export type KVStoreKey = 'chat.suggestionsEnabled' | 'rag.docsEmbedded'
 export type KVStoreValue = string | null