diff --git a/Dockerfile b/Dockerfile index 54e10bd..ade661f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,5 +31,6 @@ COPY --from=build /app/build /app # Copy root package.json for version info COPY package.json /app/version.json COPY admin/docs /app/docs +COPY README.md /app/README.md EXPOSE 8080 CMD ["node", "./bin/server.js"] \ No newline at end of file diff --git a/admin/README.md b/admin/README.md deleted file mode 100644 index c57c541..0000000 --- a/admin/README.md +++ /dev/null @@ -1,5 +0,0 @@ - -## Docker container -``` -docker run --rm -it -p 8080:8080 jturnercosmistack/projectnomad:admin-latest -e PORT=8080 -e HOST=0.0.0.0 -e APP_KEY=secretlongpasswordsecret -e LOG_LEVEL=debug -e DRIVE_DISK=fs -``` \ No newline at end of file diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index f7cd6c8..32b697e 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -35,6 +35,21 @@ export class EmbedFileJob { const ragService = new RagService(dockerService, ollamaService) try { + // Check if Ollama and Qdrant services are ready + const existingModels = await ollamaService.getModels() + if (!existingModels) { + logger.warn('[EmbedFileJob] Ollama service not ready yet. Will retry...') + throw new Error('Ollama service not ready yet') + } + + const qdrantUrl = await dockerService.getServiceURL('nomad_qdrant') + if (!qdrantUrl) { + logger.warn('[EmbedFileJob] Qdrant service not ready yet. Will retry...') + throw new Error('Qdrant service not ready yet') + } + + logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`) + // Update progress starting await job.updateProgress(0) await job.updateData({ @@ -102,10 +117,10 @@ export class EmbedFileJob { try { const job = await queue.add(this.key, params, { jobId, - attempts: 3, + attempts: 30, backoff: { - type: 'exponential', - delay: 5000, // Delay 5 seconds before retrying + type: 'fixed', + delay: 60000, // Check every 60 seconds for service readiness }, removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history removeOnFail: { count: 20 } // Keep last 20 failed jobs for debugging diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 99ad6d8..ed9949a 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -9,7 +9,8 @@ import { ZIM_STORAGE_PATH } from '../utils/fs.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { exec } from 'child_process' import { promisify } from 'util' -import { readdir } from 'fs/promises' +// import { readdir } from 'fs/promises' +import KVStore from '#models/kv_store' @inject() export class DockerService { @@ -473,34 +474,34 @@ export class DockerService { ], } } else if (gpuType === 'amd') { - this._broadcast( - service.service_name, - 'gpu-config', - `AMD GPU detected. Using ROCm image and configuring container with GPU support...` - ) + // this._broadcast( + // service.service_name, + // 'gpu-config', + // `AMD GPU detected. Using ROCm image and configuring container with GPU support...` + // ) - // Use ROCm image for AMD - finalImage = 'ollama/ollama:rocm' + // // Use ROCm image for AMD + // finalImage = 'ollama/ollama:rocm' - // Dynamically discover and add AMD GPU devices - const amdDevices = await this._discoverAMDDevices() - if (!amdDevices || amdDevices.length === 0) { - this._broadcast( - service.service_name, - 'gpu-config-error', - `Failed to discover AMD GPU devices. Proceeding with CPU-only configuration...` - ) - gpuHostConfig = { ...gpuHostConfig } // No GPU devices added - logger.warn(`[DockerService] No AMD GPU devices discovered for Ollama`) - } else { - gpuHostConfig = { - ...gpuHostConfig, - Devices: amdDevices, - } - logger.info( - `[DockerService] Configured ${amdDevices.length} AMD GPU devices for Ollama` - ) - } + // // Dynamically discover and add AMD GPU devices + // const amdDevices = await this._discoverAMDDevices() + // if (!amdDevices || amdDevices.length === 0) { + // this._broadcast( + // service.service_name, + // 'gpu-config-error', + // `Failed to discover AMD GPU devices. Proceeding with CPU-only configuration...` + // ) + // gpuHostConfig = { ...gpuHostConfig } // No GPU devices added + // logger.warn(`[DockerService] No AMD GPU devices discovered for Ollama`) + // } else { + // gpuHostConfig = { + // ...gpuHostConfig, + // Devices: amdDevices, + // } + // logger.info( + // `[DockerService] Configured ${amdDevices.length} AMD GPU devices for Ollama` + // ) + // } } else { this._broadcast( service.service_name, @@ -553,6 +554,22 @@ export class DockerService { // Remove from active installs tracking this.activeInstallations.delete(service.service_name) + // If Ollama was just installed, trigger Nomad docs discovery and embedding + if (service.service_name === SERVICE_NAMES.OLLAMA) { + logger.info('[DockerService] Ollama installation complete. Enabling chat suggestions by default.') + await KVStore.setValue('chat.suggestionsEnabled', "true") + + logger.info('[DockerService] Ollama installation complete. Triggering Nomad docs discovery...') + + // Need to use dynamic imports here to avoid circular dependency + const ollamaService = new (await import('./ollama_service.js')).OllamaService() + const ragService = new (await import('./rag_service.js')).RagService(this, ollamaService) + + ragService.discoverNomadDocs().catch((error) => { + logger.error('[DockerService] Failed to discover Nomad docs:', error) + }) + } + this._broadcast( service.service_name, 'completed', @@ -715,57 +732,57 @@ export class DockerService { * Discover AMD GPU DRI devices dynamically. * Returns an array of device configurations for Docker. */ - private async _discoverAMDDevices(): Promise< - Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }> - > { - try { - const devices: Array<{ - PathOnHost: string - PathInContainer: string - CgroupPermissions: string - }> = [] + // private async _discoverAMDDevices(): Promise< + // Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }> + // > { + // try { + // const devices: Array<{ + // PathOnHost: string + // PathInContainer: string + // CgroupPermissions: string + // }> = [] - // Always add /dev/kfd (Kernel Fusion Driver) - devices.push({ - PathOnHost: '/dev/kfd', - PathInContainer: '/dev/kfd', - CgroupPermissions: 'rwm', - }) + // // Always add /dev/kfd (Kernel Fusion Driver) + // devices.push({ + // PathOnHost: '/dev/kfd', + // PathInContainer: '/dev/kfd', + // CgroupPermissions: 'rwm', + // }) - // Discover DRI devices in /dev/dri/ - try { - const driDevices = await readdir('/dev/dri') - for (const device of driDevices) { - const devicePath = `/dev/dri/${device}` - devices.push({ - PathOnHost: devicePath, - PathInContainer: devicePath, - CgroupPermissions: 'rwm', - }) - } - logger.info( - `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}` - ) - } catch (error) { - logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`) - // Fallback to common device names if directory read fails - const fallbackDevices = ['card0', 'renderD128'] - for (const device of fallbackDevices) { - devices.push({ - PathOnHost: `/dev/dri/${device}`, - PathInContainer: `/dev/dri/${device}`, - CgroupPermissions: 'rwm', - }) - } - logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`) - } + // // Discover DRI devices in /dev/dri/ + // try { + // const driDevices = await readdir('/dev/dri') + // for (const device of driDevices) { + // const devicePath = `/dev/dri/${device}` + // devices.push({ + // PathOnHost: devicePath, + // PathInContainer: devicePath, + // CgroupPermissions: 'rwm', + // }) + // } + // logger.info( + // `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}` + // ) + // } catch (error) { + // logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`) + // // Fallback to common device names if directory read fails + // const fallbackDevices = ['card0', 'renderD128'] + // for (const device of fallbackDevices) { + // devices.push({ + // PathOnHost: `/dev/dri/${device}`, + // PathInContainer: `/dev/dri/${device}`, + // CgroupPermissions: 'rwm', + // }) + // } + // logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`) + // } - return devices - } catch (error) { - logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`) - return [] - } - } + // return devices + // } catch (error) { + // logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`) + // return [] + // } + // } private _broadcast(service: string, status: string, message: string) { transmit.broadcast('service-installation', { diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 03f8420..c2f2d86 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -42,8 +42,8 @@ export class OllamaService { } /** - * Synchronous version of model download (waits for completion). Should only be used for - * small models or in contexts where a background job is incompatible. + * Downloads a model from the Ollama service with progress tracking. Where possible, + * one should dispatch a background job instead of calling this method directly to avoid long blocking. * @param model Model name to download * @returns Success status and message */ diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index b558eb1..87a5870 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -4,7 +4,7 @@ import { inject } from '@adonisjs/core' import logger from '@adonisjs/core/services/logger' import { TokenChunker } from '@chonkiejs/core' import sharp from 'sharp' -import { deleteFileIfExists, determineFileType, getFile } from '../utils/fs.js' +import { deleteFileIfExists, determineFileType, getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js' import { PDFParse } from 'pdf-parse' import { createWorker } from 'tesseract.js' import { fromBuffer } from 'pdf2pic' @@ -12,6 +12,9 @@ import { OllamaService } from './ollama_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { removeStopwords } from 'stopword' import { randomUUID } from 'node:crypto' +import { join } from 'node:path' +import KVStore from '#models/kv_store' +import { parseBoolean } from '../utils/misc.js' @inject() export class RagService { @@ -33,7 +36,7 @@ export class RagService { constructor( private dockerService: DockerService, private ollamaService: OllamaService - ) {} + ) { } /** * Estimates token count for text. This is a conservative approximation: @@ -166,9 +169,19 @@ export class RagService { const allModels = await this.ollamaService.getModels(true) const embeddingModel = allModels.find((model) => model.name === RagService.EMBEDDING_MODEL) - // TODO: Attempt to download the embedding model if not found if (!embeddingModel) { - throw new Error(`${RagService.EMBEDDING_MODEL} does not exist and could not be downloaded.`) + try { + const downloadResult = await this.ollamaService.downloadModel(RagService.EMBEDDING_MODEL) + if (!downloadResult.success) { + throw new Error(downloadResult.message || 'Unknown error during model download') + } + } catch (modelError) { + logger.error( + `[RAG] Embedding model ${RagService.EMBEDDING_MODEL} not found locally and failed to download:`, + modelError + ) + return null + } } // TokenChunker uses character-based tokenization (1 char = 1 token) @@ -326,7 +339,8 @@ export class RagService { * This includes text extraction, chunking, embedding, and storing in Qdrant. */ public async processAndEmbedFile( - filepath: string // Should already be the full path to the uploaded file + filepath: string, // Should already be the full path to the uploaded file + deleteAfterEmbedding: boolean = false ): Promise<{ success: boolean; message: string; chunks?: number }> { try { const fileType = determineFileType(filepath) @@ -368,9 +382,15 @@ export class RagService { source: filepath }) - // Cleanup the file from disk - logger.info(`[RAG] Embedding complete, deleting uploaded file: ${filepath}`) - await deleteFileIfExists(filepath) + if (!embedResult) { + return { success: false, message: 'Failed to embed and store the extracted text.' } + } + + if (deleteAfterEmbedding) { + // Cleanup the file from disk + logger.info(`[RAG] Embedding complete, deleting uploaded file: ${filepath}`) + await deleteFileIfExists(filepath) + } return { success: true, @@ -656,4 +676,63 @@ export class RagService { return [] } } + + public async discoverNomadDocs(force?: boolean): Promise<{ success: boolean; message: string }> { + try { + const README_PATH = join(process.cwd(), 'README.md') + const DOCS_DIR = join(process.cwd(), 'docs') + + const alreadyEmbeddedRaw = await KVStore.getValue('rag.docsEmbedded') + if (parseBoolean(alreadyEmbeddedRaw) && !force) { + logger.info('[RAG] Nomad docs have already been discovered and queued. Skipping.') + return { success: true, message: 'Nomad docs have already been discovered and queued. Skipping.' } + } + + const filesToEmbed: Array<{ path: string; source: string }> = [] + + const readmeExists = await getFileStatsIfExists(README_PATH) + if (readmeExists) { + filesToEmbed.push({ path: README_PATH, source: 'README.md' }) + } + + const dirContents = await listDirectoryContentsRecursive(DOCS_DIR) + for (const entry of dirContents) { + if (entry.type === 'file') { + filesToEmbed.push({ path: entry.key, source: join('docs', entry.name) }) + } + } + + logger.info(`[RAG] Discovered ${filesToEmbed.length} Nomad doc files to embed`) + + // Import EmbedFileJob dynamically to avoid circular dependencies + const { EmbedFileJob } = await import('#jobs/embed_file_job') + + // Dispatch an EmbedFileJob for each discovered file + for (const fileInfo of filesToEmbed) { + try { + logger.info(`[RAG] Dispatching embed job for: ${fileInfo.source}`) + const stats = await getFileStatsIfExists(fileInfo.path) + await EmbedFileJob.dispatch({ + filePath: fileInfo.path, + fileName: fileInfo.source, + fileSize: stats?.size, + }) + logger.info(`[RAG] Successfully dispatched job for ${fileInfo.source}`) + } catch (fileError) { + logger.error( + `[RAG] Error dispatching job for file ${fileInfo.source}:`, + fileError + ) + } + } + + // Update KV store to mark docs as discovered so we don't redo this unnecessarily + await KVStore.setValue('rag.docsEmbedded', 'true') + + return { success: true, message: `Nomad docs discovery completed. Dispatched ${filesToEmbed.length} embedding jobs.` } + } catch (error) { + logger.error('Error discovering Nomad docs:', error) + return { success: false, message: 'Error discovering Nomad docs.' } + } + } } diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index d875cae..780acd7 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -1,3 +1,3 @@ -export type KVStoreKey = 'chat.suggestionsEnabled' +export type KVStoreKey = 'chat.suggestionsEnabled' | 'rag.docsEmbedded' export type KVStoreValue = string | null \ No newline at end of file