github-mirror/project-nomad
1
0
Fork 0
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git synced 2026-03-28 03:29:25 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: auto add NOMAD docs to KB on AI install

Browse Source
This commit is contained in:
Jake Turner 2026-02-03 23:12:52 -08:00 committed by Jake Turner
parent 907982062f
commit ab07551719
7 changed files with 201 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@ -31,5 +31,6 @@ COPY --from=build /app/build /app
# Copy root package.json for version info
COPY package.json /app/version.json
COPY admin/docs /app/docs
COPY README.md /app/README.md
EXPOSE 8080
CMD ["node", "./bin/server.js"]

5
admin/README.md
View File

@ -1,5 +0,0 @@
## Docker container
```
docker run --rm -it -p 8080:8080 jturnercosmistack/projectnomad:admin-latest -e PORT=8080 -e HOST=0.0.0.0 -e APP_KEY=secretlongpasswordsecret -e LOG_LEVEL=debug -e DRIVE_DISK=fs
```

21
admin/app/jobs/embed_file_job.ts
View File

@ -35,6 +35,21 @@ export class EmbedFileJob {
const ragService = new RagService(dockerService, ollamaService)
try {
// Check if Ollama and Qdrant services are ready
const existingModels = await ollamaService.getModels()
if (!existingModels) {
logger.warn('[EmbedFileJob] Ollama service not ready yet. Will retry...')
throw new Error('Ollama service not ready yet')
}
const qdrantUrl = await dockerService.getServiceURL('nomad_qdrant')
if (!qdrantUrl) {
logger.warn('[EmbedFileJob] Qdrant service not ready yet. Will retry...')
throw new Error('Qdrant service not ready yet')
}
logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`)
// Update progress starting
await job.updateProgress(0)
await job.updateData({
@ -102,10 +117,10 @@ export class EmbedFileJob {
try {
const job = await queue.add(this.key, params, {
jobId,
attempts: 3,
attempts: 30,
backoff: {
type: 'exponential',
delay: 5000, // Delay 5 seconds before retrying
type: 'fixed',
delay: 60000, // Check every 60 seconds for service readiness
},
removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history
removeOnFail: { count: 20 } // Keep last 20 failed jobs for debugging

167
admin/app/services/docker_service.ts
View File

@ -9,7 +9,8 @@ import { ZIM_STORAGE_PATH } from '../utils/fs.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { exec } from 'child_process'
import { promisify } from 'util'
import { readdir } from 'fs/promises'
// import { readdir } from 'fs/promises'
import KVStore from '#models/kv_store'
@inject()
export class DockerService {
@ -473,34 +474,34 @@ export class DockerService {
],
}
} else if (gpuType === 'amd') {
this._broadcast(
service.service_name,
'gpu-config',
`AMD GPU detected. Using ROCm image and configuring container with GPU support...`
)
// this._broadcast(
// service.service_name,
// 'gpu-config',
// `AMD GPU detected. Using ROCm image and configuring container with GPU support...`
// )
// Use ROCm image for AMD
finalImage = 'ollama/ollama:rocm'
// // Use ROCm image for AMD
// finalImage = 'ollama/ollama:rocm'
// Dynamically discover and add AMD GPU devices
const amdDevices = await this._discoverAMDDevices()
if (!amdDevices || amdDevices.length === 0) {
this._broadcast(
service.service_name,
'gpu-config-error',
`Failed to discover AMD GPU devices. Proceeding with CPU-only configuration...`
)
gpuHostConfig = { ...gpuHostConfig } // No GPU devices added
logger.warn(`[DockerService] No AMD GPU devices discovered for Ollama`)
} else {
gpuHostConfig = {
...gpuHostConfig,
Devices: amdDevices,
}
logger.info(
`[DockerService] Configured ${amdDevices.length} AMD GPU devices for Ollama`
)
}
// // Dynamically discover and add AMD GPU devices
// const amdDevices = await this._discoverAMDDevices()
// if (!amdDevices || amdDevices.length === 0) {
// this._broadcast(
// service.service_name,
// 'gpu-config-error',
// `Failed to discover AMD GPU devices. Proceeding with CPU-only configuration...`
// )
// gpuHostConfig = { ...gpuHostConfig } // No GPU devices added
// logger.warn(`[DockerService] No AMD GPU devices discovered for Ollama`)
// } else {
// gpuHostConfig = {
// ...gpuHostConfig,
// Devices: amdDevices,
// }
// logger.info(
// `[DockerService] Configured ${amdDevices.length} AMD GPU devices for Ollama`
// )
// }
} else {
this._broadcast(
service.service_name,
@ -553,6 +554,22 @@ export class DockerService {
// Remove from active installs tracking
this.activeInstallations.delete(service.service_name)
// If Ollama was just installed, trigger Nomad docs discovery and embedding
if (service.service_name === SERVICE_NAMES.OLLAMA) {
logger.info('[DockerService] Ollama installation complete. Enabling chat suggestions by default.')
await KVStore.setValue('chat.suggestionsEnabled', "true")
logger.info('[DockerService] Ollama installation complete. Triggering Nomad docs discovery...')
// Need to use dynamic imports here to avoid circular dependency
const ollamaService = new (await import('./ollama_service.js')).OllamaService()
const ragService = new (await import('./rag_service.js')).RagService(this, ollamaService)
ragService.discoverNomadDocs().catch((error) => {
logger.error('[DockerService] Failed to discover Nomad docs:', error)
})
}
this._broadcast(
service.service_name,
'completed',
@ -715,57 +732,57 @@ export class DockerService {
* Discover AMD GPU DRI devices dynamically.
* Returns an array of device configurations for Docker.
*/
private async _discoverAMDDevices(): Promise<
Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
> {
try {
const devices: Array<{
PathOnHost: string
PathInContainer: string
CgroupPermissions: string
}> = []
// private async _discoverAMDDevices(): Promise<
// Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
// > {
// try {
// const devices: Array<{
// PathOnHost: string
// PathInContainer: string
// CgroupPermissions: string
// }> = []
// Always add /dev/kfd (Kernel Fusion Driver)
devices.push({
PathOnHost: '/dev/kfd',
PathInContainer: '/dev/kfd',
CgroupPermissions: 'rwm',
})
// // Always add /dev/kfd (Kernel Fusion Driver)
// devices.push({
// PathOnHost: '/dev/kfd',
// PathInContainer: '/dev/kfd',
// CgroupPermissions: 'rwm',
// })
// Discover DRI devices in /dev/dri/
try {
const driDevices = await readdir('/dev/dri')
for (const device of driDevices) {
const devicePath = `/dev/dri/${device}`
devices.push({
PathOnHost: devicePath,
PathInContainer: devicePath,
CgroupPermissions: 'rwm',
})
}
logger.info(
`[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}`
)
} catch (error) {
logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`)
// Fallback to common device names if directory read fails
const fallbackDevices = ['card0', 'renderD128']
for (const device of fallbackDevices) {
devices.push({
PathOnHost: `/dev/dri/${device}`,
PathInContainer: `/dev/dri/${device}`,
CgroupPermissions: 'rwm',
})
}
logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`)
}
// // Discover DRI devices in /dev/dri/
// try {
// const driDevices = await readdir('/dev/dri')
// for (const device of driDevices) {
// const devicePath = `/dev/dri/${device}`
// devices.push({
// PathOnHost: devicePath,
// PathInContainer: devicePath,
// CgroupPermissions: 'rwm',
// })
// }
// logger.info(
// `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}`
// )
// } catch (error) {
// logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`)
// // Fallback to common device names if directory read fails
// const fallbackDevices = ['card0', 'renderD128']
// for (const device of fallbackDevices) {
// devices.push({
// PathOnHost: `/dev/dri/${device}`,
// PathInContainer: `/dev/dri/${device}`,
// CgroupPermissions: 'rwm',
// })
// }
// logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`)
// }
return devices
} catch (error) {
logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`)
return []
}
}
// return devices
// } catch (error) {
// logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`)
// return []
// }
// }
private _broadcast(service: string, status: string, message: string) {
transmit.broadcast('service-installation', {

4
admin/app/services/ollama_service.ts
View File

@ -42,8 +42,8 @@ export class OllamaService {
}
/**
* Synchronous version of model download (waits for completion). Should only be used for
* small models or in contexts where a background job is incompatible.
* Downloads a model from the Ollama service with progress tracking. Where possible,
* one should dispatch a background job instead of calling this method directly to avoid long blocking.
* @param model Model name to download
* @returns Success status and message
*/

95
admin/app/services/rag_service.ts
View File

@ -4,7 +4,7 @@ import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import { TokenChunker } from '@chonkiejs/core'
import sharp from 'sharp'
import { deleteFileIfExists, determineFileType, getFile } from '../utils/fs.js'
import { deleteFileIfExists, determineFileType, getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js'
import { PDFParse } from 'pdf-parse'
import { createWorker } from 'tesseract.js'
import { fromBuffer } from 'pdf2pic'
@ -12,6 +12,9 @@ import { OllamaService } from './ollama_service.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { removeStopwords } from 'stopword'
import { randomUUID } from 'node:crypto'
import { join } from 'node:path'
import KVStore from '#models/kv_store'
import { parseBoolean } from '../utils/misc.js'
@inject()
export class RagService {
@ -33,7 +36,7 @@ export class RagService {
constructor(
private dockerService: DockerService,
private ollamaService: OllamaService
) {}
) { }
/**
* Estimates token count for text. This is a conservative approximation:
@ -166,9 +169,19 @@ export class RagService {
const allModels = await this.ollamaService.getModels(true)
const embeddingModel = allModels.find((model) => model.name === RagService.EMBEDDING_MODEL)
// TODO: Attempt to download the embedding model if not found
if (!embeddingModel) {
throw new Error(`${RagService.EMBEDDING_MODEL} does not exist and could not be downloaded.`)
try {
const downloadResult = await this.ollamaService.downloadModel(RagService.EMBEDDING_MODEL)
if (!downloadResult.success) {
throw new Error(downloadResult.message || 'Unknown error during model download')
}
} catch (modelError) {
logger.error(
`[RAG] Embedding model ${RagService.EMBEDDING_MODEL} not found locally and failed to download:`,
modelError
)
return null
}
}
// TokenChunker uses character-based tokenization (1 char = 1 token)
@ -326,7 +339,8 @@ export class RagService {
* This includes text extraction, chunking, embedding, and storing in Qdrant.
*/
public async processAndEmbedFile(
filepath: string // Should already be the full path to the uploaded file
filepath: string, // Should already be the full path to the uploaded file
deleteAfterEmbedding: boolean = false
): Promise<{ success: boolean; message: string; chunks?: number }> {
try {
const fileType = determineFileType(filepath)
@ -368,9 +382,15 @@ export class RagService {
source: filepath
})
// Cleanup the file from disk
logger.info(`[RAG] Embedding complete, deleting uploaded file: ${filepath}`)
await deleteFileIfExists(filepath)
if (!embedResult) {
return { success: false, message: 'Failed to embed and store the extracted text.' }
}
if (deleteAfterEmbedding) {
// Cleanup the file from disk
logger.info(`[RAG] Embedding complete, deleting uploaded file: ${filepath}`)
await deleteFileIfExists(filepath)
}
return {
success: true,
@ -656,4 +676,63 @@ export class RagService {
return []
}
}
public async discoverNomadDocs(force?: boolean): Promise<{ success: boolean; message: string }> {
try {
const README_PATH = join(process.cwd(), 'README.md')
const DOCS_DIR = join(process.cwd(), 'docs')
const alreadyEmbeddedRaw = await KVStore.getValue('rag.docsEmbedded')
if (parseBoolean(alreadyEmbeddedRaw) && !force) {
logger.info('[RAG] Nomad docs have already been discovered and queued. Skipping.')
return { success: true, message: 'Nomad docs have already been discovered and queued. Skipping.' }
}
const filesToEmbed: Array<{ path: string; source: string }> = []
const readmeExists = await getFileStatsIfExists(README_PATH)
if (readmeExists) {
filesToEmbed.push({ path: README_PATH, source: 'README.md' })
}
const dirContents = await listDirectoryContentsRecursive(DOCS_DIR)
for (const entry of dirContents) {
if (entry.type === 'file') {
filesToEmbed.push({ path: entry.key, source: join('docs', entry.name) })
}
}
logger.info(`[RAG] Discovered ${filesToEmbed.length} Nomad doc files to embed`)
// Import EmbedFileJob dynamically to avoid circular dependencies
const { EmbedFileJob } = await import('#jobs/embed_file_job')
// Dispatch an EmbedFileJob for each discovered file
for (const fileInfo of filesToEmbed) {
try {
logger.info(`[RAG] Dispatching embed job for: ${fileInfo.source}`)
const stats = await getFileStatsIfExists(fileInfo.path)
await EmbedFileJob.dispatch({
filePath: fileInfo.path,
fileName: fileInfo.source,
fileSize: stats?.size,
})
logger.info(`[RAG] Successfully dispatched job for ${fileInfo.source}`)
} catch (fileError) {
logger.error(
`[RAG] Error dispatching job for file ${fileInfo.source}:`,
fileError
)
}
}
// Update KV store to mark docs as discovered so we don't redo this unnecessarily
await KVStore.setValue('rag.docsEmbedded', 'true')
return { success: true, message: `Nomad docs discovery completed. Dispatched ${filesToEmbed.length} embedding jobs.` }
} catch (error) {
logger.error('Error discovering Nomad docs:', error)
return { success: false, message: 'Error discovering Nomad docs.' }
}
}
}

2
admin/types/kv_store.ts
View File

@ -1,3 +1,3 @@
export type KVStoreKey = 'chat.suggestionsEnabled'
export type KVStoreKey = 'chat.suggestionsEnabled' | 'rag.docsEmbedded'
export type KVStoreValue = string | null