From 755807f95e79fb5f4c81ca831f7c181907b0b427 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 21 Jan 2026 10:32:50 -0800 Subject: [PATCH] feat: Add system benchmark feature with NOMAD Score Add comprehensive benchmarking capability to measure server performance: Backend: - BenchmarkService with CPU, memory, disk, and AI benchmarks using sysbench - Database migrations for benchmark_results and benchmark_settings tables - REST API endpoints for running benchmarks and retrieving results - CLI commands: benchmark:run, benchmark:results, benchmark:submit - BullMQ job for async benchmark execution with SSE progress updates - Synchronous mode option (?sync=true) for simpler local dev setup Frontend: - Benchmark settings page with circular gauges for scores - NOMAD Score display with weighted composite calculation - System Performance section (CPU, Memory, Disk Read/Write) - AI Performance section (tokens/sec, time to first token) - Hardware Information display - Expandable Benchmark Details section - Progress simulation during sync benchmark execution Easy Setup Integration: - Added System Benchmark to Additional Tools section - Built-in capability pattern for non-Docker features - Click-to-navigate behavior for built-in tools Fixes: - Docker log multiplexing issue (Tty: true) for proper output parsing - Consolidated disk benchmarks into single container execution Co-Authored-By: Claude Opus 4.5 --- admin/app/controllers/benchmark_controller.ts | 230 ++++++ admin/app/controllers/settings_controller.ts | 16 +- admin/app/jobs/run_benchmark_job.ts | 99 +++ admin/app/models/benchmark_result.ts | 82 ++ admin/app/models/benchmark_setting.ts | 60 ++ admin/app/services/benchmark_service.ts | 723 ++++++++++++++++++ admin/app/services/docker_service.ts | 1 + admin/app/validators/benchmark.ts | 13 + admin/commands/benchmark/results.ts | 96 +++ admin/commands/benchmark/run.ts | 103 +++ admin/commands/benchmark/submit.ts | 99 +++ admin/commands/queue/work.ts | 2 + ...97600001_create_benchmark_results_table.ts | 47 ++ ...7600002_create_benchmark_settings_table.ts | 19 + admin/database/seeders/service_seeder.ts | 20 + admin/inertia/pages/easy-setup/index.tsx | 94 ++- admin/inertia/pages/settings/benchmark.tsx | 618 +++++++++++++++ admin/package-lock.json | 30 +- admin/package.json | 2 + admin/start/routes.ts | 18 + admin/types/benchmark.ts | 216 ++++++ 21 files changed, 2559 insertions(+), 29 deletions(-) create mode 100644 admin/app/controllers/benchmark_controller.ts create mode 100644 admin/app/jobs/run_benchmark_job.ts create mode 100644 admin/app/models/benchmark_result.ts create mode 100644 admin/app/models/benchmark_setting.ts create mode 100644 admin/app/services/benchmark_service.ts create mode 100644 admin/app/validators/benchmark.ts create mode 100644 admin/commands/benchmark/results.ts create mode 100644 admin/commands/benchmark/run.ts create mode 100644 admin/commands/benchmark/submit.ts create mode 100644 admin/database/migrations/1769097600001_create_benchmark_results_table.ts create mode 100644 admin/database/migrations/1769097600002_create_benchmark_settings_table.ts create mode 100644 admin/inertia/pages/settings/benchmark.tsx create mode 100644 admin/types/benchmark.ts diff --git a/admin/app/controllers/benchmark_controller.ts b/admin/app/controllers/benchmark_controller.ts new file mode 100644 index 0000000..d2ac00c --- /dev/null +++ b/admin/app/controllers/benchmark_controller.ts @@ -0,0 +1,230 @@ +import { inject } from '@adonisjs/core' +import type { HttpContext } from '@adonisjs/core/http' +import { BenchmarkService } from '#services/benchmark_service' +import { runBenchmarkValidator, submitBenchmarkValidator } from '#validators/benchmark' +import { RunBenchmarkJob } from '#jobs/run_benchmark_job' +import { v4 as uuidv4 } from 'uuid' +import type { BenchmarkType } from '../../types/benchmark.js' + +@inject() +export default class BenchmarkController { + constructor(private benchmarkService: BenchmarkService) {} + + /** + * Start a benchmark run (async via job queue, or sync if specified) + */ + async run({ request, response }: HttpContext) { + const payload = await request.validateUsing(runBenchmarkValidator) + const benchmarkType: BenchmarkType = payload.benchmark_type || 'full' + const runSync = request.input('sync') === 'true' || request.input('sync') === true + + // Check if a benchmark is already running + const status = this.benchmarkService.getStatus() + if (status.status !== 'idle') { + return response.status(409).send({ + success: false, + error: 'A benchmark is already running', + current_benchmark_id: status.benchmarkId, + }) + } + + // Run synchronously if requested (useful for local dev without Redis) + if (runSync) { + try { + let result + switch (benchmarkType) { + case 'full': + result = await this.benchmarkService.runFullBenchmark() + break + case 'system': + result = await this.benchmarkService.runSystemBenchmarks() + break + case 'ai': + result = await this.benchmarkService.runAIBenchmark() + break + default: + result = await this.benchmarkService.runFullBenchmark() + } + return response.send({ + success: true, + benchmark_id: result.benchmark_id, + nomad_score: result.nomad_score, + result, + }) + } catch (error) { + return response.status(500).send({ + success: false, + error: error.message, + }) + } + } + + // Generate benchmark ID and dispatch job (async) + const benchmarkId = uuidv4() + const { job, created } = await RunBenchmarkJob.dispatch({ + benchmark_id: benchmarkId, + benchmark_type: benchmarkType, + include_ai: benchmarkType === 'full' || benchmarkType === 'ai', + }) + + return response.status(201).send({ + success: true, + job_id: job?.id || benchmarkId, + benchmark_id: benchmarkId, + message: created + ? `${benchmarkType} benchmark started` + : 'Benchmark job already exists', + }) + } + + /** + * Run a system-only benchmark (CPU, memory, disk) + */ + async runSystem({ response }: HttpContext) { + const status = this.benchmarkService.getStatus() + if (status.status !== 'idle') { + return response.status(409).send({ + success: false, + error: 'A benchmark is already running', + }) + } + + const benchmarkId = uuidv4() + await RunBenchmarkJob.dispatch({ + benchmark_id: benchmarkId, + benchmark_type: 'system', + include_ai: false, + }) + + return response.status(201).send({ + success: true, + benchmark_id: benchmarkId, + message: 'System benchmark started', + }) + } + + /** + * Run an AI-only benchmark + */ + async runAI({ response }: HttpContext) { + const status = this.benchmarkService.getStatus() + if (status.status !== 'idle') { + return response.status(409).send({ + success: false, + error: 'A benchmark is already running', + }) + } + + const benchmarkId = uuidv4() + await RunBenchmarkJob.dispatch({ + benchmark_id: benchmarkId, + benchmark_type: 'ai', + include_ai: true, + }) + + return response.status(201).send({ + success: true, + benchmark_id: benchmarkId, + message: 'AI benchmark started', + }) + } + + /** + * Get all benchmark results + */ + async results({}: HttpContext) { + const results = await this.benchmarkService.getAllResults() + return { + results, + total: results.length, + } + } + + /** + * Get the latest benchmark result + */ + async latest({}: HttpContext) { + const result = await this.benchmarkService.getLatestResult() + if (!result) { + return { result: null } + } + return { result } + } + + /** + * Get a specific benchmark result by ID + */ + async show({ params, response }: HttpContext) { + const result = await this.benchmarkService.getResultById(params.id) + if (!result) { + return response.status(404).send({ + error: 'Benchmark result not found', + }) + } + return { result } + } + + /** + * Submit benchmark results to central repository + */ + async submit({ request, response }: HttpContext) { + const payload = await request.validateUsing(submitBenchmarkValidator) + + try { + const submitResult = await this.benchmarkService.submitToRepository(payload.benchmark_id) + return response.send({ + success: true, + repository_id: submitResult.repository_id, + percentile: submitResult.percentile, + }) + } catch (error) { + return response.status(400).send({ + success: false, + error: error.message, + }) + } + } + + /** + * Get comparison stats from central repository + */ + async comparison({}: HttpContext) { + const stats = await this.benchmarkService.getComparisonStats() + return { stats } + } + + /** + * Get current benchmark status + */ + async status({}: HttpContext) { + return this.benchmarkService.getStatus() + } + + /** + * Get benchmark settings + */ + async settings({}: HttpContext) { + const { default: BenchmarkSetting } = await import('#models/benchmark_setting') + return await BenchmarkSetting.getAllSettings() + } + + /** + * Update benchmark settings + */ + async updateSettings({ request, response }: HttpContext) { + const { default: BenchmarkSetting } = await import('#models/benchmark_setting') + const body = request.body() + + if (body.allow_anonymous_submission !== undefined) { + await BenchmarkSetting.setValue( + 'allow_anonymous_submission', + body.allow_anonymous_submission ? 'true' : 'false' + ) + } + + return response.send({ + success: true, + settings: await BenchmarkSetting.getAllSettings(), + }) + } +} diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index 54142ab..3defcc0 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -1,3 +1,4 @@ +import { BenchmarkService } from '#services/benchmark_service'; import { MapService } from '#services/map_service'; import { OpenWebUIService } from '#services/openwebui_service'; import { SystemService } from '#services/system_service'; @@ -9,7 +10,8 @@ export default class SettingsController { constructor( private systemService: SystemService, private mapService: MapService, - private openWebUIService: OpenWebUIService + private openWebUIService: OpenWebUIService, + private benchmarkService: BenchmarkService ) { } async system({ inertia }: HttpContext) { @@ -74,4 +76,16 @@ export default class SettingsController { async zimRemote({ inertia }: HttpContext) { return inertia.render('settings/zim/remote-explorer'); } + + async benchmark({ inertia }: HttpContext) { + const latestResult = await this.benchmarkService.getLatestResult(); + const status = this.benchmarkService.getStatus(); + return inertia.render('settings/benchmark', { + benchmark: { + latestResult, + status: status.status, + currentBenchmarkId: status.benchmarkId + } + }); + } } \ No newline at end of file diff --git a/admin/app/jobs/run_benchmark_job.ts b/admin/app/jobs/run_benchmark_job.ts new file mode 100644 index 0000000..f975e30 --- /dev/null +++ b/admin/app/jobs/run_benchmark_job.ts @@ -0,0 +1,99 @@ +import { Job } from 'bullmq' +import { QueueService } from '#services/queue_service' +import { BenchmarkService } from '#services/benchmark_service' +import type { RunBenchmarkJobParams } from '../../types/benchmark.js' +import logger from '@adonisjs/core/services/logger' + +export class RunBenchmarkJob { + static get queue() { + return 'benchmarks' + } + + static get key() { + return 'run-benchmark' + } + + async handle(job: Job) { + const { benchmark_id, benchmark_type } = job.data as RunBenchmarkJobParams + + logger.info(`[RunBenchmarkJob] Starting benchmark ${benchmark_id} of type ${benchmark_type}`) + + const benchmarkService = new BenchmarkService() + + try { + let result + + switch (benchmark_type) { + case 'full': + result = await benchmarkService.runFullBenchmark() + break + case 'system': + result = await benchmarkService.runSystemBenchmarks() + break + case 'ai': + result = await benchmarkService.runAIBenchmark() + break + default: + throw new Error(`Unknown benchmark type: ${benchmark_type}`) + } + + logger.info(`[RunBenchmarkJob] Benchmark ${benchmark_id} completed with NOMAD score: ${result.nomad_score}`) + + return { + success: true, + benchmark_id: result.benchmark_id, + nomad_score: result.nomad_score, + } + } catch (error) { + logger.error(`[RunBenchmarkJob] Benchmark ${benchmark_id} failed: ${error.message}`) + throw error + } + } + + static async dispatch(params: RunBenchmarkJobParams) { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + + try { + const job = await queue.add(this.key, params, { + jobId: params.benchmark_id, + attempts: 1, // Benchmarks shouldn't be retried automatically + removeOnComplete: { + count: 10, // Keep last 10 completed jobs + }, + removeOnFail: { + count: 5, // Keep last 5 failed jobs + }, + }) + + logger.info(`[RunBenchmarkJob] Dispatched benchmark job ${params.benchmark_id}`) + + return { + job, + created: true, + message: `Benchmark job ${params.benchmark_id} dispatched successfully`, + } + } catch (error) { + if (error.message.includes('job already exists')) { + const existing = await queue.getJob(params.benchmark_id) + return { + job: existing, + created: false, + message: `Benchmark job ${params.benchmark_id} already exists`, + } + } + throw error + } + } + + static async getJob(benchmarkId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + return await queue.getJob(benchmarkId) + } + + static async getJobState(benchmarkId: string): Promise { + const job = await this.getJob(benchmarkId) + return job ? await job.getState() : undefined + } +} diff --git a/admin/app/models/benchmark_result.ts b/admin/app/models/benchmark_result.ts new file mode 100644 index 0000000..8aaa8ae --- /dev/null +++ b/admin/app/models/benchmark_result.ts @@ -0,0 +1,82 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import type { BenchmarkType, DiskType } from '../../types/benchmark.js' + +export default class BenchmarkResult extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare benchmark_id: string + + @column() + declare benchmark_type: BenchmarkType + + // Hardware information + @column() + declare cpu_model: string + + @column() + declare cpu_cores: number + + @column() + declare cpu_threads: number + + @column() + declare ram_bytes: number + + @column() + declare disk_type: DiskType + + @column() + declare gpu_model: string | null + + // System benchmark scores + @column() + declare cpu_score: number + + @column() + declare memory_score: number + + @column() + declare disk_read_score: number + + @column() + declare disk_write_score: number + + // AI benchmark scores (nullable for system-only benchmarks) + @column() + declare ai_tokens_per_second: number | null + + @column() + declare ai_model_used: string | null + + @column() + declare ai_time_to_first_token: number | null + + // Composite NOMAD score (0-100) + @column() + declare nomad_score: number + + // Repository submission tracking + @column({ + serialize(value) { + return Boolean(value) + }, + }) + declare submitted_to_repository: boolean + + @column.dateTime() + declare submitted_at: DateTime | null + + @column() + declare repository_id: string | null + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime +} diff --git a/admin/app/models/benchmark_setting.ts b/admin/app/models/benchmark_setting.ts new file mode 100644 index 0000000..bd6f2b6 --- /dev/null +++ b/admin/app/models/benchmark_setting.ts @@ -0,0 +1,60 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import type { BenchmarkSettingKey } from '../../types/benchmark.js' + +export default class BenchmarkSetting extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare key: BenchmarkSettingKey + + @column() + declare value: string | null + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime + + /** + * Get a setting value by key + */ + static async getValue(key: BenchmarkSettingKey): Promise { + const setting = await this.findBy('key', key) + return setting?.value ?? null + } + + /** + * Set a setting value by key (creates if not exists) + */ + static async setValue(key: BenchmarkSettingKey, value: string | null): Promise { + const setting = await this.firstOrCreate({ key }, { key, value }) + if (setting.value !== value) { + setting.value = value + await setting.save() + } + return setting + } + + /** + * Get all benchmark settings as a typed object + */ + static async getAllSettings(): Promise<{ + allow_anonymous_submission: boolean + installation_id: string | null + last_benchmark_run: string | null + }> { + const settings = await this.all() + const map = new Map(settings.map((s) => [s.key, s.value])) + + return { + allow_anonymous_submission: map.get('allow_anonymous_submission') === 'true', + installation_id: map.get('installation_id') ?? null, + last_benchmark_run: map.get('last_benchmark_run') ?? null, + } + } +} diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts new file mode 100644 index 0000000..2e9d149 --- /dev/null +++ b/admin/app/services/benchmark_service.ts @@ -0,0 +1,723 @@ +import { inject } from '@adonisjs/core' +import logger from '@adonisjs/core/services/logger' +import transmit from '@adonisjs/transmit/services/main' +import Docker from 'dockerode' +import si from 'systeminformation' +import { v4 as uuidv4 } from 'uuid' +import axios from 'axios' +import BenchmarkResult from '#models/benchmark_result' +import BenchmarkSetting from '#models/benchmark_setting' +import { SystemService } from '#services/system_service' +import type { + BenchmarkType, + BenchmarkStatus, + BenchmarkProgress, + HardwareInfo, + DiskType, + SystemScores, + AIScores, + SysbenchCpuResult, + SysbenchMemoryResult, + SysbenchDiskResult, + RepositorySubmission, + RepositorySubmitResponse, + RepositoryStats, +} from '../../types/benchmark.js' + +// Re-export default weights for use in service +const SCORE_WEIGHTS = { + ai_tokens_per_second: 0.30, + cpu: 0.25, + memory: 0.15, + ai_ttft: 0.10, + disk_read: 0.10, + disk_write: 0.10, +} + +// Benchmark configuration constants +const SYSBENCH_IMAGE = 'severalnines/sysbench:latest' +const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench' +const BENCHMARK_CHANNEL = 'benchmark-progress' + +// Reference model for AI benchmark - small but meaningful +const AI_BENCHMARK_MODEL = 'llama3.2:1b' +const AI_BENCHMARK_PROMPT = 'Explain recursion in programming in exactly 100 words.' + +// Reference scores for normalization (calibrated to 0-100 scale) +// These represent "expected" scores for a mid-range system (score ~50) +const REFERENCE_SCORES = { + cpu_events_per_second: 5000, // sysbench cpu events/sec for ~50 score + memory_ops_per_second: 5000000, // sysbench memory ops/sec for ~50 score + disk_read_mb_per_sec: 500, // 500 MB/s read for ~50 score + disk_write_mb_per_sec: 400, // 400 MB/s write for ~50 score + ai_tokens_per_second: 30, // 30 tok/s for ~50 score + ai_ttft_ms: 500, // 500ms time to first token for ~50 score (lower is better) +} + +@inject() +export class BenchmarkService { + private docker: Docker + private currentBenchmarkId: string | null = null + private currentStatus: BenchmarkStatus = 'idle' + + constructor() { + const isWindows = process.platform === 'win32' + if (isWindows) { + this.docker = new Docker({ socketPath: '//./pipe/docker_engine' }) + } else { + this.docker = new Docker({ socketPath: '/var/run/docker.sock' }) + } + } + + /** + * Run a full benchmark suite + */ + async runFullBenchmark(): Promise { + return this._runBenchmark('full', true) + } + + /** + * Run system benchmarks only (CPU, memory, disk) + */ + async runSystemBenchmarks(): Promise { + return this._runBenchmark('system', false) + } + + /** + * Run AI benchmark only + */ + async runAIBenchmark(): Promise { + return this._runBenchmark('ai', true) + } + + /** + * Get the latest benchmark result + */ + async getLatestResult(): Promise { + return await BenchmarkResult.query().orderBy('created_at', 'desc').first() + } + + /** + * Get all benchmark results + */ + async getAllResults(): Promise { + return await BenchmarkResult.query().orderBy('created_at', 'desc') + } + + /** + * Get a specific benchmark result by ID + */ + async getResultById(benchmarkId: string): Promise { + return await BenchmarkResult.findBy('benchmark_id', benchmarkId) + } + + /** + * Submit benchmark results to central repository + */ + async submitToRepository(benchmarkId?: string): Promise { + const result = benchmarkId + ? await this.getResultById(benchmarkId) + : await this.getLatestResult() + + if (!result) { + throw new Error('No benchmark result found to submit') + } + + if (result.submitted_to_repository) { + throw new Error('Benchmark result has already been submitted') + } + + const submission: RepositorySubmission = { + cpu_model: result.cpu_model, + cpu_cores: result.cpu_cores, + cpu_threads: result.cpu_threads, + ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)), + disk_type: result.disk_type, + gpu_model: result.gpu_model, + cpu_score: result.cpu_score, + memory_score: result.memory_score, + disk_read_score: result.disk_read_score, + disk_write_score: result.disk_write_score, + ai_tokens_per_second: result.ai_tokens_per_second, + ai_time_to_first_token: result.ai_time_to_first_token, + nomad_score: result.nomad_score, + nomad_version: SystemService.getAppVersion(), + benchmark_version: '1.0.0', + } + + try { + const response = await axios.post( + 'https://benchmark.projectnomad.us/api/v1/submit', + submission, + { timeout: 30000 } + ) + + if (response.data.success) { + result.submitted_to_repository = true + result.submitted_at = new Date() as any + result.repository_id = response.data.repository_id + await result.save() + + await BenchmarkSetting.setValue('last_benchmark_run', new Date().toISOString()) + } + + return response.data as RepositorySubmitResponse + } catch (error) { + logger.error(`Failed to submit benchmark to repository: ${error.message}`) + throw new Error(`Failed to submit benchmark: ${error.message}`) + } + } + + /** + * Get comparison stats from central repository + */ + async getComparisonStats(): Promise { + try { + const response = await axios.get('https://benchmark.projectnomad.us/api/v1/stats', { + timeout: 10000, + }) + return response.data as RepositoryStats + } catch (error) { + logger.warn(`Failed to fetch comparison stats: ${error.message}`) + return null + } + } + + /** + * Get current benchmark status + */ + getStatus(): { status: BenchmarkStatus; benchmarkId: string | null } { + return { + status: this.currentStatus, + benchmarkId: this.currentBenchmarkId, + } + } + + /** + * Detect system hardware information + */ + async getHardwareInfo(): Promise { + this._updateStatus('detecting_hardware', 'Detecting system hardware...') + + try { + const [cpu, mem, diskLayout, graphics] = await Promise.all([ + si.cpu(), + si.mem(), + si.diskLayout(), + si.graphics(), + ]) + + // Determine disk type from primary disk + let diskType: DiskType = 'unknown' + if (diskLayout.length > 0) { + const primaryDisk = diskLayout[0] + if (primaryDisk.type?.toLowerCase().includes('nvme')) { + diskType = 'nvme' + } else if (primaryDisk.type?.toLowerCase().includes('ssd')) { + diskType = 'ssd' + } else if (primaryDisk.type?.toLowerCase().includes('hdd') || primaryDisk.interfaceType === 'SATA') { + // SATA could be SSD or HDD, check if it's rotational + diskType = 'hdd' + } + } + + // Get GPU model (prefer discrete GPU) + let gpuModel: string | null = null + if (graphics.controllers && graphics.controllers.length > 0) { + const discreteGpu = graphics.controllers.find( + (g) => !g.vendor?.toLowerCase().includes('intel') && + !g.vendor?.toLowerCase().includes('amd') || + (g.vram && g.vram > 0) + ) + gpuModel = discreteGpu?.model || graphics.controllers[0]?.model || null + } + + return { + cpu_model: `${cpu.manufacturer} ${cpu.brand}`, + cpu_cores: cpu.physicalCores, + cpu_threads: cpu.cores, + ram_bytes: mem.total, + disk_type: diskType, + gpu_model: gpuModel, + } + } catch (error) { + logger.error(`Error detecting hardware: ${error.message}`) + throw new Error(`Failed to detect hardware: ${error.message}`) + } + } + + /** + * Main benchmark execution method + */ + private async _runBenchmark(type: BenchmarkType, includeAI: boolean): Promise { + if (this.currentStatus !== 'idle') { + throw new Error('A benchmark is already running') + } + + this.currentBenchmarkId = uuidv4() + this._updateStatus('starting', 'Starting benchmark...') + + try { + // Detect hardware + const hardware = await this.getHardwareInfo() + + // Run system benchmarks + let systemScores: SystemScores = { + cpu_score: 0, + memory_score: 0, + disk_read_score: 0, + disk_write_score: 0, + } + + if (type === 'full' || type === 'system') { + systemScores = await this._runSystemBenchmarks() + } + + // Run AI benchmark if requested and Ollama is available + let aiScores: Partial = {} + if (includeAI && (type === 'full' || type === 'ai')) { + try { + aiScores = await this._runAIBenchmark() + } catch (error) { + logger.warn(`AI benchmark skipped: ${error.message}`) + // AI benchmark is optional, continue without it + } + } + + // Calculate NOMAD score + this._updateStatus('calculating_score', 'Calculating NOMAD score...') + const nomadScore = this._calculateNomadScore(systemScores, aiScores) + + // Save result + const result = await BenchmarkResult.create({ + benchmark_id: this.currentBenchmarkId, + benchmark_type: type, + cpu_model: hardware.cpu_model, + cpu_cores: hardware.cpu_cores, + cpu_threads: hardware.cpu_threads, + ram_bytes: hardware.ram_bytes, + disk_type: hardware.disk_type, + gpu_model: hardware.gpu_model, + cpu_score: systemScores.cpu_score, + memory_score: systemScores.memory_score, + disk_read_score: systemScores.disk_read_score, + disk_write_score: systemScores.disk_write_score, + ai_tokens_per_second: aiScores.ai_tokens_per_second || null, + ai_model_used: aiScores.ai_model_used || null, + ai_time_to_first_token: aiScores.ai_time_to_first_token || null, + nomad_score: nomadScore, + submitted_to_repository: false, + }) + + this._updateStatus('completed', 'Benchmark completed successfully') + this.currentStatus = 'idle' + this.currentBenchmarkId = null + + return result + } catch (error) { + this._updateStatus('error', `Benchmark failed: ${error.message}`) + this.currentStatus = 'idle' + this.currentBenchmarkId = null + throw error + } + } + + /** + * Run system benchmarks using sysbench in Docker + */ + private async _runSystemBenchmarks(): Promise { + // Ensure sysbench image is available + await this._ensureSysbenchImage() + + // Run CPU benchmark + this._updateStatus('running_cpu', 'Running CPU benchmark...') + const cpuResult = await this._runSysbenchCpu() + + // Run memory benchmark + this._updateStatus('running_memory', 'Running memory benchmark...') + const memoryResult = await this._runSysbenchMemory() + + // Run disk benchmarks + this._updateStatus('running_disk_read', 'Running disk read benchmark...') + const diskReadResult = await this._runSysbenchDiskRead() + + this._updateStatus('running_disk_write', 'Running disk write benchmark...') + const diskWriteResult = await this._runSysbenchDiskWrite() + + // Normalize scores to 0-100 scale + return { + cpu_score: this._normalizeScore(cpuResult.events_per_second, REFERENCE_SCORES.cpu_events_per_second), + memory_score: this._normalizeScore(memoryResult.operations_per_second, REFERENCE_SCORES.memory_ops_per_second), + disk_read_score: this._normalizeScore(diskReadResult.read_mb_per_sec, REFERENCE_SCORES.disk_read_mb_per_sec), + disk_write_score: this._normalizeScore(diskWriteResult.write_mb_per_sec, REFERENCE_SCORES.disk_write_mb_per_sec), + } + } + + /** + * Run AI benchmark using Ollama + */ + private async _runAIBenchmark(): Promise { + this._updateStatus('running_ai', 'Running AI benchmark...') + + // Check if Ollama is available + try { + await axios.get('http://localhost:11434/api/tags', { timeout: 5000 }) + } catch { + throw new Error('Ollama is not running or not accessible') + } + + // Check if the benchmark model is available, pull if not + try { + const modelsResponse = await axios.get('http://localhost:11434/api/tags') + const models = modelsResponse.data.models || [] + const hasModel = models.some((m: any) => m.name === AI_BENCHMARK_MODEL || m.name.startsWith(AI_BENCHMARK_MODEL.split(':')[0])) + + if (!hasModel) { + this._updateStatus('running_ai', `Pulling benchmark model ${AI_BENCHMARK_MODEL}...`) + await axios.post('http://localhost:11434/api/pull', { name: AI_BENCHMARK_MODEL }) + } + } catch (error) { + logger.warn(`Could not check/pull model: ${error.message}`) + } + + // Run inference benchmark + const startTime = Date.now() + + try { + const response = await axios.post( + 'http://localhost:11434/api/generate', + { + model: AI_BENCHMARK_MODEL, + prompt: AI_BENCHMARK_PROMPT, + stream: false, + }, + { timeout: 120000 } + ) + + const endTime = Date.now() + const totalTime = (endTime - startTime) / 1000 // seconds + + // Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds) + if (response.data.eval_count && response.data.eval_duration) { + const tokenCount = response.data.eval_count + const evalDurationSeconds = response.data.eval_duration / 1e9 + const tokensPerSecond = tokenCount / evalDurationSeconds + + // Time to first token from prompt_eval_duration + const ttft = response.data.prompt_eval_duration + ? response.data.prompt_eval_duration / 1e6 // Convert to ms + : (totalTime * 1000) / 2 // Estimate if not available + + return { + ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, + ai_model_used: AI_BENCHMARK_MODEL, + ai_time_to_first_token: Math.round(ttft * 100) / 100, + } + } + + // Fallback calculation + const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100 + const tokensPerSecond = estimatedTokens / totalTime + + return { + ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, + ai_model_used: AI_BENCHMARK_MODEL, + ai_time_to_first_token: Math.round((totalTime * 1000) / 2), + } + } catch (error) { + throw new Error(`AI benchmark failed: ${error.message}`) + } + } + + /** + * Calculate weighted NOMAD score + */ + private _calculateNomadScore(systemScores: SystemScores, aiScores: Partial): number { + let totalWeight = 0 + let weightedSum = 0 + + // CPU score + weightedSum += systemScores.cpu_score * SCORE_WEIGHTS.cpu + totalWeight += SCORE_WEIGHTS.cpu + + // Memory score + weightedSum += systemScores.memory_score * SCORE_WEIGHTS.memory + totalWeight += SCORE_WEIGHTS.memory + + // Disk scores + weightedSum += systemScores.disk_read_score * SCORE_WEIGHTS.disk_read + totalWeight += SCORE_WEIGHTS.disk_read + weightedSum += systemScores.disk_write_score * SCORE_WEIGHTS.disk_write + totalWeight += SCORE_WEIGHTS.disk_write + + // AI scores (if available) + if (aiScores.ai_tokens_per_second !== undefined) { + const aiScore = this._normalizeScore( + aiScores.ai_tokens_per_second, + REFERENCE_SCORES.ai_tokens_per_second + ) + weightedSum += aiScore * SCORE_WEIGHTS.ai_tokens_per_second + totalWeight += SCORE_WEIGHTS.ai_tokens_per_second + } + + if (aiScores.ai_time_to_first_token !== undefined) { + // For TTFT, lower is better, so we invert the score + const ttftScore = this._normalizeScoreInverse( + aiScores.ai_time_to_first_token, + REFERENCE_SCORES.ai_ttft_ms + ) + weightedSum += ttftScore * SCORE_WEIGHTS.ai_ttft + totalWeight += SCORE_WEIGHTS.ai_ttft + } + + // Normalize by actual weight used (in case AI benchmarks were skipped) + const nomadScore = totalWeight > 0 ? (weightedSum / totalWeight) * 100 : 0 + + return Math.round(Math.min(100, Math.max(0, nomadScore)) * 100) / 100 + } + + /** + * Normalize a raw score to 0-100 scale using log scaling + * This provides diminishing returns for very high scores + */ + private _normalizeScore(value: number, reference: number): number { + if (value <= 0) return 0 + // Log scale: score = 50 * (1 + log2(value/reference)) + // This gives 50 at reference value, scales logarithmically + const ratio = value / reference + const score = 50 * (1 + Math.log2(Math.max(0.01, ratio))) + return Math.min(100, Math.max(0, score)) / 100 + } + + /** + * Normalize a score where lower is better (like latency) + */ + private _normalizeScoreInverse(value: number, reference: number): number { + if (value <= 0) return 1 + // Inverse: lower values = higher scores + const ratio = reference / value + const score = 50 * (1 + Math.log2(Math.max(0.01, ratio))) + return Math.min(100, Math.max(0, score)) / 100 + } + + /** + * Ensure sysbench Docker image is available + */ + private async _ensureSysbenchImage(): Promise { + try { + await this.docker.getImage(SYSBENCH_IMAGE).inspect() + } catch { + this._updateStatus('starting', `Pulling sysbench image...`) + const pullStream = await this.docker.pull(SYSBENCH_IMAGE) + await new Promise((resolve) => this.docker.modem.followProgress(pullStream, resolve)) + } + } + + /** + * Run sysbench CPU benchmark + */ + private async _runSysbenchCpu(): Promise { + const output = await this._runSysbenchCommand([ + 'sysbench', + 'cpu', + '--cpu-max-prime=20000', + '--threads=4', + '--time=30', + 'run', + ]) + + // Parse output for events per second + const eventsMatch = output.match(/events per second:\s*([\d.]+)/i) + const totalTimeMatch = output.match(/total time:\s*([\d.]+)s/i) + const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i) + + return { + events_per_second: eventsMatch ? parseFloat(eventsMatch[1]) : 0, + total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30, + total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0, + } + } + + /** + * Run sysbench memory benchmark + */ + private async _runSysbenchMemory(): Promise { + const output = await this._runSysbenchCommand([ + 'sysbench', + 'memory', + '--memory-block-size=1K', + '--memory-total-size=10G', + '--threads=4', + 'run', + ]) + + // Parse output + const opsMatch = output.match(/Total operations:\s*\d+\s*\(([\d.]+)\s*per second\)/i) + const transferMatch = output.match(/([\d.]+)\s*MiB\/sec/i) + const timeMatch = output.match(/total time:\s*([\d.]+)s/i) + + return { + operations_per_second: opsMatch ? parseFloat(opsMatch[1]) : 0, + transfer_rate_mb_per_sec: transferMatch ? parseFloat(transferMatch[1]) : 0, + total_time: timeMatch ? parseFloat(timeMatch[1]) : 0, + } + } + + /** + * Run sysbench disk read benchmark + */ + private async _runSysbenchDiskRead(): Promise { + // Run prepare, test, and cleanup in a single container + // This is necessary because each container has its own filesystem + const output = await this._runSysbenchCommand([ + 'sh', + '-c', + 'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' + + 'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --time=30 run && ' + + 'sysbench fileio --file-total-size=1G --file-num=4 cleanup', + ]) + + // Parse output - look for the Throughput section + const readMatch = output.match(/read,\s*MiB\/s:\s*([\d.]+)/i) + const readsPerSecMatch = output.match(/reads\/s:\s*([\d.]+)/i) + + logger.debug(`[BenchmarkService] Disk read output parsing - read: ${readMatch?.[1]}, reads/s: ${readsPerSecMatch?.[1]}`) + + return { + reads_per_second: readsPerSecMatch ? parseFloat(readsPerSecMatch[1]) : 0, + writes_per_second: 0, + read_mb_per_sec: readMatch ? parseFloat(readMatch[1]) : 0, + write_mb_per_sec: 0, + total_time: 30, + } + } + + /** + * Run sysbench disk write benchmark + */ + private async _runSysbenchDiskWrite(): Promise { + // Run prepare, test, and cleanup in a single container + // This is necessary because each container has its own filesystem + const output = await this._runSysbenchCommand([ + 'sh', + '-c', + 'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' + + 'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --time=30 run && ' + + 'sysbench fileio --file-total-size=1G --file-num=4 cleanup', + ]) + + // Parse output - look for the Throughput section + const writeMatch = output.match(/written,\s*MiB\/s:\s*([\d.]+)/i) + const writesPerSecMatch = output.match(/writes\/s:\s*([\d.]+)/i) + + logger.debug(`[BenchmarkService] Disk write output parsing - written: ${writeMatch?.[1]}, writes/s: ${writesPerSecMatch?.[1]}`) + + return { + reads_per_second: 0, + writes_per_second: writesPerSecMatch ? parseFloat(writesPerSecMatch[1]) : 0, + read_mb_per_sec: 0, + write_mb_per_sec: writeMatch ? parseFloat(writeMatch[1]) : 0, + total_time: 30, + } + } + + /** + * Run a sysbench command in a Docker container + */ + private async _runSysbenchCommand(cmd: string[]): Promise { + try { + // Create container with TTY to avoid multiplexed output + const container = await this.docker.createContainer({ + Image: SYSBENCH_IMAGE, + Cmd: cmd, + name: `${SYSBENCH_CONTAINER_NAME}_${Date.now()}`, + Tty: true, // Important: prevents multiplexed stdout/stderr headers + HostConfig: { + AutoRemove: true, + }, + }) + + // Start container + await container.start() + + // Wait for completion and get logs + await container.wait() + const logs = await container.logs({ + stdout: true, + stderr: true, + }) + + // Parse logs (Docker logs include header bytes) + const output = logs.toString('utf8') + .replace(/[\x00-\x08]/g, '') // Remove control characters + .trim() + + return output + } catch (error) { + logger.error(`Sysbench command failed: ${error.message}`) + throw new Error(`Sysbench command failed: ${error.message}`) + } + } + + /** + * Broadcast benchmark progress update + */ + private _updateStatus(status: BenchmarkStatus, message: string) { + this.currentStatus = status + + const progress: BenchmarkProgress = { + status, + progress: this._getProgressPercent(status), + message, + current_stage: this._getStageLabel(status), + timestamp: new Date().toISOString(), + } + + transmit.broadcast(BENCHMARK_CHANNEL, { + benchmark_id: this.currentBenchmarkId, + ...progress, + }) + + logger.info(`[BenchmarkService] ${status}: ${message}`) + } + + /** + * Get progress percentage for a given status + */ + private _getProgressPercent(status: BenchmarkStatus): number { + const progressMap: Record = { + idle: 0, + starting: 5, + detecting_hardware: 10, + running_cpu: 25, + running_memory: 40, + running_disk_read: 55, + running_disk_write: 70, + running_ai: 85, + calculating_score: 95, + completed: 100, + error: 0, + } + return progressMap[status] || 0 + } + + /** + * Get human-readable stage label + */ + private _getStageLabel(status: BenchmarkStatus): string { + const labelMap: Record = { + idle: 'Idle', + starting: 'Starting', + detecting_hardware: 'Detecting Hardware', + running_cpu: 'CPU Benchmark', + running_memory: 'Memory Benchmark', + running_disk_read: 'Disk Read Test', + running_disk_write: 'Disk Write Test', + running_ai: 'AI Inference Test', + calculating_score: 'Calculating Score', + completed: 'Complete', + error: 'Error', + } + return labelMap[status] || status + } +} diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 4659dd8..267e43f 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -17,6 +17,7 @@ export class DockerService { public static CYBERCHEF_SERVICE_NAME = 'nomad_cyberchef' public static FLATNOTES_SERVICE_NAME = 'nomad_flatnotes' public static KOLIBRI_SERVICE_NAME = 'nomad_kolibri' + public static BENCHMARK_SERVICE_NAME = 'nomad_benchmark' constructor() { // Support both Linux (production) and Windows (development with Docker Desktop) diff --git a/admin/app/validators/benchmark.ts b/admin/app/validators/benchmark.ts new file mode 100644 index 0000000..385ac1f --- /dev/null +++ b/admin/app/validators/benchmark.ts @@ -0,0 +1,13 @@ +import vine from '@vinejs/vine' + +export const runBenchmarkValidator = vine.compile( + vine.object({ + benchmark_type: vine.enum(['full', 'system', 'ai']).optional(), + }) +) + +export const submitBenchmarkValidator = vine.compile( + vine.object({ + benchmark_id: vine.string().optional(), + }) +) diff --git a/admin/commands/benchmark/results.ts b/admin/commands/benchmark/results.ts new file mode 100644 index 0000000..1583d20 --- /dev/null +++ b/admin/commands/benchmark/results.ts @@ -0,0 +1,96 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' + +export default class BenchmarkResults extends BaseCommand { + static commandName = 'benchmark:results' + static description = 'Display benchmark results' + + @flags.boolean({ description: 'Show only the latest result', alias: 'l' }) + declare latest: boolean + + @flags.string({ description: 'Output format (table, json)', default: 'table' }) + declare format: string + + @flags.string({ description: 'Show specific benchmark by ID', alias: 'i' }) + declare id: string + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { BenchmarkService } = await import('#services/benchmark_service') + const benchmarkService = new BenchmarkService() + + try { + let results + + if (this.id) { + const result = await benchmarkService.getResultById(this.id) + results = result ? [result] : [] + } else if (this.latest) { + const result = await benchmarkService.getLatestResult() + results = result ? [result] : [] + } else { + results = await benchmarkService.getAllResults() + } + + if (results.length === 0) { + this.logger.info('No benchmark results found.') + this.logger.info('Run "node ace benchmark:run" to create a benchmark.') + return + } + + if (this.format === 'json') { + console.log(JSON.stringify(results, null, 2)) + return + } + + // Table format + for (const result of results) { + this.logger.info('') + this.logger.info(`=== Benchmark ${result.benchmark_id} ===`) + this.logger.info(`Type: ${result.benchmark_type}`) + this.logger.info(`Date: ${result.created_at}`) + this.logger.info('') + + this.logger.info('Hardware:') + this.logger.info(` CPU: ${result.cpu_model}`) + this.logger.info(` Cores: ${result.cpu_cores} physical, ${result.cpu_threads} threads`) + this.logger.info(` RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`) + this.logger.info(` Disk: ${result.disk_type}`) + if (result.gpu_model) { + this.logger.info(` GPU: ${result.gpu_model}`) + } + this.logger.info('') + + this.logger.info('Scores:') + this.logger.info(` CPU: ${result.cpu_score.toFixed(2)}`) + this.logger.info(` Memory: ${result.memory_score.toFixed(2)}`) + this.logger.info(` Disk Read: ${result.disk_read_score.toFixed(2)}`) + this.logger.info(` Disk Write: ${result.disk_write_score.toFixed(2)}`) + + if (result.ai_tokens_per_second) { + this.logger.info(` AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`) + this.logger.info(` AI TTFT: ${result.ai_time_to_first_token?.toFixed(2)} ms`) + } + this.logger.info('') + + this.logger.info(`NOMAD Score: ${result.nomad_score.toFixed(2)} / 100`) + + if (result.submitted_to_repository) { + this.logger.info(`Submitted: Yes (${result.repository_id})`) + } else { + this.logger.info('Submitted: No') + } + this.logger.info('') + } + + this.logger.info(`Total results: ${results.length}`) + + } catch (error) { + this.logger.error(`Failed to retrieve results: ${error.message}`) + this.exitCode = 1 + } + } +} diff --git a/admin/commands/benchmark/run.ts b/admin/commands/benchmark/run.ts new file mode 100644 index 0000000..18ddb4e --- /dev/null +++ b/admin/commands/benchmark/run.ts @@ -0,0 +1,103 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' + +export default class BenchmarkRun extends BaseCommand { + static commandName = 'benchmark:run' + static description = 'Run system and/or AI benchmarks to measure server performance' + + @flags.boolean({ description: 'Run system benchmarks only (CPU, memory, disk)', alias: 's' }) + declare systemOnly: boolean + + @flags.boolean({ description: 'Run AI benchmark only', alias: 'a' }) + declare aiOnly: boolean + + @flags.boolean({ description: 'Submit results to repository after completion', alias: 'S' }) + declare submit: boolean + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { BenchmarkService } = await import('#services/benchmark_service') + const benchmarkService = new BenchmarkService() + + // Determine benchmark type + let benchmarkType: 'full' | 'system' | 'ai' = 'full' + if (this.systemOnly) { + benchmarkType = 'system' + } else if (this.aiOnly) { + benchmarkType = 'ai' + } + + this.logger.info(`Starting ${benchmarkType} benchmark...`) + this.logger.info('') + + try { + // Run the benchmark + let result + switch (benchmarkType) { + case 'system': + this.logger.info('Running system benchmarks (CPU, memory, disk)...') + result = await benchmarkService.runSystemBenchmarks() + break + case 'ai': + this.logger.info('Running AI benchmark...') + result = await benchmarkService.runAIBenchmark() + break + default: + this.logger.info('Running full benchmark suite...') + result = await benchmarkService.runFullBenchmark() + } + + // Display results + this.logger.info('') + this.logger.success('Benchmark completed!') + this.logger.info('') + + this.logger.info('=== Hardware Info ===') + this.logger.info(`CPU: ${result.cpu_model}`) + this.logger.info(`Cores: ${result.cpu_cores} physical, ${result.cpu_threads} threads`) + this.logger.info(`RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`) + this.logger.info(`Disk Type: ${result.disk_type}`) + if (result.gpu_model) { + this.logger.info(`GPU: ${result.gpu_model}`) + } + + this.logger.info('') + this.logger.info('=== Benchmark Scores ===') + this.logger.info(`CPU Score: ${result.cpu_score.toFixed(2)}`) + this.logger.info(`Memory Score: ${result.memory_score.toFixed(2)}`) + this.logger.info(`Disk Read Score: ${result.disk_read_score.toFixed(2)}`) + this.logger.info(`Disk Write Score: ${result.disk_write_score.toFixed(2)}`) + + if (result.ai_tokens_per_second) { + this.logger.info(`AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`) + this.logger.info(`AI Time to First Token: ${result.ai_time_to_first_token?.toFixed(2)} ms`) + this.logger.info(`AI Model: ${result.ai_model_used}`) + } + + this.logger.info('') + this.logger.info(`NOMAD Score: ${result.nomad_score.toFixed(2)} / 100`) + this.logger.info('') + this.logger.info(`Benchmark ID: ${result.benchmark_id}`) + + // Submit if requested + if (this.submit) { + this.logger.info('') + this.logger.info('Submitting results to repository...') + try { + const submitResult = await benchmarkService.submitToRepository(result.benchmark_id) + this.logger.success(`Results submitted! Repository ID: ${submitResult.repository_id}`) + this.logger.info(`Your percentile: ${submitResult.percentile}%`) + } catch (error) { + this.logger.error(`Failed to submit: ${error.message}`) + } + } + + } catch (error) { + this.logger.error(`Benchmark failed: ${error.message}`) + this.exitCode = 1 + } + } +} diff --git a/admin/commands/benchmark/submit.ts b/admin/commands/benchmark/submit.ts new file mode 100644 index 0000000..e5e8b02 --- /dev/null +++ b/admin/commands/benchmark/submit.ts @@ -0,0 +1,99 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' + +export default class BenchmarkSubmit extends BaseCommand { + static commandName = 'benchmark:submit' + static description = 'Submit benchmark results to the community repository' + + @flags.string({ description: 'Benchmark ID to submit (defaults to latest)', alias: 'i' }) + declare benchmarkId: string + + @flags.boolean({ description: 'Skip confirmation prompt', alias: 'y' }) + declare yes: boolean + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { BenchmarkService } = await import('#services/benchmark_service') + const benchmarkService = new BenchmarkService() + + try { + // Get the result to submit + const result = this.benchmarkId + ? await benchmarkService.getResultById(this.benchmarkId) + : await benchmarkService.getLatestResult() + + if (!result) { + this.logger.error('No benchmark result found.') + this.logger.info('Run "node ace benchmark:run" first to create a benchmark.') + this.exitCode = 1 + return + } + + if (result.submitted_to_repository) { + this.logger.warning(`Benchmark ${result.benchmark_id} has already been submitted.`) + this.logger.info(`Repository ID: ${result.repository_id}`) + return + } + + // Show what will be submitted + this.logger.info('') + this.logger.info('=== Data to be submitted ===') + this.logger.info('') + this.logger.info('Hardware Information:') + this.logger.info(` CPU Model: ${result.cpu_model}`) + this.logger.info(` CPU Cores: ${result.cpu_cores}`) + this.logger.info(` CPU Threads: ${result.cpu_threads}`) + this.logger.info(` RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`) + this.logger.info(` Disk Type: ${result.disk_type}`) + if (result.gpu_model) { + this.logger.info(` GPU: ${result.gpu_model}`) + } + this.logger.info('') + this.logger.info('Benchmark Scores:') + this.logger.info(` CPU Score: ${result.cpu_score.toFixed(2)}`) + this.logger.info(` Memory Score: ${result.memory_score.toFixed(2)}`) + this.logger.info(` Disk Read: ${result.disk_read_score.toFixed(2)}`) + this.logger.info(` Disk Write: ${result.disk_write_score.toFixed(2)}`) + if (result.ai_tokens_per_second) { + this.logger.info(` AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`) + this.logger.info(` AI TTFT: ${result.ai_time_to_first_token?.toFixed(2)} ms`) + } + this.logger.info(` NOMAD Score: ${result.nomad_score.toFixed(2)}`) + this.logger.info('') + this.logger.info('Privacy Notice:') + this.logger.info(' - Only the information shown above will be submitted') + this.logger.info(' - No IP addresses, hostnames, or personal data is collected') + this.logger.info(' - Submissions are completely anonymous') + this.logger.info('') + + // Confirm submission + if (!this.yes) { + const confirm = await this.prompt.confirm( + 'Do you want to submit this benchmark to the community repository?' + ) + if (!confirm) { + this.logger.info('Submission cancelled.') + return + } + } + + // Submit + this.logger.info('Submitting benchmark...') + const submitResult = await benchmarkService.submitToRepository(result.benchmark_id) + + this.logger.success('Benchmark submitted successfully!') + this.logger.info('') + this.logger.info(`Repository ID: ${submitResult.repository_id}`) + this.logger.info(`Your percentile: ${submitResult.percentile}%`) + this.logger.info('') + this.logger.info('Thank you for contributing to the NOMAD community!') + + } catch (error) { + this.logger.error(`Submission failed: ${error.message}`) + this.exitCode = 1 + } + } +} diff --git a/admin/commands/queue/work.ts b/admin/commands/queue/work.ts index 8fc4cd2..9dcdb9b 100644 --- a/admin/commands/queue/work.ts +++ b/admin/commands/queue/work.ts @@ -61,8 +61,10 @@ export default class QueueWork extends BaseCommand { const { RunDownloadJob } = await import('#jobs/run_download_job') const { DownloadModelJob } = await import('#jobs/download_model_job') + const { RunBenchmarkJob } = await import('#jobs/run_benchmark_job') handlers.set(RunDownloadJob.key, new RunDownloadJob()) handlers.set(DownloadModelJob.key, new DownloadModelJob()) + handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob()) return handlers } diff --git a/admin/database/migrations/1769097600001_create_benchmark_results_table.ts b/admin/database/migrations/1769097600001_create_benchmark_results_table.ts new file mode 100644 index 0000000..9cbe9e1 --- /dev/null +++ b/admin/database/migrations/1769097600001_create_benchmark_results_table.ts @@ -0,0 +1,47 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'benchmark_results' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id') + table.string('benchmark_id').unique().notNullable() + table.enum('benchmark_type', ['full', 'system', 'ai']).notNullable() + + // Hardware information + table.string('cpu_model').notNullable() + table.integer('cpu_cores').notNullable() + table.integer('cpu_threads').notNullable() + table.bigInteger('ram_bytes').notNullable() + table.enum('disk_type', ['ssd', 'hdd', 'nvme', 'unknown']).notNullable() + table.string('gpu_model').nullable() + + // System benchmark scores + table.float('cpu_score').notNullable() + table.float('memory_score').notNullable() + table.float('disk_read_score').notNullable() + table.float('disk_write_score').notNullable() + + // AI benchmark scores (nullable for system-only benchmarks) + table.float('ai_tokens_per_second').nullable() + table.string('ai_model_used').nullable() + table.float('ai_time_to_first_token').nullable() + + // Composite NOMAD score (0-100) + table.float('nomad_score').notNullable() + + // Repository submission tracking + table.boolean('submitted_to_repository').defaultTo(false) + table.timestamp('submitted_at').nullable() + table.string('repository_id').nullable() + + table.timestamp('created_at') + table.timestamp('updated_at') + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/database/migrations/1769097600002_create_benchmark_settings_table.ts b/admin/database/migrations/1769097600002_create_benchmark_settings_table.ts new file mode 100644 index 0000000..e263b24 --- /dev/null +++ b/admin/database/migrations/1769097600002_create_benchmark_settings_table.ts @@ -0,0 +1,19 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'benchmark_settings' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id') + table.string('key').unique().notNullable() + table.text('value').nullable() + table.timestamp('created_at') + table.timestamp('updated_at') + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/database/seeders/service_seeder.ts b/admin/database/seeders/service_seeder.ts index 7697816..8604c7f 100644 --- a/admin/database/seeders/service_seeder.ts +++ b/admin/database/seeders/service_seeder.ts @@ -147,6 +147,26 @@ export default class ServiceSeeder extends BaseSeeder { is_dependency_service: false, depends_on: null, }, + { + service_name: DockerService.BENCHMARK_SERVICE_NAME, + friendly_name: 'System Benchmark', + description: 'Measure your server performance and compare with the NOMAD community', + icon: 'IconChartBar', + container_image: 'severalnines/sysbench:latest', + container_command: null, + container_config: JSON.stringify({ + HostConfig: { + AutoRemove: true, + Binds: [`${ServiceSeeder.NOMAD_STORAGE_ABS_PATH}/benchmark:/tmp/benchmark`] + }, + WorkingDir: '/tmp/benchmark', + }), + ui_location: null, // UI is integrated into Command Center + installed: false, + installation_status: 'idle', + is_dependency_service: false, + depends_on: null, + }, ] async run() { diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index 48c3f30..13c857d 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -10,7 +10,7 @@ import CategoryCard from '~/components/CategoryCard' import TierSelectionModal from '~/components/TierSelectionModal' import LoadingSpinner from '~/components/LoadingSpinner' import Alert from '~/components/Alert' -import { IconCheck, IconChevronDown, IconChevronUp } from '@tabler/icons-react' +import { IconCheck, IconChevronDown, IconChevronUp, IconArrowRight } from '@tabler/icons-react' import StorageProjectionBar from '~/components/StorageProjectionBar' import { useNotifications } from '~/context/NotificationContext' import useInternetStatus from '~/hooks/useInternetStatus' @@ -98,6 +98,19 @@ const ADDITIONAL_TOOLS: Capability[] = [ services: ['nomad_cyberchef'], icon: 'IconChefHat', }, + { + id: 'benchmark', + name: 'System Benchmark', + technicalName: 'Built-in', + description: 'Measure your server performance and compare with the NOMAD community', + features: [ + 'CPU, memory, and disk benchmarks', + 'AI inference performance testing', + 'NOMAD Score for easy comparison', + ], + services: ['__builtin_benchmark'], // Special marker for built-in features + icon: 'IconChartBar', + }, ] type WizardStep = 1 | 2 | 3 | 4 @@ -487,13 +500,20 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim ) } + // Check if a capability is a built-in feature (not a Docker service) + const isBuiltInCapability = (capability: Capability) => { + return capability.services.some((service) => service.startsWith('__builtin_')) + } + // Check if a capability is selected (all its services are in selectedServices) const isCapabilitySelected = (capability: Capability) => { + if (isBuiltInCapability(capability)) return false // Built-ins can't be selected return capability.services.every((service) => selectedServices.includes(service)) } // Check if a capability is already installed (all its services are installed) const isCapabilityInstalled = (capability: Capability) => { + if (isBuiltInCapability(capability)) return true // Built-ins are always "installed" return capability.services.every((service) => installedServices.some((s) => s.service_name === service) ) @@ -501,6 +521,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim // Check if a capability exists in the system (has at least one matching service) const capabilityExists = (capability: Capability) => { + if (isBuiltInCapability(capability)) return true // Built-ins always exist return capability.services.some((service) => allServices.some((s) => s.service_name === service) ) @@ -528,23 +549,38 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim const selected = isCapabilitySelected(capability) const installed = isCapabilityInstalled(capability) const exists = capabilityExists(capability) + const isBuiltIn = isBuiltInCapability(capability) if (!exists) return null // Determine visual state: installed (locked), selected (user chose it), or default const isChecked = installed || selected + // Handle click - built-in features navigate to their page, others toggle selection + const handleClick = () => { + if (isBuiltIn) { + // Navigate to the appropriate settings page for built-in features + if (capability.id === 'benchmark') { + router.visit('/settings/benchmark') + } + } else { + toggleCapability(capability) + } + } + return (
toggleCapability(capability)} + onClick={handleClick} className={classNames( 'p-6 rounded-lg border-2 transition-all', - installed - ? 'border-desert-green bg-desert-green/20 cursor-default' - : selected - ? 'border-desert-green bg-desert-green shadow-md cursor-pointer' - : 'border-desert-stone-light bg-white hover:border-desert-green hover:shadow-sm cursor-pointer' + isBuiltIn + ? 'border-desert-stone bg-desert-stone-lighter/50 hover:border-desert-green hover:shadow-sm cursor-pointer' + : installed + ? 'border-desert-green bg-desert-green/20 cursor-default' + : selected + ? 'border-desert-green bg-desert-green shadow-md cursor-pointer' + : 'border-desert-stone-light bg-white hover:border-desert-green hover:shadow-sm cursor-pointer' )} >
@@ -553,12 +589,16 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim

{capability.name}

- {installed && ( + {isBuiltIn ? ( + + Built-in + + ) : installed && ( Installed @@ -567,15 +607,15 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim

- Powered by {capability.technicalName} + {isBuiltIn ? 'Click to open' : `Powered by ${capability.technicalName}`}

{capability.description} @@ -584,7 +624,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim

    {capability.features.map((feature, idx) => ( @@ -592,11 +632,13 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim • @@ -610,14 +652,18 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
    - {isChecked && ( + {isBuiltIn ? ( + + ) : isChecked && ( )}
    diff --git a/admin/inertia/pages/settings/benchmark.tsx b/admin/inertia/pages/settings/benchmark.tsx new file mode 100644 index 0000000..57a5072 --- /dev/null +++ b/admin/inertia/pages/settings/benchmark.tsx @@ -0,0 +1,618 @@ +import { Head } from '@inertiajs/react' +import { useState, useEffect } from 'react' +import SettingsLayout from '~/layouts/SettingsLayout' +import { useQuery, useMutation } from '@tanstack/react-query' +import CircularGauge from '~/components/systeminfo/CircularGauge' +import InfoCard from '~/components/systeminfo/InfoCard' +import Alert from '~/components/Alert' +import StyledButton from '~/components/StyledButton' +import { + ChartBarIcon, + CpuChipIcon, + CircleStackIcon, + ServerIcon, + CloudArrowUpIcon, + PlayIcon, + ChevronDownIcon, +} from '@heroicons/react/24/outline' +import { IconRobot } from '@tabler/icons-react' +import { useTransmit } from 'react-adonis-transmit' + +type BenchmarkResult = { + id: number + benchmark_id: string + benchmark_type: 'full' | 'system' | 'ai' + cpu_model: string + cpu_cores: number + cpu_threads: number + ram_bytes: number + disk_type: string + gpu_model: string | null + cpu_score: number + memory_score: number + disk_read_score: number + disk_write_score: number + ai_tokens_per_second: number | null + ai_model_used: string | null + ai_time_to_first_token: number | null + nomad_score: number + submitted_to_repository: boolean + repository_id: string | null + created_at: string +} + +type BenchmarkStatus = 'idle' | 'starting' | 'detecting_hardware' | 'running_cpu' | 'running_memory' | 'running_disk_read' | 'running_disk_write' | 'running_ai' | 'calculating_score' | 'completed' | 'error' + +type BenchmarkProgress = { + status: BenchmarkStatus + progress: number + message: string + current_stage: string + benchmark_id: string +} + +export default function BenchmarkPage(props: { + benchmark: { + latestResult: BenchmarkResult | null + status: BenchmarkStatus + currentBenchmarkId: string | null + } +}) { + const { subscribe } = useTransmit() + const [progress, setProgress] = useState(null) + const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle') + const [showDetails, setShowDetails] = useState(false) + + // Fetch latest result + const { data: latestResult, refetch: refetchLatest } = useQuery({ + queryKey: ['benchmark', 'latest'], + queryFn: async () => { + const res = await fetch('/api/benchmark/results/latest') + const data = await res.json() + return data.result as BenchmarkResult | null + }, + initialData: props.benchmark.latestResult, + }) + + // Run benchmark mutation (uses sync mode by default for simpler local dev) + const runBenchmark = useMutation({ + mutationFn: async (type: 'full' | 'system' | 'ai') => { + setIsRunning(true) + setProgress({ + status: 'starting', + progress: 5, + message: 'Starting benchmark... This takes 2-5 minutes.', + current_stage: 'Starting', + benchmark_id: '', + }) + + // Use sync mode - runs inline without needing Redis/queue worker + const res = await fetch('/api/benchmark/run?sync=true', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ benchmark_type: type }), + }) + return res.json() + }, + onSuccess: (data) => { + if (data.success) { + setProgress({ + status: 'completed', + progress: 100, + message: 'Benchmark completed!', + current_stage: 'Complete', + benchmark_id: data.benchmark_id, + }) + refetchLatest() + } else { + setProgress({ + status: 'error', + progress: 0, + message: data.error || 'Benchmark failed', + current_stage: 'Error', + benchmark_id: '', + }) + } + setIsRunning(false) + }, + onError: (error) => { + setProgress({ + status: 'error', + progress: 0, + message: error.message || 'Benchmark failed', + current_stage: 'Error', + benchmark_id: '', + }) + setIsRunning(false) + }, + }) + + // Submit to repository mutation + const submitResult = useMutation({ + mutationFn: async (benchmarkId?: string) => { + const res = await fetch('/api/benchmark/submit', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ benchmark_id: benchmarkId }), + }) + return res.json() + }, + onSuccess: () => { + refetchLatest() + }, + }) + + // Simulate progress during sync benchmark (since we don't get SSE updates) + useEffect(() => { + if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return + + const stages: { status: BenchmarkStatus; progress: number; message: string; label: string; duration: number }[] = [ + { status: 'detecting_hardware', progress: 10, message: 'Detecting system hardware...', label: 'Detecting Hardware', duration: 2000 }, + { status: 'running_cpu', progress: 25, message: 'Running CPU benchmark (30s)...', label: 'CPU Benchmark', duration: 32000 }, + { status: 'running_memory', progress: 40, message: 'Running memory benchmark...', label: 'Memory Benchmark', duration: 8000 }, + { status: 'running_disk_read', progress: 55, message: 'Running disk read benchmark (30s)...', label: 'Disk Read Test', duration: 35000 }, + { status: 'running_disk_write', progress: 70, message: 'Running disk write benchmark (30s)...', label: 'Disk Write Test', duration: 35000 }, + { status: 'calculating_score', progress: 95, message: 'Calculating NOMAD score...', label: 'Calculating Score', duration: 2000 }, + ] + + let currentStage = 0 + const advanceStage = () => { + if (currentStage < stages.length && isRunning) { + const stage = stages[currentStage] + setProgress({ + status: stage.status, + progress: stage.progress, + message: stage.message, + current_stage: stage.label, + benchmark_id: '', + }) + currentStage++ + } + } + + // Start the first stage after a short delay + const timers: NodeJS.Timeout[] = [] + let elapsed = 1000 + stages.forEach((stage, index) => { + timers.push(setTimeout(() => advanceStage(), elapsed)) + elapsed += stage.duration + }) + + return () => { + timers.forEach(t => clearTimeout(t)) + } + }, [isRunning]) + + // Listen for benchmark progress via SSE (backup for async mode) + useEffect(() => { + const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgress) => { + setProgress(data) + if (data.status === 'completed' || data.status === 'error') { + setIsRunning(false) + refetchLatest() + } + }) + + return () => { + unsubscribe() + } + }, [subscribe, refetchLatest]) + + const formatBytes = (bytes: number) => { + const gb = bytes / (1024 * 1024 * 1024) + return `${gb.toFixed(1)} GB` + } + + const getScoreColor = (score: number) => { + if (score >= 70) return 'text-green-600' + if (score >= 40) return 'text-yellow-600' + return 'text-red-600' + } + + const getProgressPercent = () => { + if (!progress) return 0 + const stages: Record = { + idle: 0, + starting: 5, + detecting_hardware: 10, + running_cpu: 25, + running_memory: 40, + running_disk_read: 55, + running_disk_write: 70, + running_ai: 85, + calculating_score: 95, + completed: 100, + error: 0, + } + return stages[progress.status] || 0 + } + + // Calculate AI score from tokens per second (normalized to 0-100) + // Reference: 30 tok/s = 50 score, 60 tok/s = 100 score + const getAIScore = (tokensPerSecond: number | null): number => { + if (!tokensPerSecond) return 0 + const score = (tokensPerSecond / 60) * 100 + return Math.min(100, Math.max(0, score)) + } + + return ( + + +
    +
    +
    +

    System Benchmark

    +

    + Measure your server's performance and compare with the NOMAD community +

    +
    + + {/* Run Benchmark Section */} +
    +

    +
    + Run Benchmark +

    + +
    + {isRunning ? ( +
    +
    +
    + + {progress?.current_stage || 'Running benchmark...'} + +
    +
    +
    +
    +

    {progress?.message}

    +
    + ) : ( +
    +

    + Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance. + The benchmark takes approximately 2-5 minutes to complete. +

    +
    + runBenchmark.mutate('full')} + disabled={runBenchmark.isPending} + leftIcon={} + > + Run Full Benchmark + + runBenchmark.mutate('system')} + disabled={runBenchmark.isPending} + leftIcon={} + > + System Only + + runBenchmark.mutate('ai')} + disabled={runBenchmark.isPending} + leftIcon={} + > + AI Only + +
    +
    + )} +
    +
    + + {/* Results Section */} + {latestResult && ( + <> +
    +

    +
    + NOMAD Score +

    + +
    +
    +
    + } + /> +
    +
    +
    + {latestResult.nomad_score.toFixed(1)} +
    +

    + Your NOMAD Score is a weighted composite of all benchmark results. +

    + {!latestResult.submitted_to_repository && ( + submitResult.mutate(latestResult.benchmark_id)} + disabled={submitResult.isPending} + leftIcon={} + > + Share with Community + + )} + {latestResult.submitted_to_repository && ( + + )} +
    +
    +
    +
    + +
    +

    +
    + System Performance +

    + +
    +
    + } + /> +
    +
    + } + /> +
    +
    + } + /> +
    +
    + } + /> +
    +
    +
    + + {/* AI Performance Section */} +
    +

    +
    + AI Performance +

    + + {latestResult.ai_tokens_per_second ? ( +
    +
    + } + /> +
    +
    +
    + +
    +
    + {latestResult.ai_tokens_per_second.toFixed(1)} +
    +
    Tokens per Second
    +
    +
    +
    +
    +
    + +
    +
    + {latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms +
    +
    Time to First Token
    +
    +
    +
    +
    + ) : ( +
    +
    + +

    No AI Benchmark Data

    +

    + Run a Full Benchmark or AI Only benchmark to measure AI inference performance. +

    +
    +
    + )} +
    + +
    +

    +
    + Hardware Information +

    + +
    + } + variant="elevated" + data={[ + { label: 'Model', value: latestResult.cpu_model }, + { label: 'Cores', value: latestResult.cpu_cores }, + { label: 'Threads', value: latestResult.cpu_threads }, + ]} + /> + } + variant="elevated" + data={[ + { label: 'RAM', value: formatBytes(latestResult.ram_bytes) }, + { label: 'Disk Type', value: latestResult.disk_type.toUpperCase() }, + { label: 'GPU', value: latestResult.gpu_model || 'Not detected' }, + ]} + /> +
    +
    + +
    +

    +
    + Benchmark Details +

    + +
    + {/* Summary row - always visible */} + + + {/* Expanded details */} + {showDetails && ( +
    +
    + {/* Raw Scores */} +
    +

    Raw Scores

    +
    +
    + CPU Score + {(latestResult.cpu_score * 100).toFixed(1)}% +
    +
    + Memory Score + {(latestResult.memory_score * 100).toFixed(1)}% +
    +
    + Disk Read Score + {(latestResult.disk_read_score * 100).toFixed(1)}% +
    +
    + Disk Write Score + {(latestResult.disk_write_score * 100).toFixed(1)}% +
    + {latestResult.ai_tokens_per_second && ( + <> +
    + AI Tokens/sec + {latestResult.ai_tokens_per_second.toFixed(1)} +
    +
    + AI Time to First Token + {latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms +
    + + )} +
    +
    + + {/* Benchmark Info */} +
    +

    Benchmark Info

    +
    +
    + Full Benchmark ID + {latestResult.benchmark_id} +
    +
    + Benchmark Type + {latestResult.benchmark_type} +
    +
    + Run Date + {new Date(latestResult.created_at).toLocaleString()} +
    + {latestResult.ai_model_used && ( +
    + AI Model Used + {latestResult.ai_model_used} +
    + )} +
    + Submitted to Repository + {latestResult.submitted_to_repository ? 'Yes' : 'No'} +
    + {latestResult.repository_id && ( +
    + Repository ID + {latestResult.repository_id} +
    + )} +
    +
    +
    +
    + )} +
    +
    + + )} + + {!latestResult && !isRunning && ( + + )} +
    +
    +     + ) +} diff --git a/admin/package-lock.json b/admin/package-lock.json index d5595f4..4e07866 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -55,6 +55,7 @@ "tar": "^7.5.6", "url-join": "^5.0.0", "usehooks-ts": "^3.1.1", + "uuid": "^13.0.0", "yaml": "^2.8.0" }, "devDependencies": { @@ -72,6 +73,7 @@ "@types/node": "^22.15.18", "@types/react": "^19.1.8", "@types/react-dom": "^19.1.6", + "@types/uuid": "^10.0.0", "eslint": "^9.26.0", "hot-hook": "^0.4.0", "prettier": "^3.5.3", @@ -4304,6 +4306,13 @@ "@types/geojson": "*" } }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/validator": { "version": "13.15.2", "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.2.tgz", @@ -5172,6 +5181,19 @@ "uuid": "^11.1.0" } }, + "node_modules/bullmq/node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -11982,16 +12004,16 @@ "license": "MIT" }, "node_modules/uuid": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", - "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", + "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], "license": "MIT", "bin": { - "uuid": "dist/esm/bin/uuid" + "uuid": "dist-node/bin/uuid" } }, "node_modules/v8-compile-cache-lib": { diff --git a/admin/package.json b/admin/package.json index b09fe6b..9809731 100644 --- a/admin/package.json +++ b/admin/package.json @@ -50,6 +50,7 @@ "@types/node": "^22.15.18", "@types/react": "^19.1.8", "@types/react-dom": "^19.1.6", + "@types/uuid": "^10.0.0", "eslint": "^9.26.0", "hot-hook": "^0.4.0", "prettier": "^3.5.3", @@ -104,6 +105,7 @@ "tar": "^7.5.6", "url-join": "^5.0.0", "usehooks-ts": "^3.1.1", + "uuid": "^13.0.0", "yaml": "^2.8.0" }, "hotHook": { diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 98e93b0..ac16800 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -6,6 +6,7 @@ | The routes file is used for defining the HTTP routes. | */ +import BenchmarkController from '#controllers/benchmark_controller' import DocsController from '#controllers/docs_controller' import DownloadsController from '#controllers/downloads_controller' import EasySetupController from '#controllers/easy_setup_controller' @@ -38,6 +39,7 @@ router router.get('/update', [SettingsController, 'update']) router.get('/zim', [SettingsController, 'zim']) router.get('/zim/remote-explorer', [SettingsController, 'zimRemote']) + router.get('/benchmark', [SettingsController, 'benchmark']) }) .prefix('/settings') @@ -122,3 +124,19 @@ router router.delete('/:filename', [ZimController, 'delete']) }) .prefix('/api/zim') + +router + .group(() => { + router.post('/run', [BenchmarkController, 'run']) + router.post('/run/system', [BenchmarkController, 'runSystem']) + router.post('/run/ai', [BenchmarkController, 'runAI']) + router.get('/results', [BenchmarkController, 'results']) + router.get('/results/latest', [BenchmarkController, 'latest']) + router.get('/results/:id', [BenchmarkController, 'show']) + router.post('/submit', [BenchmarkController, 'submit']) + router.get('/comparison', [BenchmarkController, 'comparison']) + router.get('/status', [BenchmarkController, 'status']) + router.get('/settings', [BenchmarkController, 'settings']) + router.post('/settings', [BenchmarkController, 'updateSettings']) + }) + .prefix('/api/benchmark') diff --git a/admin/types/benchmark.ts b/admin/types/benchmark.ts new file mode 100644 index 0000000..80e3097 --- /dev/null +++ b/admin/types/benchmark.ts @@ -0,0 +1,216 @@ +// Benchmark type identifiers +export type BenchmarkType = 'full' | 'system' | 'ai' + +// Benchmark execution status +export type BenchmarkStatus = + | 'idle' + | 'starting' + | 'detecting_hardware' + | 'running_cpu' + | 'running_memory' + | 'running_disk_read' + | 'running_disk_write' + | 'running_ai' + | 'calculating_score' + | 'completed' + | 'error' + +// Hardware detection types +export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown' + +export type HardwareInfo = { + cpu_model: string + cpu_cores: number + cpu_threads: number + ram_bytes: number + disk_type: DiskType + gpu_model: string | null +} + +// Individual benchmark scores +export type SystemScores = { + cpu_score: number + memory_score: number + disk_read_score: number + disk_write_score: number +} + +export type AIScores = { + ai_tokens_per_second: number + ai_model_used: string + ai_time_to_first_token: number +} + +// Complete benchmark result +export type BenchmarkResult = { + id: number + benchmark_id: string + benchmark_type: BenchmarkType + hardware: HardwareInfo + scores: SystemScores & Partial + nomad_score: number + submitted_to_repository: boolean + submitted_at: string | null + repository_id: string | null + created_at: string + updated_at: string +} + +// Slim version for lists +export type BenchmarkResultSlim = Pick< + BenchmarkResult, + | 'id' + | 'benchmark_id' + | 'benchmark_type' + | 'nomad_score' + | 'submitted_to_repository' + | 'created_at' +> & { + cpu_model: string + gpu_model: string | null +} + +// Benchmark settings key-value store +export type BenchmarkSettingKey = + | 'allow_anonymous_submission' + | 'installation_id' + | 'last_benchmark_run' + +export type BenchmarkSettings = { + allow_anonymous_submission: boolean + installation_id: string | null + last_benchmark_run: string | null +} + +// Progress update for real-time feedback +export type BenchmarkProgress = { + status: BenchmarkStatus + progress: number + message: string + current_stage: string + timestamp: string +} + +// API request types +export type RunBenchmarkParams = { + benchmark_type: BenchmarkType +} + +export type SubmitBenchmarkParams = { + benchmark_id?: string +} + +// API response types +export type RunBenchmarkResponse = { + success: boolean + job_id: string + benchmark_id: string + message: string +} + +export type BenchmarkResultsResponse = { + results: BenchmarkResult[] + total: number +} + +// Central repository submission payload (privacy-first) +export type RepositorySubmission = { + cpu_model: string + cpu_cores: number + cpu_threads: number + ram_gb: number + disk_type: DiskType + gpu_model: string | null + cpu_score: number + memory_score: number + disk_read_score: number + disk_write_score: number + ai_tokens_per_second: number | null + ai_time_to_first_token: number | null + nomad_score: number + nomad_version: string + benchmark_version: string +} + +// Central repository response types +export type RepositorySubmitResponse = { + success: boolean + repository_id: string + percentile: number +} + +export type RepositoryStats = { + total_submissions: number + average_score: number + median_score: number + top_score: number + percentiles: { + p10: number + p25: number + p50: number + p75: number + p90: number + } +} + +export type LeaderboardEntry = { + rank: number + cpu_model: string + gpu_model: string | null + nomad_score: number + submitted_at: string +} + +export type ComparisonResponse = { + matching_submissions: number + average_score: number + your_percentile: number | null +} + +// Score calculation weights (for reference in UI) +export type ScoreWeights = { + ai_tokens_per_second: number + cpu: number + memory: number + ai_ttft: number + disk_read: number + disk_write: number +} + +// Default weights as defined in plan +export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = { + ai_tokens_per_second: 0.30, + cpu: 0.25, + memory: 0.15, + ai_ttft: 0.10, + disk_read: 0.10, + disk_write: 0.10, +} + +// Benchmark job parameters +export type RunBenchmarkJobParams = { + benchmark_id: string + benchmark_type: BenchmarkType + include_ai: boolean +} + +// sysbench result parsing types +export type SysbenchCpuResult = { + events_per_second: number + total_time: number + total_events: number +} + +export type SysbenchMemoryResult = { + operations_per_second: number + transfer_rate_mb_per_sec: number + total_time: number +} + +export type SysbenchDiskResult = { + reads_per_second: number + writes_per_second: number + read_mb_per_sec: number + write_mb_per_sec: number + total_time: number +}