feat: Add system benchmark feature with NOMAD Score

Add comprehensive benchmarking capability to measure server performance:

Backend:
- BenchmarkService with CPU, memory, disk, and AI benchmarks using sysbench
- Database migrations for benchmark_results and benchmark_settings tables
- REST API endpoints for running benchmarks and retrieving results
- CLI commands: benchmark:run, benchmark:results, benchmark:submit
- BullMQ job for async benchmark execution with SSE progress updates
- Synchronous mode option (?sync=true) for simpler local dev setup

Frontend:
- Benchmark settings page with circular gauges for scores
- NOMAD Score display with weighted composite calculation
- System Performance section (CPU, Memory, Disk Read/Write)
- AI Performance section (tokens/sec, time to first token)
- Hardware Information display
- Expandable Benchmark Details section
- Progress simulation during sync benchmark execution

Easy Setup Integration:
- Added System Benchmark to Additional Tools section
- Built-in capability pattern for non-Docker features
- Click-to-navigate behavior for built-in tools

Fixes:
- Docker log multiplexing issue (Tty: true) for proper output parsing
- Consolidated disk benchmarks into single container execution

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood 2026-01-21 10:32:50 -08:00 committed by Jake Turner
parent 6bee84f367
commit 755807f95e
21 changed files with 2559 additions and 29 deletions

View File

@ -0,0 +1,230 @@
import { inject } from '@adonisjs/core'
import type { HttpContext } from '@adonisjs/core/http'
import { BenchmarkService } from '#services/benchmark_service'
import { runBenchmarkValidator, submitBenchmarkValidator } from '#validators/benchmark'
import { RunBenchmarkJob } from '#jobs/run_benchmark_job'
import { v4 as uuidv4 } from 'uuid'
import type { BenchmarkType } from '../../types/benchmark.js'
@inject()
export default class BenchmarkController {
constructor(private benchmarkService: BenchmarkService) {}
/**
* Start a benchmark run (async via job queue, or sync if specified)
*/
async run({ request, response }: HttpContext) {
const payload = await request.validateUsing(runBenchmarkValidator)
const benchmarkType: BenchmarkType = payload.benchmark_type || 'full'
const runSync = request.input('sync') === 'true' || request.input('sync') === true
// Check if a benchmark is already running
const status = this.benchmarkService.getStatus()
if (status.status !== 'idle') {
return response.status(409).send({
success: false,
error: 'A benchmark is already running',
current_benchmark_id: status.benchmarkId,
})
}
// Run synchronously if requested (useful for local dev without Redis)
if (runSync) {
try {
let result
switch (benchmarkType) {
case 'full':
result = await this.benchmarkService.runFullBenchmark()
break
case 'system':
result = await this.benchmarkService.runSystemBenchmarks()
break
case 'ai':
result = await this.benchmarkService.runAIBenchmark()
break
default:
result = await this.benchmarkService.runFullBenchmark()
}
return response.send({
success: true,
benchmark_id: result.benchmark_id,
nomad_score: result.nomad_score,
result,
})
} catch (error) {
return response.status(500).send({
success: false,
error: error.message,
})
}
}
// Generate benchmark ID and dispatch job (async)
const benchmarkId = uuidv4()
const { job, created } = await RunBenchmarkJob.dispatch({
benchmark_id: benchmarkId,
benchmark_type: benchmarkType,
include_ai: benchmarkType === 'full' || benchmarkType === 'ai',
})
return response.status(201).send({
success: true,
job_id: job?.id || benchmarkId,
benchmark_id: benchmarkId,
message: created
? `${benchmarkType} benchmark started`
: 'Benchmark job already exists',
})
}
/**
* Run a system-only benchmark (CPU, memory, disk)
*/
async runSystem({ response }: HttpContext) {
const status = this.benchmarkService.getStatus()
if (status.status !== 'idle') {
return response.status(409).send({
success: false,
error: 'A benchmark is already running',
})
}
const benchmarkId = uuidv4()
await RunBenchmarkJob.dispatch({
benchmark_id: benchmarkId,
benchmark_type: 'system',
include_ai: false,
})
return response.status(201).send({
success: true,
benchmark_id: benchmarkId,
message: 'System benchmark started',
})
}
/**
* Run an AI-only benchmark
*/
async runAI({ response }: HttpContext) {
const status = this.benchmarkService.getStatus()
if (status.status !== 'idle') {
return response.status(409).send({
success: false,
error: 'A benchmark is already running',
})
}
const benchmarkId = uuidv4()
await RunBenchmarkJob.dispatch({
benchmark_id: benchmarkId,
benchmark_type: 'ai',
include_ai: true,
})
return response.status(201).send({
success: true,
benchmark_id: benchmarkId,
message: 'AI benchmark started',
})
}
/**
* Get all benchmark results
*/
async results({}: HttpContext) {
const results = await this.benchmarkService.getAllResults()
return {
results,
total: results.length,
}
}
/**
* Get the latest benchmark result
*/
async latest({}: HttpContext) {
const result = await this.benchmarkService.getLatestResult()
if (!result) {
return { result: null }
}
return { result }
}
/**
* Get a specific benchmark result by ID
*/
async show({ params, response }: HttpContext) {
const result = await this.benchmarkService.getResultById(params.id)
if (!result) {
return response.status(404).send({
error: 'Benchmark result not found',
})
}
return { result }
}
/**
* Submit benchmark results to central repository
*/
async submit({ request, response }: HttpContext) {
const payload = await request.validateUsing(submitBenchmarkValidator)
try {
const submitResult = await this.benchmarkService.submitToRepository(payload.benchmark_id)
return response.send({
success: true,
repository_id: submitResult.repository_id,
percentile: submitResult.percentile,
})
} catch (error) {
return response.status(400).send({
success: false,
error: error.message,
})
}
}
/**
* Get comparison stats from central repository
*/
async comparison({}: HttpContext) {
const stats = await this.benchmarkService.getComparisonStats()
return { stats }
}
/**
* Get current benchmark status
*/
async status({}: HttpContext) {
return this.benchmarkService.getStatus()
}
/**
* Get benchmark settings
*/
async settings({}: HttpContext) {
const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
return await BenchmarkSetting.getAllSettings()
}
/**
* Update benchmark settings
*/
async updateSettings({ request, response }: HttpContext) {
const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
const body = request.body()
if (body.allow_anonymous_submission !== undefined) {
await BenchmarkSetting.setValue(
'allow_anonymous_submission',
body.allow_anonymous_submission ? 'true' : 'false'
)
}
return response.send({
success: true,
settings: await BenchmarkSetting.getAllSettings(),
})
}
}

View File

@ -1,3 +1,4 @@
import { BenchmarkService } from '#services/benchmark_service';
import { MapService } from '#services/map_service';
import { OpenWebUIService } from '#services/openwebui_service';
import { SystemService } from '#services/system_service';
@ -9,7 +10,8 @@ export default class SettingsController {
constructor(
private systemService: SystemService,
private mapService: MapService,
private openWebUIService: OpenWebUIService
private openWebUIService: OpenWebUIService,
private benchmarkService: BenchmarkService
) { }
async system({ inertia }: HttpContext) {
@ -74,4 +76,16 @@ export default class SettingsController {
async zimRemote({ inertia }: HttpContext) {
return inertia.render('settings/zim/remote-explorer');
}
async benchmark({ inertia }: HttpContext) {
const latestResult = await this.benchmarkService.getLatestResult();
const status = this.benchmarkService.getStatus();
return inertia.render('settings/benchmark', {
benchmark: {
latestResult,
status: status.status,
currentBenchmarkId: status.benchmarkId
}
});
}
}

View File

@ -0,0 +1,99 @@
import { Job } from 'bullmq'
import { QueueService } from '#services/queue_service'
import { BenchmarkService } from '#services/benchmark_service'
import type { RunBenchmarkJobParams } from '../../types/benchmark.js'
import logger from '@adonisjs/core/services/logger'
export class RunBenchmarkJob {
static get queue() {
return 'benchmarks'
}
static get key() {
return 'run-benchmark'
}
async handle(job: Job) {
const { benchmark_id, benchmark_type } = job.data as RunBenchmarkJobParams
logger.info(`[RunBenchmarkJob] Starting benchmark ${benchmark_id} of type ${benchmark_type}`)
const benchmarkService = new BenchmarkService()
try {
let result
switch (benchmark_type) {
case 'full':
result = await benchmarkService.runFullBenchmark()
break
case 'system':
result = await benchmarkService.runSystemBenchmarks()
break
case 'ai':
result = await benchmarkService.runAIBenchmark()
break
default:
throw new Error(`Unknown benchmark type: ${benchmark_type}`)
}
logger.info(`[RunBenchmarkJob] Benchmark ${benchmark_id} completed with NOMAD score: ${result.nomad_score}`)
return {
success: true,
benchmark_id: result.benchmark_id,
nomad_score: result.nomad_score,
}
} catch (error) {
logger.error(`[RunBenchmarkJob] Benchmark ${benchmark_id} failed: ${error.message}`)
throw error
}
}
static async dispatch(params: RunBenchmarkJobParams) {
const queueService = new QueueService()
const queue = queueService.getQueue(this.queue)
try {
const job = await queue.add(this.key, params, {
jobId: params.benchmark_id,
attempts: 1, // Benchmarks shouldn't be retried automatically
removeOnComplete: {
count: 10, // Keep last 10 completed jobs
},
removeOnFail: {
count: 5, // Keep last 5 failed jobs
},
})
logger.info(`[RunBenchmarkJob] Dispatched benchmark job ${params.benchmark_id}`)
return {
job,
created: true,
message: `Benchmark job ${params.benchmark_id} dispatched successfully`,
}
} catch (error) {
if (error.message.includes('job already exists')) {
const existing = await queue.getJob(params.benchmark_id)
return {
job: existing,
created: false,
message: `Benchmark job ${params.benchmark_id} already exists`,
}
}
throw error
}
}
static async getJob(benchmarkId: string): Promise<Job | undefined> {
const queueService = new QueueService()
const queue = queueService.getQueue(this.queue)
return await queue.getJob(benchmarkId)
}
static async getJobState(benchmarkId: string): Promise<string | undefined> {
const job = await this.getJob(benchmarkId)
return job ? await job.getState() : undefined
}
}

View File

@ -0,0 +1,82 @@
import { DateTime } from 'luxon'
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
import type { BenchmarkType, DiskType } from '../../types/benchmark.js'
export default class BenchmarkResult extends BaseModel {
static namingStrategy = new SnakeCaseNamingStrategy()
@column({ isPrimary: true })
declare id: number
@column()
declare benchmark_id: string
@column()
declare benchmark_type: BenchmarkType
// Hardware information
@column()
declare cpu_model: string
@column()
declare cpu_cores: number
@column()
declare cpu_threads: number
@column()
declare ram_bytes: number
@column()
declare disk_type: DiskType
@column()
declare gpu_model: string | null
// System benchmark scores
@column()
declare cpu_score: number
@column()
declare memory_score: number
@column()
declare disk_read_score: number
@column()
declare disk_write_score: number
// AI benchmark scores (nullable for system-only benchmarks)
@column()
declare ai_tokens_per_second: number | null
@column()
declare ai_model_used: string | null
@column()
declare ai_time_to_first_token: number | null
// Composite NOMAD score (0-100)
@column()
declare nomad_score: number
// Repository submission tracking
@column({
serialize(value) {
return Boolean(value)
},
})
declare submitted_to_repository: boolean
@column.dateTime()
declare submitted_at: DateTime | null
@column()
declare repository_id: string | null
@column.dateTime({ autoCreate: true })
declare created_at: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updated_at: DateTime
}

View File

@ -0,0 +1,60 @@
import { DateTime } from 'luxon'
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
import type { BenchmarkSettingKey } from '../../types/benchmark.js'
export default class BenchmarkSetting extends BaseModel {
static namingStrategy = new SnakeCaseNamingStrategy()
@column({ isPrimary: true })
declare id: number
@column()
declare key: BenchmarkSettingKey
@column()
declare value: string | null
@column.dateTime({ autoCreate: true })
declare created_at: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updated_at: DateTime
/**
* Get a setting value by key
*/
static async getValue(key: BenchmarkSettingKey): Promise<string | null> {
const setting = await this.findBy('key', key)
return setting?.value ?? null
}
/**
* Set a setting value by key (creates if not exists)
*/
static async setValue(key: BenchmarkSettingKey, value: string | null): Promise<BenchmarkSetting> {
const setting = await this.firstOrCreate({ key }, { key, value })
if (setting.value !== value) {
setting.value = value
await setting.save()
}
return setting
}
/**
* Get all benchmark settings as a typed object
*/
static async getAllSettings(): Promise<{
allow_anonymous_submission: boolean
installation_id: string | null
last_benchmark_run: string | null
}> {
const settings = await this.all()
const map = new Map(settings.map((s) => [s.key, s.value]))
return {
allow_anonymous_submission: map.get('allow_anonymous_submission') === 'true',
installation_id: map.get('installation_id') ?? null,
last_benchmark_run: map.get('last_benchmark_run') ?? null,
}
}
}

View File

@ -0,0 +1,723 @@
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import transmit from '@adonisjs/transmit/services/main'
import Docker from 'dockerode'
import si from 'systeminformation'
import { v4 as uuidv4 } from 'uuid'
import axios from 'axios'
import BenchmarkResult from '#models/benchmark_result'
import BenchmarkSetting from '#models/benchmark_setting'
import { SystemService } from '#services/system_service'
import type {
BenchmarkType,
BenchmarkStatus,
BenchmarkProgress,
HardwareInfo,
DiskType,
SystemScores,
AIScores,
SysbenchCpuResult,
SysbenchMemoryResult,
SysbenchDiskResult,
RepositorySubmission,
RepositorySubmitResponse,
RepositoryStats,
} from '../../types/benchmark.js'
// Re-export default weights for use in service
const SCORE_WEIGHTS = {
ai_tokens_per_second: 0.30,
cpu: 0.25,
memory: 0.15,
ai_ttft: 0.10,
disk_read: 0.10,
disk_write: 0.10,
}
// Benchmark configuration constants
const SYSBENCH_IMAGE = 'severalnines/sysbench:latest'
const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench'
const BENCHMARK_CHANNEL = 'benchmark-progress'
// Reference model for AI benchmark - small but meaningful
const AI_BENCHMARK_MODEL = 'llama3.2:1b'
const AI_BENCHMARK_PROMPT = 'Explain recursion in programming in exactly 100 words.'
// Reference scores for normalization (calibrated to 0-100 scale)
// These represent "expected" scores for a mid-range system (score ~50)
const REFERENCE_SCORES = {
cpu_events_per_second: 5000, // sysbench cpu events/sec for ~50 score
memory_ops_per_second: 5000000, // sysbench memory ops/sec for ~50 score
disk_read_mb_per_sec: 500, // 500 MB/s read for ~50 score
disk_write_mb_per_sec: 400, // 400 MB/s write for ~50 score
ai_tokens_per_second: 30, // 30 tok/s for ~50 score
ai_ttft_ms: 500, // 500ms time to first token for ~50 score (lower is better)
}
@inject()
export class BenchmarkService {
private docker: Docker
private currentBenchmarkId: string | null = null
private currentStatus: BenchmarkStatus = 'idle'
constructor() {
const isWindows = process.platform === 'win32'
if (isWindows) {
this.docker = new Docker({ socketPath: '//./pipe/docker_engine' })
} else {
this.docker = new Docker({ socketPath: '/var/run/docker.sock' })
}
}
/**
* Run a full benchmark suite
*/
async runFullBenchmark(): Promise<BenchmarkResult> {
return this._runBenchmark('full', true)
}
/**
* Run system benchmarks only (CPU, memory, disk)
*/
async runSystemBenchmarks(): Promise<BenchmarkResult> {
return this._runBenchmark('system', false)
}
/**
* Run AI benchmark only
*/
async runAIBenchmark(): Promise<BenchmarkResult> {
return this._runBenchmark('ai', true)
}
/**
* Get the latest benchmark result
*/
async getLatestResult(): Promise<BenchmarkResult | null> {
return await BenchmarkResult.query().orderBy('created_at', 'desc').first()
}
/**
* Get all benchmark results
*/
async getAllResults(): Promise<BenchmarkResult[]> {
return await BenchmarkResult.query().orderBy('created_at', 'desc')
}
/**
* Get a specific benchmark result by ID
*/
async getResultById(benchmarkId: string): Promise<BenchmarkResult | null> {
return await BenchmarkResult.findBy('benchmark_id', benchmarkId)
}
/**
* Submit benchmark results to central repository
*/
async submitToRepository(benchmarkId?: string): Promise<RepositorySubmitResponse> {
const result = benchmarkId
? await this.getResultById(benchmarkId)
: await this.getLatestResult()
if (!result) {
throw new Error('No benchmark result found to submit')
}
if (result.submitted_to_repository) {
throw new Error('Benchmark result has already been submitted')
}
const submission: RepositorySubmission = {
cpu_model: result.cpu_model,
cpu_cores: result.cpu_cores,
cpu_threads: result.cpu_threads,
ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)),
disk_type: result.disk_type,
gpu_model: result.gpu_model,
cpu_score: result.cpu_score,
memory_score: result.memory_score,
disk_read_score: result.disk_read_score,
disk_write_score: result.disk_write_score,
ai_tokens_per_second: result.ai_tokens_per_second,
ai_time_to_first_token: result.ai_time_to_first_token,
nomad_score: result.nomad_score,
nomad_version: SystemService.getAppVersion(),
benchmark_version: '1.0.0',
}
try {
const response = await axios.post(
'https://benchmark.projectnomad.us/api/v1/submit',
submission,
{ timeout: 30000 }
)
if (response.data.success) {
result.submitted_to_repository = true
result.submitted_at = new Date() as any
result.repository_id = response.data.repository_id
await result.save()
await BenchmarkSetting.setValue('last_benchmark_run', new Date().toISOString())
}
return response.data as RepositorySubmitResponse
} catch (error) {
logger.error(`Failed to submit benchmark to repository: ${error.message}`)
throw new Error(`Failed to submit benchmark: ${error.message}`)
}
}
/**
* Get comparison stats from central repository
*/
async getComparisonStats(): Promise<RepositoryStats | null> {
try {
const response = await axios.get('https://benchmark.projectnomad.us/api/v1/stats', {
timeout: 10000,
})
return response.data as RepositoryStats
} catch (error) {
logger.warn(`Failed to fetch comparison stats: ${error.message}`)
return null
}
}
/**
* Get current benchmark status
*/
getStatus(): { status: BenchmarkStatus; benchmarkId: string | null } {
return {
status: this.currentStatus,
benchmarkId: this.currentBenchmarkId,
}
}
/**
* Detect system hardware information
*/
async getHardwareInfo(): Promise<HardwareInfo> {
this._updateStatus('detecting_hardware', 'Detecting system hardware...')
try {
const [cpu, mem, diskLayout, graphics] = await Promise.all([
si.cpu(),
si.mem(),
si.diskLayout(),
si.graphics(),
])
// Determine disk type from primary disk
let diskType: DiskType = 'unknown'
if (diskLayout.length > 0) {
const primaryDisk = diskLayout[0]
if (primaryDisk.type?.toLowerCase().includes('nvme')) {
diskType = 'nvme'
} else if (primaryDisk.type?.toLowerCase().includes('ssd')) {
diskType = 'ssd'
} else if (primaryDisk.type?.toLowerCase().includes('hdd') || primaryDisk.interfaceType === 'SATA') {
// SATA could be SSD or HDD, check if it's rotational
diskType = 'hdd'
}
}
// Get GPU model (prefer discrete GPU)
let gpuModel: string | null = null
if (graphics.controllers && graphics.controllers.length > 0) {
const discreteGpu = graphics.controllers.find(
(g) => !g.vendor?.toLowerCase().includes('intel') &&
!g.vendor?.toLowerCase().includes('amd') ||
(g.vram && g.vram > 0)
)
gpuModel = discreteGpu?.model || graphics.controllers[0]?.model || null
}
return {
cpu_model: `${cpu.manufacturer} ${cpu.brand}`,
cpu_cores: cpu.physicalCores,
cpu_threads: cpu.cores,
ram_bytes: mem.total,
disk_type: diskType,
gpu_model: gpuModel,
}
} catch (error) {
logger.error(`Error detecting hardware: ${error.message}`)
throw new Error(`Failed to detect hardware: ${error.message}`)
}
}
/**
* Main benchmark execution method
*/
private async _runBenchmark(type: BenchmarkType, includeAI: boolean): Promise<BenchmarkResult> {
if (this.currentStatus !== 'idle') {
throw new Error('A benchmark is already running')
}
this.currentBenchmarkId = uuidv4()
this._updateStatus('starting', 'Starting benchmark...')
try {
// Detect hardware
const hardware = await this.getHardwareInfo()
// Run system benchmarks
let systemScores: SystemScores = {
cpu_score: 0,
memory_score: 0,
disk_read_score: 0,
disk_write_score: 0,
}
if (type === 'full' || type === 'system') {
systemScores = await this._runSystemBenchmarks()
}
// Run AI benchmark if requested and Ollama is available
let aiScores: Partial<AIScores> = {}
if (includeAI && (type === 'full' || type === 'ai')) {
try {
aiScores = await this._runAIBenchmark()
} catch (error) {
logger.warn(`AI benchmark skipped: ${error.message}`)
// AI benchmark is optional, continue without it
}
}
// Calculate NOMAD score
this._updateStatus('calculating_score', 'Calculating NOMAD score...')
const nomadScore = this._calculateNomadScore(systemScores, aiScores)
// Save result
const result = await BenchmarkResult.create({
benchmark_id: this.currentBenchmarkId,
benchmark_type: type,
cpu_model: hardware.cpu_model,
cpu_cores: hardware.cpu_cores,
cpu_threads: hardware.cpu_threads,
ram_bytes: hardware.ram_bytes,
disk_type: hardware.disk_type,
gpu_model: hardware.gpu_model,
cpu_score: systemScores.cpu_score,
memory_score: systemScores.memory_score,
disk_read_score: systemScores.disk_read_score,
disk_write_score: systemScores.disk_write_score,
ai_tokens_per_second: aiScores.ai_tokens_per_second || null,
ai_model_used: aiScores.ai_model_used || null,
ai_time_to_first_token: aiScores.ai_time_to_first_token || null,
nomad_score: nomadScore,
submitted_to_repository: false,
})
this._updateStatus('completed', 'Benchmark completed successfully')
this.currentStatus = 'idle'
this.currentBenchmarkId = null
return result
} catch (error) {
this._updateStatus('error', `Benchmark failed: ${error.message}`)
this.currentStatus = 'idle'
this.currentBenchmarkId = null
throw error
}
}
/**
* Run system benchmarks using sysbench in Docker
*/
private async _runSystemBenchmarks(): Promise<SystemScores> {
// Ensure sysbench image is available
await this._ensureSysbenchImage()
// Run CPU benchmark
this._updateStatus('running_cpu', 'Running CPU benchmark...')
const cpuResult = await this._runSysbenchCpu()
// Run memory benchmark
this._updateStatus('running_memory', 'Running memory benchmark...')
const memoryResult = await this._runSysbenchMemory()
// Run disk benchmarks
this._updateStatus('running_disk_read', 'Running disk read benchmark...')
const diskReadResult = await this._runSysbenchDiskRead()
this._updateStatus('running_disk_write', 'Running disk write benchmark...')
const diskWriteResult = await this._runSysbenchDiskWrite()
// Normalize scores to 0-100 scale
return {
cpu_score: this._normalizeScore(cpuResult.events_per_second, REFERENCE_SCORES.cpu_events_per_second),
memory_score: this._normalizeScore(memoryResult.operations_per_second, REFERENCE_SCORES.memory_ops_per_second),
disk_read_score: this._normalizeScore(diskReadResult.read_mb_per_sec, REFERENCE_SCORES.disk_read_mb_per_sec),
disk_write_score: this._normalizeScore(diskWriteResult.write_mb_per_sec, REFERENCE_SCORES.disk_write_mb_per_sec),
}
}
/**
* Run AI benchmark using Ollama
*/
private async _runAIBenchmark(): Promise<AIScores> {
this._updateStatus('running_ai', 'Running AI benchmark...')
// Check if Ollama is available
try {
await axios.get('http://localhost:11434/api/tags', { timeout: 5000 })
} catch {
throw new Error('Ollama is not running or not accessible')
}
// Check if the benchmark model is available, pull if not
try {
const modelsResponse = await axios.get('http://localhost:11434/api/tags')
const models = modelsResponse.data.models || []
const hasModel = models.some((m: any) => m.name === AI_BENCHMARK_MODEL || m.name.startsWith(AI_BENCHMARK_MODEL.split(':')[0]))
if (!hasModel) {
this._updateStatus('running_ai', `Pulling benchmark model ${AI_BENCHMARK_MODEL}...`)
await axios.post('http://localhost:11434/api/pull', { name: AI_BENCHMARK_MODEL })
}
} catch (error) {
logger.warn(`Could not check/pull model: ${error.message}`)
}
// Run inference benchmark
const startTime = Date.now()
try {
const response = await axios.post(
'http://localhost:11434/api/generate',
{
model: AI_BENCHMARK_MODEL,
prompt: AI_BENCHMARK_PROMPT,
stream: false,
},
{ timeout: 120000 }
)
const endTime = Date.now()
const totalTime = (endTime - startTime) / 1000 // seconds
// Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds)
if (response.data.eval_count && response.data.eval_duration) {
const tokenCount = response.data.eval_count
const evalDurationSeconds = response.data.eval_duration / 1e9
const tokensPerSecond = tokenCount / evalDurationSeconds
// Time to first token from prompt_eval_duration
const ttft = response.data.prompt_eval_duration
? response.data.prompt_eval_duration / 1e6 // Convert to ms
: (totalTime * 1000) / 2 // Estimate if not available
return {
ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
ai_model_used: AI_BENCHMARK_MODEL,
ai_time_to_first_token: Math.round(ttft * 100) / 100,
}
}
// Fallback calculation
const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100
const tokensPerSecond = estimatedTokens / totalTime
return {
ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
ai_model_used: AI_BENCHMARK_MODEL,
ai_time_to_first_token: Math.round((totalTime * 1000) / 2),
}
} catch (error) {
throw new Error(`AI benchmark failed: ${error.message}`)
}
}
/**
* Calculate weighted NOMAD score
*/
private _calculateNomadScore(systemScores: SystemScores, aiScores: Partial<AIScores>): number {
let totalWeight = 0
let weightedSum = 0
// CPU score
weightedSum += systemScores.cpu_score * SCORE_WEIGHTS.cpu
totalWeight += SCORE_WEIGHTS.cpu
// Memory score
weightedSum += systemScores.memory_score * SCORE_WEIGHTS.memory
totalWeight += SCORE_WEIGHTS.memory
// Disk scores
weightedSum += systemScores.disk_read_score * SCORE_WEIGHTS.disk_read
totalWeight += SCORE_WEIGHTS.disk_read
weightedSum += systemScores.disk_write_score * SCORE_WEIGHTS.disk_write
totalWeight += SCORE_WEIGHTS.disk_write
// AI scores (if available)
if (aiScores.ai_tokens_per_second !== undefined) {
const aiScore = this._normalizeScore(
aiScores.ai_tokens_per_second,
REFERENCE_SCORES.ai_tokens_per_second
)
weightedSum += aiScore * SCORE_WEIGHTS.ai_tokens_per_second
totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
}
if (aiScores.ai_time_to_first_token !== undefined) {
// For TTFT, lower is better, so we invert the score
const ttftScore = this._normalizeScoreInverse(
aiScores.ai_time_to_first_token,
REFERENCE_SCORES.ai_ttft_ms
)
weightedSum += ttftScore * SCORE_WEIGHTS.ai_ttft
totalWeight += SCORE_WEIGHTS.ai_ttft
}
// Normalize by actual weight used (in case AI benchmarks were skipped)
const nomadScore = totalWeight > 0 ? (weightedSum / totalWeight) * 100 : 0
return Math.round(Math.min(100, Math.max(0, nomadScore)) * 100) / 100
}
/**
* Normalize a raw score to 0-100 scale using log scaling
* This provides diminishing returns for very high scores
*/
private _normalizeScore(value: number, reference: number): number {
if (value <= 0) return 0
// Log scale: score = 50 * (1 + log2(value/reference))
// This gives 50 at reference value, scales logarithmically
const ratio = value / reference
const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)))
return Math.min(100, Math.max(0, score)) / 100
}
/**
* Normalize a score where lower is better (like latency)
*/
private _normalizeScoreInverse(value: number, reference: number): number {
if (value <= 0) return 1
// Inverse: lower values = higher scores
const ratio = reference / value
const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)))
return Math.min(100, Math.max(0, score)) / 100
}
/**
* Ensure sysbench Docker image is available
*/
private async _ensureSysbenchImage(): Promise<void> {
try {
await this.docker.getImage(SYSBENCH_IMAGE).inspect()
} catch {
this._updateStatus('starting', `Pulling sysbench image...`)
const pullStream = await this.docker.pull(SYSBENCH_IMAGE)
await new Promise((resolve) => this.docker.modem.followProgress(pullStream, resolve))
}
}
/**
* Run sysbench CPU benchmark
*/
private async _runSysbenchCpu(): Promise<SysbenchCpuResult> {
const output = await this._runSysbenchCommand([
'sysbench',
'cpu',
'--cpu-max-prime=20000',
'--threads=4',
'--time=30',
'run',
])
// Parse output for events per second
const eventsMatch = output.match(/events per second:\s*([\d.]+)/i)
const totalTimeMatch = output.match(/total time:\s*([\d.]+)s/i)
const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i)
return {
events_per_second: eventsMatch ? parseFloat(eventsMatch[1]) : 0,
total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30,
total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0,
}
}
/**
* Run sysbench memory benchmark
*/
private async _runSysbenchMemory(): Promise<SysbenchMemoryResult> {
const output = await this._runSysbenchCommand([
'sysbench',
'memory',
'--memory-block-size=1K',
'--memory-total-size=10G',
'--threads=4',
'run',
])
// Parse output
const opsMatch = output.match(/Total operations:\s*\d+\s*\(([\d.]+)\s*per second\)/i)
const transferMatch = output.match(/([\d.]+)\s*MiB\/sec/i)
const timeMatch = output.match(/total time:\s*([\d.]+)s/i)
return {
operations_per_second: opsMatch ? parseFloat(opsMatch[1]) : 0,
transfer_rate_mb_per_sec: transferMatch ? parseFloat(transferMatch[1]) : 0,
total_time: timeMatch ? parseFloat(timeMatch[1]) : 0,
}
}
/**
* Run sysbench disk read benchmark
*/
private async _runSysbenchDiskRead(): Promise<SysbenchDiskResult> {
// Run prepare, test, and cleanup in a single container
// This is necessary because each container has its own filesystem
const output = await this._runSysbenchCommand([
'sh',
'-c',
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --time=30 run && ' +
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
])
// Parse output - look for the Throughput section
const readMatch = output.match(/read,\s*MiB\/s:\s*([\d.]+)/i)
const readsPerSecMatch = output.match(/reads\/s:\s*([\d.]+)/i)
logger.debug(`[BenchmarkService] Disk read output parsing - read: ${readMatch?.[1]}, reads/s: ${readsPerSecMatch?.[1]}`)
return {
reads_per_second: readsPerSecMatch ? parseFloat(readsPerSecMatch[1]) : 0,
writes_per_second: 0,
read_mb_per_sec: readMatch ? parseFloat(readMatch[1]) : 0,
write_mb_per_sec: 0,
total_time: 30,
}
}
/**
* Run sysbench disk write benchmark
*/
private async _runSysbenchDiskWrite(): Promise<SysbenchDiskResult> {
// Run prepare, test, and cleanup in a single container
// This is necessary because each container has its own filesystem
const output = await this._runSysbenchCommand([
'sh',
'-c',
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --time=30 run && ' +
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
])
// Parse output - look for the Throughput section
const writeMatch = output.match(/written,\s*MiB\/s:\s*([\d.]+)/i)
const writesPerSecMatch = output.match(/writes\/s:\s*([\d.]+)/i)
logger.debug(`[BenchmarkService] Disk write output parsing - written: ${writeMatch?.[1]}, writes/s: ${writesPerSecMatch?.[1]}`)
return {
reads_per_second: 0,
writes_per_second: writesPerSecMatch ? parseFloat(writesPerSecMatch[1]) : 0,
read_mb_per_sec: 0,
write_mb_per_sec: writeMatch ? parseFloat(writeMatch[1]) : 0,
total_time: 30,
}
}
/**
* Run a sysbench command in a Docker container
*/
private async _runSysbenchCommand(cmd: string[]): Promise<string> {
try {
// Create container with TTY to avoid multiplexed output
const container = await this.docker.createContainer({
Image: SYSBENCH_IMAGE,
Cmd: cmd,
name: `${SYSBENCH_CONTAINER_NAME}_${Date.now()}`,
Tty: true, // Important: prevents multiplexed stdout/stderr headers
HostConfig: {
AutoRemove: true,
},
})
// Start container
await container.start()
// Wait for completion and get logs
await container.wait()
const logs = await container.logs({
stdout: true,
stderr: true,
})
// Parse logs (Docker logs include header bytes)
const output = logs.toString('utf8')
.replace(/[\x00-\x08]/g, '') // Remove control characters
.trim()
return output
} catch (error) {
logger.error(`Sysbench command failed: ${error.message}`)
throw new Error(`Sysbench command failed: ${error.message}`)
}
}
/**
* Broadcast benchmark progress update
*/
private _updateStatus(status: BenchmarkStatus, message: string) {
this.currentStatus = status
const progress: BenchmarkProgress = {
status,
progress: this._getProgressPercent(status),
message,
current_stage: this._getStageLabel(status),
timestamp: new Date().toISOString(),
}
transmit.broadcast(BENCHMARK_CHANNEL, {
benchmark_id: this.currentBenchmarkId,
...progress,
})
logger.info(`[BenchmarkService] ${status}: ${message}`)
}
/**
* Get progress percentage for a given status
*/
private _getProgressPercent(status: BenchmarkStatus): number {
const progressMap: Record<BenchmarkStatus, number> = {
idle: 0,
starting: 5,
detecting_hardware: 10,
running_cpu: 25,
running_memory: 40,
running_disk_read: 55,
running_disk_write: 70,
running_ai: 85,
calculating_score: 95,
completed: 100,
error: 0,
}
return progressMap[status] || 0
}
/**
* Get human-readable stage label
*/
private _getStageLabel(status: BenchmarkStatus): string {
const labelMap: Record<BenchmarkStatus, string> = {
idle: 'Idle',
starting: 'Starting',
detecting_hardware: 'Detecting Hardware',
running_cpu: 'CPU Benchmark',
running_memory: 'Memory Benchmark',
running_disk_read: 'Disk Read Test',
running_disk_write: 'Disk Write Test',
running_ai: 'AI Inference Test',
calculating_score: 'Calculating Score',
completed: 'Complete',
error: 'Error',
}
return labelMap[status] || status
}
}

View File

@ -17,6 +17,7 @@ export class DockerService {
public static CYBERCHEF_SERVICE_NAME = 'nomad_cyberchef'
public static FLATNOTES_SERVICE_NAME = 'nomad_flatnotes'
public static KOLIBRI_SERVICE_NAME = 'nomad_kolibri'
public static BENCHMARK_SERVICE_NAME = 'nomad_benchmark'
constructor() {
// Support both Linux (production) and Windows (development with Docker Desktop)

View File

@ -0,0 +1,13 @@
import vine from '@vinejs/vine'
export const runBenchmarkValidator = vine.compile(
vine.object({
benchmark_type: vine.enum(['full', 'system', 'ai']).optional(),
})
)
export const submitBenchmarkValidator = vine.compile(
vine.object({
benchmark_id: vine.string().optional(),
})
)

View File

@ -0,0 +1,96 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
export default class BenchmarkResults extends BaseCommand {
static commandName = 'benchmark:results'
static description = 'Display benchmark results'
@flags.boolean({ description: 'Show only the latest result', alias: 'l' })
declare latest: boolean
@flags.string({ description: 'Output format (table, json)', default: 'table' })
declare format: string
@flags.string({ description: 'Show specific benchmark by ID', alias: 'i' })
declare id: string
static options: CommandOptions = {
startApp: true,
}
async run() {
const { BenchmarkService } = await import('#services/benchmark_service')
const benchmarkService = new BenchmarkService()
try {
let results
if (this.id) {
const result = await benchmarkService.getResultById(this.id)
results = result ? [result] : []
} else if (this.latest) {
const result = await benchmarkService.getLatestResult()
results = result ? [result] : []
} else {
results = await benchmarkService.getAllResults()
}
if (results.length === 0) {
this.logger.info('No benchmark results found.')
this.logger.info('Run "node ace benchmark:run" to create a benchmark.')
return
}
if (this.format === 'json') {
console.log(JSON.stringify(results, null, 2))
return
}
// Table format
for (const result of results) {
this.logger.info('')
this.logger.info(`=== Benchmark ${result.benchmark_id} ===`)
this.logger.info(`Type: ${result.benchmark_type}`)
this.logger.info(`Date: ${result.created_at}`)
this.logger.info('')
this.logger.info('Hardware:')
this.logger.info(` CPU: ${result.cpu_model}`)
this.logger.info(` Cores: ${result.cpu_cores} physical, ${result.cpu_threads} threads`)
this.logger.info(` RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`)
this.logger.info(` Disk: ${result.disk_type}`)
if (result.gpu_model) {
this.logger.info(` GPU: ${result.gpu_model}`)
}
this.logger.info('')
this.logger.info('Scores:')
this.logger.info(` CPU: ${result.cpu_score.toFixed(2)}`)
this.logger.info(` Memory: ${result.memory_score.toFixed(2)}`)
this.logger.info(` Disk Read: ${result.disk_read_score.toFixed(2)}`)
this.logger.info(` Disk Write: ${result.disk_write_score.toFixed(2)}`)
if (result.ai_tokens_per_second) {
this.logger.info(` AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`)
this.logger.info(` AI TTFT: ${result.ai_time_to_first_token?.toFixed(2)} ms`)
}
this.logger.info('')
this.logger.info(`NOMAD Score: ${result.nomad_score.toFixed(2)} / 100`)
if (result.submitted_to_repository) {
this.logger.info(`Submitted: Yes (${result.repository_id})`)
} else {
this.logger.info('Submitted: No')
}
this.logger.info('')
}
this.logger.info(`Total results: ${results.length}`)
} catch (error) {
this.logger.error(`Failed to retrieve results: ${error.message}`)
this.exitCode = 1
}
}
}

View File

@ -0,0 +1,103 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
export default class BenchmarkRun extends BaseCommand {
static commandName = 'benchmark:run'
static description = 'Run system and/or AI benchmarks to measure server performance'
@flags.boolean({ description: 'Run system benchmarks only (CPU, memory, disk)', alias: 's' })
declare systemOnly: boolean
@flags.boolean({ description: 'Run AI benchmark only', alias: 'a' })
declare aiOnly: boolean
@flags.boolean({ description: 'Submit results to repository after completion', alias: 'S' })
declare submit: boolean
static options: CommandOptions = {
startApp: true,
}
async run() {
const { BenchmarkService } = await import('#services/benchmark_service')
const benchmarkService = new BenchmarkService()
// Determine benchmark type
let benchmarkType: 'full' | 'system' | 'ai' = 'full'
if (this.systemOnly) {
benchmarkType = 'system'
} else if (this.aiOnly) {
benchmarkType = 'ai'
}
this.logger.info(`Starting ${benchmarkType} benchmark...`)
this.logger.info('')
try {
// Run the benchmark
let result
switch (benchmarkType) {
case 'system':
this.logger.info('Running system benchmarks (CPU, memory, disk)...')
result = await benchmarkService.runSystemBenchmarks()
break
case 'ai':
this.logger.info('Running AI benchmark...')
result = await benchmarkService.runAIBenchmark()
break
default:
this.logger.info('Running full benchmark suite...')
result = await benchmarkService.runFullBenchmark()
}
// Display results
this.logger.info('')
this.logger.success('Benchmark completed!')
this.logger.info('')
this.logger.info('=== Hardware Info ===')
this.logger.info(`CPU: ${result.cpu_model}`)
this.logger.info(`Cores: ${result.cpu_cores} physical, ${result.cpu_threads} threads`)
this.logger.info(`RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`)
this.logger.info(`Disk Type: ${result.disk_type}`)
if (result.gpu_model) {
this.logger.info(`GPU: ${result.gpu_model}`)
}
this.logger.info('')
this.logger.info('=== Benchmark Scores ===')
this.logger.info(`CPU Score: ${result.cpu_score.toFixed(2)}`)
this.logger.info(`Memory Score: ${result.memory_score.toFixed(2)}`)
this.logger.info(`Disk Read Score: ${result.disk_read_score.toFixed(2)}`)
this.logger.info(`Disk Write Score: ${result.disk_write_score.toFixed(2)}`)
if (result.ai_tokens_per_second) {
this.logger.info(`AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`)
this.logger.info(`AI Time to First Token: ${result.ai_time_to_first_token?.toFixed(2)} ms`)
this.logger.info(`AI Model: ${result.ai_model_used}`)
}
this.logger.info('')
this.logger.info(`NOMAD Score: ${result.nomad_score.toFixed(2)} / 100`)
this.logger.info('')
this.logger.info(`Benchmark ID: ${result.benchmark_id}`)
// Submit if requested
if (this.submit) {
this.logger.info('')
this.logger.info('Submitting results to repository...')
try {
const submitResult = await benchmarkService.submitToRepository(result.benchmark_id)
this.logger.success(`Results submitted! Repository ID: ${submitResult.repository_id}`)
this.logger.info(`Your percentile: ${submitResult.percentile}%`)
} catch (error) {
this.logger.error(`Failed to submit: ${error.message}`)
}
}
} catch (error) {
this.logger.error(`Benchmark failed: ${error.message}`)
this.exitCode = 1
}
}
}

View File

@ -0,0 +1,99 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
export default class BenchmarkSubmit extends BaseCommand {
static commandName = 'benchmark:submit'
static description = 'Submit benchmark results to the community repository'
@flags.string({ description: 'Benchmark ID to submit (defaults to latest)', alias: 'i' })
declare benchmarkId: string
@flags.boolean({ description: 'Skip confirmation prompt', alias: 'y' })
declare yes: boolean
static options: CommandOptions = {
startApp: true,
}
async run() {
const { BenchmarkService } = await import('#services/benchmark_service')
const benchmarkService = new BenchmarkService()
try {
// Get the result to submit
const result = this.benchmarkId
? await benchmarkService.getResultById(this.benchmarkId)
: await benchmarkService.getLatestResult()
if (!result) {
this.logger.error('No benchmark result found.')
this.logger.info('Run "node ace benchmark:run" first to create a benchmark.')
this.exitCode = 1
return
}
if (result.submitted_to_repository) {
this.logger.warning(`Benchmark ${result.benchmark_id} has already been submitted.`)
this.logger.info(`Repository ID: ${result.repository_id}`)
return
}
// Show what will be submitted
this.logger.info('')
this.logger.info('=== Data to be submitted ===')
this.logger.info('')
this.logger.info('Hardware Information:')
this.logger.info(` CPU Model: ${result.cpu_model}`)
this.logger.info(` CPU Cores: ${result.cpu_cores}`)
this.logger.info(` CPU Threads: ${result.cpu_threads}`)
this.logger.info(` RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`)
this.logger.info(` Disk Type: ${result.disk_type}`)
if (result.gpu_model) {
this.logger.info(` GPU: ${result.gpu_model}`)
}
this.logger.info('')
this.logger.info('Benchmark Scores:')
this.logger.info(` CPU Score: ${result.cpu_score.toFixed(2)}`)
this.logger.info(` Memory Score: ${result.memory_score.toFixed(2)}`)
this.logger.info(` Disk Read: ${result.disk_read_score.toFixed(2)}`)
this.logger.info(` Disk Write: ${result.disk_write_score.toFixed(2)}`)
if (result.ai_tokens_per_second) {
this.logger.info(` AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`)
this.logger.info(` AI TTFT: ${result.ai_time_to_first_token?.toFixed(2)} ms`)
}
this.logger.info(` NOMAD Score: ${result.nomad_score.toFixed(2)}`)
this.logger.info('')
this.logger.info('Privacy Notice:')
this.logger.info(' - Only the information shown above will be submitted')
this.logger.info(' - No IP addresses, hostnames, or personal data is collected')
this.logger.info(' - Submissions are completely anonymous')
this.logger.info('')
// Confirm submission
if (!this.yes) {
const confirm = await this.prompt.confirm(
'Do you want to submit this benchmark to the community repository?'
)
if (!confirm) {
this.logger.info('Submission cancelled.')
return
}
}
// Submit
this.logger.info('Submitting benchmark...')
const submitResult = await benchmarkService.submitToRepository(result.benchmark_id)
this.logger.success('Benchmark submitted successfully!')
this.logger.info('')
this.logger.info(`Repository ID: ${submitResult.repository_id}`)
this.logger.info(`Your percentile: ${submitResult.percentile}%`)
this.logger.info('')
this.logger.info('Thank you for contributing to the NOMAD community!')
} catch (error) {
this.logger.error(`Submission failed: ${error.message}`)
this.exitCode = 1
}
}
}

View File

@ -61,8 +61,10 @@ export default class QueueWork extends BaseCommand {
const { RunDownloadJob } = await import('#jobs/run_download_job')
const { DownloadModelJob } = await import('#jobs/download_model_job')
const { RunBenchmarkJob } = await import('#jobs/run_benchmark_job')
handlers.set(RunDownloadJob.key, new RunDownloadJob())
handlers.set(DownloadModelJob.key, new DownloadModelJob())
handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob())
return handlers
}

View File

@ -0,0 +1,47 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'benchmark_results'
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id')
table.string('benchmark_id').unique().notNullable()
table.enum('benchmark_type', ['full', 'system', 'ai']).notNullable()
// Hardware information
table.string('cpu_model').notNullable()
table.integer('cpu_cores').notNullable()
table.integer('cpu_threads').notNullable()
table.bigInteger('ram_bytes').notNullable()
table.enum('disk_type', ['ssd', 'hdd', 'nvme', 'unknown']).notNullable()
table.string('gpu_model').nullable()
// System benchmark scores
table.float('cpu_score').notNullable()
table.float('memory_score').notNullable()
table.float('disk_read_score').notNullable()
table.float('disk_write_score').notNullable()
// AI benchmark scores (nullable for system-only benchmarks)
table.float('ai_tokens_per_second').nullable()
table.string('ai_model_used').nullable()
table.float('ai_time_to_first_token').nullable()
// Composite NOMAD score (0-100)
table.float('nomad_score').notNullable()
// Repository submission tracking
table.boolean('submitted_to_repository').defaultTo(false)
table.timestamp('submitted_at').nullable()
table.string('repository_id').nullable()
table.timestamp('created_at')
table.timestamp('updated_at')
})
}
async down() {
this.schema.dropTable(this.tableName)
}
}

View File

@ -0,0 +1,19 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'benchmark_settings'
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id')
table.string('key').unique().notNullable()
table.text('value').nullable()
table.timestamp('created_at')
table.timestamp('updated_at')
})
}
async down() {
this.schema.dropTable(this.tableName)
}
}

View File

@ -147,6 +147,26 @@ export default class ServiceSeeder extends BaseSeeder {
is_dependency_service: false,
depends_on: null,
},
{
service_name: DockerService.BENCHMARK_SERVICE_NAME,
friendly_name: 'System Benchmark',
description: 'Measure your server performance and compare with the NOMAD community',
icon: 'IconChartBar',
container_image: 'severalnines/sysbench:latest',
container_command: null,
container_config: JSON.stringify({
HostConfig: {
AutoRemove: true,
Binds: [`${ServiceSeeder.NOMAD_STORAGE_ABS_PATH}/benchmark:/tmp/benchmark`]
},
WorkingDir: '/tmp/benchmark',
}),
ui_location: null, // UI is integrated into Command Center
installed: false,
installation_status: 'idle',
is_dependency_service: false,
depends_on: null,
},
]
async run() {

View File

@ -10,7 +10,7 @@ import CategoryCard from '~/components/CategoryCard'
import TierSelectionModal from '~/components/TierSelectionModal'
import LoadingSpinner from '~/components/LoadingSpinner'
import Alert from '~/components/Alert'
import { IconCheck, IconChevronDown, IconChevronUp } from '@tabler/icons-react'
import { IconCheck, IconChevronDown, IconChevronUp, IconArrowRight } from '@tabler/icons-react'
import StorageProjectionBar from '~/components/StorageProjectionBar'
import { useNotifications } from '~/context/NotificationContext'
import useInternetStatus from '~/hooks/useInternetStatus'
@ -98,6 +98,19 @@ const ADDITIONAL_TOOLS: Capability[] = [
services: ['nomad_cyberchef'],
icon: 'IconChefHat',
},
{
id: 'benchmark',
name: 'System Benchmark',
technicalName: 'Built-in',
description: 'Measure your server performance and compare with the NOMAD community',
features: [
'CPU, memory, and disk benchmarks',
'AI inference performance testing',
'NOMAD Score for easy comparison',
],
services: ['__builtin_benchmark'], // Special marker for built-in features
icon: 'IconChartBar',
},
]
type WizardStep = 1 | 2 | 3 | 4
@ -487,13 +500,20 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
)
}
// Check if a capability is a built-in feature (not a Docker service)
const isBuiltInCapability = (capability: Capability) => {
return capability.services.some((service) => service.startsWith('__builtin_'))
}
// Check if a capability is selected (all its services are in selectedServices)
const isCapabilitySelected = (capability: Capability) => {
if (isBuiltInCapability(capability)) return false // Built-ins can't be selected
return capability.services.every((service) => selectedServices.includes(service))
}
// Check if a capability is already installed (all its services are installed)
const isCapabilityInstalled = (capability: Capability) => {
if (isBuiltInCapability(capability)) return true // Built-ins are always "installed"
return capability.services.every((service) =>
installedServices.some((s) => s.service_name === service)
)
@ -501,6 +521,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
// Check if a capability exists in the system (has at least one matching service)
const capabilityExists = (capability: Capability) => {
if (isBuiltInCapability(capability)) return true // Built-ins always exist
return capability.services.some((service) =>
allServices.some((s) => s.service_name === service)
)
@ -528,23 +549,38 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
const selected = isCapabilitySelected(capability)
const installed = isCapabilityInstalled(capability)
const exists = capabilityExists(capability)
const isBuiltIn = isBuiltInCapability(capability)
if (!exists) return null
// Determine visual state: installed (locked), selected (user chose it), or default
const isChecked = installed || selected
// Handle click - built-in features navigate to their page, others toggle selection
const handleClick = () => {
if (isBuiltIn) {
// Navigate to the appropriate settings page for built-in features
if (capability.id === 'benchmark') {
router.visit('/settings/benchmark')
}
} else {
toggleCapability(capability)
}
}
return (
<div
key={capability.id}
onClick={() => toggleCapability(capability)}
onClick={handleClick}
className={classNames(
'p-6 rounded-lg border-2 transition-all',
installed
? 'border-desert-green bg-desert-green/20 cursor-default'
: selected
? 'border-desert-green bg-desert-green shadow-md cursor-pointer'
: 'border-desert-stone-light bg-white hover:border-desert-green hover:shadow-sm cursor-pointer'
isBuiltIn
? 'border-desert-stone bg-desert-stone-lighter/50 hover:border-desert-green hover:shadow-sm cursor-pointer'
: installed
? 'border-desert-green bg-desert-green/20 cursor-default'
: selected
? 'border-desert-green bg-desert-green shadow-md cursor-pointer'
: 'border-desert-stone-light bg-white hover:border-desert-green hover:shadow-sm cursor-pointer'
)}
>
<div className="flex items-start justify-between">
@ -553,12 +589,16 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
<h3
className={classNames(
'text-xl font-bold',
installed ? 'text-gray-700' : selected ? 'text-white' : 'text-gray-900'
isBuiltIn ? 'text-gray-700' : installed ? 'text-gray-700' : selected ? 'text-white' : 'text-gray-900'
)}
>
{capability.name}
</h3>
{installed && (
{isBuiltIn ? (
<span className="text-xs bg-desert-stone text-white px-2 py-0.5 rounded-full">
Built-in
</span>
) : installed && (
<span className="text-xs bg-desert-green text-white px-2 py-0.5 rounded-full">
Installed
</span>
@ -567,15 +607,15 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
<p
className={classNames(
'text-sm mt-0.5',
installed ? 'text-gray-500' : selected ? 'text-green-100' : 'text-gray-500'
isBuiltIn ? 'text-gray-500' : installed ? 'text-gray-500' : selected ? 'text-green-100' : 'text-gray-500'
)}
>
Powered by {capability.technicalName}
{isBuiltIn ? 'Click to open' : `Powered by ${capability.technicalName}`}
</p>
<p
className={classNames(
'text-sm mt-3',
installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
isBuiltIn ? 'text-gray-600' : installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
)}
>
{capability.description}
@ -584,7 +624,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
<ul
className={classNames(
'mt-3 space-y-1',
installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
isBuiltIn ? 'text-gray-600' : installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
)}
>
{capability.features.map((feature, idx) => (
@ -592,11 +632,13 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
<span
className={classNames(
'mr-2',
installed
? 'text-desert-green'
: selected
? 'text-white'
: 'text-desert-green'
isBuiltIn
? 'text-desert-stone'
: installed
? 'text-desert-green'
: selected
? 'text-white'
: 'text-desert-green'
)}
>
@ -610,14 +652,18 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
<div
className={classNames(
'ml-4 w-7 h-7 rounded-full border-2 flex items-center justify-center transition-all flex-shrink-0',
isChecked
? installed
? 'border-desert-green bg-desert-green'
: 'border-white bg-white'
: 'border-desert-stone'
isBuiltIn
? 'border-desert-stone bg-desert-stone'
: isChecked
? installed
? 'border-desert-green bg-desert-green'
: 'border-white bg-white'
: 'border-desert-stone'
)}
>
{isChecked && (
{isBuiltIn ? (
<IconArrowRight size={16} className="text-white" />
) : isChecked && (
<IconCheck size={20} className={installed ? 'text-white' : 'text-desert-green'} />
)}
</div>

View File

@ -0,0 +1,618 @@
import { Head } from '@inertiajs/react'
import { useState, useEffect } from 'react'
import SettingsLayout from '~/layouts/SettingsLayout'
import { useQuery, useMutation } from '@tanstack/react-query'
import CircularGauge from '~/components/systeminfo/CircularGauge'
import InfoCard from '~/components/systeminfo/InfoCard'
import Alert from '~/components/Alert'
import StyledButton from '~/components/StyledButton'
import {
ChartBarIcon,
CpuChipIcon,
CircleStackIcon,
ServerIcon,
CloudArrowUpIcon,
PlayIcon,
ChevronDownIcon,
} from '@heroicons/react/24/outline'
import { IconRobot } from '@tabler/icons-react'
import { useTransmit } from 'react-adonis-transmit'
type BenchmarkResult = {
id: number
benchmark_id: string
benchmark_type: 'full' | 'system' | 'ai'
cpu_model: string
cpu_cores: number
cpu_threads: number
ram_bytes: number
disk_type: string
gpu_model: string | null
cpu_score: number
memory_score: number
disk_read_score: number
disk_write_score: number
ai_tokens_per_second: number | null
ai_model_used: string | null
ai_time_to_first_token: number | null
nomad_score: number
submitted_to_repository: boolean
repository_id: string | null
created_at: string
}
type BenchmarkStatus = 'idle' | 'starting' | 'detecting_hardware' | 'running_cpu' | 'running_memory' | 'running_disk_read' | 'running_disk_write' | 'running_ai' | 'calculating_score' | 'completed' | 'error'
type BenchmarkProgress = {
status: BenchmarkStatus
progress: number
message: string
current_stage: string
benchmark_id: string
}
export default function BenchmarkPage(props: {
benchmark: {
latestResult: BenchmarkResult | null
status: BenchmarkStatus
currentBenchmarkId: string | null
}
}) {
const { subscribe } = useTransmit()
const [progress, setProgress] = useState<BenchmarkProgress | null>(null)
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
const [showDetails, setShowDetails] = useState(false)
// Fetch latest result
const { data: latestResult, refetch: refetchLatest } = useQuery({
queryKey: ['benchmark', 'latest'],
queryFn: async () => {
const res = await fetch('/api/benchmark/results/latest')
const data = await res.json()
return data.result as BenchmarkResult | null
},
initialData: props.benchmark.latestResult,
})
// Run benchmark mutation (uses sync mode by default for simpler local dev)
const runBenchmark = useMutation({
mutationFn: async (type: 'full' | 'system' | 'ai') => {
setIsRunning(true)
setProgress({
status: 'starting',
progress: 5,
message: 'Starting benchmark... This takes 2-5 minutes.',
current_stage: 'Starting',
benchmark_id: '',
})
// Use sync mode - runs inline without needing Redis/queue worker
const res = await fetch('/api/benchmark/run?sync=true', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ benchmark_type: type }),
})
return res.json()
},
onSuccess: (data) => {
if (data.success) {
setProgress({
status: 'completed',
progress: 100,
message: 'Benchmark completed!',
current_stage: 'Complete',
benchmark_id: data.benchmark_id,
})
refetchLatest()
} else {
setProgress({
status: 'error',
progress: 0,
message: data.error || 'Benchmark failed',
current_stage: 'Error',
benchmark_id: '',
})
}
setIsRunning(false)
},
onError: (error) => {
setProgress({
status: 'error',
progress: 0,
message: error.message || 'Benchmark failed',
current_stage: 'Error',
benchmark_id: '',
})
setIsRunning(false)
},
})
// Submit to repository mutation
const submitResult = useMutation({
mutationFn: async (benchmarkId?: string) => {
const res = await fetch('/api/benchmark/submit', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ benchmark_id: benchmarkId }),
})
return res.json()
},
onSuccess: () => {
refetchLatest()
},
})
// Simulate progress during sync benchmark (since we don't get SSE updates)
useEffect(() => {
if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return
const stages: { status: BenchmarkStatus; progress: number; message: string; label: string; duration: number }[] = [
{ status: 'detecting_hardware', progress: 10, message: 'Detecting system hardware...', label: 'Detecting Hardware', duration: 2000 },
{ status: 'running_cpu', progress: 25, message: 'Running CPU benchmark (30s)...', label: 'CPU Benchmark', duration: 32000 },
{ status: 'running_memory', progress: 40, message: 'Running memory benchmark...', label: 'Memory Benchmark', duration: 8000 },
{ status: 'running_disk_read', progress: 55, message: 'Running disk read benchmark (30s)...', label: 'Disk Read Test', duration: 35000 },
{ status: 'running_disk_write', progress: 70, message: 'Running disk write benchmark (30s)...', label: 'Disk Write Test', duration: 35000 },
{ status: 'calculating_score', progress: 95, message: 'Calculating NOMAD score...', label: 'Calculating Score', duration: 2000 },
]
let currentStage = 0
const advanceStage = () => {
if (currentStage < stages.length && isRunning) {
const stage = stages[currentStage]
setProgress({
status: stage.status,
progress: stage.progress,
message: stage.message,
current_stage: stage.label,
benchmark_id: '',
})
currentStage++
}
}
// Start the first stage after a short delay
const timers: NodeJS.Timeout[] = []
let elapsed = 1000
stages.forEach((stage, index) => {
timers.push(setTimeout(() => advanceStage(), elapsed))
elapsed += stage.duration
})
return () => {
timers.forEach(t => clearTimeout(t))
}
}, [isRunning])
// Listen for benchmark progress via SSE (backup for async mode)
useEffect(() => {
const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgress) => {
setProgress(data)
if (data.status === 'completed' || data.status === 'error') {
setIsRunning(false)
refetchLatest()
}
})
return () => {
unsubscribe()
}
}, [subscribe, refetchLatest])
const formatBytes = (bytes: number) => {
const gb = bytes / (1024 * 1024 * 1024)
return `${gb.toFixed(1)} GB`
}
const getScoreColor = (score: number) => {
if (score >= 70) return 'text-green-600'
if (score >= 40) return 'text-yellow-600'
return 'text-red-600'
}
const getProgressPercent = () => {
if (!progress) return 0
const stages: Record<BenchmarkStatus, number> = {
idle: 0,
starting: 5,
detecting_hardware: 10,
running_cpu: 25,
running_memory: 40,
running_disk_read: 55,
running_disk_write: 70,
running_ai: 85,
calculating_score: 95,
completed: 100,
error: 0,
}
return stages[progress.status] || 0
}
// Calculate AI score from tokens per second (normalized to 0-100)
// Reference: 30 tok/s = 50 score, 60 tok/s = 100 score
const getAIScore = (tokensPerSecond: number | null): number => {
if (!tokensPerSecond) return 0
const score = (tokensPerSecond / 60) * 100
return Math.min(100, Math.max(0, score))
}
return (
<SettingsLayout>
<Head title="System Benchmark" />
<div className="xl:pl-72 w-full">
<main className="px-6 lg:px-12 py-6 lg:py-8">
<div className="mb-8">
<h1 className="text-4xl font-bold text-desert-green mb-2">System Benchmark</h1>
<p className="text-desert-stone-dark">
Measure your server's performance and compare with the NOMAD community
</p>
</div>
{/* Run Benchmark Section */}
<section className="mb-12">
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
<div className="w-1 h-6 bg-desert-green" />
Run Benchmark
</h2>
<div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm">
{isRunning ? (
<div className="space-y-4">
<div className="flex items-center gap-3">
<div className="animate-spin h-6 w-6 border-2 border-desert-green border-t-transparent rounded-full" />
<span className="text-lg font-medium">
{progress?.current_stage || 'Running benchmark...'}
</span>
</div>
<div className="w-full bg-desert-stone-lighter rounded-full h-4 overflow-hidden">
<div
className="bg-desert-green h-full transition-all duration-500"
style={{ width: `${getProgressPercent()}%` }}
/>
</div>
<p className="text-sm text-desert-stone-dark">{progress?.message}</p>
</div>
) : (
<div className="space-y-6">
<p className="text-desert-stone-dark">
Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance.
The benchmark takes approximately 2-5 minutes to complete.
</p>
<div className="flex flex-wrap gap-4">
<StyledButton
onClick={() => runBenchmark.mutate('full')}
disabled={runBenchmark.isPending}
leftIcon={<PlayIcon className="w-5 h-5" />}
>
Run Full Benchmark
</StyledButton>
<StyledButton
variant="secondary"
onClick={() => runBenchmark.mutate('system')}
disabled={runBenchmark.isPending}
leftIcon={<CpuChipIcon className="w-5 h-5" />}
>
System Only
</StyledButton>
<StyledButton
variant="secondary"
onClick={() => runBenchmark.mutate('ai')}
disabled={runBenchmark.isPending}
leftIcon={<IconRobot className="w-5 h-5" />}
>
AI Only
</StyledButton>
</div>
</div>
)}
</div>
</section>
{/* Results Section */}
{latestResult && (
<>
<section className="mb-12">
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
<div className="w-1 h-6 bg-desert-green" />
NOMAD Score
</h2>
<div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm">
<div className="flex flex-col md:flex-row items-center gap-8">
<div className="flex-shrink-0">
<CircularGauge
value={latestResult.nomad_score}
label="NOMAD Score"
size="lg"
variant="cpu"
subtext="out of 100"
icon={<ChartBarIcon className="w-8 h-8" />}
/>
</div>
<div className="flex-1 space-y-4">
<div className={`text-5xl font-bold ${getScoreColor(latestResult.nomad_score)}`}>
{latestResult.nomad_score.toFixed(1)}
</div>
<p className="text-desert-stone-dark">
Your NOMAD Score is a weighted composite of all benchmark results.
</p>
{!latestResult.submitted_to_repository && (
<StyledButton
onClick={() => submitResult.mutate(latestResult.benchmark_id)}
disabled={submitResult.isPending}
leftIcon={<CloudArrowUpIcon className="w-5 h-5" />}
>
Share with Community
</StyledButton>
)}
{latestResult.submitted_to_repository && (
<Alert
type="success"
title="Shared with Community"
message={`Repository ID: ${latestResult.repository_id}`}
variant="bordered"
/>
)}
</div>
</div>
</div>
</section>
<section className="mb-12">
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
<div className="w-1 h-6 bg-desert-green" />
System Performance
</h2>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
<CircularGauge
value={latestResult.cpu_score * 100}
label="CPU"
size="md"
variant="cpu"
icon={<CpuChipIcon className="w-6 h-6" />}
/>
</div>
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
<CircularGauge
value={latestResult.memory_score * 100}
label="Memory"
size="md"
variant="memory"
icon={<CircleStackIcon className="w-6 h-6" />}
/>
</div>
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
<CircularGauge
value={latestResult.disk_read_score * 100}
label="Disk Read"
size="md"
variant="disk"
icon={<ServerIcon className="w-6 h-6" />}
/>
</div>
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
<CircularGauge
value={latestResult.disk_write_score * 100}
label="Disk Write"
size="md"
variant="disk"
icon={<ServerIcon className="w-6 h-6" />}
/>
</div>
</div>
</section>
{/* AI Performance Section */}
<section className="mb-12">
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
<div className="w-1 h-6 bg-desert-green" />
AI Performance
</h2>
{latestResult.ai_tokens_per_second ? (
<div className="grid grid-cols-1 md:grid-cols-3 gap-6">
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
<CircularGauge
value={getAIScore(latestResult.ai_tokens_per_second)}
label="AI Score"
size="md"
variant="cpu"
icon={<IconRobot className="w-6 h-6" />}
/>
</div>
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm flex items-center justify-center">
<div className="flex items-center gap-4">
<IconRobot className="w-10 h-10 text-desert-green" />
<div>
<div className="text-3xl font-bold text-desert-green">
{latestResult.ai_tokens_per_second.toFixed(1)}
</div>
<div className="text-sm text-desert-stone-dark">Tokens per Second</div>
</div>
</div>
</div>
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm flex items-center justify-center">
<div className="flex items-center gap-4">
<IconRobot className="w-10 h-10 text-desert-green" />
<div>
<div className="text-3xl font-bold text-desert-green">
{latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms
</div>
<div className="text-sm text-desert-stone-dark">Time to First Token</div>
</div>
</div>
</div>
</div>
) : (
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
<div className="text-center text-desert-stone-dark">
<IconRobot className="w-12 h-12 mx-auto mb-3 opacity-40" />
<p className="font-medium">No AI Benchmark Data</p>
<p className="text-sm mt-1">
Run a Full Benchmark or AI Only benchmark to measure AI inference performance.
</p>
</div>
</div>
)}
</section>
<section className="mb-12">
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
<div className="w-1 h-6 bg-desert-green" />
Hardware Information
</h2>
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6">
<InfoCard
title="Processor"
icon={<CpuChipIcon className="w-6 h-6" />}
variant="elevated"
data={[
{ label: 'Model', value: latestResult.cpu_model },
{ label: 'Cores', value: latestResult.cpu_cores },
{ label: 'Threads', value: latestResult.cpu_threads },
]}
/>
<InfoCard
title="System"
icon={<ServerIcon className="w-6 h-6" />}
variant="elevated"
data={[
{ label: 'RAM', value: formatBytes(latestResult.ram_bytes) },
{ label: 'Disk Type', value: latestResult.disk_type.toUpperCase() },
{ label: 'GPU', value: latestResult.gpu_model || 'Not detected' },
]}
/>
</div>
</section>
<section>
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
<div className="w-1 h-6 bg-desert-green" />
Benchmark Details
</h2>
<div className="bg-desert-white rounded-lg border border-desert-stone-light shadow-sm overflow-hidden">
{/* Summary row - always visible */}
<button
onClick={() => setShowDetails(!showDetails)}
className="w-full p-6 flex items-center justify-between hover:bg-desert-stone-lighter/30 transition-colors"
>
<div className="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm text-left flex-1">
<div>
<div className="text-desert-stone-dark">Benchmark ID</div>
<div className="font-mono text-xs">{latestResult.benchmark_id.slice(0, 8)}...</div>
</div>
<div>
<div className="text-desert-stone-dark">Type</div>
<div className="capitalize">{latestResult.benchmark_type}</div>
</div>
<div>
<div className="text-desert-stone-dark">Date</div>
<div>{new Date(latestResult.created_at).toLocaleDateString()}</div>
</div>
<div>
<div className="text-desert-stone-dark">NOMAD Score</div>
<div className="font-bold text-desert-green">{latestResult.nomad_score.toFixed(1)}</div>
</div>
</div>
<ChevronDownIcon
className={`w-5 h-5 text-desert-stone-dark transition-transform ${showDetails ? 'rotate-180' : ''}`}
/>
</button>
{/* Expanded details */}
{showDetails && (
<div className="border-t border-desert-stone-light p-6 bg-desert-stone-lighter/20">
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
{/* Raw Scores */}
<div>
<h4 className="font-semibold text-desert-green mb-3">Raw Scores</h4>
<div className="space-y-2 text-sm">
<div className="flex justify-between">
<span className="text-desert-stone-dark">CPU Score</span>
<span className="font-mono">{(latestResult.cpu_score * 100).toFixed(1)}%</span>
</div>
<div className="flex justify-between">
<span className="text-desert-stone-dark">Memory Score</span>
<span className="font-mono">{(latestResult.memory_score * 100).toFixed(1)}%</span>
</div>
<div className="flex justify-between">
<span className="text-desert-stone-dark">Disk Read Score</span>
<span className="font-mono">{(latestResult.disk_read_score * 100).toFixed(1)}%</span>
</div>
<div className="flex justify-between">
<span className="text-desert-stone-dark">Disk Write Score</span>
<span className="font-mono">{(latestResult.disk_write_score * 100).toFixed(1)}%</span>
</div>
{latestResult.ai_tokens_per_second && (
<>
<div className="flex justify-between">
<span className="text-desert-stone-dark">AI Tokens/sec</span>
<span className="font-mono">{latestResult.ai_tokens_per_second.toFixed(1)}</span>
</div>
<div className="flex justify-between">
<span className="text-desert-stone-dark">AI Time to First Token</span>
<span className="font-mono">{latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms</span>
</div>
</>
)}
</div>
</div>
{/* Benchmark Info */}
<div>
<h4 className="font-semibold text-desert-green mb-3">Benchmark Info</h4>
<div className="space-y-2 text-sm">
<div className="flex justify-between">
<span className="text-desert-stone-dark">Full Benchmark ID</span>
<span className="font-mono text-xs">{latestResult.benchmark_id}</span>
</div>
<div className="flex justify-between">
<span className="text-desert-stone-dark">Benchmark Type</span>
<span className="capitalize">{latestResult.benchmark_type}</span>
</div>
<div className="flex justify-between">
<span className="text-desert-stone-dark">Run Date</span>
<span>{new Date(latestResult.created_at).toLocaleString()}</span>
</div>
{latestResult.ai_model_used && (
<div className="flex justify-between">
<span className="text-desert-stone-dark">AI Model Used</span>
<span>{latestResult.ai_model_used}</span>
</div>
)}
<div className="flex justify-between">
<span className="text-desert-stone-dark">Submitted to Repository</span>
<span>{latestResult.submitted_to_repository ? 'Yes' : 'No'}</span>
</div>
{latestResult.repository_id && (
<div className="flex justify-between">
<span className="text-desert-stone-dark">Repository ID</span>
<span className="font-mono text-xs">{latestResult.repository_id}</span>
</div>
)}
</div>
</div>
</div>
</div>
)}
</div>
</section>
</>
)}
{!latestResult && !isRunning && (
<Alert
type="info"
title="No Benchmark Results"
message="Run your first benchmark to see your server's performance scores."
variant="bordered"
/>
)}
</main>
</div>
</SettingsLayout>
)
}

View File

@ -55,6 +55,7 @@
"tar": "^7.5.6",
"url-join": "^5.0.0",
"usehooks-ts": "^3.1.1",
"uuid": "^13.0.0",
"yaml": "^2.8.0"
},
"devDependencies": {
@ -72,6 +73,7 @@
"@types/node": "^22.15.18",
"@types/react": "^19.1.8",
"@types/react-dom": "^19.1.6",
"@types/uuid": "^10.0.0",
"eslint": "^9.26.0",
"hot-hook": "^0.4.0",
"prettier": "^3.5.3",
@ -4304,6 +4306,13 @@
"@types/geojson": "*"
}
},
"node_modules/@types/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/validator": {
"version": "13.15.2",
"resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.2.tgz",
@ -5172,6 +5181,19 @@
"uuid": "^11.1.0"
}
},
"node_modules/bullmq/node_modules/uuid": {
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
"integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/esm/bin/uuid"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@ -11982,16 +12004,16 @@
"license": "MIT"
},
"node_modules/uuid": {
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
"integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
"version": "13.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz",
"integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/esm/bin/uuid"
"uuid": "dist-node/bin/uuid"
}
},
"node_modules/v8-compile-cache-lib": {

View File

@ -50,6 +50,7 @@
"@types/node": "^22.15.18",
"@types/react": "^19.1.8",
"@types/react-dom": "^19.1.6",
"@types/uuid": "^10.0.0",
"eslint": "^9.26.0",
"hot-hook": "^0.4.0",
"prettier": "^3.5.3",
@ -104,6 +105,7 @@
"tar": "^7.5.6",
"url-join": "^5.0.0",
"usehooks-ts": "^3.1.1",
"uuid": "^13.0.0",
"yaml": "^2.8.0"
},
"hotHook": {

View File

@ -6,6 +6,7 @@
| The routes file is used for defining the HTTP routes.
|
*/
import BenchmarkController from '#controllers/benchmark_controller'
import DocsController from '#controllers/docs_controller'
import DownloadsController from '#controllers/downloads_controller'
import EasySetupController from '#controllers/easy_setup_controller'
@ -38,6 +39,7 @@ router
router.get('/update', [SettingsController, 'update'])
router.get('/zim', [SettingsController, 'zim'])
router.get('/zim/remote-explorer', [SettingsController, 'zimRemote'])
router.get('/benchmark', [SettingsController, 'benchmark'])
})
.prefix('/settings')
@ -122,3 +124,19 @@ router
router.delete('/:filename', [ZimController, 'delete'])
})
.prefix('/api/zim')
router
.group(() => {
router.post('/run', [BenchmarkController, 'run'])
router.post('/run/system', [BenchmarkController, 'runSystem'])
router.post('/run/ai', [BenchmarkController, 'runAI'])
router.get('/results', [BenchmarkController, 'results'])
router.get('/results/latest', [BenchmarkController, 'latest'])
router.get('/results/:id', [BenchmarkController, 'show'])
router.post('/submit', [BenchmarkController, 'submit'])
router.get('/comparison', [BenchmarkController, 'comparison'])
router.get('/status', [BenchmarkController, 'status'])
router.get('/settings', [BenchmarkController, 'settings'])
router.post('/settings', [BenchmarkController, 'updateSettings'])
})
.prefix('/api/benchmark')

216
admin/types/benchmark.ts Normal file
View File

@ -0,0 +1,216 @@
// Benchmark type identifiers
export type BenchmarkType = 'full' | 'system' | 'ai'
// Benchmark execution status
export type BenchmarkStatus =
| 'idle'
| 'starting'
| 'detecting_hardware'
| 'running_cpu'
| 'running_memory'
| 'running_disk_read'
| 'running_disk_write'
| 'running_ai'
| 'calculating_score'
| 'completed'
| 'error'
// Hardware detection types
export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown'
export type HardwareInfo = {
cpu_model: string
cpu_cores: number
cpu_threads: number
ram_bytes: number
disk_type: DiskType
gpu_model: string | null
}
// Individual benchmark scores
export type SystemScores = {
cpu_score: number
memory_score: number
disk_read_score: number
disk_write_score: number
}
export type AIScores = {
ai_tokens_per_second: number
ai_model_used: string
ai_time_to_first_token: number
}
// Complete benchmark result
export type BenchmarkResult = {
id: number
benchmark_id: string
benchmark_type: BenchmarkType
hardware: HardwareInfo
scores: SystemScores & Partial<AIScores>
nomad_score: number
submitted_to_repository: boolean
submitted_at: string | null
repository_id: string | null
created_at: string
updated_at: string
}
// Slim version for lists
export type BenchmarkResultSlim = Pick<
BenchmarkResult,
| 'id'
| 'benchmark_id'
| 'benchmark_type'
| 'nomad_score'
| 'submitted_to_repository'
| 'created_at'
> & {
cpu_model: string
gpu_model: string | null
}
// Benchmark settings key-value store
export type BenchmarkSettingKey =
| 'allow_anonymous_submission'
| 'installation_id'
| 'last_benchmark_run'
export type BenchmarkSettings = {
allow_anonymous_submission: boolean
installation_id: string | null
last_benchmark_run: string | null
}
// Progress update for real-time feedback
export type BenchmarkProgress = {
status: BenchmarkStatus
progress: number
message: string
current_stage: string
timestamp: string
}
// API request types
export type RunBenchmarkParams = {
benchmark_type: BenchmarkType
}
export type SubmitBenchmarkParams = {
benchmark_id?: string
}
// API response types
export type RunBenchmarkResponse = {
success: boolean
job_id: string
benchmark_id: string
message: string
}
export type BenchmarkResultsResponse = {
results: BenchmarkResult[]
total: number
}
// Central repository submission payload (privacy-first)
export type RepositorySubmission = {
cpu_model: string
cpu_cores: number
cpu_threads: number
ram_gb: number
disk_type: DiskType
gpu_model: string | null
cpu_score: number
memory_score: number
disk_read_score: number
disk_write_score: number
ai_tokens_per_second: number | null
ai_time_to_first_token: number | null
nomad_score: number
nomad_version: string
benchmark_version: string
}
// Central repository response types
export type RepositorySubmitResponse = {
success: boolean
repository_id: string
percentile: number
}
export type RepositoryStats = {
total_submissions: number
average_score: number
median_score: number
top_score: number
percentiles: {
p10: number
p25: number
p50: number
p75: number
p90: number
}
}
export type LeaderboardEntry = {
rank: number
cpu_model: string
gpu_model: string | null
nomad_score: number
submitted_at: string
}
export type ComparisonResponse = {
matching_submissions: number
average_score: number
your_percentile: number | null
}
// Score calculation weights (for reference in UI)
export type ScoreWeights = {
ai_tokens_per_second: number
cpu: number
memory: number
ai_ttft: number
disk_read: number
disk_write: number
}
// Default weights as defined in plan
export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = {
ai_tokens_per_second: 0.30,
cpu: 0.25,
memory: 0.15,
ai_ttft: 0.10,
disk_read: 0.10,
disk_write: 0.10,
}
// Benchmark job parameters
export type RunBenchmarkJobParams = {
benchmark_id: string
benchmark_type: BenchmarkType
include_ai: boolean
}
// sysbench result parsing types
export type SysbenchCpuResult = {
events_per_second: number
total_time: number
total_events: number
}
export type SysbenchMemoryResult = {
operations_per_second: number
transfer_rate_mb_per_sec: number
total_time: number
}
export type SysbenchDiskResult = {
reads_per_second: number
writes_per_second: number
read_mb_per_sec: number
write_mb_per_sec: number
total_time: number
}