mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-03-28 03:29:25 +01:00
feat: Add system benchmark feature with NOMAD Score
Add comprehensive benchmarking capability to measure server performance: Backend: - BenchmarkService with CPU, memory, disk, and AI benchmarks using sysbench - Database migrations for benchmark_results and benchmark_settings tables - REST API endpoints for running benchmarks and retrieving results - CLI commands: benchmark:run, benchmark:results, benchmark:submit - BullMQ job for async benchmark execution with SSE progress updates - Synchronous mode option (?sync=true) for simpler local dev setup Frontend: - Benchmark settings page with circular gauges for scores - NOMAD Score display with weighted composite calculation - System Performance section (CPU, Memory, Disk Read/Write) - AI Performance section (tokens/sec, time to first token) - Hardware Information display - Expandable Benchmark Details section - Progress simulation during sync benchmark execution Easy Setup Integration: - Added System Benchmark to Additional Tools section - Built-in capability pattern for non-Docker features - Click-to-navigate behavior for built-in tools Fixes: - Docker log multiplexing issue (Tty: true) for proper output parsing - Consolidated disk benchmarks into single container execution Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
6bee84f367
commit
755807f95e
230
admin/app/controllers/benchmark_controller.ts
Normal file
230
admin/app/controllers/benchmark_controller.ts
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
import { inject } from '@adonisjs/core'
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import { BenchmarkService } from '#services/benchmark_service'
|
||||
import { runBenchmarkValidator, submitBenchmarkValidator } from '#validators/benchmark'
|
||||
import { RunBenchmarkJob } from '#jobs/run_benchmark_job'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import type { BenchmarkType } from '../../types/benchmark.js'
|
||||
|
||||
@inject()
|
||||
export default class BenchmarkController {
|
||||
constructor(private benchmarkService: BenchmarkService) {}
|
||||
|
||||
/**
|
||||
* Start a benchmark run (async via job queue, or sync if specified)
|
||||
*/
|
||||
async run({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(runBenchmarkValidator)
|
||||
const benchmarkType: BenchmarkType = payload.benchmark_type || 'full'
|
||||
const runSync = request.input('sync') === 'true' || request.input('sync') === true
|
||||
|
||||
// Check if a benchmark is already running
|
||||
const status = this.benchmarkService.getStatus()
|
||||
if (status.status !== 'idle') {
|
||||
return response.status(409).send({
|
||||
success: false,
|
||||
error: 'A benchmark is already running',
|
||||
current_benchmark_id: status.benchmarkId,
|
||||
})
|
||||
}
|
||||
|
||||
// Run synchronously if requested (useful for local dev without Redis)
|
||||
if (runSync) {
|
||||
try {
|
||||
let result
|
||||
switch (benchmarkType) {
|
||||
case 'full':
|
||||
result = await this.benchmarkService.runFullBenchmark()
|
||||
break
|
||||
case 'system':
|
||||
result = await this.benchmarkService.runSystemBenchmarks()
|
||||
break
|
||||
case 'ai':
|
||||
result = await this.benchmarkService.runAIBenchmark()
|
||||
break
|
||||
default:
|
||||
result = await this.benchmarkService.runFullBenchmark()
|
||||
}
|
||||
return response.send({
|
||||
success: true,
|
||||
benchmark_id: result.benchmark_id,
|
||||
nomad_score: result.nomad_score,
|
||||
result,
|
||||
})
|
||||
} catch (error) {
|
||||
return response.status(500).send({
|
||||
success: false,
|
||||
error: error.message,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Generate benchmark ID and dispatch job (async)
|
||||
const benchmarkId = uuidv4()
|
||||
const { job, created } = await RunBenchmarkJob.dispatch({
|
||||
benchmark_id: benchmarkId,
|
||||
benchmark_type: benchmarkType,
|
||||
include_ai: benchmarkType === 'full' || benchmarkType === 'ai',
|
||||
})
|
||||
|
||||
return response.status(201).send({
|
||||
success: true,
|
||||
job_id: job?.id || benchmarkId,
|
||||
benchmark_id: benchmarkId,
|
||||
message: created
|
||||
? `${benchmarkType} benchmark started`
|
||||
: 'Benchmark job already exists',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a system-only benchmark (CPU, memory, disk)
|
||||
*/
|
||||
async runSystem({ response }: HttpContext) {
|
||||
const status = this.benchmarkService.getStatus()
|
||||
if (status.status !== 'idle') {
|
||||
return response.status(409).send({
|
||||
success: false,
|
||||
error: 'A benchmark is already running',
|
||||
})
|
||||
}
|
||||
|
||||
const benchmarkId = uuidv4()
|
||||
await RunBenchmarkJob.dispatch({
|
||||
benchmark_id: benchmarkId,
|
||||
benchmark_type: 'system',
|
||||
include_ai: false,
|
||||
})
|
||||
|
||||
return response.status(201).send({
|
||||
success: true,
|
||||
benchmark_id: benchmarkId,
|
||||
message: 'System benchmark started',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Run an AI-only benchmark
|
||||
*/
|
||||
async runAI({ response }: HttpContext) {
|
||||
const status = this.benchmarkService.getStatus()
|
||||
if (status.status !== 'idle') {
|
||||
return response.status(409).send({
|
||||
success: false,
|
||||
error: 'A benchmark is already running',
|
||||
})
|
||||
}
|
||||
|
||||
const benchmarkId = uuidv4()
|
||||
await RunBenchmarkJob.dispatch({
|
||||
benchmark_id: benchmarkId,
|
||||
benchmark_type: 'ai',
|
||||
include_ai: true,
|
||||
})
|
||||
|
||||
return response.status(201).send({
|
||||
success: true,
|
||||
benchmark_id: benchmarkId,
|
||||
message: 'AI benchmark started',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all benchmark results
|
||||
*/
|
||||
async results({}: HttpContext) {
|
||||
const results = await this.benchmarkService.getAllResults()
|
||||
return {
|
||||
results,
|
||||
total: results.length,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the latest benchmark result
|
||||
*/
|
||||
async latest({}: HttpContext) {
|
||||
const result = await this.benchmarkService.getLatestResult()
|
||||
if (!result) {
|
||||
return { result: null }
|
||||
}
|
||||
return { result }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific benchmark result by ID
|
||||
*/
|
||||
async show({ params, response }: HttpContext) {
|
||||
const result = await this.benchmarkService.getResultById(params.id)
|
||||
if (!result) {
|
||||
return response.status(404).send({
|
||||
error: 'Benchmark result not found',
|
||||
})
|
||||
}
|
||||
return { result }
|
||||
}
|
||||
|
||||
/**
|
||||
* Submit benchmark results to central repository
|
||||
*/
|
||||
async submit({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(submitBenchmarkValidator)
|
||||
|
||||
try {
|
||||
const submitResult = await this.benchmarkService.submitToRepository(payload.benchmark_id)
|
||||
return response.send({
|
||||
success: true,
|
||||
repository_id: submitResult.repository_id,
|
||||
percentile: submitResult.percentile,
|
||||
})
|
||||
} catch (error) {
|
||||
return response.status(400).send({
|
||||
success: false,
|
||||
error: error.message,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get comparison stats from central repository
|
||||
*/
|
||||
async comparison({}: HttpContext) {
|
||||
const stats = await this.benchmarkService.getComparisonStats()
|
||||
return { stats }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current benchmark status
|
||||
*/
|
||||
async status({}: HttpContext) {
|
||||
return this.benchmarkService.getStatus()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get benchmark settings
|
||||
*/
|
||||
async settings({}: HttpContext) {
|
||||
const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
|
||||
return await BenchmarkSetting.getAllSettings()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update benchmark settings
|
||||
*/
|
||||
async updateSettings({ request, response }: HttpContext) {
|
||||
const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
|
||||
const body = request.body()
|
||||
|
||||
if (body.allow_anonymous_submission !== undefined) {
|
||||
await BenchmarkSetting.setValue(
|
||||
'allow_anonymous_submission',
|
||||
body.allow_anonymous_submission ? 'true' : 'false'
|
||||
)
|
||||
}
|
||||
|
||||
return response.send({
|
||||
success: true,
|
||||
settings: await BenchmarkSetting.getAllSettings(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { BenchmarkService } from '#services/benchmark_service';
|
||||
import { MapService } from '#services/map_service';
|
||||
import { OpenWebUIService } from '#services/openwebui_service';
|
||||
import { SystemService } from '#services/system_service';
|
||||
|
|
@ -9,7 +10,8 @@ export default class SettingsController {
|
|||
constructor(
|
||||
private systemService: SystemService,
|
||||
private mapService: MapService,
|
||||
private openWebUIService: OpenWebUIService
|
||||
private openWebUIService: OpenWebUIService,
|
||||
private benchmarkService: BenchmarkService
|
||||
) { }
|
||||
|
||||
async system({ inertia }: HttpContext) {
|
||||
|
|
@ -74,4 +76,16 @@ export default class SettingsController {
|
|||
async zimRemote({ inertia }: HttpContext) {
|
||||
return inertia.render('settings/zim/remote-explorer');
|
||||
}
|
||||
|
||||
async benchmark({ inertia }: HttpContext) {
|
||||
const latestResult = await this.benchmarkService.getLatestResult();
|
||||
const status = this.benchmarkService.getStatus();
|
||||
return inertia.render('settings/benchmark', {
|
||||
benchmark: {
|
||||
latestResult,
|
||||
status: status.status,
|
||||
currentBenchmarkId: status.benchmarkId
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
99
admin/app/jobs/run_benchmark_job.ts
Normal file
99
admin/app/jobs/run_benchmark_job.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { Job } from 'bullmq'
|
||||
import { QueueService } from '#services/queue_service'
|
||||
import { BenchmarkService } from '#services/benchmark_service'
|
||||
import type { RunBenchmarkJobParams } from '../../types/benchmark.js'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
export class RunBenchmarkJob {
|
||||
static get queue() {
|
||||
return 'benchmarks'
|
||||
}
|
||||
|
||||
static get key() {
|
||||
return 'run-benchmark'
|
||||
}
|
||||
|
||||
async handle(job: Job) {
|
||||
const { benchmark_id, benchmark_type } = job.data as RunBenchmarkJobParams
|
||||
|
||||
logger.info(`[RunBenchmarkJob] Starting benchmark ${benchmark_id} of type ${benchmark_type}`)
|
||||
|
||||
const benchmarkService = new BenchmarkService()
|
||||
|
||||
try {
|
||||
let result
|
||||
|
||||
switch (benchmark_type) {
|
||||
case 'full':
|
||||
result = await benchmarkService.runFullBenchmark()
|
||||
break
|
||||
case 'system':
|
||||
result = await benchmarkService.runSystemBenchmarks()
|
||||
break
|
||||
case 'ai':
|
||||
result = await benchmarkService.runAIBenchmark()
|
||||
break
|
||||
default:
|
||||
throw new Error(`Unknown benchmark type: ${benchmark_type}`)
|
||||
}
|
||||
|
||||
logger.info(`[RunBenchmarkJob] Benchmark ${benchmark_id} completed with NOMAD score: ${result.nomad_score}`)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
benchmark_id: result.benchmark_id,
|
||||
nomad_score: result.nomad_score,
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[RunBenchmarkJob] Benchmark ${benchmark_id} failed: ${error.message}`)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
static async dispatch(params: RunBenchmarkJobParams) {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
|
||||
try {
|
||||
const job = await queue.add(this.key, params, {
|
||||
jobId: params.benchmark_id,
|
||||
attempts: 1, // Benchmarks shouldn't be retried automatically
|
||||
removeOnComplete: {
|
||||
count: 10, // Keep last 10 completed jobs
|
||||
},
|
||||
removeOnFail: {
|
||||
count: 5, // Keep last 5 failed jobs
|
||||
},
|
||||
})
|
||||
|
||||
logger.info(`[RunBenchmarkJob] Dispatched benchmark job ${params.benchmark_id}`)
|
||||
|
||||
return {
|
||||
job,
|
||||
created: true,
|
||||
message: `Benchmark job ${params.benchmark_id} dispatched successfully`,
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.message.includes('job already exists')) {
|
||||
const existing = await queue.getJob(params.benchmark_id)
|
||||
return {
|
||||
job: existing,
|
||||
created: false,
|
||||
message: `Benchmark job ${params.benchmark_id} already exists`,
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
static async getJob(benchmarkId: string): Promise<Job | undefined> {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
return await queue.getJob(benchmarkId)
|
||||
}
|
||||
|
||||
static async getJobState(benchmarkId: string): Promise<string | undefined> {
|
||||
const job = await this.getJob(benchmarkId)
|
||||
return job ? await job.getState() : undefined
|
||||
}
|
||||
}
|
||||
82
admin/app/models/benchmark_result.ts
Normal file
82
admin/app/models/benchmark_result.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
|
||||
import type { BenchmarkType, DiskType } from '../../types/benchmark.js'
|
||||
|
||||
export default class BenchmarkResult extends BaseModel {
|
||||
static namingStrategy = new SnakeCaseNamingStrategy()
|
||||
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare benchmark_id: string
|
||||
|
||||
@column()
|
||||
declare benchmark_type: BenchmarkType
|
||||
|
||||
// Hardware information
|
||||
@column()
|
||||
declare cpu_model: string
|
||||
|
||||
@column()
|
||||
declare cpu_cores: number
|
||||
|
||||
@column()
|
||||
declare cpu_threads: number
|
||||
|
||||
@column()
|
||||
declare ram_bytes: number
|
||||
|
||||
@column()
|
||||
declare disk_type: DiskType
|
||||
|
||||
@column()
|
||||
declare gpu_model: string | null
|
||||
|
||||
// System benchmark scores
|
||||
@column()
|
||||
declare cpu_score: number
|
||||
|
||||
@column()
|
||||
declare memory_score: number
|
||||
|
||||
@column()
|
||||
declare disk_read_score: number
|
||||
|
||||
@column()
|
||||
declare disk_write_score: number
|
||||
|
||||
// AI benchmark scores (nullable for system-only benchmarks)
|
||||
@column()
|
||||
declare ai_tokens_per_second: number | null
|
||||
|
||||
@column()
|
||||
declare ai_model_used: string | null
|
||||
|
||||
@column()
|
||||
declare ai_time_to_first_token: number | null
|
||||
|
||||
// Composite NOMAD score (0-100)
|
||||
@column()
|
||||
declare nomad_score: number
|
||||
|
||||
// Repository submission tracking
|
||||
@column({
|
||||
serialize(value) {
|
||||
return Boolean(value)
|
||||
},
|
||||
})
|
||||
declare submitted_to_repository: boolean
|
||||
|
||||
@column.dateTime()
|
||||
declare submitted_at: DateTime | null
|
||||
|
||||
@column()
|
||||
declare repository_id: string | null
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare created_at: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updated_at: DateTime
|
||||
}
|
||||
60
admin/app/models/benchmark_setting.ts
Normal file
60
admin/app/models/benchmark_setting.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
|
||||
import type { BenchmarkSettingKey } from '../../types/benchmark.js'
|
||||
|
||||
export default class BenchmarkSetting extends BaseModel {
|
||||
static namingStrategy = new SnakeCaseNamingStrategy()
|
||||
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare key: BenchmarkSettingKey
|
||||
|
||||
@column()
|
||||
declare value: string | null
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare created_at: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updated_at: DateTime
|
||||
|
||||
/**
|
||||
* Get a setting value by key
|
||||
*/
|
||||
static async getValue(key: BenchmarkSettingKey): Promise<string | null> {
|
||||
const setting = await this.findBy('key', key)
|
||||
return setting?.value ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a setting value by key (creates if not exists)
|
||||
*/
|
||||
static async setValue(key: BenchmarkSettingKey, value: string | null): Promise<BenchmarkSetting> {
|
||||
const setting = await this.firstOrCreate({ key }, { key, value })
|
||||
if (setting.value !== value) {
|
||||
setting.value = value
|
||||
await setting.save()
|
||||
}
|
||||
return setting
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all benchmark settings as a typed object
|
||||
*/
|
||||
static async getAllSettings(): Promise<{
|
||||
allow_anonymous_submission: boolean
|
||||
installation_id: string | null
|
||||
last_benchmark_run: string | null
|
||||
}> {
|
||||
const settings = await this.all()
|
||||
const map = new Map(settings.map((s) => [s.key, s.value]))
|
||||
|
||||
return {
|
||||
allow_anonymous_submission: map.get('allow_anonymous_submission') === 'true',
|
||||
installation_id: map.get('installation_id') ?? null,
|
||||
last_benchmark_run: map.get('last_benchmark_run') ?? null,
|
||||
}
|
||||
}
|
||||
}
|
||||
723
admin/app/services/benchmark_service.ts
Normal file
723
admin/app/services/benchmark_service.ts
Normal file
|
|
@ -0,0 +1,723 @@
|
|||
import { inject } from '@adonisjs/core'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import transmit from '@adonisjs/transmit/services/main'
|
||||
import Docker from 'dockerode'
|
||||
import si from 'systeminformation'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import axios from 'axios'
|
||||
import BenchmarkResult from '#models/benchmark_result'
|
||||
import BenchmarkSetting from '#models/benchmark_setting'
|
||||
import { SystemService } from '#services/system_service'
|
||||
import type {
|
||||
BenchmarkType,
|
||||
BenchmarkStatus,
|
||||
BenchmarkProgress,
|
||||
HardwareInfo,
|
||||
DiskType,
|
||||
SystemScores,
|
||||
AIScores,
|
||||
SysbenchCpuResult,
|
||||
SysbenchMemoryResult,
|
||||
SysbenchDiskResult,
|
||||
RepositorySubmission,
|
||||
RepositorySubmitResponse,
|
||||
RepositoryStats,
|
||||
} from '../../types/benchmark.js'
|
||||
|
||||
// Re-export default weights for use in service
|
||||
const SCORE_WEIGHTS = {
|
||||
ai_tokens_per_second: 0.30,
|
||||
cpu: 0.25,
|
||||
memory: 0.15,
|
||||
ai_ttft: 0.10,
|
||||
disk_read: 0.10,
|
||||
disk_write: 0.10,
|
||||
}
|
||||
|
||||
// Benchmark configuration constants
|
||||
const SYSBENCH_IMAGE = 'severalnines/sysbench:latest'
|
||||
const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench'
|
||||
const BENCHMARK_CHANNEL = 'benchmark-progress'
|
||||
|
||||
// Reference model for AI benchmark - small but meaningful
|
||||
const AI_BENCHMARK_MODEL = 'llama3.2:1b'
|
||||
const AI_BENCHMARK_PROMPT = 'Explain recursion in programming in exactly 100 words.'
|
||||
|
||||
// Reference scores for normalization (calibrated to 0-100 scale)
|
||||
// These represent "expected" scores for a mid-range system (score ~50)
|
||||
const REFERENCE_SCORES = {
|
||||
cpu_events_per_second: 5000, // sysbench cpu events/sec for ~50 score
|
||||
memory_ops_per_second: 5000000, // sysbench memory ops/sec for ~50 score
|
||||
disk_read_mb_per_sec: 500, // 500 MB/s read for ~50 score
|
||||
disk_write_mb_per_sec: 400, // 400 MB/s write for ~50 score
|
||||
ai_tokens_per_second: 30, // 30 tok/s for ~50 score
|
||||
ai_ttft_ms: 500, // 500ms time to first token for ~50 score (lower is better)
|
||||
}
|
||||
|
||||
@inject()
|
||||
export class BenchmarkService {
|
||||
private docker: Docker
|
||||
private currentBenchmarkId: string | null = null
|
||||
private currentStatus: BenchmarkStatus = 'idle'
|
||||
|
||||
constructor() {
|
||||
const isWindows = process.platform === 'win32'
|
||||
if (isWindows) {
|
||||
this.docker = new Docker({ socketPath: '//./pipe/docker_engine' })
|
||||
} else {
|
||||
this.docker = new Docker({ socketPath: '/var/run/docker.sock' })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a full benchmark suite
|
||||
*/
|
||||
async runFullBenchmark(): Promise<BenchmarkResult> {
|
||||
return this._runBenchmark('full', true)
|
||||
}
|
||||
|
||||
/**
|
||||
* Run system benchmarks only (CPU, memory, disk)
|
||||
*/
|
||||
async runSystemBenchmarks(): Promise<BenchmarkResult> {
|
||||
return this._runBenchmark('system', false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Run AI benchmark only
|
||||
*/
|
||||
async runAIBenchmark(): Promise<BenchmarkResult> {
|
||||
return this._runBenchmark('ai', true)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the latest benchmark result
|
||||
*/
|
||||
async getLatestResult(): Promise<BenchmarkResult | null> {
|
||||
return await BenchmarkResult.query().orderBy('created_at', 'desc').first()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all benchmark results
|
||||
*/
|
||||
async getAllResults(): Promise<BenchmarkResult[]> {
|
||||
return await BenchmarkResult.query().orderBy('created_at', 'desc')
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific benchmark result by ID
|
||||
*/
|
||||
async getResultById(benchmarkId: string): Promise<BenchmarkResult | null> {
|
||||
return await BenchmarkResult.findBy('benchmark_id', benchmarkId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Submit benchmark results to central repository
|
||||
*/
|
||||
async submitToRepository(benchmarkId?: string): Promise<RepositorySubmitResponse> {
|
||||
const result = benchmarkId
|
||||
? await this.getResultById(benchmarkId)
|
||||
: await this.getLatestResult()
|
||||
|
||||
if (!result) {
|
||||
throw new Error('No benchmark result found to submit')
|
||||
}
|
||||
|
||||
if (result.submitted_to_repository) {
|
||||
throw new Error('Benchmark result has already been submitted')
|
||||
}
|
||||
|
||||
const submission: RepositorySubmission = {
|
||||
cpu_model: result.cpu_model,
|
||||
cpu_cores: result.cpu_cores,
|
||||
cpu_threads: result.cpu_threads,
|
||||
ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)),
|
||||
disk_type: result.disk_type,
|
||||
gpu_model: result.gpu_model,
|
||||
cpu_score: result.cpu_score,
|
||||
memory_score: result.memory_score,
|
||||
disk_read_score: result.disk_read_score,
|
||||
disk_write_score: result.disk_write_score,
|
||||
ai_tokens_per_second: result.ai_tokens_per_second,
|
||||
ai_time_to_first_token: result.ai_time_to_first_token,
|
||||
nomad_score: result.nomad_score,
|
||||
nomad_version: SystemService.getAppVersion(),
|
||||
benchmark_version: '1.0.0',
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await axios.post(
|
||||
'https://benchmark.projectnomad.us/api/v1/submit',
|
||||
submission,
|
||||
{ timeout: 30000 }
|
||||
)
|
||||
|
||||
if (response.data.success) {
|
||||
result.submitted_to_repository = true
|
||||
result.submitted_at = new Date() as any
|
||||
result.repository_id = response.data.repository_id
|
||||
await result.save()
|
||||
|
||||
await BenchmarkSetting.setValue('last_benchmark_run', new Date().toISOString())
|
||||
}
|
||||
|
||||
return response.data as RepositorySubmitResponse
|
||||
} catch (error) {
|
||||
logger.error(`Failed to submit benchmark to repository: ${error.message}`)
|
||||
throw new Error(`Failed to submit benchmark: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get comparison stats from central repository
|
||||
*/
|
||||
async getComparisonStats(): Promise<RepositoryStats | null> {
|
||||
try {
|
||||
const response = await axios.get('https://benchmark.projectnomad.us/api/v1/stats', {
|
||||
timeout: 10000,
|
||||
})
|
||||
return response.data as RepositoryStats
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch comparison stats: ${error.message}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current benchmark status
|
||||
*/
|
||||
getStatus(): { status: BenchmarkStatus; benchmarkId: string | null } {
|
||||
return {
|
||||
status: this.currentStatus,
|
||||
benchmarkId: this.currentBenchmarkId,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect system hardware information
|
||||
*/
|
||||
async getHardwareInfo(): Promise<HardwareInfo> {
|
||||
this._updateStatus('detecting_hardware', 'Detecting system hardware...')
|
||||
|
||||
try {
|
||||
const [cpu, mem, diskLayout, graphics] = await Promise.all([
|
||||
si.cpu(),
|
||||
si.mem(),
|
||||
si.diskLayout(),
|
||||
si.graphics(),
|
||||
])
|
||||
|
||||
// Determine disk type from primary disk
|
||||
let diskType: DiskType = 'unknown'
|
||||
if (diskLayout.length > 0) {
|
||||
const primaryDisk = diskLayout[0]
|
||||
if (primaryDisk.type?.toLowerCase().includes('nvme')) {
|
||||
diskType = 'nvme'
|
||||
} else if (primaryDisk.type?.toLowerCase().includes('ssd')) {
|
||||
diskType = 'ssd'
|
||||
} else if (primaryDisk.type?.toLowerCase().includes('hdd') || primaryDisk.interfaceType === 'SATA') {
|
||||
// SATA could be SSD or HDD, check if it's rotational
|
||||
diskType = 'hdd'
|
||||
}
|
||||
}
|
||||
|
||||
// Get GPU model (prefer discrete GPU)
|
||||
let gpuModel: string | null = null
|
||||
if (graphics.controllers && graphics.controllers.length > 0) {
|
||||
const discreteGpu = graphics.controllers.find(
|
||||
(g) => !g.vendor?.toLowerCase().includes('intel') &&
|
||||
!g.vendor?.toLowerCase().includes('amd') ||
|
||||
(g.vram && g.vram > 0)
|
||||
)
|
||||
gpuModel = discreteGpu?.model || graphics.controllers[0]?.model || null
|
||||
}
|
||||
|
||||
return {
|
||||
cpu_model: `${cpu.manufacturer} ${cpu.brand}`,
|
||||
cpu_cores: cpu.physicalCores,
|
||||
cpu_threads: cpu.cores,
|
||||
ram_bytes: mem.total,
|
||||
disk_type: diskType,
|
||||
gpu_model: gpuModel,
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Error detecting hardware: ${error.message}`)
|
||||
throw new Error(`Failed to detect hardware: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main benchmark execution method
|
||||
*/
|
||||
private async _runBenchmark(type: BenchmarkType, includeAI: boolean): Promise<BenchmarkResult> {
|
||||
if (this.currentStatus !== 'idle') {
|
||||
throw new Error('A benchmark is already running')
|
||||
}
|
||||
|
||||
this.currentBenchmarkId = uuidv4()
|
||||
this._updateStatus('starting', 'Starting benchmark...')
|
||||
|
||||
try {
|
||||
// Detect hardware
|
||||
const hardware = await this.getHardwareInfo()
|
||||
|
||||
// Run system benchmarks
|
||||
let systemScores: SystemScores = {
|
||||
cpu_score: 0,
|
||||
memory_score: 0,
|
||||
disk_read_score: 0,
|
||||
disk_write_score: 0,
|
||||
}
|
||||
|
||||
if (type === 'full' || type === 'system') {
|
||||
systemScores = await this._runSystemBenchmarks()
|
||||
}
|
||||
|
||||
// Run AI benchmark if requested and Ollama is available
|
||||
let aiScores: Partial<AIScores> = {}
|
||||
if (includeAI && (type === 'full' || type === 'ai')) {
|
||||
try {
|
||||
aiScores = await this._runAIBenchmark()
|
||||
} catch (error) {
|
||||
logger.warn(`AI benchmark skipped: ${error.message}`)
|
||||
// AI benchmark is optional, continue without it
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate NOMAD score
|
||||
this._updateStatus('calculating_score', 'Calculating NOMAD score...')
|
||||
const nomadScore = this._calculateNomadScore(systemScores, aiScores)
|
||||
|
||||
// Save result
|
||||
const result = await BenchmarkResult.create({
|
||||
benchmark_id: this.currentBenchmarkId,
|
||||
benchmark_type: type,
|
||||
cpu_model: hardware.cpu_model,
|
||||
cpu_cores: hardware.cpu_cores,
|
||||
cpu_threads: hardware.cpu_threads,
|
||||
ram_bytes: hardware.ram_bytes,
|
||||
disk_type: hardware.disk_type,
|
||||
gpu_model: hardware.gpu_model,
|
||||
cpu_score: systemScores.cpu_score,
|
||||
memory_score: systemScores.memory_score,
|
||||
disk_read_score: systemScores.disk_read_score,
|
||||
disk_write_score: systemScores.disk_write_score,
|
||||
ai_tokens_per_second: aiScores.ai_tokens_per_second || null,
|
||||
ai_model_used: aiScores.ai_model_used || null,
|
||||
ai_time_to_first_token: aiScores.ai_time_to_first_token || null,
|
||||
nomad_score: nomadScore,
|
||||
submitted_to_repository: false,
|
||||
})
|
||||
|
||||
this._updateStatus('completed', 'Benchmark completed successfully')
|
||||
this.currentStatus = 'idle'
|
||||
this.currentBenchmarkId = null
|
||||
|
||||
return result
|
||||
} catch (error) {
|
||||
this._updateStatus('error', `Benchmark failed: ${error.message}`)
|
||||
this.currentStatus = 'idle'
|
||||
this.currentBenchmarkId = null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run system benchmarks using sysbench in Docker
|
||||
*/
|
||||
private async _runSystemBenchmarks(): Promise<SystemScores> {
|
||||
// Ensure sysbench image is available
|
||||
await this._ensureSysbenchImage()
|
||||
|
||||
// Run CPU benchmark
|
||||
this._updateStatus('running_cpu', 'Running CPU benchmark...')
|
||||
const cpuResult = await this._runSysbenchCpu()
|
||||
|
||||
// Run memory benchmark
|
||||
this._updateStatus('running_memory', 'Running memory benchmark...')
|
||||
const memoryResult = await this._runSysbenchMemory()
|
||||
|
||||
// Run disk benchmarks
|
||||
this._updateStatus('running_disk_read', 'Running disk read benchmark...')
|
||||
const diskReadResult = await this._runSysbenchDiskRead()
|
||||
|
||||
this._updateStatus('running_disk_write', 'Running disk write benchmark...')
|
||||
const diskWriteResult = await this._runSysbenchDiskWrite()
|
||||
|
||||
// Normalize scores to 0-100 scale
|
||||
return {
|
||||
cpu_score: this._normalizeScore(cpuResult.events_per_second, REFERENCE_SCORES.cpu_events_per_second),
|
||||
memory_score: this._normalizeScore(memoryResult.operations_per_second, REFERENCE_SCORES.memory_ops_per_second),
|
||||
disk_read_score: this._normalizeScore(diskReadResult.read_mb_per_sec, REFERENCE_SCORES.disk_read_mb_per_sec),
|
||||
disk_write_score: this._normalizeScore(diskWriteResult.write_mb_per_sec, REFERENCE_SCORES.disk_write_mb_per_sec),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run AI benchmark using Ollama
|
||||
*/
|
||||
private async _runAIBenchmark(): Promise<AIScores> {
|
||||
this._updateStatus('running_ai', 'Running AI benchmark...')
|
||||
|
||||
// Check if Ollama is available
|
||||
try {
|
||||
await axios.get('http://localhost:11434/api/tags', { timeout: 5000 })
|
||||
} catch {
|
||||
throw new Error('Ollama is not running or not accessible')
|
||||
}
|
||||
|
||||
// Check if the benchmark model is available, pull if not
|
||||
try {
|
||||
const modelsResponse = await axios.get('http://localhost:11434/api/tags')
|
||||
const models = modelsResponse.data.models || []
|
||||
const hasModel = models.some((m: any) => m.name === AI_BENCHMARK_MODEL || m.name.startsWith(AI_BENCHMARK_MODEL.split(':')[0]))
|
||||
|
||||
if (!hasModel) {
|
||||
this._updateStatus('running_ai', `Pulling benchmark model ${AI_BENCHMARK_MODEL}...`)
|
||||
await axios.post('http://localhost:11434/api/pull', { name: AI_BENCHMARK_MODEL })
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Could not check/pull model: ${error.message}`)
|
||||
}
|
||||
|
||||
// Run inference benchmark
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const response = await axios.post(
|
||||
'http://localhost:11434/api/generate',
|
||||
{
|
||||
model: AI_BENCHMARK_MODEL,
|
||||
prompt: AI_BENCHMARK_PROMPT,
|
||||
stream: false,
|
||||
},
|
||||
{ timeout: 120000 }
|
||||
)
|
||||
|
||||
const endTime = Date.now()
|
||||
const totalTime = (endTime - startTime) / 1000 // seconds
|
||||
|
||||
// Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds)
|
||||
if (response.data.eval_count && response.data.eval_duration) {
|
||||
const tokenCount = response.data.eval_count
|
||||
const evalDurationSeconds = response.data.eval_duration / 1e9
|
||||
const tokensPerSecond = tokenCount / evalDurationSeconds
|
||||
|
||||
// Time to first token from prompt_eval_duration
|
||||
const ttft = response.data.prompt_eval_duration
|
||||
? response.data.prompt_eval_duration / 1e6 // Convert to ms
|
||||
: (totalTime * 1000) / 2 // Estimate if not available
|
||||
|
||||
return {
|
||||
ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
|
||||
ai_model_used: AI_BENCHMARK_MODEL,
|
||||
ai_time_to_first_token: Math.round(ttft * 100) / 100,
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback calculation
|
||||
const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100
|
||||
const tokensPerSecond = estimatedTokens / totalTime
|
||||
|
||||
return {
|
||||
ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
|
||||
ai_model_used: AI_BENCHMARK_MODEL,
|
||||
ai_time_to_first_token: Math.round((totalTime * 1000) / 2),
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`AI benchmark failed: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate weighted NOMAD score
|
||||
*/
|
||||
private _calculateNomadScore(systemScores: SystemScores, aiScores: Partial<AIScores>): number {
|
||||
let totalWeight = 0
|
||||
let weightedSum = 0
|
||||
|
||||
// CPU score
|
||||
weightedSum += systemScores.cpu_score * SCORE_WEIGHTS.cpu
|
||||
totalWeight += SCORE_WEIGHTS.cpu
|
||||
|
||||
// Memory score
|
||||
weightedSum += systemScores.memory_score * SCORE_WEIGHTS.memory
|
||||
totalWeight += SCORE_WEIGHTS.memory
|
||||
|
||||
// Disk scores
|
||||
weightedSum += systemScores.disk_read_score * SCORE_WEIGHTS.disk_read
|
||||
totalWeight += SCORE_WEIGHTS.disk_read
|
||||
weightedSum += systemScores.disk_write_score * SCORE_WEIGHTS.disk_write
|
||||
totalWeight += SCORE_WEIGHTS.disk_write
|
||||
|
||||
// AI scores (if available)
|
||||
if (aiScores.ai_tokens_per_second !== undefined) {
|
||||
const aiScore = this._normalizeScore(
|
||||
aiScores.ai_tokens_per_second,
|
||||
REFERENCE_SCORES.ai_tokens_per_second
|
||||
)
|
||||
weightedSum += aiScore * SCORE_WEIGHTS.ai_tokens_per_second
|
||||
totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
|
||||
}
|
||||
|
||||
if (aiScores.ai_time_to_first_token !== undefined) {
|
||||
// For TTFT, lower is better, so we invert the score
|
||||
const ttftScore = this._normalizeScoreInverse(
|
||||
aiScores.ai_time_to_first_token,
|
||||
REFERENCE_SCORES.ai_ttft_ms
|
||||
)
|
||||
weightedSum += ttftScore * SCORE_WEIGHTS.ai_ttft
|
||||
totalWeight += SCORE_WEIGHTS.ai_ttft
|
||||
}
|
||||
|
||||
// Normalize by actual weight used (in case AI benchmarks were skipped)
|
||||
const nomadScore = totalWeight > 0 ? (weightedSum / totalWeight) * 100 : 0
|
||||
|
||||
return Math.round(Math.min(100, Math.max(0, nomadScore)) * 100) / 100
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a raw score to 0-100 scale using log scaling
|
||||
* This provides diminishing returns for very high scores
|
||||
*/
|
||||
private _normalizeScore(value: number, reference: number): number {
|
||||
if (value <= 0) return 0
|
||||
// Log scale: score = 50 * (1 + log2(value/reference))
|
||||
// This gives 50 at reference value, scales logarithmically
|
||||
const ratio = value / reference
|
||||
const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)))
|
||||
return Math.min(100, Math.max(0, score)) / 100
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a score where lower is better (like latency)
|
||||
*/
|
||||
private _normalizeScoreInverse(value: number, reference: number): number {
|
||||
if (value <= 0) return 1
|
||||
// Inverse: lower values = higher scores
|
||||
const ratio = reference / value
|
||||
const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)))
|
||||
return Math.min(100, Math.max(0, score)) / 100
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure sysbench Docker image is available
|
||||
*/
|
||||
private async _ensureSysbenchImage(): Promise<void> {
|
||||
try {
|
||||
await this.docker.getImage(SYSBENCH_IMAGE).inspect()
|
||||
} catch {
|
||||
this._updateStatus('starting', `Pulling sysbench image...`)
|
||||
const pullStream = await this.docker.pull(SYSBENCH_IMAGE)
|
||||
await new Promise((resolve) => this.docker.modem.followProgress(pullStream, resolve))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run sysbench CPU benchmark
|
||||
*/
|
||||
private async _runSysbenchCpu(): Promise<SysbenchCpuResult> {
|
||||
const output = await this._runSysbenchCommand([
|
||||
'sysbench',
|
||||
'cpu',
|
||||
'--cpu-max-prime=20000',
|
||||
'--threads=4',
|
||||
'--time=30',
|
||||
'run',
|
||||
])
|
||||
|
||||
// Parse output for events per second
|
||||
const eventsMatch = output.match(/events per second:\s*([\d.]+)/i)
|
||||
const totalTimeMatch = output.match(/total time:\s*([\d.]+)s/i)
|
||||
const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i)
|
||||
|
||||
return {
|
||||
events_per_second: eventsMatch ? parseFloat(eventsMatch[1]) : 0,
|
||||
total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30,
|
||||
total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run sysbench memory benchmark
|
||||
*/
|
||||
private async _runSysbenchMemory(): Promise<SysbenchMemoryResult> {
|
||||
const output = await this._runSysbenchCommand([
|
||||
'sysbench',
|
||||
'memory',
|
||||
'--memory-block-size=1K',
|
||||
'--memory-total-size=10G',
|
||||
'--threads=4',
|
||||
'run',
|
||||
])
|
||||
|
||||
// Parse output
|
||||
const opsMatch = output.match(/Total operations:\s*\d+\s*\(([\d.]+)\s*per second\)/i)
|
||||
const transferMatch = output.match(/([\d.]+)\s*MiB\/sec/i)
|
||||
const timeMatch = output.match(/total time:\s*([\d.]+)s/i)
|
||||
|
||||
return {
|
||||
operations_per_second: opsMatch ? parseFloat(opsMatch[1]) : 0,
|
||||
transfer_rate_mb_per_sec: transferMatch ? parseFloat(transferMatch[1]) : 0,
|
||||
total_time: timeMatch ? parseFloat(timeMatch[1]) : 0,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run sysbench disk read benchmark
|
||||
*/
|
||||
private async _runSysbenchDiskRead(): Promise<SysbenchDiskResult> {
|
||||
// Run prepare, test, and cleanup in a single container
|
||||
// This is necessary because each container has its own filesystem
|
||||
const output = await this._runSysbenchCommand([
|
||||
'sh',
|
||||
'-c',
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --time=30 run && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||||
])
|
||||
|
||||
// Parse output - look for the Throughput section
|
||||
const readMatch = output.match(/read,\s*MiB\/s:\s*([\d.]+)/i)
|
||||
const readsPerSecMatch = output.match(/reads\/s:\s*([\d.]+)/i)
|
||||
|
||||
logger.debug(`[BenchmarkService] Disk read output parsing - read: ${readMatch?.[1]}, reads/s: ${readsPerSecMatch?.[1]}`)
|
||||
|
||||
return {
|
||||
reads_per_second: readsPerSecMatch ? parseFloat(readsPerSecMatch[1]) : 0,
|
||||
writes_per_second: 0,
|
||||
read_mb_per_sec: readMatch ? parseFloat(readMatch[1]) : 0,
|
||||
write_mb_per_sec: 0,
|
||||
total_time: 30,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run sysbench disk write benchmark
|
||||
*/
|
||||
private async _runSysbenchDiskWrite(): Promise<SysbenchDiskResult> {
|
||||
// Run prepare, test, and cleanup in a single container
|
||||
// This is necessary because each container has its own filesystem
|
||||
const output = await this._runSysbenchCommand([
|
||||
'sh',
|
||||
'-c',
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --time=30 run && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||||
])
|
||||
|
||||
// Parse output - look for the Throughput section
|
||||
const writeMatch = output.match(/written,\s*MiB\/s:\s*([\d.]+)/i)
|
||||
const writesPerSecMatch = output.match(/writes\/s:\s*([\d.]+)/i)
|
||||
|
||||
logger.debug(`[BenchmarkService] Disk write output parsing - written: ${writeMatch?.[1]}, writes/s: ${writesPerSecMatch?.[1]}`)
|
||||
|
||||
return {
|
||||
reads_per_second: 0,
|
||||
writes_per_second: writesPerSecMatch ? parseFloat(writesPerSecMatch[1]) : 0,
|
||||
read_mb_per_sec: 0,
|
||||
write_mb_per_sec: writeMatch ? parseFloat(writeMatch[1]) : 0,
|
||||
total_time: 30,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a sysbench command in a Docker container
|
||||
*/
|
||||
private async _runSysbenchCommand(cmd: string[]): Promise<string> {
|
||||
try {
|
||||
// Create container with TTY to avoid multiplexed output
|
||||
const container = await this.docker.createContainer({
|
||||
Image: SYSBENCH_IMAGE,
|
||||
Cmd: cmd,
|
||||
name: `${SYSBENCH_CONTAINER_NAME}_${Date.now()}`,
|
||||
Tty: true, // Important: prevents multiplexed stdout/stderr headers
|
||||
HostConfig: {
|
||||
AutoRemove: true,
|
||||
},
|
||||
})
|
||||
|
||||
// Start container
|
||||
await container.start()
|
||||
|
||||
// Wait for completion and get logs
|
||||
await container.wait()
|
||||
const logs = await container.logs({
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
})
|
||||
|
||||
// Parse logs (Docker logs include header bytes)
|
||||
const output = logs.toString('utf8')
|
||||
.replace(/[\x00-\x08]/g, '') // Remove control characters
|
||||
.trim()
|
||||
|
||||
return output
|
||||
} catch (error) {
|
||||
logger.error(`Sysbench command failed: ${error.message}`)
|
||||
throw new Error(`Sysbench command failed: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Broadcast benchmark progress update
|
||||
*/
|
||||
private _updateStatus(status: BenchmarkStatus, message: string) {
|
||||
this.currentStatus = status
|
||||
|
||||
const progress: BenchmarkProgress = {
|
||||
status,
|
||||
progress: this._getProgressPercent(status),
|
||||
message,
|
||||
current_stage: this._getStageLabel(status),
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
|
||||
transmit.broadcast(BENCHMARK_CHANNEL, {
|
||||
benchmark_id: this.currentBenchmarkId,
|
||||
...progress,
|
||||
})
|
||||
|
||||
logger.info(`[BenchmarkService] ${status}: ${message}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get progress percentage for a given status
|
||||
*/
|
||||
private _getProgressPercent(status: BenchmarkStatus): number {
|
||||
const progressMap: Record<BenchmarkStatus, number> = {
|
||||
idle: 0,
|
||||
starting: 5,
|
||||
detecting_hardware: 10,
|
||||
running_cpu: 25,
|
||||
running_memory: 40,
|
||||
running_disk_read: 55,
|
||||
running_disk_write: 70,
|
||||
running_ai: 85,
|
||||
calculating_score: 95,
|
||||
completed: 100,
|
||||
error: 0,
|
||||
}
|
||||
return progressMap[status] || 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Get human-readable stage label
|
||||
*/
|
||||
private _getStageLabel(status: BenchmarkStatus): string {
|
||||
const labelMap: Record<BenchmarkStatus, string> = {
|
||||
idle: 'Idle',
|
||||
starting: 'Starting',
|
||||
detecting_hardware: 'Detecting Hardware',
|
||||
running_cpu: 'CPU Benchmark',
|
||||
running_memory: 'Memory Benchmark',
|
||||
running_disk_read: 'Disk Read Test',
|
||||
running_disk_write: 'Disk Write Test',
|
||||
running_ai: 'AI Inference Test',
|
||||
calculating_score: 'Calculating Score',
|
||||
completed: 'Complete',
|
||||
error: 'Error',
|
||||
}
|
||||
return labelMap[status] || status
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ export class DockerService {
|
|||
public static CYBERCHEF_SERVICE_NAME = 'nomad_cyberchef'
|
||||
public static FLATNOTES_SERVICE_NAME = 'nomad_flatnotes'
|
||||
public static KOLIBRI_SERVICE_NAME = 'nomad_kolibri'
|
||||
public static BENCHMARK_SERVICE_NAME = 'nomad_benchmark'
|
||||
|
||||
constructor() {
|
||||
// Support both Linux (production) and Windows (development with Docker Desktop)
|
||||
|
|
|
|||
13
admin/app/validators/benchmark.ts
Normal file
13
admin/app/validators/benchmark.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import vine from '@vinejs/vine'
|
||||
|
||||
export const runBenchmarkValidator = vine.compile(
|
||||
vine.object({
|
||||
benchmark_type: vine.enum(['full', 'system', 'ai']).optional(),
|
||||
})
|
||||
)
|
||||
|
||||
export const submitBenchmarkValidator = vine.compile(
|
||||
vine.object({
|
||||
benchmark_id: vine.string().optional(),
|
||||
})
|
||||
)
|
||||
96
admin/commands/benchmark/results.ts
Normal file
96
admin/commands/benchmark/results.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
|
||||
export default class BenchmarkResults extends BaseCommand {
|
||||
static commandName = 'benchmark:results'
|
||||
static description = 'Display benchmark results'
|
||||
|
||||
@flags.boolean({ description: 'Show only the latest result', alias: 'l' })
|
||||
declare latest: boolean
|
||||
|
||||
@flags.string({ description: 'Output format (table, json)', default: 'table' })
|
||||
declare format: string
|
||||
|
||||
@flags.string({ description: 'Show specific benchmark by ID', alias: 'i' })
|
||||
declare id: string
|
||||
|
||||
static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
async run() {
|
||||
const { BenchmarkService } = await import('#services/benchmark_service')
|
||||
const benchmarkService = new BenchmarkService()
|
||||
|
||||
try {
|
||||
let results
|
||||
|
||||
if (this.id) {
|
||||
const result = await benchmarkService.getResultById(this.id)
|
||||
results = result ? [result] : []
|
||||
} else if (this.latest) {
|
||||
const result = await benchmarkService.getLatestResult()
|
||||
results = result ? [result] : []
|
||||
} else {
|
||||
results = await benchmarkService.getAllResults()
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
this.logger.info('No benchmark results found.')
|
||||
this.logger.info('Run "node ace benchmark:run" to create a benchmark.')
|
||||
return
|
||||
}
|
||||
|
||||
if (this.format === 'json') {
|
||||
console.log(JSON.stringify(results, null, 2))
|
||||
return
|
||||
}
|
||||
|
||||
// Table format
|
||||
for (const result of results) {
|
||||
this.logger.info('')
|
||||
this.logger.info(`=== Benchmark ${result.benchmark_id} ===`)
|
||||
this.logger.info(`Type: ${result.benchmark_type}`)
|
||||
this.logger.info(`Date: ${result.created_at}`)
|
||||
this.logger.info('')
|
||||
|
||||
this.logger.info('Hardware:')
|
||||
this.logger.info(` CPU: ${result.cpu_model}`)
|
||||
this.logger.info(` Cores: ${result.cpu_cores} physical, ${result.cpu_threads} threads`)
|
||||
this.logger.info(` RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`)
|
||||
this.logger.info(` Disk: ${result.disk_type}`)
|
||||
if (result.gpu_model) {
|
||||
this.logger.info(` GPU: ${result.gpu_model}`)
|
||||
}
|
||||
this.logger.info('')
|
||||
|
||||
this.logger.info('Scores:')
|
||||
this.logger.info(` CPU: ${result.cpu_score.toFixed(2)}`)
|
||||
this.logger.info(` Memory: ${result.memory_score.toFixed(2)}`)
|
||||
this.logger.info(` Disk Read: ${result.disk_read_score.toFixed(2)}`)
|
||||
this.logger.info(` Disk Write: ${result.disk_write_score.toFixed(2)}`)
|
||||
|
||||
if (result.ai_tokens_per_second) {
|
||||
this.logger.info(` AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`)
|
||||
this.logger.info(` AI TTFT: ${result.ai_time_to_first_token?.toFixed(2)} ms`)
|
||||
}
|
||||
this.logger.info('')
|
||||
|
||||
this.logger.info(`NOMAD Score: ${result.nomad_score.toFixed(2)} / 100`)
|
||||
|
||||
if (result.submitted_to_repository) {
|
||||
this.logger.info(`Submitted: Yes (${result.repository_id})`)
|
||||
} else {
|
||||
this.logger.info('Submitted: No')
|
||||
}
|
||||
this.logger.info('')
|
||||
}
|
||||
|
||||
this.logger.info(`Total results: ${results.length}`)
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to retrieve results: ${error.message}`)
|
||||
this.exitCode = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
103
admin/commands/benchmark/run.ts
Normal file
103
admin/commands/benchmark/run.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
|
||||
export default class BenchmarkRun extends BaseCommand {
|
||||
static commandName = 'benchmark:run'
|
||||
static description = 'Run system and/or AI benchmarks to measure server performance'
|
||||
|
||||
@flags.boolean({ description: 'Run system benchmarks only (CPU, memory, disk)', alias: 's' })
|
||||
declare systemOnly: boolean
|
||||
|
||||
@flags.boolean({ description: 'Run AI benchmark only', alias: 'a' })
|
||||
declare aiOnly: boolean
|
||||
|
||||
@flags.boolean({ description: 'Submit results to repository after completion', alias: 'S' })
|
||||
declare submit: boolean
|
||||
|
||||
static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
async run() {
|
||||
const { BenchmarkService } = await import('#services/benchmark_service')
|
||||
const benchmarkService = new BenchmarkService()
|
||||
|
||||
// Determine benchmark type
|
||||
let benchmarkType: 'full' | 'system' | 'ai' = 'full'
|
||||
if (this.systemOnly) {
|
||||
benchmarkType = 'system'
|
||||
} else if (this.aiOnly) {
|
||||
benchmarkType = 'ai'
|
||||
}
|
||||
|
||||
this.logger.info(`Starting ${benchmarkType} benchmark...`)
|
||||
this.logger.info('')
|
||||
|
||||
try {
|
||||
// Run the benchmark
|
||||
let result
|
||||
switch (benchmarkType) {
|
||||
case 'system':
|
||||
this.logger.info('Running system benchmarks (CPU, memory, disk)...')
|
||||
result = await benchmarkService.runSystemBenchmarks()
|
||||
break
|
||||
case 'ai':
|
||||
this.logger.info('Running AI benchmark...')
|
||||
result = await benchmarkService.runAIBenchmark()
|
||||
break
|
||||
default:
|
||||
this.logger.info('Running full benchmark suite...')
|
||||
result = await benchmarkService.runFullBenchmark()
|
||||
}
|
||||
|
||||
// Display results
|
||||
this.logger.info('')
|
||||
this.logger.success('Benchmark completed!')
|
||||
this.logger.info('')
|
||||
|
||||
this.logger.info('=== Hardware Info ===')
|
||||
this.logger.info(`CPU: ${result.cpu_model}`)
|
||||
this.logger.info(`Cores: ${result.cpu_cores} physical, ${result.cpu_threads} threads`)
|
||||
this.logger.info(`RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`)
|
||||
this.logger.info(`Disk Type: ${result.disk_type}`)
|
||||
if (result.gpu_model) {
|
||||
this.logger.info(`GPU: ${result.gpu_model}`)
|
||||
}
|
||||
|
||||
this.logger.info('')
|
||||
this.logger.info('=== Benchmark Scores ===')
|
||||
this.logger.info(`CPU Score: ${result.cpu_score.toFixed(2)}`)
|
||||
this.logger.info(`Memory Score: ${result.memory_score.toFixed(2)}`)
|
||||
this.logger.info(`Disk Read Score: ${result.disk_read_score.toFixed(2)}`)
|
||||
this.logger.info(`Disk Write Score: ${result.disk_write_score.toFixed(2)}`)
|
||||
|
||||
if (result.ai_tokens_per_second) {
|
||||
this.logger.info(`AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`)
|
||||
this.logger.info(`AI Time to First Token: ${result.ai_time_to_first_token?.toFixed(2)} ms`)
|
||||
this.logger.info(`AI Model: ${result.ai_model_used}`)
|
||||
}
|
||||
|
||||
this.logger.info('')
|
||||
this.logger.info(`NOMAD Score: ${result.nomad_score.toFixed(2)} / 100`)
|
||||
this.logger.info('')
|
||||
this.logger.info(`Benchmark ID: ${result.benchmark_id}`)
|
||||
|
||||
// Submit if requested
|
||||
if (this.submit) {
|
||||
this.logger.info('')
|
||||
this.logger.info('Submitting results to repository...')
|
||||
try {
|
||||
const submitResult = await benchmarkService.submitToRepository(result.benchmark_id)
|
||||
this.logger.success(`Results submitted! Repository ID: ${submitResult.repository_id}`)
|
||||
this.logger.info(`Your percentile: ${submitResult.percentile}%`)
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to submit: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error(`Benchmark failed: ${error.message}`)
|
||||
this.exitCode = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
99
admin/commands/benchmark/submit.ts
Normal file
99
admin/commands/benchmark/submit.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
|
||||
export default class BenchmarkSubmit extends BaseCommand {
|
||||
static commandName = 'benchmark:submit'
|
||||
static description = 'Submit benchmark results to the community repository'
|
||||
|
||||
@flags.string({ description: 'Benchmark ID to submit (defaults to latest)', alias: 'i' })
|
||||
declare benchmarkId: string
|
||||
|
||||
@flags.boolean({ description: 'Skip confirmation prompt', alias: 'y' })
|
||||
declare yes: boolean
|
||||
|
||||
static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
async run() {
|
||||
const { BenchmarkService } = await import('#services/benchmark_service')
|
||||
const benchmarkService = new BenchmarkService()
|
||||
|
||||
try {
|
||||
// Get the result to submit
|
||||
const result = this.benchmarkId
|
||||
? await benchmarkService.getResultById(this.benchmarkId)
|
||||
: await benchmarkService.getLatestResult()
|
||||
|
||||
if (!result) {
|
||||
this.logger.error('No benchmark result found.')
|
||||
this.logger.info('Run "node ace benchmark:run" first to create a benchmark.')
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
|
||||
if (result.submitted_to_repository) {
|
||||
this.logger.warning(`Benchmark ${result.benchmark_id} has already been submitted.`)
|
||||
this.logger.info(`Repository ID: ${result.repository_id}`)
|
||||
return
|
||||
}
|
||||
|
||||
// Show what will be submitted
|
||||
this.logger.info('')
|
||||
this.logger.info('=== Data to be submitted ===')
|
||||
this.logger.info('')
|
||||
this.logger.info('Hardware Information:')
|
||||
this.logger.info(` CPU Model: ${result.cpu_model}`)
|
||||
this.logger.info(` CPU Cores: ${result.cpu_cores}`)
|
||||
this.logger.info(` CPU Threads: ${result.cpu_threads}`)
|
||||
this.logger.info(` RAM: ${Math.round(result.ram_bytes / (1024 * 1024 * 1024))} GB`)
|
||||
this.logger.info(` Disk Type: ${result.disk_type}`)
|
||||
if (result.gpu_model) {
|
||||
this.logger.info(` GPU: ${result.gpu_model}`)
|
||||
}
|
||||
this.logger.info('')
|
||||
this.logger.info('Benchmark Scores:')
|
||||
this.logger.info(` CPU Score: ${result.cpu_score.toFixed(2)}`)
|
||||
this.logger.info(` Memory Score: ${result.memory_score.toFixed(2)}`)
|
||||
this.logger.info(` Disk Read: ${result.disk_read_score.toFixed(2)}`)
|
||||
this.logger.info(` Disk Write: ${result.disk_write_score.toFixed(2)}`)
|
||||
if (result.ai_tokens_per_second) {
|
||||
this.logger.info(` AI Tokens/sec: ${result.ai_tokens_per_second.toFixed(2)}`)
|
||||
this.logger.info(` AI TTFT: ${result.ai_time_to_first_token?.toFixed(2)} ms`)
|
||||
}
|
||||
this.logger.info(` NOMAD Score: ${result.nomad_score.toFixed(2)}`)
|
||||
this.logger.info('')
|
||||
this.logger.info('Privacy Notice:')
|
||||
this.logger.info(' - Only the information shown above will be submitted')
|
||||
this.logger.info(' - No IP addresses, hostnames, or personal data is collected')
|
||||
this.logger.info(' - Submissions are completely anonymous')
|
||||
this.logger.info('')
|
||||
|
||||
// Confirm submission
|
||||
if (!this.yes) {
|
||||
const confirm = await this.prompt.confirm(
|
||||
'Do you want to submit this benchmark to the community repository?'
|
||||
)
|
||||
if (!confirm) {
|
||||
this.logger.info('Submission cancelled.')
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Submit
|
||||
this.logger.info('Submitting benchmark...')
|
||||
const submitResult = await benchmarkService.submitToRepository(result.benchmark_id)
|
||||
|
||||
this.logger.success('Benchmark submitted successfully!')
|
||||
this.logger.info('')
|
||||
this.logger.info(`Repository ID: ${submitResult.repository_id}`)
|
||||
this.logger.info(`Your percentile: ${submitResult.percentile}%`)
|
||||
this.logger.info('')
|
||||
this.logger.info('Thank you for contributing to the NOMAD community!')
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error(`Submission failed: ${error.message}`)
|
||||
this.exitCode = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -61,8 +61,10 @@ export default class QueueWork extends BaseCommand {
|
|||
|
||||
const { RunDownloadJob } = await import('#jobs/run_download_job')
|
||||
const { DownloadModelJob } = await import('#jobs/download_model_job')
|
||||
const { RunBenchmarkJob } = await import('#jobs/run_benchmark_job')
|
||||
handlers.set(RunDownloadJob.key, new RunDownloadJob())
|
||||
handlers.set(DownloadModelJob.key, new DownloadModelJob())
|
||||
handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob())
|
||||
|
||||
return handlers
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'benchmark_results'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id')
|
||||
table.string('benchmark_id').unique().notNullable()
|
||||
table.enum('benchmark_type', ['full', 'system', 'ai']).notNullable()
|
||||
|
||||
// Hardware information
|
||||
table.string('cpu_model').notNullable()
|
||||
table.integer('cpu_cores').notNullable()
|
||||
table.integer('cpu_threads').notNullable()
|
||||
table.bigInteger('ram_bytes').notNullable()
|
||||
table.enum('disk_type', ['ssd', 'hdd', 'nvme', 'unknown']).notNullable()
|
||||
table.string('gpu_model').nullable()
|
||||
|
||||
// System benchmark scores
|
||||
table.float('cpu_score').notNullable()
|
||||
table.float('memory_score').notNullable()
|
||||
table.float('disk_read_score').notNullable()
|
||||
table.float('disk_write_score').notNullable()
|
||||
|
||||
// AI benchmark scores (nullable for system-only benchmarks)
|
||||
table.float('ai_tokens_per_second').nullable()
|
||||
table.string('ai_model_used').nullable()
|
||||
table.float('ai_time_to_first_token').nullable()
|
||||
|
||||
// Composite NOMAD score (0-100)
|
||||
table.float('nomad_score').notNullable()
|
||||
|
||||
// Repository submission tracking
|
||||
table.boolean('submitted_to_repository').defaultTo(false)
|
||||
table.timestamp('submitted_at').nullable()
|
||||
table.string('repository_id').nullable()
|
||||
|
||||
table.timestamp('created_at')
|
||||
table.timestamp('updated_at')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'benchmark_settings'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id')
|
||||
table.string('key').unique().notNullable()
|
||||
table.text('value').nullable()
|
||||
table.timestamp('created_at')
|
||||
table.timestamp('updated_at')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
|
|
@ -147,6 +147,26 @@ export default class ServiceSeeder extends BaseSeeder {
|
|||
is_dependency_service: false,
|
||||
depends_on: null,
|
||||
},
|
||||
{
|
||||
service_name: DockerService.BENCHMARK_SERVICE_NAME,
|
||||
friendly_name: 'System Benchmark',
|
||||
description: 'Measure your server performance and compare with the NOMAD community',
|
||||
icon: 'IconChartBar',
|
||||
container_image: 'severalnines/sysbench:latest',
|
||||
container_command: null,
|
||||
container_config: JSON.stringify({
|
||||
HostConfig: {
|
||||
AutoRemove: true,
|
||||
Binds: [`${ServiceSeeder.NOMAD_STORAGE_ABS_PATH}/benchmark:/tmp/benchmark`]
|
||||
},
|
||||
WorkingDir: '/tmp/benchmark',
|
||||
}),
|
||||
ui_location: null, // UI is integrated into Command Center
|
||||
installed: false,
|
||||
installation_status: 'idle',
|
||||
is_dependency_service: false,
|
||||
depends_on: null,
|
||||
},
|
||||
]
|
||||
|
||||
async run() {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import CategoryCard from '~/components/CategoryCard'
|
|||
import TierSelectionModal from '~/components/TierSelectionModal'
|
||||
import LoadingSpinner from '~/components/LoadingSpinner'
|
||||
import Alert from '~/components/Alert'
|
||||
import { IconCheck, IconChevronDown, IconChevronUp } from '@tabler/icons-react'
|
||||
import { IconCheck, IconChevronDown, IconChevronUp, IconArrowRight } from '@tabler/icons-react'
|
||||
import StorageProjectionBar from '~/components/StorageProjectionBar'
|
||||
import { useNotifications } from '~/context/NotificationContext'
|
||||
import useInternetStatus from '~/hooks/useInternetStatus'
|
||||
|
|
@ -98,6 +98,19 @@ const ADDITIONAL_TOOLS: Capability[] = [
|
|||
services: ['nomad_cyberchef'],
|
||||
icon: 'IconChefHat',
|
||||
},
|
||||
{
|
||||
id: 'benchmark',
|
||||
name: 'System Benchmark',
|
||||
technicalName: 'Built-in',
|
||||
description: 'Measure your server performance and compare with the NOMAD community',
|
||||
features: [
|
||||
'CPU, memory, and disk benchmarks',
|
||||
'AI inference performance testing',
|
||||
'NOMAD Score for easy comparison',
|
||||
],
|
||||
services: ['__builtin_benchmark'], // Special marker for built-in features
|
||||
icon: 'IconChartBar',
|
||||
},
|
||||
]
|
||||
|
||||
type WizardStep = 1 | 2 | 3 | 4
|
||||
|
|
@ -487,13 +500,20 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
)
|
||||
}
|
||||
|
||||
// Check if a capability is a built-in feature (not a Docker service)
|
||||
const isBuiltInCapability = (capability: Capability) => {
|
||||
return capability.services.some((service) => service.startsWith('__builtin_'))
|
||||
}
|
||||
|
||||
// Check if a capability is selected (all its services are in selectedServices)
|
||||
const isCapabilitySelected = (capability: Capability) => {
|
||||
if (isBuiltInCapability(capability)) return false // Built-ins can't be selected
|
||||
return capability.services.every((service) => selectedServices.includes(service))
|
||||
}
|
||||
|
||||
// Check if a capability is already installed (all its services are installed)
|
||||
const isCapabilityInstalled = (capability: Capability) => {
|
||||
if (isBuiltInCapability(capability)) return true // Built-ins are always "installed"
|
||||
return capability.services.every((service) =>
|
||||
installedServices.some((s) => s.service_name === service)
|
||||
)
|
||||
|
|
@ -501,6 +521,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
|
||||
// Check if a capability exists in the system (has at least one matching service)
|
||||
const capabilityExists = (capability: Capability) => {
|
||||
if (isBuiltInCapability(capability)) return true // Built-ins always exist
|
||||
return capability.services.some((service) =>
|
||||
allServices.some((s) => s.service_name === service)
|
||||
)
|
||||
|
|
@ -528,23 +549,38 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
const selected = isCapabilitySelected(capability)
|
||||
const installed = isCapabilityInstalled(capability)
|
||||
const exists = capabilityExists(capability)
|
||||
const isBuiltIn = isBuiltInCapability(capability)
|
||||
|
||||
if (!exists) return null
|
||||
|
||||
// Determine visual state: installed (locked), selected (user chose it), or default
|
||||
const isChecked = installed || selected
|
||||
|
||||
// Handle click - built-in features navigate to their page, others toggle selection
|
||||
const handleClick = () => {
|
||||
if (isBuiltIn) {
|
||||
// Navigate to the appropriate settings page for built-in features
|
||||
if (capability.id === 'benchmark') {
|
||||
router.visit('/settings/benchmark')
|
||||
}
|
||||
} else {
|
||||
toggleCapability(capability)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
key={capability.id}
|
||||
onClick={() => toggleCapability(capability)}
|
||||
onClick={handleClick}
|
||||
className={classNames(
|
||||
'p-6 rounded-lg border-2 transition-all',
|
||||
installed
|
||||
? 'border-desert-green bg-desert-green/20 cursor-default'
|
||||
: selected
|
||||
? 'border-desert-green bg-desert-green shadow-md cursor-pointer'
|
||||
: 'border-desert-stone-light bg-white hover:border-desert-green hover:shadow-sm cursor-pointer'
|
||||
isBuiltIn
|
||||
? 'border-desert-stone bg-desert-stone-lighter/50 hover:border-desert-green hover:shadow-sm cursor-pointer'
|
||||
: installed
|
||||
? 'border-desert-green bg-desert-green/20 cursor-default'
|
||||
: selected
|
||||
? 'border-desert-green bg-desert-green shadow-md cursor-pointer'
|
||||
: 'border-desert-stone-light bg-white hover:border-desert-green hover:shadow-sm cursor-pointer'
|
||||
)}
|
||||
>
|
||||
<div className="flex items-start justify-between">
|
||||
|
|
@ -553,12 +589,16 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
<h3
|
||||
className={classNames(
|
||||
'text-xl font-bold',
|
||||
installed ? 'text-gray-700' : selected ? 'text-white' : 'text-gray-900'
|
||||
isBuiltIn ? 'text-gray-700' : installed ? 'text-gray-700' : selected ? 'text-white' : 'text-gray-900'
|
||||
)}
|
||||
>
|
||||
{capability.name}
|
||||
</h3>
|
||||
{installed && (
|
||||
{isBuiltIn ? (
|
||||
<span className="text-xs bg-desert-stone text-white px-2 py-0.5 rounded-full">
|
||||
Built-in
|
||||
</span>
|
||||
) : installed && (
|
||||
<span className="text-xs bg-desert-green text-white px-2 py-0.5 rounded-full">
|
||||
Installed
|
||||
</span>
|
||||
|
|
@ -567,15 +607,15 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
<p
|
||||
className={classNames(
|
||||
'text-sm mt-0.5',
|
||||
installed ? 'text-gray-500' : selected ? 'text-green-100' : 'text-gray-500'
|
||||
isBuiltIn ? 'text-gray-500' : installed ? 'text-gray-500' : selected ? 'text-green-100' : 'text-gray-500'
|
||||
)}
|
||||
>
|
||||
Powered by {capability.technicalName}
|
||||
{isBuiltIn ? 'Click to open' : `Powered by ${capability.technicalName}`}
|
||||
</p>
|
||||
<p
|
||||
className={classNames(
|
||||
'text-sm mt-3',
|
||||
installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
|
||||
isBuiltIn ? 'text-gray-600' : installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
|
||||
)}
|
||||
>
|
||||
{capability.description}
|
||||
|
|
@ -584,7 +624,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
<ul
|
||||
className={classNames(
|
||||
'mt-3 space-y-1',
|
||||
installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
|
||||
isBuiltIn ? 'text-gray-600' : installed ? 'text-gray-600' : selected ? 'text-white' : 'text-gray-600'
|
||||
)}
|
||||
>
|
||||
{capability.features.map((feature, idx) => (
|
||||
|
|
@ -592,11 +632,13 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
<span
|
||||
className={classNames(
|
||||
'mr-2',
|
||||
installed
|
||||
? 'text-desert-green'
|
||||
: selected
|
||||
? 'text-white'
|
||||
: 'text-desert-green'
|
||||
isBuiltIn
|
||||
? 'text-desert-stone'
|
||||
: installed
|
||||
? 'text-desert-green'
|
||||
: selected
|
||||
? 'text-white'
|
||||
: 'text-desert-green'
|
||||
)}
|
||||
>
|
||||
•
|
||||
|
|
@ -610,14 +652,18 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
<div
|
||||
className={classNames(
|
||||
'ml-4 w-7 h-7 rounded-full border-2 flex items-center justify-center transition-all flex-shrink-0',
|
||||
isChecked
|
||||
? installed
|
||||
? 'border-desert-green bg-desert-green'
|
||||
: 'border-white bg-white'
|
||||
: 'border-desert-stone'
|
||||
isBuiltIn
|
||||
? 'border-desert-stone bg-desert-stone'
|
||||
: isChecked
|
||||
? installed
|
||||
? 'border-desert-green bg-desert-green'
|
||||
: 'border-white bg-white'
|
||||
: 'border-desert-stone'
|
||||
)}
|
||||
>
|
||||
{isChecked && (
|
||||
{isBuiltIn ? (
|
||||
<IconArrowRight size={16} className="text-white" />
|
||||
) : isChecked && (
|
||||
<IconCheck size={20} className={installed ? 'text-white' : 'text-desert-green'} />
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
618
admin/inertia/pages/settings/benchmark.tsx
Normal file
618
admin/inertia/pages/settings/benchmark.tsx
Normal file
|
|
@ -0,0 +1,618 @@
|
|||
import { Head } from '@inertiajs/react'
|
||||
import { useState, useEffect } from 'react'
|
||||
import SettingsLayout from '~/layouts/SettingsLayout'
|
||||
import { useQuery, useMutation } from '@tanstack/react-query'
|
||||
import CircularGauge from '~/components/systeminfo/CircularGauge'
|
||||
import InfoCard from '~/components/systeminfo/InfoCard'
|
||||
import Alert from '~/components/Alert'
|
||||
import StyledButton from '~/components/StyledButton'
|
||||
import {
|
||||
ChartBarIcon,
|
||||
CpuChipIcon,
|
||||
CircleStackIcon,
|
||||
ServerIcon,
|
||||
CloudArrowUpIcon,
|
||||
PlayIcon,
|
||||
ChevronDownIcon,
|
||||
} from '@heroicons/react/24/outline'
|
||||
import { IconRobot } from '@tabler/icons-react'
|
||||
import { useTransmit } from 'react-adonis-transmit'
|
||||
|
||||
type BenchmarkResult = {
|
||||
id: number
|
||||
benchmark_id: string
|
||||
benchmark_type: 'full' | 'system' | 'ai'
|
||||
cpu_model: string
|
||||
cpu_cores: number
|
||||
cpu_threads: number
|
||||
ram_bytes: number
|
||||
disk_type: string
|
||||
gpu_model: string | null
|
||||
cpu_score: number
|
||||
memory_score: number
|
||||
disk_read_score: number
|
||||
disk_write_score: number
|
||||
ai_tokens_per_second: number | null
|
||||
ai_model_used: string | null
|
||||
ai_time_to_first_token: number | null
|
||||
nomad_score: number
|
||||
submitted_to_repository: boolean
|
||||
repository_id: string | null
|
||||
created_at: string
|
||||
}
|
||||
|
||||
type BenchmarkStatus = 'idle' | 'starting' | 'detecting_hardware' | 'running_cpu' | 'running_memory' | 'running_disk_read' | 'running_disk_write' | 'running_ai' | 'calculating_score' | 'completed' | 'error'
|
||||
|
||||
type BenchmarkProgress = {
|
||||
status: BenchmarkStatus
|
||||
progress: number
|
||||
message: string
|
||||
current_stage: string
|
||||
benchmark_id: string
|
||||
}
|
||||
|
||||
export default function BenchmarkPage(props: {
|
||||
benchmark: {
|
||||
latestResult: BenchmarkResult | null
|
||||
status: BenchmarkStatus
|
||||
currentBenchmarkId: string | null
|
||||
}
|
||||
}) {
|
||||
const { subscribe } = useTransmit()
|
||||
const [progress, setProgress] = useState<BenchmarkProgress | null>(null)
|
||||
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
|
||||
const [showDetails, setShowDetails] = useState(false)
|
||||
|
||||
// Fetch latest result
|
||||
const { data: latestResult, refetch: refetchLatest } = useQuery({
|
||||
queryKey: ['benchmark', 'latest'],
|
||||
queryFn: async () => {
|
||||
const res = await fetch('/api/benchmark/results/latest')
|
||||
const data = await res.json()
|
||||
return data.result as BenchmarkResult | null
|
||||
},
|
||||
initialData: props.benchmark.latestResult,
|
||||
})
|
||||
|
||||
// Run benchmark mutation (uses sync mode by default for simpler local dev)
|
||||
const runBenchmark = useMutation({
|
||||
mutationFn: async (type: 'full' | 'system' | 'ai') => {
|
||||
setIsRunning(true)
|
||||
setProgress({
|
||||
status: 'starting',
|
||||
progress: 5,
|
||||
message: 'Starting benchmark... This takes 2-5 minutes.',
|
||||
current_stage: 'Starting',
|
||||
benchmark_id: '',
|
||||
})
|
||||
|
||||
// Use sync mode - runs inline without needing Redis/queue worker
|
||||
const res = await fetch('/api/benchmark/run?sync=true', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ benchmark_type: type }),
|
||||
})
|
||||
return res.json()
|
||||
},
|
||||
onSuccess: (data) => {
|
||||
if (data.success) {
|
||||
setProgress({
|
||||
status: 'completed',
|
||||
progress: 100,
|
||||
message: 'Benchmark completed!',
|
||||
current_stage: 'Complete',
|
||||
benchmark_id: data.benchmark_id,
|
||||
})
|
||||
refetchLatest()
|
||||
} else {
|
||||
setProgress({
|
||||
status: 'error',
|
||||
progress: 0,
|
||||
message: data.error || 'Benchmark failed',
|
||||
current_stage: 'Error',
|
||||
benchmark_id: '',
|
||||
})
|
||||
}
|
||||
setIsRunning(false)
|
||||
},
|
||||
onError: (error) => {
|
||||
setProgress({
|
||||
status: 'error',
|
||||
progress: 0,
|
||||
message: error.message || 'Benchmark failed',
|
||||
current_stage: 'Error',
|
||||
benchmark_id: '',
|
||||
})
|
||||
setIsRunning(false)
|
||||
},
|
||||
})
|
||||
|
||||
// Submit to repository mutation
|
||||
const submitResult = useMutation({
|
||||
mutationFn: async (benchmarkId?: string) => {
|
||||
const res = await fetch('/api/benchmark/submit', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ benchmark_id: benchmarkId }),
|
||||
})
|
||||
return res.json()
|
||||
},
|
||||
onSuccess: () => {
|
||||
refetchLatest()
|
||||
},
|
||||
})
|
||||
|
||||
// Simulate progress during sync benchmark (since we don't get SSE updates)
|
||||
useEffect(() => {
|
||||
if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return
|
||||
|
||||
const stages: { status: BenchmarkStatus; progress: number; message: string; label: string; duration: number }[] = [
|
||||
{ status: 'detecting_hardware', progress: 10, message: 'Detecting system hardware...', label: 'Detecting Hardware', duration: 2000 },
|
||||
{ status: 'running_cpu', progress: 25, message: 'Running CPU benchmark (30s)...', label: 'CPU Benchmark', duration: 32000 },
|
||||
{ status: 'running_memory', progress: 40, message: 'Running memory benchmark...', label: 'Memory Benchmark', duration: 8000 },
|
||||
{ status: 'running_disk_read', progress: 55, message: 'Running disk read benchmark (30s)...', label: 'Disk Read Test', duration: 35000 },
|
||||
{ status: 'running_disk_write', progress: 70, message: 'Running disk write benchmark (30s)...', label: 'Disk Write Test', duration: 35000 },
|
||||
{ status: 'calculating_score', progress: 95, message: 'Calculating NOMAD score...', label: 'Calculating Score', duration: 2000 },
|
||||
]
|
||||
|
||||
let currentStage = 0
|
||||
const advanceStage = () => {
|
||||
if (currentStage < stages.length && isRunning) {
|
||||
const stage = stages[currentStage]
|
||||
setProgress({
|
||||
status: stage.status,
|
||||
progress: stage.progress,
|
||||
message: stage.message,
|
||||
current_stage: stage.label,
|
||||
benchmark_id: '',
|
||||
})
|
||||
currentStage++
|
||||
}
|
||||
}
|
||||
|
||||
// Start the first stage after a short delay
|
||||
const timers: NodeJS.Timeout[] = []
|
||||
let elapsed = 1000
|
||||
stages.forEach((stage, index) => {
|
||||
timers.push(setTimeout(() => advanceStage(), elapsed))
|
||||
elapsed += stage.duration
|
||||
})
|
||||
|
||||
return () => {
|
||||
timers.forEach(t => clearTimeout(t))
|
||||
}
|
||||
}, [isRunning])
|
||||
|
||||
// Listen for benchmark progress via SSE (backup for async mode)
|
||||
useEffect(() => {
|
||||
const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgress) => {
|
||||
setProgress(data)
|
||||
if (data.status === 'completed' || data.status === 'error') {
|
||||
setIsRunning(false)
|
||||
refetchLatest()
|
||||
}
|
||||
})
|
||||
|
||||
return () => {
|
||||
unsubscribe()
|
||||
}
|
||||
}, [subscribe, refetchLatest])
|
||||
|
||||
const formatBytes = (bytes: number) => {
|
||||
const gb = bytes / (1024 * 1024 * 1024)
|
||||
return `${gb.toFixed(1)} GB`
|
||||
}
|
||||
|
||||
const getScoreColor = (score: number) => {
|
||||
if (score >= 70) return 'text-green-600'
|
||||
if (score >= 40) return 'text-yellow-600'
|
||||
return 'text-red-600'
|
||||
}
|
||||
|
||||
const getProgressPercent = () => {
|
||||
if (!progress) return 0
|
||||
const stages: Record<BenchmarkStatus, number> = {
|
||||
idle: 0,
|
||||
starting: 5,
|
||||
detecting_hardware: 10,
|
||||
running_cpu: 25,
|
||||
running_memory: 40,
|
||||
running_disk_read: 55,
|
||||
running_disk_write: 70,
|
||||
running_ai: 85,
|
||||
calculating_score: 95,
|
||||
completed: 100,
|
||||
error: 0,
|
||||
}
|
||||
return stages[progress.status] || 0
|
||||
}
|
||||
|
||||
// Calculate AI score from tokens per second (normalized to 0-100)
|
||||
// Reference: 30 tok/s = 50 score, 60 tok/s = 100 score
|
||||
const getAIScore = (tokensPerSecond: number | null): number => {
|
||||
if (!tokensPerSecond) return 0
|
||||
const score = (tokensPerSecond / 60) * 100
|
||||
return Math.min(100, Math.max(0, score))
|
||||
}
|
||||
|
||||
return (
|
||||
<SettingsLayout>
|
||||
<Head title="System Benchmark" />
|
||||
<div className="xl:pl-72 w-full">
|
||||
<main className="px-6 lg:px-12 py-6 lg:py-8">
|
||||
<div className="mb-8">
|
||||
<h1 className="text-4xl font-bold text-desert-green mb-2">System Benchmark</h1>
|
||||
<p className="text-desert-stone-dark">
|
||||
Measure your server's performance and compare with the NOMAD community
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Run Benchmark Section */}
|
||||
<section className="mb-12">
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
|
||||
<div className="w-1 h-6 bg-desert-green" />
|
||||
Run Benchmark
|
||||
</h2>
|
||||
|
||||
<div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm">
|
||||
{isRunning ? (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="animate-spin h-6 w-6 border-2 border-desert-green border-t-transparent rounded-full" />
|
||||
<span className="text-lg font-medium">
|
||||
{progress?.current_stage || 'Running benchmark...'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="w-full bg-desert-stone-lighter rounded-full h-4 overflow-hidden">
|
||||
<div
|
||||
className="bg-desert-green h-full transition-all duration-500"
|
||||
style={{ width: `${getProgressPercent()}%` }}
|
||||
/>
|
||||
</div>
|
||||
<p className="text-sm text-desert-stone-dark">{progress?.message}</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-6">
|
||||
<p className="text-desert-stone-dark">
|
||||
Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance.
|
||||
The benchmark takes approximately 2-5 minutes to complete.
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-4">
|
||||
<StyledButton
|
||||
onClick={() => runBenchmark.mutate('full')}
|
||||
disabled={runBenchmark.isPending}
|
||||
leftIcon={<PlayIcon className="w-5 h-5" />}
|
||||
>
|
||||
Run Full Benchmark
|
||||
</StyledButton>
|
||||
<StyledButton
|
||||
variant="secondary"
|
||||
onClick={() => runBenchmark.mutate('system')}
|
||||
disabled={runBenchmark.isPending}
|
||||
leftIcon={<CpuChipIcon className="w-5 h-5" />}
|
||||
>
|
||||
System Only
|
||||
</StyledButton>
|
||||
<StyledButton
|
||||
variant="secondary"
|
||||
onClick={() => runBenchmark.mutate('ai')}
|
||||
disabled={runBenchmark.isPending}
|
||||
leftIcon={<IconRobot className="w-5 h-5" />}
|
||||
>
|
||||
AI Only
|
||||
</StyledButton>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* Results Section */}
|
||||
{latestResult && (
|
||||
<>
|
||||
<section className="mb-12">
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
|
||||
<div className="w-1 h-6 bg-desert-green" />
|
||||
NOMAD Score
|
||||
</h2>
|
||||
|
||||
<div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm">
|
||||
<div className="flex flex-col md:flex-row items-center gap-8">
|
||||
<div className="flex-shrink-0">
|
||||
<CircularGauge
|
||||
value={latestResult.nomad_score}
|
||||
label="NOMAD Score"
|
||||
size="lg"
|
||||
variant="cpu"
|
||||
subtext="out of 100"
|
||||
icon={<ChartBarIcon className="w-8 h-8" />}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex-1 space-y-4">
|
||||
<div className={`text-5xl font-bold ${getScoreColor(latestResult.nomad_score)}`}>
|
||||
{latestResult.nomad_score.toFixed(1)}
|
||||
</div>
|
||||
<p className="text-desert-stone-dark">
|
||||
Your NOMAD Score is a weighted composite of all benchmark results.
|
||||
</p>
|
||||
{!latestResult.submitted_to_repository && (
|
||||
<StyledButton
|
||||
onClick={() => submitResult.mutate(latestResult.benchmark_id)}
|
||||
disabled={submitResult.isPending}
|
||||
leftIcon={<CloudArrowUpIcon className="w-5 h-5" />}
|
||||
>
|
||||
Share with Community
|
||||
</StyledButton>
|
||||
)}
|
||||
{latestResult.submitted_to_repository && (
|
||||
<Alert
|
||||
type="success"
|
||||
title="Shared with Community"
|
||||
message={`Repository ID: ${latestResult.repository_id}`}
|
||||
variant="bordered"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="mb-12">
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
|
||||
<div className="w-1 h-6 bg-desert-green" />
|
||||
System Performance
|
||||
</h2>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
|
||||
<CircularGauge
|
||||
value={latestResult.cpu_score * 100}
|
||||
label="CPU"
|
||||
size="md"
|
||||
variant="cpu"
|
||||
icon={<CpuChipIcon className="w-6 h-6" />}
|
||||
/>
|
||||
</div>
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
|
||||
<CircularGauge
|
||||
value={latestResult.memory_score * 100}
|
||||
label="Memory"
|
||||
size="md"
|
||||
variant="memory"
|
||||
icon={<CircleStackIcon className="w-6 h-6" />}
|
||||
/>
|
||||
</div>
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
|
||||
<CircularGauge
|
||||
value={latestResult.disk_read_score * 100}
|
||||
label="Disk Read"
|
||||
size="md"
|
||||
variant="disk"
|
||||
icon={<ServerIcon className="w-6 h-6" />}
|
||||
/>
|
||||
</div>
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
|
||||
<CircularGauge
|
||||
value={latestResult.disk_write_score * 100}
|
||||
label="Disk Write"
|
||||
size="md"
|
||||
variant="disk"
|
||||
icon={<ServerIcon className="w-6 h-6" />}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* AI Performance Section */}
|
||||
<section className="mb-12">
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
|
||||
<div className="w-1 h-6 bg-desert-green" />
|
||||
AI Performance
|
||||
</h2>
|
||||
|
||||
{latestResult.ai_tokens_per_second ? (
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-6">
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
|
||||
<CircularGauge
|
||||
value={getAIScore(latestResult.ai_tokens_per_second)}
|
||||
label="AI Score"
|
||||
size="md"
|
||||
variant="cpu"
|
||||
icon={<IconRobot className="w-6 h-6" />}
|
||||
/>
|
||||
</div>
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm flex items-center justify-center">
|
||||
<div className="flex items-center gap-4">
|
||||
<IconRobot className="w-10 h-10 text-desert-green" />
|
||||
<div>
|
||||
<div className="text-3xl font-bold text-desert-green">
|
||||
{latestResult.ai_tokens_per_second.toFixed(1)}
|
||||
</div>
|
||||
<div className="text-sm text-desert-stone-dark">Tokens per Second</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm flex items-center justify-center">
|
||||
<div className="flex items-center gap-4">
|
||||
<IconRobot className="w-10 h-10 text-desert-green" />
|
||||
<div>
|
||||
<div className="text-3xl font-bold text-desert-green">
|
||||
{latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms
|
||||
</div>
|
||||
<div className="text-sm text-desert-stone-dark">Time to First Token</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="bg-desert-white rounded-lg p-6 border border-desert-stone-light shadow-sm">
|
||||
<div className="text-center text-desert-stone-dark">
|
||||
<IconRobot className="w-12 h-12 mx-auto mb-3 opacity-40" />
|
||||
<p className="font-medium">No AI Benchmark Data</p>
|
||||
<p className="text-sm mt-1">
|
||||
Run a Full Benchmark or AI Only benchmark to measure AI inference performance.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className="mb-12">
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
|
||||
<div className="w-1 h-6 bg-desert-green" />
|
||||
Hardware Information
|
||||
</h2>
|
||||
|
||||
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6">
|
||||
<InfoCard
|
||||
title="Processor"
|
||||
icon={<CpuChipIcon className="w-6 h-6" />}
|
||||
variant="elevated"
|
||||
data={[
|
||||
{ label: 'Model', value: latestResult.cpu_model },
|
||||
{ label: 'Cores', value: latestResult.cpu_cores },
|
||||
{ label: 'Threads', value: latestResult.cpu_threads },
|
||||
]}
|
||||
/>
|
||||
<InfoCard
|
||||
title="System"
|
||||
icon={<ServerIcon className="w-6 h-6" />}
|
||||
variant="elevated"
|
||||
data={[
|
||||
{ label: 'RAM', value: formatBytes(latestResult.ram_bytes) },
|
||||
{ label: 'Disk Type', value: latestResult.disk_type.toUpperCase() },
|
||||
{ label: 'GPU', value: latestResult.gpu_model || 'Not detected' },
|
||||
]}
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
|
||||
<div className="w-1 h-6 bg-desert-green" />
|
||||
Benchmark Details
|
||||
</h2>
|
||||
|
||||
<div className="bg-desert-white rounded-lg border border-desert-stone-light shadow-sm overflow-hidden">
|
||||
{/* Summary row - always visible */}
|
||||
<button
|
||||
onClick={() => setShowDetails(!showDetails)}
|
||||
className="w-full p-6 flex items-center justify-between hover:bg-desert-stone-lighter/30 transition-colors"
|
||||
>
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm text-left flex-1">
|
||||
<div>
|
||||
<div className="text-desert-stone-dark">Benchmark ID</div>
|
||||
<div className="font-mono text-xs">{latestResult.benchmark_id.slice(0, 8)}...</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-desert-stone-dark">Type</div>
|
||||
<div className="capitalize">{latestResult.benchmark_type}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-desert-stone-dark">Date</div>
|
||||
<div>{new Date(latestResult.created_at).toLocaleDateString()}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-desert-stone-dark">NOMAD Score</div>
|
||||
<div className="font-bold text-desert-green">{latestResult.nomad_score.toFixed(1)}</div>
|
||||
</div>
|
||||
</div>
|
||||
<ChevronDownIcon
|
||||
className={`w-5 h-5 text-desert-stone-dark transition-transform ${showDetails ? 'rotate-180' : ''}`}
|
||||
/>
|
||||
</button>
|
||||
|
||||
{/* Expanded details */}
|
||||
{showDetails && (
|
||||
<div className="border-t border-desert-stone-light p-6 bg-desert-stone-lighter/20">
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
{/* Raw Scores */}
|
||||
<div>
|
||||
<h4 className="font-semibold text-desert-green mb-3">Raw Scores</h4>
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">CPU Score</span>
|
||||
<span className="font-mono">{(latestResult.cpu_score * 100).toFixed(1)}%</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Memory Score</span>
|
||||
<span className="font-mono">{(latestResult.memory_score * 100).toFixed(1)}%</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Disk Read Score</span>
|
||||
<span className="font-mono">{(latestResult.disk_read_score * 100).toFixed(1)}%</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Disk Write Score</span>
|
||||
<span className="font-mono">{(latestResult.disk_write_score * 100).toFixed(1)}%</span>
|
||||
</div>
|
||||
{latestResult.ai_tokens_per_second && (
|
||||
<>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">AI Tokens/sec</span>
|
||||
<span className="font-mono">{latestResult.ai_tokens_per_second.toFixed(1)}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">AI Time to First Token</span>
|
||||
<span className="font-mono">{latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms</span>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Benchmark Info */}
|
||||
<div>
|
||||
<h4 className="font-semibold text-desert-green mb-3">Benchmark Info</h4>
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Full Benchmark ID</span>
|
||||
<span className="font-mono text-xs">{latestResult.benchmark_id}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Benchmark Type</span>
|
||||
<span className="capitalize">{latestResult.benchmark_type}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Run Date</span>
|
||||
<span>{new Date(latestResult.created_at).toLocaleString()}</span>
|
||||
</div>
|
||||
{latestResult.ai_model_used && (
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">AI Model Used</span>
|
||||
<span>{latestResult.ai_model_used}</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Submitted to Repository</span>
|
||||
<span>{latestResult.submitted_to_repository ? 'Yes' : 'No'}</span>
|
||||
</div>
|
||||
{latestResult.repository_id && (
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Repository ID</span>
|
||||
<span className="font-mono text-xs">{latestResult.repository_id}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
</>
|
||||
)}
|
||||
|
||||
{!latestResult && !isRunning && (
|
||||
<Alert
|
||||
type="info"
|
||||
title="No Benchmark Results"
|
||||
message="Run your first benchmark to see your server's performance scores."
|
||||
variant="bordered"
|
||||
/>
|
||||
)}
|
||||
</main>
|
||||
</div>
|
||||
</SettingsLayout>
|
||||
)
|
||||
}
|
||||
30
admin/package-lock.json
generated
30
admin/package-lock.json
generated
|
|
@ -55,6 +55,7 @@
|
|||
"tar": "^7.5.6",
|
||||
"url-join": "^5.0.0",
|
||||
"usehooks-ts": "^3.1.1",
|
||||
"uuid": "^13.0.0",
|
||||
"yaml": "^2.8.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -72,6 +73,7 @@
|
|||
"@types/node": "^22.15.18",
|
||||
"@types/react": "^19.1.8",
|
||||
"@types/react-dom": "^19.1.6",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.26.0",
|
||||
"hot-hook": "^0.4.0",
|
||||
"prettier": "^3.5.3",
|
||||
|
|
@ -4304,6 +4306,13 @@
|
|||
"@types/geojson": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/uuid": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
|
||||
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/validator": {
|
||||
"version": "13.15.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.2.tgz",
|
||||
|
|
@ -5172,6 +5181,19 @@
|
|||
"uuid": "^11.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bullmq/node_modules/uuid": {
|
||||
"version": "11.1.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
|
||||
"integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/esm/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/bytes": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
|
||||
|
|
@ -11982,16 +12004,16 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "11.1.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
|
||||
"integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
|
||||
"version": "13.0.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz",
|
||||
"integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/esm/bin/uuid"
|
||||
"uuid": "dist-node/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/v8-compile-cache-lib": {
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@
|
|||
"@types/node": "^22.15.18",
|
||||
"@types/react": "^19.1.8",
|
||||
"@types/react-dom": "^19.1.6",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.26.0",
|
||||
"hot-hook": "^0.4.0",
|
||||
"prettier": "^3.5.3",
|
||||
|
|
@ -104,6 +105,7 @@
|
|||
"tar": "^7.5.6",
|
||||
"url-join": "^5.0.0",
|
||||
"usehooks-ts": "^3.1.1",
|
||||
"uuid": "^13.0.0",
|
||||
"yaml": "^2.8.0"
|
||||
},
|
||||
"hotHook": {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
| The routes file is used for defining the HTTP routes.
|
||||
|
|
||||
*/
|
||||
import BenchmarkController from '#controllers/benchmark_controller'
|
||||
import DocsController from '#controllers/docs_controller'
|
||||
import DownloadsController from '#controllers/downloads_controller'
|
||||
import EasySetupController from '#controllers/easy_setup_controller'
|
||||
|
|
@ -38,6 +39,7 @@ router
|
|||
router.get('/update', [SettingsController, 'update'])
|
||||
router.get('/zim', [SettingsController, 'zim'])
|
||||
router.get('/zim/remote-explorer', [SettingsController, 'zimRemote'])
|
||||
router.get('/benchmark', [SettingsController, 'benchmark'])
|
||||
})
|
||||
.prefix('/settings')
|
||||
|
||||
|
|
@ -122,3 +124,19 @@ router
|
|||
router.delete('/:filename', [ZimController, 'delete'])
|
||||
})
|
||||
.prefix('/api/zim')
|
||||
|
||||
router
|
||||
.group(() => {
|
||||
router.post('/run', [BenchmarkController, 'run'])
|
||||
router.post('/run/system', [BenchmarkController, 'runSystem'])
|
||||
router.post('/run/ai', [BenchmarkController, 'runAI'])
|
||||
router.get('/results', [BenchmarkController, 'results'])
|
||||
router.get('/results/latest', [BenchmarkController, 'latest'])
|
||||
router.get('/results/:id', [BenchmarkController, 'show'])
|
||||
router.post('/submit', [BenchmarkController, 'submit'])
|
||||
router.get('/comparison', [BenchmarkController, 'comparison'])
|
||||
router.get('/status', [BenchmarkController, 'status'])
|
||||
router.get('/settings', [BenchmarkController, 'settings'])
|
||||
router.post('/settings', [BenchmarkController, 'updateSettings'])
|
||||
})
|
||||
.prefix('/api/benchmark')
|
||||
|
|
|
|||
216
admin/types/benchmark.ts
Normal file
216
admin/types/benchmark.ts
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
// Benchmark type identifiers
|
||||
export type BenchmarkType = 'full' | 'system' | 'ai'
|
||||
|
||||
// Benchmark execution status
|
||||
export type BenchmarkStatus =
|
||||
| 'idle'
|
||||
| 'starting'
|
||||
| 'detecting_hardware'
|
||||
| 'running_cpu'
|
||||
| 'running_memory'
|
||||
| 'running_disk_read'
|
||||
| 'running_disk_write'
|
||||
| 'running_ai'
|
||||
| 'calculating_score'
|
||||
| 'completed'
|
||||
| 'error'
|
||||
|
||||
// Hardware detection types
|
||||
export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown'
|
||||
|
||||
export type HardwareInfo = {
|
||||
cpu_model: string
|
||||
cpu_cores: number
|
||||
cpu_threads: number
|
||||
ram_bytes: number
|
||||
disk_type: DiskType
|
||||
gpu_model: string | null
|
||||
}
|
||||
|
||||
// Individual benchmark scores
|
||||
export type SystemScores = {
|
||||
cpu_score: number
|
||||
memory_score: number
|
||||
disk_read_score: number
|
||||
disk_write_score: number
|
||||
}
|
||||
|
||||
export type AIScores = {
|
||||
ai_tokens_per_second: number
|
||||
ai_model_used: string
|
||||
ai_time_to_first_token: number
|
||||
}
|
||||
|
||||
// Complete benchmark result
|
||||
export type BenchmarkResult = {
|
||||
id: number
|
||||
benchmark_id: string
|
||||
benchmark_type: BenchmarkType
|
||||
hardware: HardwareInfo
|
||||
scores: SystemScores & Partial<AIScores>
|
||||
nomad_score: number
|
||||
submitted_to_repository: boolean
|
||||
submitted_at: string | null
|
||||
repository_id: string | null
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
|
||||
// Slim version for lists
|
||||
export type BenchmarkResultSlim = Pick<
|
||||
BenchmarkResult,
|
||||
| 'id'
|
||||
| 'benchmark_id'
|
||||
| 'benchmark_type'
|
||||
| 'nomad_score'
|
||||
| 'submitted_to_repository'
|
||||
| 'created_at'
|
||||
> & {
|
||||
cpu_model: string
|
||||
gpu_model: string | null
|
||||
}
|
||||
|
||||
// Benchmark settings key-value store
|
||||
export type BenchmarkSettingKey =
|
||||
| 'allow_anonymous_submission'
|
||||
| 'installation_id'
|
||||
| 'last_benchmark_run'
|
||||
|
||||
export type BenchmarkSettings = {
|
||||
allow_anonymous_submission: boolean
|
||||
installation_id: string | null
|
||||
last_benchmark_run: string | null
|
||||
}
|
||||
|
||||
// Progress update for real-time feedback
|
||||
export type BenchmarkProgress = {
|
||||
status: BenchmarkStatus
|
||||
progress: number
|
||||
message: string
|
||||
current_stage: string
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
// API request types
|
||||
export type RunBenchmarkParams = {
|
||||
benchmark_type: BenchmarkType
|
||||
}
|
||||
|
||||
export type SubmitBenchmarkParams = {
|
||||
benchmark_id?: string
|
||||
}
|
||||
|
||||
// API response types
|
||||
export type RunBenchmarkResponse = {
|
||||
success: boolean
|
||||
job_id: string
|
||||
benchmark_id: string
|
||||
message: string
|
||||
}
|
||||
|
||||
export type BenchmarkResultsResponse = {
|
||||
results: BenchmarkResult[]
|
||||
total: number
|
||||
}
|
||||
|
||||
// Central repository submission payload (privacy-first)
|
||||
export type RepositorySubmission = {
|
||||
cpu_model: string
|
||||
cpu_cores: number
|
||||
cpu_threads: number
|
||||
ram_gb: number
|
||||
disk_type: DiskType
|
||||
gpu_model: string | null
|
||||
cpu_score: number
|
||||
memory_score: number
|
||||
disk_read_score: number
|
||||
disk_write_score: number
|
||||
ai_tokens_per_second: number | null
|
||||
ai_time_to_first_token: number | null
|
||||
nomad_score: number
|
||||
nomad_version: string
|
||||
benchmark_version: string
|
||||
}
|
||||
|
||||
// Central repository response types
|
||||
export type RepositorySubmitResponse = {
|
||||
success: boolean
|
||||
repository_id: string
|
||||
percentile: number
|
||||
}
|
||||
|
||||
export type RepositoryStats = {
|
||||
total_submissions: number
|
||||
average_score: number
|
||||
median_score: number
|
||||
top_score: number
|
||||
percentiles: {
|
||||
p10: number
|
||||
p25: number
|
||||
p50: number
|
||||
p75: number
|
||||
p90: number
|
||||
}
|
||||
}
|
||||
|
||||
export type LeaderboardEntry = {
|
||||
rank: number
|
||||
cpu_model: string
|
||||
gpu_model: string | null
|
||||
nomad_score: number
|
||||
submitted_at: string
|
||||
}
|
||||
|
||||
export type ComparisonResponse = {
|
||||
matching_submissions: number
|
||||
average_score: number
|
||||
your_percentile: number | null
|
||||
}
|
||||
|
||||
// Score calculation weights (for reference in UI)
|
||||
export type ScoreWeights = {
|
||||
ai_tokens_per_second: number
|
||||
cpu: number
|
||||
memory: number
|
||||
ai_ttft: number
|
||||
disk_read: number
|
||||
disk_write: number
|
||||
}
|
||||
|
||||
// Default weights as defined in plan
|
||||
export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = {
|
||||
ai_tokens_per_second: 0.30,
|
||||
cpu: 0.25,
|
||||
memory: 0.15,
|
||||
ai_ttft: 0.10,
|
||||
disk_read: 0.10,
|
||||
disk_write: 0.10,
|
||||
}
|
||||
|
||||
// Benchmark job parameters
|
||||
export type RunBenchmarkJobParams = {
|
||||
benchmark_id: string
|
||||
benchmark_type: BenchmarkType
|
||||
include_ai: boolean
|
||||
}
|
||||
|
||||
// sysbench result parsing types
|
||||
export type SysbenchCpuResult = {
|
||||
events_per_second: number
|
||||
total_time: number
|
||||
total_events: number
|
||||
}
|
||||
|
||||
export type SysbenchMemoryResult = {
|
||||
operations_per_second: number
|
||||
transfer_rate_mb_per_sec: number
|
||||
total_time: number
|
||||
}
|
||||
|
||||
export type SysbenchDiskResult = {
|
||||
reads_per_second: number
|
||||
writes_per_second: number
|
||||
read_mb_per_sec: number
|
||||
write_mb_per_sec: number
|
||||
total_time: number
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user