fix(Benchmark): cleanup types for SSOT

This commit is contained in:
Jake Turner 2026-01-22 17:30:14 -08:00 committed by Jake Turner
parent 6efd049424
commit 438d683bac
4 changed files with 48 additions and 92 deletions

View File

@ -452,7 +452,7 @@ export class BenchmarkService {
totalWeight += SCORE_WEIGHTS.disk_write totalWeight += SCORE_WEIGHTS.disk_write
// AI scores (if available) // AI scores (if available)
if (aiScores.ai_tokens_per_second !== undefined) { if (aiScores.ai_tokens_per_second !== undefined && aiScores.ai_tokens_per_second !== null) {
const aiScore = this._normalizeScore( const aiScore = this._normalizeScore(
aiScores.ai_tokens_per_second, aiScores.ai_tokens_per_second,
REFERENCE_SCORES.ai_tokens_per_second REFERENCE_SCORES.ai_tokens_per_second
@ -461,7 +461,7 @@ export class BenchmarkService {
totalWeight += SCORE_WEIGHTS.ai_tokens_per_second totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
} }
if (aiScores.ai_time_to_first_token !== undefined) { if (aiScores.ai_time_to_first_token !== undefined && aiScores.ai_time_to_first_token !== null) {
// For TTFT, lower is better, so we invert the score // For TTFT, lower is better, so we invert the score
const ttftScore = this._normalizeScoreInverse( const ttftScore = this._normalizeScoreInverse(
aiScores.ai_time_to_first_token, aiScores.ai_time_to_first_token,

View File

@ -17,39 +17,10 @@ import {
} from '@heroicons/react/24/outline' } from '@heroicons/react/24/outline'
import { IconRobot } from '@tabler/icons-react' import { IconRobot } from '@tabler/icons-react'
import { useTransmit } from 'react-adonis-transmit' import { useTransmit } from 'react-adonis-transmit'
import { BenchmarkProgress, BenchmarkStatus } from '../../../types/benchmark'
import BenchmarkResult from '#models/benchmark_result'
type BenchmarkResult = { type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string }
id: number
benchmark_id: string
benchmark_type: 'full' | 'system' | 'ai'
cpu_model: string
cpu_cores: number
cpu_threads: number
ram_bytes: number
disk_type: string
gpu_model: string | null
cpu_score: number
memory_score: number
disk_read_score: number
disk_write_score: number
ai_tokens_per_second: number | null
ai_model_used: string | null
ai_time_to_first_token: number | null
nomad_score: number
submitted_to_repository: boolean
repository_id: string | null
created_at: string
}
type BenchmarkStatus = 'idle' | 'starting' | 'detecting_hardware' | 'running_cpu' | 'running_memory' | 'running_disk_read' | 'running_disk_write' | 'running_ai' | 'calculating_score' | 'completed' | 'error'
type BenchmarkProgress = {
status: BenchmarkStatus
progress: number
message: string
current_stage: string
benchmark_id: string
}
export default function BenchmarkPage(props: { export default function BenchmarkPage(props: {
benchmark: { benchmark: {
@ -59,7 +30,7 @@ export default function BenchmarkPage(props: {
} }
}) { }) {
const { subscribe } = useTransmit() const { subscribe } = useTransmit()
const [progress, setProgress] = useState<BenchmarkProgress | null>(null) const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle') const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
const [showDetails, setShowDetails] = useState(false) const [showDetails, setShowDetails] = useState(false)
@ -84,6 +55,7 @@ export default function BenchmarkPage(props: {
message: 'Starting benchmark... This takes 2-5 minutes.', message: 'Starting benchmark... This takes 2-5 minutes.',
current_stage: 'Starting', current_stage: 'Starting',
benchmark_id: '', benchmark_id: '',
timestamp: new Date().toISOString(),
}) })
// Use sync mode - runs inline without needing Redis/queue worker // Use sync mode - runs inline without needing Redis/queue worker
@ -102,6 +74,7 @@ export default function BenchmarkPage(props: {
message: 'Benchmark completed!', message: 'Benchmark completed!',
current_stage: 'Complete', current_stage: 'Complete',
benchmark_id: data.benchmark_id, benchmark_id: data.benchmark_id,
timestamp: new Date().toISOString(),
}) })
refetchLatest() refetchLatest()
} else { } else {
@ -111,6 +84,7 @@ export default function BenchmarkPage(props: {
message: data.error || 'Benchmark failed', message: data.error || 'Benchmark failed',
current_stage: 'Error', current_stage: 'Error',
benchmark_id: '', benchmark_id: '',
timestamp: new Date().toISOString(),
}) })
} }
setIsRunning(false) setIsRunning(false)
@ -122,6 +96,7 @@ export default function BenchmarkPage(props: {
message: error.message || 'Benchmark failed', message: error.message || 'Benchmark failed',
current_stage: 'Error', current_stage: 'Error',
benchmark_id: '', benchmark_id: '',
timestamp: new Date().toISOString(),
}) })
setIsRunning(false) setIsRunning(false)
}, },
@ -174,6 +149,7 @@ export default function BenchmarkPage(props: {
message: stage.message, message: stage.message,
current_stage: stage.label, current_stage: stage.label,
benchmark_id: '', benchmark_id: '',
timestamp: new Date().toISOString(),
}) })
currentStage++ currentStage++
} }
@ -194,7 +170,7 @@ export default function BenchmarkPage(props: {
// Listen for benchmark progress via SSE (backup for async mode) // Listen for benchmark progress via SSE (backup for async mode)
useEffect(() => { useEffect(() => {
const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgress) => { const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgressWithID) => {
setProgress(data) setProgress(data)
if (data.status === 'completed' || data.status === 'error') { if (data.status === 'completed' || data.status === 'error') {
setIsRunning(false) setIsRunning(false)

View File

@ -1,3 +1,5 @@
import BenchmarkResult from '#models/benchmark_result'
// Benchmark type identifiers // Benchmark type identifiers
export type BenchmarkType = 'full' | 'system' | 'ai' export type BenchmarkType = 'full' | 'system' | 'ai'
@ -18,43 +20,21 @@ export type BenchmarkStatus =
// Hardware detection types // Hardware detection types
export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown' export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown'
export type HardwareInfo = { export type HardwareInfo = Pick<
cpu_model: string BenchmarkResult,
cpu_cores: number 'cpu_model' | 'cpu_cores' | 'cpu_threads' | 'ram_bytes' | 'disk_type' | 'gpu_model'
cpu_threads: number >
ram_bytes: number
disk_type: DiskType
gpu_model: string | null
}
// Individual benchmark scores // Individual benchmark scores
export type SystemScores = { export type SystemScores = Pick<
cpu_score: number BenchmarkResult,
memory_score: number 'cpu_score' | 'memory_score' | 'disk_read_score' | 'disk_write_score'
disk_read_score: number >
disk_write_score: number
}
export type AIScores = { export type AIScores = Pick<
ai_tokens_per_second: number BenchmarkResult,
ai_model_used: string 'ai_tokens_per_second' | 'ai_model_used' | 'ai_time_to_first_token'
ai_time_to_first_token: number >
}
// Complete benchmark result
export type BenchmarkResult = {
id: number
benchmark_id: string
benchmark_type: BenchmarkType
hardware: HardwareInfo
scores: SystemScores & Partial<AIScores>
nomad_score: number
submitted_to_repository: boolean
submitted_at: string | null
repository_id: string | null
created_at: string
updated_at: string
}
// Slim version for lists // Slim version for lists
export type BenchmarkResultSlim = Pick< export type BenchmarkResultSlim = Pick<
@ -114,22 +94,24 @@ export type BenchmarkResultsResponse = {
} }
// Central repository submission payload (privacy-first) // Central repository submission payload (privacy-first)
export type RepositorySubmission = { export type RepositorySubmission = Pick<
cpu_model: string BenchmarkResult,
cpu_cores: number | 'cpu_model'
cpu_threads: number | 'cpu_cores'
ram_gb: number | 'cpu_threads'
disk_type: DiskType | 'disk_type'
gpu_model: string | null | 'gpu_model'
cpu_score: number | 'cpu_score'
memory_score: number | 'memory_score'
disk_read_score: number | 'disk_read_score'
disk_write_score: number | 'disk_write_score'
ai_tokens_per_second: number | null | 'ai_tokens_per_second'
ai_time_to_first_token: number | null | 'ai_time_to_first_token'
nomad_score: number | 'nomad_score'
> & {
nomad_version: string nomad_version: string
benchmark_version: string benchmark_version: string
ram_gb: number
} }
// Central repository response types // Central repository response types
@ -153,11 +135,8 @@ export type RepositoryStats = {
} }
} }
export type LeaderboardEntry = { export type LeaderboardEntry = Pick<BenchmarkResult, 'cpu_model' | 'gpu_model' | 'nomad_score'> & {
rank: number rank: number
cpu_model: string
gpu_model: string | null
nomad_score: number
submitted_at: string submitted_at: string
} }
@ -179,12 +158,12 @@ export type ScoreWeights = {
// Default weights as defined in plan // Default weights as defined in plan
export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = { export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = {
ai_tokens_per_second: 0.30, ai_tokens_per_second: 0.3,
cpu: 0.25, cpu: 0.25,
memory: 0.15, memory: 0.15,
ai_ttft: 0.10, ai_ttft: 0.1,
disk_read: 0.10, disk_read: 0.1,
disk_write: 0.10, disk_write: 0.1,
} }
// Benchmark job parameters // Benchmark job parameters

View File

@ -21,6 +21,7 @@ node ace db:seed
echo "Starting background workers for queues..." echo "Starting background workers for queues..."
node ace queue:work --queue=downloads & node ace queue:work --queue=downloads &
node ace queue:work --queue=model-downloads & node ace queue:work --queue=model-downloads &
node ace queue:work --queue
# Start the AdonisJS application # Start the AdonisJS application
echo "Starting AdonisJS application..." echo "Starting AdonisJS application..."