mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-03-28 03:29:25 +01:00
fix(Benchmark): cleanup types for SSOT
This commit is contained in:
parent
6efd049424
commit
438d683bac
|
|
@ -452,7 +452,7 @@ export class BenchmarkService {
|
|||
totalWeight += SCORE_WEIGHTS.disk_write
|
||||
|
||||
// AI scores (if available)
|
||||
if (aiScores.ai_tokens_per_second !== undefined) {
|
||||
if (aiScores.ai_tokens_per_second !== undefined && aiScores.ai_tokens_per_second !== null) {
|
||||
const aiScore = this._normalizeScore(
|
||||
aiScores.ai_tokens_per_second,
|
||||
REFERENCE_SCORES.ai_tokens_per_second
|
||||
|
|
@ -461,7 +461,7 @@ export class BenchmarkService {
|
|||
totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
|
||||
}
|
||||
|
||||
if (aiScores.ai_time_to_first_token !== undefined) {
|
||||
if (aiScores.ai_time_to_first_token !== undefined && aiScores.ai_time_to_first_token !== null) {
|
||||
// For TTFT, lower is better, so we invert the score
|
||||
const ttftScore = this._normalizeScoreInverse(
|
||||
aiScores.ai_time_to_first_token,
|
||||
|
|
|
|||
|
|
@ -17,39 +17,10 @@ import {
|
|||
} from '@heroicons/react/24/outline'
|
||||
import { IconRobot } from '@tabler/icons-react'
|
||||
import { useTransmit } from 'react-adonis-transmit'
|
||||
import { BenchmarkProgress, BenchmarkStatus } from '../../../types/benchmark'
|
||||
import BenchmarkResult from '#models/benchmark_result'
|
||||
|
||||
type BenchmarkResult = {
|
||||
id: number
|
||||
benchmark_id: string
|
||||
benchmark_type: 'full' | 'system' | 'ai'
|
||||
cpu_model: string
|
||||
cpu_cores: number
|
||||
cpu_threads: number
|
||||
ram_bytes: number
|
||||
disk_type: string
|
||||
gpu_model: string | null
|
||||
cpu_score: number
|
||||
memory_score: number
|
||||
disk_read_score: number
|
||||
disk_write_score: number
|
||||
ai_tokens_per_second: number | null
|
||||
ai_model_used: string | null
|
||||
ai_time_to_first_token: number | null
|
||||
nomad_score: number
|
||||
submitted_to_repository: boolean
|
||||
repository_id: string | null
|
||||
created_at: string
|
||||
}
|
||||
|
||||
type BenchmarkStatus = 'idle' | 'starting' | 'detecting_hardware' | 'running_cpu' | 'running_memory' | 'running_disk_read' | 'running_disk_write' | 'running_ai' | 'calculating_score' | 'completed' | 'error'
|
||||
|
||||
type BenchmarkProgress = {
|
||||
status: BenchmarkStatus
|
||||
progress: number
|
||||
message: string
|
||||
current_stage: string
|
||||
benchmark_id: string
|
||||
}
|
||||
type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string }
|
||||
|
||||
export default function BenchmarkPage(props: {
|
||||
benchmark: {
|
||||
|
|
@ -59,7 +30,7 @@ export default function BenchmarkPage(props: {
|
|||
}
|
||||
}) {
|
||||
const { subscribe } = useTransmit()
|
||||
const [progress, setProgress] = useState<BenchmarkProgress | null>(null)
|
||||
const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
|
||||
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
|
||||
const [showDetails, setShowDetails] = useState(false)
|
||||
|
||||
|
|
@ -84,6 +55,7 @@ export default function BenchmarkPage(props: {
|
|||
message: 'Starting benchmark... This takes 2-5 minutes.',
|
||||
current_stage: 'Starting',
|
||||
benchmark_id: '',
|
||||
timestamp: new Date().toISOString(),
|
||||
})
|
||||
|
||||
// Use sync mode - runs inline without needing Redis/queue worker
|
||||
|
|
@ -102,6 +74,7 @@ export default function BenchmarkPage(props: {
|
|||
message: 'Benchmark completed!',
|
||||
current_stage: 'Complete',
|
||||
benchmark_id: data.benchmark_id,
|
||||
timestamp: new Date().toISOString(),
|
||||
})
|
||||
refetchLatest()
|
||||
} else {
|
||||
|
|
@ -111,6 +84,7 @@ export default function BenchmarkPage(props: {
|
|||
message: data.error || 'Benchmark failed',
|
||||
current_stage: 'Error',
|
||||
benchmark_id: '',
|
||||
timestamp: new Date().toISOString(),
|
||||
})
|
||||
}
|
||||
setIsRunning(false)
|
||||
|
|
@ -122,6 +96,7 @@ export default function BenchmarkPage(props: {
|
|||
message: error.message || 'Benchmark failed',
|
||||
current_stage: 'Error',
|
||||
benchmark_id: '',
|
||||
timestamp: new Date().toISOString(),
|
||||
})
|
||||
setIsRunning(false)
|
||||
},
|
||||
|
|
@ -174,6 +149,7 @@ export default function BenchmarkPage(props: {
|
|||
message: stage.message,
|
||||
current_stage: stage.label,
|
||||
benchmark_id: '',
|
||||
timestamp: new Date().toISOString(),
|
||||
})
|
||||
currentStage++
|
||||
}
|
||||
|
|
@ -194,7 +170,7 @@ export default function BenchmarkPage(props: {
|
|||
|
||||
// Listen for benchmark progress via SSE (backup for async mode)
|
||||
useEffect(() => {
|
||||
const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgress) => {
|
||||
const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgressWithID) => {
|
||||
setProgress(data)
|
||||
if (data.status === 'completed' || data.status === 'error') {
|
||||
setIsRunning(false)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import BenchmarkResult from '#models/benchmark_result'
|
||||
|
||||
// Benchmark type identifiers
|
||||
export type BenchmarkType = 'full' | 'system' | 'ai'
|
||||
|
||||
|
|
@ -18,43 +20,21 @@ export type BenchmarkStatus =
|
|||
// Hardware detection types
|
||||
export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown'
|
||||
|
||||
export type HardwareInfo = {
|
||||
cpu_model: string
|
||||
cpu_cores: number
|
||||
cpu_threads: number
|
||||
ram_bytes: number
|
||||
disk_type: DiskType
|
||||
gpu_model: string | null
|
||||
}
|
||||
export type HardwareInfo = Pick<
|
||||
BenchmarkResult,
|
||||
'cpu_model' | 'cpu_cores' | 'cpu_threads' | 'ram_bytes' | 'disk_type' | 'gpu_model'
|
||||
>
|
||||
|
||||
// Individual benchmark scores
|
||||
export type SystemScores = {
|
||||
cpu_score: number
|
||||
memory_score: number
|
||||
disk_read_score: number
|
||||
disk_write_score: number
|
||||
}
|
||||
export type SystemScores = Pick<
|
||||
BenchmarkResult,
|
||||
'cpu_score' | 'memory_score' | 'disk_read_score' | 'disk_write_score'
|
||||
>
|
||||
|
||||
export type AIScores = {
|
||||
ai_tokens_per_second: number
|
||||
ai_model_used: string
|
||||
ai_time_to_first_token: number
|
||||
}
|
||||
|
||||
// Complete benchmark result
|
||||
export type BenchmarkResult = {
|
||||
id: number
|
||||
benchmark_id: string
|
||||
benchmark_type: BenchmarkType
|
||||
hardware: HardwareInfo
|
||||
scores: SystemScores & Partial<AIScores>
|
||||
nomad_score: number
|
||||
submitted_to_repository: boolean
|
||||
submitted_at: string | null
|
||||
repository_id: string | null
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
export type AIScores = Pick<
|
||||
BenchmarkResult,
|
||||
'ai_tokens_per_second' | 'ai_model_used' | 'ai_time_to_first_token'
|
||||
>
|
||||
|
||||
// Slim version for lists
|
||||
export type BenchmarkResultSlim = Pick<
|
||||
|
|
@ -114,22 +94,24 @@ export type BenchmarkResultsResponse = {
|
|||
}
|
||||
|
||||
// Central repository submission payload (privacy-first)
|
||||
export type RepositorySubmission = {
|
||||
cpu_model: string
|
||||
cpu_cores: number
|
||||
cpu_threads: number
|
||||
ram_gb: number
|
||||
disk_type: DiskType
|
||||
gpu_model: string | null
|
||||
cpu_score: number
|
||||
memory_score: number
|
||||
disk_read_score: number
|
||||
disk_write_score: number
|
||||
ai_tokens_per_second: number | null
|
||||
ai_time_to_first_token: number | null
|
||||
nomad_score: number
|
||||
export type RepositorySubmission = Pick<
|
||||
BenchmarkResult,
|
||||
| 'cpu_model'
|
||||
| 'cpu_cores'
|
||||
| 'cpu_threads'
|
||||
| 'disk_type'
|
||||
| 'gpu_model'
|
||||
| 'cpu_score'
|
||||
| 'memory_score'
|
||||
| 'disk_read_score'
|
||||
| 'disk_write_score'
|
||||
| 'ai_tokens_per_second'
|
||||
| 'ai_time_to_first_token'
|
||||
| 'nomad_score'
|
||||
> & {
|
||||
nomad_version: string
|
||||
benchmark_version: string
|
||||
ram_gb: number
|
||||
}
|
||||
|
||||
// Central repository response types
|
||||
|
|
@ -153,11 +135,8 @@ export type RepositoryStats = {
|
|||
}
|
||||
}
|
||||
|
||||
export type LeaderboardEntry = {
|
||||
export type LeaderboardEntry = Pick<BenchmarkResult, 'cpu_model' | 'gpu_model' | 'nomad_score'> & {
|
||||
rank: number
|
||||
cpu_model: string
|
||||
gpu_model: string | null
|
||||
nomad_score: number
|
||||
submitted_at: string
|
||||
}
|
||||
|
||||
|
|
@ -179,12 +158,12 @@ export type ScoreWeights = {
|
|||
|
||||
// Default weights as defined in plan
|
||||
export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = {
|
||||
ai_tokens_per_second: 0.30,
|
||||
ai_tokens_per_second: 0.3,
|
||||
cpu: 0.25,
|
||||
memory: 0.15,
|
||||
ai_ttft: 0.10,
|
||||
disk_read: 0.10,
|
||||
disk_write: 0.10,
|
||||
ai_ttft: 0.1,
|
||||
disk_read: 0.1,
|
||||
disk_write: 0.1,
|
||||
}
|
||||
|
||||
// Benchmark job parameters
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ node ace db:seed
|
|||
echo "Starting background workers for queues..."
|
||||
node ace queue:work --queue=downloads &
|
||||
node ace queue:work --queue=model-downloads &
|
||||
node ace queue:work --queue
|
||||
|
||||
# Start the AdonisJS application
|
||||
echo "Starting AdonisJS application..."
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user