mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-04-04 15:56:16 +02:00
fix(Benchmark): cleanup types for SSOT
This commit is contained in:
parent
6efd049424
commit
438d683bac
|
|
@ -452,7 +452,7 @@ export class BenchmarkService {
|
||||||
totalWeight += SCORE_WEIGHTS.disk_write
|
totalWeight += SCORE_WEIGHTS.disk_write
|
||||||
|
|
||||||
// AI scores (if available)
|
// AI scores (if available)
|
||||||
if (aiScores.ai_tokens_per_second !== undefined) {
|
if (aiScores.ai_tokens_per_second !== undefined && aiScores.ai_tokens_per_second !== null) {
|
||||||
const aiScore = this._normalizeScore(
|
const aiScore = this._normalizeScore(
|
||||||
aiScores.ai_tokens_per_second,
|
aiScores.ai_tokens_per_second,
|
||||||
REFERENCE_SCORES.ai_tokens_per_second
|
REFERENCE_SCORES.ai_tokens_per_second
|
||||||
|
|
@ -461,7 +461,7 @@ export class BenchmarkService {
|
||||||
totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
|
totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
|
||||||
}
|
}
|
||||||
|
|
||||||
if (aiScores.ai_time_to_first_token !== undefined) {
|
if (aiScores.ai_time_to_first_token !== undefined && aiScores.ai_time_to_first_token !== null) {
|
||||||
// For TTFT, lower is better, so we invert the score
|
// For TTFT, lower is better, so we invert the score
|
||||||
const ttftScore = this._normalizeScoreInverse(
|
const ttftScore = this._normalizeScoreInverse(
|
||||||
aiScores.ai_time_to_first_token,
|
aiScores.ai_time_to_first_token,
|
||||||
|
|
|
||||||
|
|
@ -17,39 +17,10 @@ import {
|
||||||
} from '@heroicons/react/24/outline'
|
} from '@heroicons/react/24/outline'
|
||||||
import { IconRobot } from '@tabler/icons-react'
|
import { IconRobot } from '@tabler/icons-react'
|
||||||
import { useTransmit } from 'react-adonis-transmit'
|
import { useTransmit } from 'react-adonis-transmit'
|
||||||
|
import { BenchmarkProgress, BenchmarkStatus } from '../../../types/benchmark'
|
||||||
|
import BenchmarkResult from '#models/benchmark_result'
|
||||||
|
|
||||||
type BenchmarkResult = {
|
type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string }
|
||||||
id: number
|
|
||||||
benchmark_id: string
|
|
||||||
benchmark_type: 'full' | 'system' | 'ai'
|
|
||||||
cpu_model: string
|
|
||||||
cpu_cores: number
|
|
||||||
cpu_threads: number
|
|
||||||
ram_bytes: number
|
|
||||||
disk_type: string
|
|
||||||
gpu_model: string | null
|
|
||||||
cpu_score: number
|
|
||||||
memory_score: number
|
|
||||||
disk_read_score: number
|
|
||||||
disk_write_score: number
|
|
||||||
ai_tokens_per_second: number | null
|
|
||||||
ai_model_used: string | null
|
|
||||||
ai_time_to_first_token: number | null
|
|
||||||
nomad_score: number
|
|
||||||
submitted_to_repository: boolean
|
|
||||||
repository_id: string | null
|
|
||||||
created_at: string
|
|
||||||
}
|
|
||||||
|
|
||||||
type BenchmarkStatus = 'idle' | 'starting' | 'detecting_hardware' | 'running_cpu' | 'running_memory' | 'running_disk_read' | 'running_disk_write' | 'running_ai' | 'calculating_score' | 'completed' | 'error'
|
|
||||||
|
|
||||||
type BenchmarkProgress = {
|
|
||||||
status: BenchmarkStatus
|
|
||||||
progress: number
|
|
||||||
message: string
|
|
||||||
current_stage: string
|
|
||||||
benchmark_id: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export default function BenchmarkPage(props: {
|
export default function BenchmarkPage(props: {
|
||||||
benchmark: {
|
benchmark: {
|
||||||
|
|
@ -59,7 +30,7 @@ export default function BenchmarkPage(props: {
|
||||||
}
|
}
|
||||||
}) {
|
}) {
|
||||||
const { subscribe } = useTransmit()
|
const { subscribe } = useTransmit()
|
||||||
const [progress, setProgress] = useState<BenchmarkProgress | null>(null)
|
const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
|
||||||
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
|
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
|
||||||
const [showDetails, setShowDetails] = useState(false)
|
const [showDetails, setShowDetails] = useState(false)
|
||||||
|
|
||||||
|
|
@ -84,6 +55,7 @@ export default function BenchmarkPage(props: {
|
||||||
message: 'Starting benchmark... This takes 2-5 minutes.',
|
message: 'Starting benchmark... This takes 2-5 minutes.',
|
||||||
current_stage: 'Starting',
|
current_stage: 'Starting',
|
||||||
benchmark_id: '',
|
benchmark_id: '',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
})
|
})
|
||||||
|
|
||||||
// Use sync mode - runs inline without needing Redis/queue worker
|
// Use sync mode - runs inline without needing Redis/queue worker
|
||||||
|
|
@ -102,6 +74,7 @@ export default function BenchmarkPage(props: {
|
||||||
message: 'Benchmark completed!',
|
message: 'Benchmark completed!',
|
||||||
current_stage: 'Complete',
|
current_stage: 'Complete',
|
||||||
benchmark_id: data.benchmark_id,
|
benchmark_id: data.benchmark_id,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
})
|
})
|
||||||
refetchLatest()
|
refetchLatest()
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -111,6 +84,7 @@ export default function BenchmarkPage(props: {
|
||||||
message: data.error || 'Benchmark failed',
|
message: data.error || 'Benchmark failed',
|
||||||
current_stage: 'Error',
|
current_stage: 'Error',
|
||||||
benchmark_id: '',
|
benchmark_id: '',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
setIsRunning(false)
|
setIsRunning(false)
|
||||||
|
|
@ -122,6 +96,7 @@ export default function BenchmarkPage(props: {
|
||||||
message: error.message || 'Benchmark failed',
|
message: error.message || 'Benchmark failed',
|
||||||
current_stage: 'Error',
|
current_stage: 'Error',
|
||||||
benchmark_id: '',
|
benchmark_id: '',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
})
|
})
|
||||||
setIsRunning(false)
|
setIsRunning(false)
|
||||||
},
|
},
|
||||||
|
|
@ -174,6 +149,7 @@ export default function BenchmarkPage(props: {
|
||||||
message: stage.message,
|
message: stage.message,
|
||||||
current_stage: stage.label,
|
current_stage: stage.label,
|
||||||
benchmark_id: '',
|
benchmark_id: '',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
})
|
})
|
||||||
currentStage++
|
currentStage++
|
||||||
}
|
}
|
||||||
|
|
@ -194,7 +170,7 @@ export default function BenchmarkPage(props: {
|
||||||
|
|
||||||
// Listen for benchmark progress via SSE (backup for async mode)
|
// Listen for benchmark progress via SSE (backup for async mode)
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgress) => {
|
const unsubscribe = subscribe('benchmark-progress', (data: BenchmarkProgressWithID) => {
|
||||||
setProgress(data)
|
setProgress(data)
|
||||||
if (data.status === 'completed' || data.status === 'error') {
|
if (data.status === 'completed' || data.status === 'error') {
|
||||||
setIsRunning(false)
|
setIsRunning(false)
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
import BenchmarkResult from '#models/benchmark_result'
|
||||||
|
|
||||||
// Benchmark type identifiers
|
// Benchmark type identifiers
|
||||||
export type BenchmarkType = 'full' | 'system' | 'ai'
|
export type BenchmarkType = 'full' | 'system' | 'ai'
|
||||||
|
|
||||||
|
|
@ -18,43 +20,21 @@ export type BenchmarkStatus =
|
||||||
// Hardware detection types
|
// Hardware detection types
|
||||||
export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown'
|
export type DiskType = 'ssd' | 'hdd' | 'nvme' | 'unknown'
|
||||||
|
|
||||||
export type HardwareInfo = {
|
export type HardwareInfo = Pick<
|
||||||
cpu_model: string
|
BenchmarkResult,
|
||||||
cpu_cores: number
|
'cpu_model' | 'cpu_cores' | 'cpu_threads' | 'ram_bytes' | 'disk_type' | 'gpu_model'
|
||||||
cpu_threads: number
|
>
|
||||||
ram_bytes: number
|
|
||||||
disk_type: DiskType
|
|
||||||
gpu_model: string | null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Individual benchmark scores
|
// Individual benchmark scores
|
||||||
export type SystemScores = {
|
export type SystemScores = Pick<
|
||||||
cpu_score: number
|
BenchmarkResult,
|
||||||
memory_score: number
|
'cpu_score' | 'memory_score' | 'disk_read_score' | 'disk_write_score'
|
||||||
disk_read_score: number
|
>
|
||||||
disk_write_score: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export type AIScores = {
|
export type AIScores = Pick<
|
||||||
ai_tokens_per_second: number
|
BenchmarkResult,
|
||||||
ai_model_used: string
|
'ai_tokens_per_second' | 'ai_model_used' | 'ai_time_to_first_token'
|
||||||
ai_time_to_first_token: number
|
>
|
||||||
}
|
|
||||||
|
|
||||||
// Complete benchmark result
|
|
||||||
export type BenchmarkResult = {
|
|
||||||
id: number
|
|
||||||
benchmark_id: string
|
|
||||||
benchmark_type: BenchmarkType
|
|
||||||
hardware: HardwareInfo
|
|
||||||
scores: SystemScores & Partial<AIScores>
|
|
||||||
nomad_score: number
|
|
||||||
submitted_to_repository: boolean
|
|
||||||
submitted_at: string | null
|
|
||||||
repository_id: string | null
|
|
||||||
created_at: string
|
|
||||||
updated_at: string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Slim version for lists
|
// Slim version for lists
|
||||||
export type BenchmarkResultSlim = Pick<
|
export type BenchmarkResultSlim = Pick<
|
||||||
|
|
@ -114,22 +94,24 @@ export type BenchmarkResultsResponse = {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Central repository submission payload (privacy-first)
|
// Central repository submission payload (privacy-first)
|
||||||
export type RepositorySubmission = {
|
export type RepositorySubmission = Pick<
|
||||||
cpu_model: string
|
BenchmarkResult,
|
||||||
cpu_cores: number
|
| 'cpu_model'
|
||||||
cpu_threads: number
|
| 'cpu_cores'
|
||||||
ram_gb: number
|
| 'cpu_threads'
|
||||||
disk_type: DiskType
|
| 'disk_type'
|
||||||
gpu_model: string | null
|
| 'gpu_model'
|
||||||
cpu_score: number
|
| 'cpu_score'
|
||||||
memory_score: number
|
| 'memory_score'
|
||||||
disk_read_score: number
|
| 'disk_read_score'
|
||||||
disk_write_score: number
|
| 'disk_write_score'
|
||||||
ai_tokens_per_second: number | null
|
| 'ai_tokens_per_second'
|
||||||
ai_time_to_first_token: number | null
|
| 'ai_time_to_first_token'
|
||||||
nomad_score: number
|
| 'nomad_score'
|
||||||
|
> & {
|
||||||
nomad_version: string
|
nomad_version: string
|
||||||
benchmark_version: string
|
benchmark_version: string
|
||||||
|
ram_gb: number
|
||||||
}
|
}
|
||||||
|
|
||||||
// Central repository response types
|
// Central repository response types
|
||||||
|
|
@ -153,11 +135,8 @@ export type RepositoryStats = {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export type LeaderboardEntry = {
|
export type LeaderboardEntry = Pick<BenchmarkResult, 'cpu_model' | 'gpu_model' | 'nomad_score'> & {
|
||||||
rank: number
|
rank: number
|
||||||
cpu_model: string
|
|
||||||
gpu_model: string | null
|
|
||||||
nomad_score: number
|
|
||||||
submitted_at: string
|
submitted_at: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -179,12 +158,12 @@ export type ScoreWeights = {
|
||||||
|
|
||||||
// Default weights as defined in plan
|
// Default weights as defined in plan
|
||||||
export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = {
|
export const DEFAULT_SCORE_WEIGHTS: ScoreWeights = {
|
||||||
ai_tokens_per_second: 0.30,
|
ai_tokens_per_second: 0.3,
|
||||||
cpu: 0.25,
|
cpu: 0.25,
|
||||||
memory: 0.15,
|
memory: 0.15,
|
||||||
ai_ttft: 0.10,
|
ai_ttft: 0.1,
|
||||||
disk_read: 0.10,
|
disk_read: 0.1,
|
||||||
disk_write: 0.10,
|
disk_write: 0.1,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Benchmark job parameters
|
// Benchmark job parameters
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ node ace db:seed
|
||||||
echo "Starting background workers for queues..."
|
echo "Starting background workers for queues..."
|
||||||
node ace queue:work --queue=downloads &
|
node ace queue:work --queue=downloads &
|
||||||
node ace queue:work --queue=model-downloads &
|
node ace queue:work --queue=model-downloads &
|
||||||
|
node ace queue:work --queue
|
||||||
|
|
||||||
# Start the AdonisJS application
|
# Start the AdonisJS application
|
||||||
echo "Starting AdonisJS application..."
|
echo "Starting AdonisJS application..."
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user