From 4584844ca6919b33c90ad0a3f9b2191ac3e5997c Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sun, 1 Feb 2026 05:23:11 +0000 Subject: [PATCH] refactor(Benchmarks): cleanup api calls --- admin/app/services/zim_service.ts | 4 - admin/inertia/lib/api.ts | 57 +++- admin/inertia/pages/settings/benchmark.tsx | 331 +++++++++++++-------- admin/types/benchmark.ts | 17 ++ 4 files changed, 282 insertions(+), 127 deletions(-) diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 27f4297..a0a7bcb 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -517,10 +517,6 @@ export class ZimService implements IZimService { } if (success) { - // Get the old filename before updating (if there was a previous Wikipedia installed) - const options = await this.getWikipediaOptions() - const previousOption = options.find((opt) => opt.id !== selection.option_id && opt.id !== 'none') - // Update status to installed selection.status = 'installed' await selection.save() diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 393e6ba..23a2ce5 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -12,6 +12,8 @@ import { import { catchInternal } from './util' import { NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' import { ChatResponse, ModelResponse } from 'ollama' +import BenchmarkResult from '#models/benchmark_result' +import { BenchmarkType, RunBenchmarkResponse, SubmitBenchmarkResponse, UpdateBuilderTagResponse } from '../../types/benchmark' class API { private client: AxiosInstance @@ -163,6 +165,20 @@ class API { })() } + async getBenchmarkResults() { + return catchInternal(async () => { + const response = await this.client.get('/benchmark/results') + return response.data + })() + } + + async getLatestBenchmarkResult() { + return catchInternal(async () => { + const response = await this.client.get('/benchmark/results/latest') + return response.data + })() + } + async getChatSessions() { return catchInternal(async () => { const response = await this.client.get< @@ -261,6 +277,13 @@ class API { })() } + async getSystemServices() { + return catchInternal(async () => { + const response = await this.client.get>('/system/services') + return response.data + })() + } + async getSystemUpdateStatus() { return catchInternal(async () => { const response = await this.client.get('/system/update/status') @@ -343,13 +366,6 @@ class API { })() } - async listServices() { - return catchInternal(async () => { - const response = await this.client.get>('/system/services') - return response.data - })() - } - async listRemoteZimFiles({ start = 0, count = 12, @@ -384,6 +400,16 @@ class API { })() } + async runBenchmark(type: BenchmarkType, sync: boolean = false) { + return catchInternal(async () => { + const response = await this.client.post( + `/benchmark/run${sync ? '?sync=true' : ''}`, + { benchmark_type: type }, + ) + return response.data + })() + } + async startSystemUpdate() { return catchInternal(async () => { const response = await this.client.post<{ success: boolean; message: string }>( @@ -393,6 +419,13 @@ class API { })() } + async submitBenchmark(benchmark_id: string, anonymous: boolean) { + return catchInternal(async () => { + const response = await this.client.post('/benchmark/submit', { benchmark_id, anonymous }) + return response.data + })() + } + async subscribeToReleaseNotes(email: string) { return catchInternal(async () => { const response = await this.client.post<{ success: boolean; message: string }>( @@ -425,6 +458,16 @@ class API { })() } + async updateBuilderTag(benchmark_id: string, builder_tag: string) { + return catchInternal(async () => { + const response = await this.client.post( + '/benchmark/builder-tag', + { benchmark_id, builder_tag } + ) + return response.data + })() + } + async uploadDocument(file: File) { return catchInternal(async () => { const formData = new FormData() diff --git a/admin/inertia/pages/settings/benchmark.tsx b/admin/inertia/pages/settings/benchmark.tsx index 4c6fd8a..318555b 100644 --- a/admin/inertia/pages/settings/benchmark.tsx +++ b/admin/inertia/pages/settings/benchmark.tsx @@ -8,10 +8,20 @@ import Alert from '~/components/Alert' import StyledButton from '~/components/StyledButton' import InfoTooltip from '~/components/InfoTooltip' import BuilderTagSelector from '~/components/BuilderTagSelector' -import { IconRobot, IconChartBar, IconCpu, IconDatabase, IconServer, IconChevronDown, IconClock } from '@tabler/icons-react' +import { + IconRobot, + IconChartBar, + IconCpu, + IconDatabase, + IconServer, + IconChevronDown, + IconClock, +} from '@tabler/icons-react' import { useTransmit } from 'react-adonis-transmit' import { BenchmarkProgress, BenchmarkStatus } from '../../../types/benchmark' import BenchmarkResult from '#models/benchmark_result' +import api from '~/lib/api' +import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string } @@ -24,6 +34,7 @@ export default function BenchmarkPage(props: { }) { const { subscribe } = useTransmit() const queryClient = useQueryClient() + const aiInstalled = useServiceInstalledStatus('nomad_ollama') const [progress, setProgress] = useState(null) const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle') const [showDetails, setShowDetails] = useState(false) @@ -34,29 +45,12 @@ export default function BenchmarkPage(props: { props.benchmark.latestResult?.builder_tag || null ) - // Check if AI Assistant is installed - const { data: aiInstalled } = useQuery({ - queryKey: ['services', 'ai-installed'], - queryFn: async () => { - const res = await fetch('/api/system/services') - const data = await res.json() - const services = Array.isArray(data) ? data : (data.services || []) - const openWebUI = services.find((s: any) => - s.service_name === 'nomad_open_webui' || s.serviceName === 'nomad_open_webui' - ) - return openWebUI?.installed === true || openWebUI?.installed === 1 - }, - staleTime: 0, - refetchOnMount: true, - }) - // Fetch latest result const { data: latestResult, refetch: refetchLatest } = useQuery({ queryKey: ['benchmark', 'latest'], queryFn: async () => { - const res = await fetch('/api/benchmark/results/latest') - const data = await res.json() - return data.result as BenchmarkResult | null + const res = await api.getLatestBenchmarkResult() + return res ?? null }, initialData: props.benchmark.latestResult, }) @@ -65,9 +59,8 @@ export default function BenchmarkPage(props: { const { data: benchmarkHistory } = useQuery({ queryKey: ['benchmark', 'history'], queryFn: async () => { - const res = await fetch('/api/benchmark/results') - const data = await res.json() - return data.results as BenchmarkResult[] + const res = await api.getBenchmarkResults() + return res ?? [] }, }) @@ -85,15 +78,10 @@ export default function BenchmarkPage(props: { }) // Use sync mode - runs inline without needing Redis/queue worker - const res = await fetch('/api/benchmark/run?sync=true', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ benchmark_type: type }), - }) - return res.json() + return await api.runBenchmark(type, true) }, onSuccess: (data) => { - if (data.success) { + if (data?.success) { setProgress({ status: 'completed', progress: 100, @@ -107,7 +95,7 @@ export default function BenchmarkPage(props: { setProgress({ status: 'error', progress: 0, - message: data.error || 'Benchmark failed', + message: 'Benchmark failed', current_stage: 'Error', benchmark_id: '', timestamp: new Date().toISOString(), @@ -130,21 +118,25 @@ export default function BenchmarkPage(props: { // Update builder tag mutation const updateBuilderTag = useMutation({ - mutationFn: async ({ benchmarkId, builderTag }: { benchmarkId: string; builderTag: string }) => { - const res = await fetch('/api/benchmark/builder-tag', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ benchmark_id: benchmarkId, builder_tag: builderTag }), - }) - const data = await res.json() - if (!data.success) { - throw new Error(data.error || 'Failed to update builder tag') + mutationFn: async ({ + benchmarkId, + builderTag + }: { + benchmarkId: string + builderTag: string + invalidate?: boolean + }) => { + const res = await api.updateBuilderTag(benchmarkId, builderTag) + if (!res || !res.success) { + throw new Error(res?.error || 'Failed to update builder tag') } - return data + return res }, - onSuccess: () => { - refetchLatest() - queryClient.invalidateQueries({ queryKey: ['benchmark', 'history'] }) + onSuccess: (_, variables) => { + if (variables.invalidate) { + refetchLatest() + queryClient.invalidateQueries({ queryKey: ['benchmark', 'history'] }) + } }, }) @@ -154,25 +146,16 @@ export default function BenchmarkPage(props: { mutationFn: async ({ benchmarkId, anonymous }: { benchmarkId: string; anonymous: boolean }) => { setSubmitError(null) - // First, save the current builder tag to the benchmark + // First, save the current builder tag to the benchmark (don't refetch yet) if (currentBuilderTag && !anonymous) { - await fetch('/api/benchmark/builder-tag', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ benchmark_id: benchmarkId, builder_tag: currentBuilderTag }), - }) + await updateBuilderTag.mutateAsync({ benchmarkId, builderTag: currentBuilderTag, invalidate: false }) } - const res = await fetch('/api/benchmark/submit', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ benchmark_id: benchmarkId, anonymous }), - }) - const data = await res.json() - if (!data.success) { - throw new Error(data.error || 'Failed to submit benchmark') + const res = await api.submitBenchmark(benchmarkId, anonymous) + if (!res || !res.success) { + throw new Error(res?.error || 'Failed to submit benchmark') } - return data + return res }, onSuccess: () => { refetchLatest() @@ -184,7 +167,8 @@ export default function BenchmarkPage(props: { }) // Check if the latest result is a full benchmark with AI data (eligible for sharing) - const canShareBenchmark = latestResult && + const canShareBenchmark = + latestResult && latestResult.benchmark_type === 'full' && latestResult.ai_tokens_per_second !== null && latestResult.ai_tokens_per_second > 0 && @@ -204,15 +188,69 @@ export default function BenchmarkPage(props: { useEffect(() => { if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return - const stages: { status: BenchmarkStatus; progress: number; message: string; label: string; duration: number }[] = [ - { status: 'detecting_hardware', progress: 10, message: 'Detecting system hardware...', label: 'Detecting Hardware', duration: 2000 }, - { status: 'running_cpu', progress: 25, message: 'Running CPU benchmark (30s)...', label: 'CPU Benchmark', duration: 32000 }, - { status: 'running_memory', progress: 40, message: 'Running memory benchmark...', label: 'Memory Benchmark', duration: 8000 }, - { status: 'running_disk_read', progress: 55, message: 'Running disk read benchmark (30s)...', label: 'Disk Read Test', duration: 35000 }, - { status: 'running_disk_write', progress: 70, message: 'Running disk write benchmark (30s)...', label: 'Disk Write Test', duration: 35000 }, - { status: 'downloading_ai_model', progress: 80, message: 'Downloading AI benchmark model (first run only)...', label: 'Downloading AI Model', duration: 5000 }, - { status: 'running_ai', progress: 85, message: 'Running AI inference benchmark...', label: 'AI Inference Test', duration: 15000 }, - { status: 'calculating_score', progress: 95, message: 'Calculating NOMAD score...', label: 'Calculating Score', duration: 2000 }, + const stages: { + status: BenchmarkStatus + progress: number + message: string + label: string + duration: number + }[] = [ + { + status: 'detecting_hardware', + progress: 10, + message: 'Detecting system hardware...', + label: 'Detecting Hardware', + duration: 2000, + }, + { + status: 'running_cpu', + progress: 25, + message: 'Running CPU benchmark (30s)...', + label: 'CPU Benchmark', + duration: 32000, + }, + { + status: 'running_memory', + progress: 40, + message: 'Running memory benchmark...', + label: 'Memory Benchmark', + duration: 8000, + }, + { + status: 'running_disk_read', + progress: 55, + message: 'Running disk read benchmark (30s)...', + label: 'Disk Read Test', + duration: 35000, + }, + { + status: 'running_disk_write', + progress: 70, + message: 'Running disk write benchmark (30s)...', + label: 'Disk Write Test', + duration: 35000, + }, + { + status: 'downloading_ai_model', + progress: 80, + message: 'Downloading AI benchmark model (first run only)...', + label: 'Downloading AI Model', + duration: 5000, + }, + { + status: 'running_ai', + progress: 85, + message: 'Running AI inference benchmark...', + label: 'AI Inference Test', + duration: 15000, + }, + { + status: 'calculating_score', + progress: 95, + message: 'Calculating NOMAD score...', + label: 'Calculating Score', + duration: 2000, + }, ] let currentStage = 0 @@ -240,7 +278,7 @@ export default function BenchmarkPage(props: { }) return () => { - timers.forEach(t => clearTimeout(t)) + timers.forEach((t) => clearTimeout(t)) } }, [isRunning]) @@ -363,14 +401,14 @@ export default function BenchmarkPage(props: { )}

- Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance. - The benchmark takes approximately 2-5 minutes to complete. + Run a benchmark to measure your system's CPU, memory, disk, and AI inference + performance. The benchmark takes approximately 2-5 minutes to complete.

Run Full Benchmark @@ -378,7 +416,7 @@ export default function BenchmarkPage(props: { variant="secondary" onClick={() => runBenchmark.mutate('system')} disabled={runBenchmark.isPending} - icon='IconCpu' + icon="IconCpu" > System Only @@ -386,8 +424,12 @@ export default function BenchmarkPage(props: { variant="secondary" onClick={() => runBenchmark.mutate('ai')} disabled={runBenchmark.isPending || !aiInstalled} - icon='IconWand' - title={!aiInstalled ? 'AI Assistant must be installed to run AI benchmark' : undefined} + icon="IconWand" + title={ + !aiInstalled + ? 'AI Assistant must be installed to run AI benchmark' + : undefined + } > AI Only @@ -395,9 +437,13 @@ export default function BenchmarkPage(props: { {!aiInstalled && (

Note: AI Assistant is not installed. - + Install it - to run full benchmarks and share results with the community. + {' '} + to run full benchmarks and share results with the community.

)}
@@ -416,7 +462,7 @@ export default function BenchmarkPage(props: {
-
+
-
+
{latestResult.nomad_score.toFixed(1)}

@@ -439,7 +487,8 @@ export default function BenchmarkPage(props: {

Share with Community

- Share your benchmark on the community leaderboard. Choose a Builder Tag to claim your spot, or share anonymously. + Share your benchmark on the community leaderboard. Choose a Builder Tag + to claim your spot, or share anonymously.

{/* Builder Tag Selector */} @@ -469,12 +518,14 @@ export default function BenchmarkPage(props: { submitResult.mutate({ - benchmarkId: latestResult.benchmark_id, - anonymous: shareAnonymously - })} + onClick={() => + submitResult.mutate({ + benchmarkId: latestResult.benchmark_id, + anonymous: shareAnonymously, + }) + } disabled={submitResult.isPending} - icon='IconCloudUpload' + icon="IconCloudUpload" > {submitResult.isPending ? 'Submitting...' : 'Share with Community'} @@ -492,14 +543,16 @@ export default function BenchmarkPage(props: { )} {/* Show message for partial benchmarks */} - {latestResult && !latestResult.submitted_to_repository && !canShareBenchmark && ( - - )} + {latestResult && + !latestResult.submitted_to_repository && + !canShareBenchmark && ( + + )} {latestResult.submitted_to_repository && (

No AI Benchmark Data

- Run a Full Benchmark or AI Only benchmark to measure AI inference performance. + Run a Full Benchmark or AI Only benchmark to measure AI inference + performance.

@@ -674,7 +728,9 @@ export default function BenchmarkPage(props: {
Benchmark ID
-
{latestResult.benchmark_id.slice(0, 8)}...
+
+ {latestResult.benchmark_id.slice(0, 8)}... +
Type
@@ -682,11 +738,17 @@ export default function BenchmarkPage(props: {
Date
-
{new Date(latestResult.created_at as unknown as string).toLocaleDateString()}
+
+ {new Date( + latestResult.created_at as unknown as string + ).toLocaleDateString()} +
NOMAD Score
-
{latestResult.nomad_score.toFixed(1)}
+
+ {latestResult.nomad_score.toFixed(1)} +
CPU Score - {(latestResult.cpu_score * 100).toFixed(1)}% + + {(latestResult.cpu_score * 100).toFixed(1)}% +
Memory Score - {(latestResult.memory_score * 100).toFixed(1)}% + + {(latestResult.memory_score * 100).toFixed(1)}% +
Disk Read Score - {(latestResult.disk_read_score * 100).toFixed(1)}% + + {(latestResult.disk_read_score * 100).toFixed(1)}% +
Disk Write Score - {(latestResult.disk_write_score * 100).toFixed(1)}% + + {(latestResult.disk_write_score * 100).toFixed(1)}% +
{latestResult.ai_tokens_per_second && ( <>
AI Tokens/sec - {latestResult.ai_tokens_per_second.toFixed(1)} + + {latestResult.ai_tokens_per_second.toFixed(1)} +
- AI Time to First Token - {latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms + + AI Time to First Token + + + {latestResult.ai_time_to_first_token?.toFixed(0) || 'N/A'} ms +
)} @@ -747,11 +823,17 @@ export default function BenchmarkPage(props: {
Run Date - {new Date(latestResult.created_at as unknown as string).toLocaleString()} + + {new Date( + latestResult.created_at as unknown as string + ).toLocaleString()} +
Builder Tag - {latestResult.builder_tag || 'Not set'} + + {latestResult.builder_tag || 'Not set'} +
{latestResult.ai_model_used && (
@@ -760,13 +842,17 @@ export default function BenchmarkPage(props: {
)}
- Submitted to Repository + + Submitted to Repository + {latestResult.submitted_to_repository ? 'Yes' : 'No'}
{latestResult.repository_id && (
Repository ID - {latestResult.repository_id} + + {latestResult.repository_id} +
)}
@@ -793,7 +879,8 @@ export default function BenchmarkPage(props: {
- {benchmarkHistory.length} benchmark{benchmarkHistory.length !== 1 ? 's' : ''} recorded + {benchmarkHistory.length} benchmark + {benchmarkHistory.length !== 1 ? 's' : ''} recorded
- Date - Type - Score - Builder Tag - Shared + + Date + + + Type + + + Score + + + Builder Tag + + + Shared + @@ -825,7 +922,9 @@ export default function BenchmarkPage(props: { }`} > - {new Date(result.created_at as unknown as string).toLocaleDateString()} + {new Date( + result.created_at as unknown as string + ).toLocaleDateString()} {result.benchmark_type} diff --git a/admin/types/benchmark.ts b/admin/types/benchmark.ts index 91e4db8..9eda94b 100644 --- a/admin/types/benchmark.ts +++ b/admin/types/benchmark.ts @@ -95,6 +95,23 @@ export type BenchmarkResultsResponse = { total: number } +export type SubmitBenchmarkResponse = { + success: true + repository_id: string + percentile: number +} | { + success: false + error: string +} + +export type UpdateBuilderTagResponse = { + success: true, + builder_tag: string | null +} | { + success: false, + error: string +} + // Central repository submission payload (privacy-first) export type RepositorySubmission = Pick< BenchmarkResult,