feat(benchmark): Require full benchmark with AI for community sharing

Only allow users to share benchmark results with the community leaderboard
when they have completed a full benchmark that includes AI performance data.

Frontend changes:
- Add AI Assistant installation check via service API query
- Show pre-flight warning when clicking Full Benchmark without AI installed
- Disable AI Only button when AI Assistant not installed
- Show "Partial Benchmark" info alert for non-shareable results
- Only display "Share with Community" for full benchmarks with AI data
- Add note about AI installation requirement with link to Apps page

Backend changes:
- Validate benchmark_type is 'full' before allowing submission
- Require ai_tokens_per_second > 0 for community submission
- Return clear error messages explaining requirements

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood 2026-01-24 22:19:51 -08:00
parent 8f1b8de792
commit a9b8a906a6
2 changed files with 86 additions and 4 deletions

View File

@ -116,6 +116,15 @@ export class BenchmarkService {
throw new Error('No benchmark result found to submit')
}
// Only allow full benchmarks with AI data to be submitted to repository
if (result.benchmark_type !== 'full') {
throw new Error('Only full benchmarks can be shared with the community. Run a Full Benchmark to share your results.')
}
if (!result.ai_tokens_per_second || result.ai_tokens_per_second <= 0) {
throw new Error('Benchmark must include AI performance data. Ensure AI Assistant is installed and run a Full Benchmark.')
}
if (result.submitted_to_repository) {
throw new Error('Benchmark result has already been submitted')
}

View File

@ -1,4 +1,4 @@
import { Head } from '@inertiajs/react'
import { Head, Link } from '@inertiajs/react'
import { useState, useEffect } from 'react'
import SettingsLayout from '~/layouts/SettingsLayout'
import { useQuery, useMutation } from '@tanstack/react-query'
@ -31,6 +31,23 @@ export default function BenchmarkPage(props: {
const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
const [showDetails, setShowDetails] = useState(false)
const [showAIRequiredAlert, setShowAIRequiredAlert] = useState(false)
// Check if AI Assistant is installed
const { data: aiInstalled } = useQuery({
queryKey: ['services', 'ai-installed'],
queryFn: async () => {
const res = await fetch('/api/system/services')
const data = await res.json()
const services = Array.isArray(data) ? data : (data.services || [])
const openWebUI = services.find((s: any) =>
s.service_name === 'nomad_open_webui' || s.serviceName === 'nomad_open_webui'
)
return openWebUI?.installed === true || openWebUI?.installed === 1
},
staleTime: 0,
refetchOnMount: true,
})
// Fetch latest result
const { data: latestResult, refetch: refetchLatest } = useQuery({
@ -124,6 +141,23 @@ export default function BenchmarkPage(props: {
},
})
// Check if the latest result is a full benchmark with AI data (eligible for sharing)
const canShareBenchmark = latestResult &&
latestResult.benchmark_type === 'full' &&
latestResult.ai_tokens_per_second !== null &&
latestResult.ai_tokens_per_second > 0 &&
!latestResult.submitted_to_repository
// Handle Full Benchmark click with pre-flight check
const handleFullBenchmarkClick = () => {
if (!aiInstalled) {
setShowAIRequiredAlert(true)
return
}
setShowAIRequiredAlert(false)
runBenchmark.mutate('full')
}
// Simulate progress during sync benchmark (since we don't get SSE updates)
useEffect(() => {
if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return
@ -269,13 +303,30 @@ export default function BenchmarkPage(props: {
onDismiss={() => setProgress(null)}
/>
)}
{showAIRequiredAlert && (
<Alert
type="warning"
title="AI Assistant Required"
message="Full benchmark requires AI Assistant to be installed. Install it to measure your complete NOMAD capability and share results with the community."
variant="bordered"
dismissible
onDismiss={() => setShowAIRequiredAlert(false)}
>
<Link
href="/settings/apps"
className="text-sm text-desert-green hover:underline mt-2 inline-block font-medium"
>
Go to Apps to install AI Assistant
</Link>
</Alert>
)}
<p className="text-desert-stone-dark">
Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance.
The benchmark takes approximately 2-5 minutes to complete.
</p>
<div className="flex flex-wrap gap-4">
<StyledButton
onClick={() => runBenchmark.mutate('full')}
onClick={handleFullBenchmarkClick}
disabled={runBenchmark.isPending}
icon='PlayIcon'
>
@ -292,12 +343,21 @@ export default function BenchmarkPage(props: {
<StyledButton
variant="secondary"
onClick={() => runBenchmark.mutate('ai')}
disabled={runBenchmark.isPending}
disabled={runBenchmark.isPending || !aiInstalled}
icon='SparklesIcon'
title={!aiInstalled ? 'AI Assistant must be installed to run AI benchmark' : undefined}
>
AI Only
</StyledButton>
</div>
{!aiInstalled && (
<p className="text-sm text-desert-stone-dark">
<span className="text-amber-600">Note:</span> AI Assistant is not installed.
<Link href="/settings/apps" className="text-desert-green hover:underline ml-1">
Install it
</Link> to run full benchmarks and share results with the community.
</p>
)}
</div>
)}
</div>
@ -331,7 +391,9 @@ export default function BenchmarkPage(props: {
<p className="text-desert-stone-dark">
Your NOMAD Score is a weighted composite of all benchmark results.
</p>
{!latestResult.submitted_to_repository && (
{/* Share with Community - Only for full benchmarks with AI data */}
{canShareBenchmark && (
<div className="space-y-3">
<p className="text-sm text-desert-stone-dark">
Share your benchmark score anonymously with the NOMAD community. Only your hardware specs and scores are sent no identifying information.
@ -355,6 +417,17 @@ export default function BenchmarkPage(props: {
)}
</div>
)}
{/* Show message for partial benchmarks */}
{latestResult && !latestResult.submitted_to_repository && !canShareBenchmark && (
<Alert
type="info"
title="Partial Benchmark"
message={`This ${latestResult.benchmark_type} benchmark cannot be shared with the community. Run a Full Benchmark with AI Assistant installed to share your results.`}
variant="bordered"
/>
)}
{latestResult.submitted_to_repository && (
<Alert
type="success"