feat(benchmark): Require full benchmark with AI for community sharing

Only allow users to share benchmark results with the community leaderboard
when they have completed a full benchmark that includes AI performance data.

Frontend changes:
- Add AI Assistant installation check via service API query
- Show pre-flight warning when clicking Full Benchmark without AI installed
- Disable AI Only button when AI Assistant not installed
- Show "Partial Benchmark" info alert for non-shareable results
- Only display "Share with Community" for full benchmarks with AI data
- Add note about AI installation requirement with link to Apps page

Backend changes:
- Validate benchmark_type is 'full' before allowing submission
- Require ai_tokens_per_second > 0 for community submission
- Return clear error messages explaining requirements

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood 2026-01-24 22:19:51 -08:00
parent 8f1b8de792
commit a9b8a906a6
2 changed files with 86 additions and 4 deletions

View File

@ -116,6 +116,15 @@ export class BenchmarkService {
throw new Error('No benchmark result found to submit') throw new Error('No benchmark result found to submit')
} }
// Only allow full benchmarks with AI data to be submitted to repository
if (result.benchmark_type !== 'full') {
throw new Error('Only full benchmarks can be shared with the community. Run a Full Benchmark to share your results.')
}
if (!result.ai_tokens_per_second || result.ai_tokens_per_second <= 0) {
throw new Error('Benchmark must include AI performance data. Ensure AI Assistant is installed and run a Full Benchmark.')
}
if (result.submitted_to_repository) { if (result.submitted_to_repository) {
throw new Error('Benchmark result has already been submitted') throw new Error('Benchmark result has already been submitted')
} }

View File

@ -1,4 +1,4 @@
import { Head } from '@inertiajs/react' import { Head, Link } from '@inertiajs/react'
import { useState, useEffect } from 'react' import { useState, useEffect } from 'react'
import SettingsLayout from '~/layouts/SettingsLayout' import SettingsLayout from '~/layouts/SettingsLayout'
import { useQuery, useMutation } from '@tanstack/react-query' import { useQuery, useMutation } from '@tanstack/react-query'
@ -31,6 +31,23 @@ export default function BenchmarkPage(props: {
const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null) const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle') const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
const [showDetails, setShowDetails] = useState(false) const [showDetails, setShowDetails] = useState(false)
const [showAIRequiredAlert, setShowAIRequiredAlert] = useState(false)
// Check if AI Assistant is installed
const { data: aiInstalled } = useQuery({
queryKey: ['services', 'ai-installed'],
queryFn: async () => {
const res = await fetch('/api/system/services')
const data = await res.json()
const services = Array.isArray(data) ? data : (data.services || [])
const openWebUI = services.find((s: any) =>
s.service_name === 'nomad_open_webui' || s.serviceName === 'nomad_open_webui'
)
return openWebUI?.installed === true || openWebUI?.installed === 1
},
staleTime: 0,
refetchOnMount: true,
})
// Fetch latest result // Fetch latest result
const { data: latestResult, refetch: refetchLatest } = useQuery({ const { data: latestResult, refetch: refetchLatest } = useQuery({
@ -124,6 +141,23 @@ export default function BenchmarkPage(props: {
}, },
}) })
// Check if the latest result is a full benchmark with AI data (eligible for sharing)
const canShareBenchmark = latestResult &&
latestResult.benchmark_type === 'full' &&
latestResult.ai_tokens_per_second !== null &&
latestResult.ai_tokens_per_second > 0 &&
!latestResult.submitted_to_repository
// Handle Full Benchmark click with pre-flight check
const handleFullBenchmarkClick = () => {
if (!aiInstalled) {
setShowAIRequiredAlert(true)
return
}
setShowAIRequiredAlert(false)
runBenchmark.mutate('full')
}
// Simulate progress during sync benchmark (since we don't get SSE updates) // Simulate progress during sync benchmark (since we don't get SSE updates)
useEffect(() => { useEffect(() => {
if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return
@ -269,13 +303,30 @@ export default function BenchmarkPage(props: {
onDismiss={() => setProgress(null)} onDismiss={() => setProgress(null)}
/> />
)} )}
{showAIRequiredAlert && (
<Alert
type="warning"
title="AI Assistant Required"
message="Full benchmark requires AI Assistant to be installed. Install it to measure your complete NOMAD capability and share results with the community."
variant="bordered"
dismissible
onDismiss={() => setShowAIRequiredAlert(false)}
>
<Link
href="/settings/apps"
className="text-sm text-desert-green hover:underline mt-2 inline-block font-medium"
>
Go to Apps to install AI Assistant
</Link>
</Alert>
)}
<p className="text-desert-stone-dark"> <p className="text-desert-stone-dark">
Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance. Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance.
The benchmark takes approximately 2-5 minutes to complete. The benchmark takes approximately 2-5 minutes to complete.
</p> </p>
<div className="flex flex-wrap gap-4"> <div className="flex flex-wrap gap-4">
<StyledButton <StyledButton
onClick={() => runBenchmark.mutate('full')} onClick={handleFullBenchmarkClick}
disabled={runBenchmark.isPending} disabled={runBenchmark.isPending}
icon='PlayIcon' icon='PlayIcon'
> >
@ -292,12 +343,21 @@ export default function BenchmarkPage(props: {
<StyledButton <StyledButton
variant="secondary" variant="secondary"
onClick={() => runBenchmark.mutate('ai')} onClick={() => runBenchmark.mutate('ai')}
disabled={runBenchmark.isPending} disabled={runBenchmark.isPending || !aiInstalled}
icon='SparklesIcon' icon='SparklesIcon'
title={!aiInstalled ? 'AI Assistant must be installed to run AI benchmark' : undefined}
> >
AI Only AI Only
</StyledButton> </StyledButton>
</div> </div>
{!aiInstalled && (
<p className="text-sm text-desert-stone-dark">
<span className="text-amber-600">Note:</span> AI Assistant is not installed.
<Link href="/settings/apps" className="text-desert-green hover:underline ml-1">
Install it
</Link> to run full benchmarks and share results with the community.
</p>
)}
</div> </div>
)} )}
</div> </div>
@ -331,7 +391,9 @@ export default function BenchmarkPage(props: {
<p className="text-desert-stone-dark"> <p className="text-desert-stone-dark">
Your NOMAD Score is a weighted composite of all benchmark results. Your NOMAD Score is a weighted composite of all benchmark results.
</p> </p>
{!latestResult.submitted_to_repository && (
{/* Share with Community - Only for full benchmarks with AI data */}
{canShareBenchmark && (
<div className="space-y-3"> <div className="space-y-3">
<p className="text-sm text-desert-stone-dark"> <p className="text-sm text-desert-stone-dark">
Share your benchmark score anonymously with the NOMAD community. Only your hardware specs and scores are sent no identifying information. Share your benchmark score anonymously with the NOMAD community. Only your hardware specs and scores are sent no identifying information.
@ -355,6 +417,17 @@ export default function BenchmarkPage(props: {
)} )}
</div> </div>
)} )}
{/* Show message for partial benchmarks */}
{latestResult && !latestResult.submitted_to_repository && !canShareBenchmark && (
<Alert
type="info"
title="Partial Benchmark"
message={`This ${latestResult.benchmark_type} benchmark cannot be shared with the community. Run a Full Benchmark with AI Assistant installed to share your results.`}
variant="bordered"
/>
)}
{latestResult.submitted_to_repository && ( {latestResult.submitted_to_repository && (
<Alert <Alert
type="success" type="success"