mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-04-05 16:26:15 +02:00
feat(benchmark): Require full benchmark with AI for community sharing
Only allow users to share benchmark results with the community leaderboard when they have completed a full benchmark that includes AI performance data. Frontend changes: - Add AI Assistant installation check via service API query - Show pre-flight warning when clicking Full Benchmark without AI installed - Disable AI Only button when AI Assistant not installed - Show "Partial Benchmark" info alert for non-shareable results - Only display "Share with Community" for full benchmarks with AI data - Add note about AI installation requirement with link to Apps page Backend changes: - Validate benchmark_type is 'full' before allowing submission - Require ai_tokens_per_second > 0 for community submission - Return clear error messages explaining requirements Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
8f1b8de792
commit
a9b8a906a6
|
|
@ -116,6 +116,15 @@ export class BenchmarkService {
|
||||||
throw new Error('No benchmark result found to submit')
|
throw new Error('No benchmark result found to submit')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only allow full benchmarks with AI data to be submitted to repository
|
||||||
|
if (result.benchmark_type !== 'full') {
|
||||||
|
throw new Error('Only full benchmarks can be shared with the community. Run a Full Benchmark to share your results.')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.ai_tokens_per_second || result.ai_tokens_per_second <= 0) {
|
||||||
|
throw new Error('Benchmark must include AI performance data. Ensure AI Assistant is installed and run a Full Benchmark.')
|
||||||
|
}
|
||||||
|
|
||||||
if (result.submitted_to_repository) {
|
if (result.submitted_to_repository) {
|
||||||
throw new Error('Benchmark result has already been submitted')
|
throw new Error('Benchmark result has already been submitted')
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { Head } from '@inertiajs/react'
|
import { Head, Link } from '@inertiajs/react'
|
||||||
import { useState, useEffect } from 'react'
|
import { useState, useEffect } from 'react'
|
||||||
import SettingsLayout from '~/layouts/SettingsLayout'
|
import SettingsLayout from '~/layouts/SettingsLayout'
|
||||||
import { useQuery, useMutation } from '@tanstack/react-query'
|
import { useQuery, useMutation } from '@tanstack/react-query'
|
||||||
|
|
@ -31,6 +31,23 @@ export default function BenchmarkPage(props: {
|
||||||
const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
|
const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
|
||||||
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
|
const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
|
||||||
const [showDetails, setShowDetails] = useState(false)
|
const [showDetails, setShowDetails] = useState(false)
|
||||||
|
const [showAIRequiredAlert, setShowAIRequiredAlert] = useState(false)
|
||||||
|
|
||||||
|
// Check if AI Assistant is installed
|
||||||
|
const { data: aiInstalled } = useQuery({
|
||||||
|
queryKey: ['services', 'ai-installed'],
|
||||||
|
queryFn: async () => {
|
||||||
|
const res = await fetch('/api/system/services')
|
||||||
|
const data = await res.json()
|
||||||
|
const services = Array.isArray(data) ? data : (data.services || [])
|
||||||
|
const openWebUI = services.find((s: any) =>
|
||||||
|
s.service_name === 'nomad_open_webui' || s.serviceName === 'nomad_open_webui'
|
||||||
|
)
|
||||||
|
return openWebUI?.installed === true || openWebUI?.installed === 1
|
||||||
|
},
|
||||||
|
staleTime: 0,
|
||||||
|
refetchOnMount: true,
|
||||||
|
})
|
||||||
|
|
||||||
// Fetch latest result
|
// Fetch latest result
|
||||||
const { data: latestResult, refetch: refetchLatest } = useQuery({
|
const { data: latestResult, refetch: refetchLatest } = useQuery({
|
||||||
|
|
@ -124,6 +141,23 @@ export default function BenchmarkPage(props: {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Check if the latest result is a full benchmark with AI data (eligible for sharing)
|
||||||
|
const canShareBenchmark = latestResult &&
|
||||||
|
latestResult.benchmark_type === 'full' &&
|
||||||
|
latestResult.ai_tokens_per_second !== null &&
|
||||||
|
latestResult.ai_tokens_per_second > 0 &&
|
||||||
|
!latestResult.submitted_to_repository
|
||||||
|
|
||||||
|
// Handle Full Benchmark click with pre-flight check
|
||||||
|
const handleFullBenchmarkClick = () => {
|
||||||
|
if (!aiInstalled) {
|
||||||
|
setShowAIRequiredAlert(true)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
setShowAIRequiredAlert(false)
|
||||||
|
runBenchmark.mutate('full')
|
||||||
|
}
|
||||||
|
|
||||||
// Simulate progress during sync benchmark (since we don't get SSE updates)
|
// Simulate progress during sync benchmark (since we don't get SSE updates)
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return
|
if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return
|
||||||
|
|
@ -269,13 +303,30 @@ export default function BenchmarkPage(props: {
|
||||||
onDismiss={() => setProgress(null)}
|
onDismiss={() => setProgress(null)}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
{showAIRequiredAlert && (
|
||||||
|
<Alert
|
||||||
|
type="warning"
|
||||||
|
title="AI Assistant Required"
|
||||||
|
message="Full benchmark requires AI Assistant to be installed. Install it to measure your complete NOMAD capability and share results with the community."
|
||||||
|
variant="bordered"
|
||||||
|
dismissible
|
||||||
|
onDismiss={() => setShowAIRequiredAlert(false)}
|
||||||
|
>
|
||||||
|
<Link
|
||||||
|
href="/settings/apps"
|
||||||
|
className="text-sm text-desert-green hover:underline mt-2 inline-block font-medium"
|
||||||
|
>
|
||||||
|
Go to Apps to install AI Assistant →
|
||||||
|
</Link>
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
<p className="text-desert-stone-dark">
|
<p className="text-desert-stone-dark">
|
||||||
Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance.
|
Run a benchmark to measure your system's CPU, memory, disk, and AI inference performance.
|
||||||
The benchmark takes approximately 2-5 minutes to complete.
|
The benchmark takes approximately 2-5 minutes to complete.
|
||||||
</p>
|
</p>
|
||||||
<div className="flex flex-wrap gap-4">
|
<div className="flex flex-wrap gap-4">
|
||||||
<StyledButton
|
<StyledButton
|
||||||
onClick={() => runBenchmark.mutate('full')}
|
onClick={handleFullBenchmarkClick}
|
||||||
disabled={runBenchmark.isPending}
|
disabled={runBenchmark.isPending}
|
||||||
icon='PlayIcon'
|
icon='PlayIcon'
|
||||||
>
|
>
|
||||||
|
|
@ -292,12 +343,21 @@ export default function BenchmarkPage(props: {
|
||||||
<StyledButton
|
<StyledButton
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
onClick={() => runBenchmark.mutate('ai')}
|
onClick={() => runBenchmark.mutate('ai')}
|
||||||
disabled={runBenchmark.isPending}
|
disabled={runBenchmark.isPending || !aiInstalled}
|
||||||
icon='SparklesIcon'
|
icon='SparklesIcon'
|
||||||
|
title={!aiInstalled ? 'AI Assistant must be installed to run AI benchmark' : undefined}
|
||||||
>
|
>
|
||||||
AI Only
|
AI Only
|
||||||
</StyledButton>
|
</StyledButton>
|
||||||
</div>
|
</div>
|
||||||
|
{!aiInstalled && (
|
||||||
|
<p className="text-sm text-desert-stone-dark">
|
||||||
|
<span className="text-amber-600">Note:</span> AI Assistant is not installed.
|
||||||
|
<Link href="/settings/apps" className="text-desert-green hover:underline ml-1">
|
||||||
|
Install it
|
||||||
|
</Link> to run full benchmarks and share results with the community.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -331,7 +391,9 @@ export default function BenchmarkPage(props: {
|
||||||
<p className="text-desert-stone-dark">
|
<p className="text-desert-stone-dark">
|
||||||
Your NOMAD Score is a weighted composite of all benchmark results.
|
Your NOMAD Score is a weighted composite of all benchmark results.
|
||||||
</p>
|
</p>
|
||||||
{!latestResult.submitted_to_repository && (
|
|
||||||
|
{/* Share with Community - Only for full benchmarks with AI data */}
|
||||||
|
{canShareBenchmark && (
|
||||||
<div className="space-y-3">
|
<div className="space-y-3">
|
||||||
<p className="text-sm text-desert-stone-dark">
|
<p className="text-sm text-desert-stone-dark">
|
||||||
Share your benchmark score anonymously with the NOMAD community. Only your hardware specs and scores are sent — no identifying information.
|
Share your benchmark score anonymously with the NOMAD community. Only your hardware specs and scores are sent — no identifying information.
|
||||||
|
|
@ -355,6 +417,17 @@ export default function BenchmarkPage(props: {
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Show message for partial benchmarks */}
|
||||||
|
{latestResult && !latestResult.submitted_to_repository && !canShareBenchmark && (
|
||||||
|
<Alert
|
||||||
|
type="info"
|
||||||
|
title="Partial Benchmark"
|
||||||
|
message={`This ${latestResult.benchmark_type} benchmark cannot be shared with the community. Run a Full Benchmark with AI Assistant installed to share your results.`}
|
||||||
|
variant="bordered"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
{latestResult.submitted_to_repository && (
|
{latestResult.submitted_to_repository && (
|
||||||
<Alert
|
<Alert
|
||||||
type="success"
|
type="success"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user