feat: curated content update checking

This commit is contained in:
Jake Turner 2026-02-11 21:47:38 -08:00
parent c4514e8c3d
commit d55ff7b466
No known key found for this signature in database
GPG Key ID: D11724A09ED19E59
14 changed files with 470 additions and 126 deletions

View File

@ -1,9 +1,25 @@
import { CollectionUpdateService } from '#services/collection_update_service'
import {
applyContentUpdateValidator,
applyAllContentUpdatesValidator,
} from '#validators/common'
import type { HttpContext } from '@adonisjs/core/http'
export default class CollectionUpdatesController {
async checkForUpdates({}: HttpContext) {
const collectionUpdateService = new CollectionUpdateService()
return await collectionUpdateService.checkForUpdates()
const service = new CollectionUpdateService()
return await service.checkForUpdates()
}
async applyUpdate({ request }: HttpContext) {
const update = await request.validateUsing(applyContentUpdateValidator)
const service = new CollectionUpdateService()
return await service.applyUpdate(update)
}
async applyAllUpdates({ request }: HttpContext) {
const { updates } = await request.validateUsing(applyAllContentUpdatesValidator)
const service = new CollectionUpdateService()
return await service.applyAllUpdates(updates)
}
}

View File

@ -41,9 +41,16 @@ export class RunDownloadJob {
if (resourceMetadata) {
const { default: InstalledResource } = await import('#models/installed_resource')
const { DateTime } = await import('luxon')
const { getFileStatsIfExists } = await import('../utils/fs.js')
const { getFileStatsIfExists, deleteFileIfExists } = await import('../utils/fs.js')
const stats = await getFileStatsIfExists(filepath)
// Look up the old entry so we can clean up the previous file after updating
const oldEntry = await InstalledResource.query()
.where('resource_id', resourceMetadata.resource_id)
.where('resource_type', filetype as 'zim' | 'map')
.first()
const oldFilePath = oldEntry?.file_path ?? null
await InstalledResource.updateOrCreate(
{ resource_id: resourceMetadata.resource_id, resource_type: filetype as 'zim' | 'map' },
{
@ -55,6 +62,19 @@ export class RunDownloadJob {
installed_at: DateTime.now(),
}
)
// Delete the old file if it differs from the new one
if (oldFilePath && oldFilePath !== filepath) {
try {
await deleteFileIfExists(oldFilePath)
console.log(`[RunDownloadJob] Deleted old file: ${oldFilePath}`)
} catch (deleteError) {
console.warn(
`[RunDownloadJob] Failed to delete old file ${oldFilePath}:`,
deleteError
)
}
}
}
if (filetype === 'zim') {

View File

@ -192,6 +192,8 @@ export class CollectionManifestService {
let zimCount = 0
let mapCount = 0
console.log("RECONCILING FILESYSTEM MANIFESTS...")
// Reconcile ZIM files
try {
const zimDir = join(process.cwd(), ZIM_STORAGE_PATH)
@ -199,6 +201,8 @@ export class CollectionManifestService {
const zimItems = await listDirectoryContents(zimDir)
const zimFiles = zimItems.filter((f) => f.name.endsWith('.zim'))
console.log(`Found ${zimFiles.length} ZIM files on disk. Reconciling with database...`)
// Get spec for URL lookup
const zimSpec = await this.getCachedSpec<ZimCategoriesSpec>('zim_categories')
const specResourceMap = new Map<string, SpecResource>()
@ -215,10 +219,12 @@ export class CollectionManifestService {
const seenZimIds = new Set<string>()
for (const file of zimFiles) {
console.log(`Processing ZIM file: ${file.name}`)
// Skip Wikipedia files (managed by WikipediaSelection model)
if (file.name.startsWith('wikipedia_en_')) continue
const parsed = CollectionManifestService.parseZimFilename(file.name)
console.log(`Parsed ZIM filename:`, parsed)
if (!parsed) continue
seenZimIds.add(parsed.resource_id)

View File

@ -1,130 +1,157 @@
import logger from '@adonisjs/core/services/logger'
import env from '#start/env'
import axios from 'axios'
import InstalledResource from '#models/installed_resource'
import { CollectionManifestService } from './collection_manifest_service.js'
import { RunDownloadJob } from '../jobs/run_download_job.js'
import { ZIM_STORAGE_PATH } from '../utils/fs.js'
import { join } from 'path'
import type {
ZimCategoriesSpec,
MapsSpec,
CollectionResourceUpdateInfo,
CollectionUpdateCheckResult,
SpecResource,
ResourceUpdateCheckRequest,
ResourceUpdateInfo,
ContentUpdateCheckResult,
} from '../../types/collections.js'
import { NOMAD_API_DEFAULT_BASE_URL } from '../../constants/misc.js'
const MAP_STORAGE_PATH = '/storage/maps'
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
export class CollectionUpdateService {
private manifestService = new CollectionManifestService()
async checkForUpdates(): Promise<CollectionUpdateCheckResult> {
const resourceUpdates: CollectionResourceUpdateInfo[] = []
let specChanged = false
// Check if specs have changed
try {
const [zimChanged, mapsChanged] = await Promise.all([
this.manifestService.fetchAndCacheSpec('zim_categories'),
this.manifestService.fetchAndCacheSpec('maps'),
])
specChanged = zimChanged || mapsChanged
} catch (error) {
logger.error('[CollectionUpdateService] Failed to fetch latest specs:', error)
async checkForUpdates(): Promise<ContentUpdateCheckResult> {
const nomadAPIURL = env.get('NOMAD_API_URL') || NOMAD_API_DEFAULT_BASE_URL
if (!nomadAPIURL) {
return {
updates: [],
checked_at: new Date().toISOString(),
error: 'Nomad API is not configured. Set the NOMAD_API_URL environment variable.',
}
}
// Check for ZIM resource version updates
const zimUpdates = await this.checkZimUpdates()
resourceUpdates.push(...zimUpdates)
const installed = await InstalledResource.all()
if (installed.length === 0) {
return {
updates: [],
checked_at: new Date().toISOString(),
}
}
// Check for map resource version updates
const mapUpdates = await this.checkMapUpdates()
resourceUpdates.push(...mapUpdates)
const requestBody: ResourceUpdateCheckRequest = {
resources: installed.map((r) => ({
resource_id: r.resource_id,
resource_type: r.resource_type,
installed_version: r.version,
})),
}
try {
const response = await axios.post<ResourceUpdateInfo[]>(`${nomadAPIURL}/api/v1/resources/check-updates`, requestBody, {
timeout: 15000,
})
logger.info(
`[CollectionUpdateService] Update check complete: ${response.data.length} update(s) available`
)
return {
updates: response.data,
checked_at: new Date().toISOString(),
}
} catch (error) {
if (axios.isAxiosError(error) && error.response) {
logger.error(
`[CollectionUpdateService] Nomad API returned ${error.response.status}: ${JSON.stringify(error.response.data)}`
)
return {
updates: [],
checked_at: new Date().toISOString(),
error: `Nomad API returned status ${error.response.status}`,
}
}
const message =
error instanceof Error ? error.message : 'Unknown error contacting Nomad API'
logger.error(`[CollectionUpdateService] Failed to check for updates: ${message}`)
return {
updates: [],
checked_at: new Date().toISOString(),
error: `Failed to contact Nomad API: ${message}`,
}
}
}
async applyUpdate(
update: ResourceUpdateInfo
): Promise<{ success: boolean; jobId?: string; error?: string }> {
// Check if a download is already in progress for this URL
const existingJob = await RunDownloadJob.getByUrl(update.download_url)
if (existingJob) {
const state = await existingJob.getState()
if (state === 'active' || state === 'waiting' || state === 'delayed') {
return {
success: false,
error: `A download is already in progress for ${update.resource_id}`,
}
}
}
const filename = this.buildFilename(update)
const filepath = this.buildFilepath(update, filename)
const result = await RunDownloadJob.dispatch({
url: update.download_url,
filepath,
timeout: 30000,
allowedMimeTypes:
update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES,
forceNew: true,
filetype: update.resource_type,
resourceMetadata: {
resource_id: update.resource_id,
version: update.latest_version,
collection_ref: null,
},
})
if (!result || !result.job) {
return { success: false, error: 'Failed to dispatch download job' }
}
logger.info(
`[CollectionUpdateService] Update check complete: spec_changed=${specChanged}, resource_updates=${resourceUpdates.length}`
`[CollectionUpdateService] Dispatched update download for ${update.resource_id}: ${update.installed_version}${update.latest_version}`
)
return { spec_changed: specChanged, resource_updates: resourceUpdates }
return { success: true, jobId: result.job.id }
}
private async checkZimUpdates(): Promise<CollectionResourceUpdateInfo[]> {
const updates: CollectionResourceUpdateInfo[] = []
async applyAllUpdates(
updates: ResourceUpdateInfo[]
): Promise<{ results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }> }> {
const results: Array<{
resource_id: string
success: boolean
jobId?: string
error?: string
}> = []
try {
const spec = await this.manifestService.getCachedSpec<ZimCategoriesSpec>('zim_categories')
if (!spec) return updates
const installed = await InstalledResource.query().where('resource_type', 'zim')
if (installed.length === 0) return updates
// Build a map of spec resources by ID for quick lookup
const specResourceMap = new Map<string, SpecResource>()
for (const category of spec.categories) {
for (const tier of category.tiers) {
for (const resource of tier.resources) {
// Only keep the latest version if there are duplicates
const existing = specResourceMap.get(resource.id)
if (!existing || resource.version > existing.version) {
specResourceMap.set(resource.id, resource)
}
}
}
}
// Compare installed versions against spec versions
for (const entry of installed) {
const specResource = specResourceMap.get(entry.resource_id)
if (!specResource) continue
if (specResource.version > entry.version) {
updates.push({
resource_id: entry.resource_id,
installed_version: entry.version,
latest_version: specResource.version,
latest_url: specResource.url,
latest_size_mb: specResource.size_mb,
})
}
}
} catch (error) {
logger.error('[CollectionUpdateService] Error checking ZIM updates:', error)
for (const update of updates) {
const result = await this.applyUpdate(update)
results.push({ resource_id: update.resource_id, ...result })
}
return updates
return { results }
}
private async checkMapUpdates(): Promise<CollectionResourceUpdateInfo[]> {
const updates: CollectionResourceUpdateInfo[] = []
try {
const spec = await this.manifestService.getCachedSpec<MapsSpec>('maps')
if (!spec) return updates
const installed = await InstalledResource.query().where('resource_type', 'map')
if (installed.length === 0) return updates
// Build a map of spec resources by ID
const specResourceMap = new Map<string, SpecResource>()
for (const collection of spec.collections) {
for (const resource of collection.resources) {
specResourceMap.set(resource.id, resource)
}
}
// Compare installed versions against spec versions
for (const entry of installed) {
const specResource = specResourceMap.get(entry.resource_id)
if (!specResource) continue
if (specResource.version > entry.version) {
updates.push({
resource_id: entry.resource_id,
installed_version: entry.version,
latest_version: specResource.version,
latest_url: specResource.url,
latest_size_mb: specResource.size_mb,
})
}
}
} catch (error) {
logger.error('[CollectionUpdateService] Error checking map updates:', error)
private buildFilename(update: ResourceUpdateInfo): string {
if (update.resource_type === 'zim') {
return `${update.resource_id}_${update.latest_version}.zim`
}
return `${update.resource_id}_${update.latest_version}.pmtiles`
}
return updates
private buildFilepath(update: ResourceUpdateInfo, filename: string): string {
if (update.resource_type === 'zim') {
return join(process.cwd(), ZIM_STORAGE_PATH, filename)
}
return join(process.cwd(), MAP_STORAGE_PATH, 'pmtiles', filename)
}
}

View File

@ -11,8 +11,10 @@ import { SERVICE_NAMES } from '../../constants/service_names.js'
import transmit from '@adonisjs/transmit/services/main'
import Fuse, { IFuseOptions } from 'fuse.js'
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
import env from '#start/env'
import { NOMAD_API_DEFAULT_BASE_URL } from '../../constants/misc.js'
const NOMAD_MODELS_API_BASE_URL = 'https://api.projectnomad.us/api/v1/ollama/models'
const NOMAD_MODELS_API_PATH = '/api/v1/ollama/models'
const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json')
const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000 // 24 hours
@ -214,7 +216,11 @@ export class OllamaService {
}
logger.info('[OllamaService] Fetching fresh available models from API')
const response = await axios.get(NOMAD_MODELS_API_BASE_URL)
const baseUrl = env.get('NOMAD_API_URL') || NOMAD_API_DEFAULT_BASE_URL
const fullUrl = new URL(NOMAD_MODELS_API_PATH, baseUrl).toString()
const response = await axios.get(fullUrl)
if (!response.data || !Array.isArray(response.data.models)) {
logger.warn(
`[OllamaService] Invalid response format when fetching available models: ${JSON.stringify(response.data)}`

View File

@ -68,3 +68,21 @@ export const selectWikipediaValidator = vine.compile(
optionId: vine.string().trim().minLength(1),
})
)
const resourceUpdateInfoBase = vine.object({
resource_id: vine.string().trim().minLength(1),
resource_type: vine.enum(['zim', 'map'] as const),
installed_version: vine.string().trim(),
latest_version: vine.string().trim().minLength(1),
download_url: vine.string().url().trim(),
})
export const applyContentUpdateValidator = vine.compile(resourceUpdateInfoBase)
export const applyAllContentUpdatesValidator = vine.compile(
vine.object({
updates: vine
.array(resourceUpdateInfoBase)
.minLength(1),
})
)

2
admin/constants/misc.ts Normal file
View File

@ -0,0 +1,2 @@
export const NOMAD_API_DEFAULT_BASE_URL = 'https://api.projectnomad.us'

View File

@ -3,7 +3,8 @@
## Unreleased
### Features
- **Collections**: Complete overhaul of collection management with dynamic manifests, database tracking of installed resources, and improved UI for managing ZIM files and map assets
- **Collections**: Added support for checking if newer versions of installed resources are available based on manifest data
### Bug Fixes
### Improvements

View File

@ -4,7 +4,7 @@ import { ServiceSlim } from '../../types/services'
import { FileEntry } from '../../types/files'
import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system'
import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads'
import type { CategoryWithStatus, CollectionWithStatus, CollectionUpdateCheckResult } from '../../types/collections'
import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections'
import { catchInternal } from './util'
import { NomadOllamaModel, OllamaChatRequest } from '../../types/ollama'
import { ChatResponse, ModelResponse } from 'ollama'
@ -127,9 +127,28 @@ class API {
})()
}
async checkForCollectionUpdates() {
async checkForContentUpdates() {
return catchInternal(async () => {
const response = await this.client.post<CollectionUpdateCheckResult>('/collection-updates/check')
const response = await this.client.post<ContentUpdateCheckResult>('/content-updates/check')
return response.data
})()
}
async applyContentUpdate(update: ResourceUpdateInfo) {
return catchInternal(async () => {
const response = await this.client.post<{ success: boolean; jobId?: string; error?: string }>(
'/content-updates/apply',
update
)
return response.data
})()
}
async applyAllContentUpdates(updates: ResourceUpdateInfo[]) {
return catchInternal(async () => {
const response = await this.client.post<{
results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }>
}>('/content-updates/apply-all', { updates })
return response.data
})()
}

View File

@ -95,7 +95,7 @@ export default function Home(props: {
const { data: easySetupVisited } = useSystemSetting({
key: 'ui.hasVisitedEasySetup'
})
const shouldHighlightEasySetup = easySetupVisited?.value !== 'true'
const shouldHighlightEasySetup = easySetupVisited?.value ? easySetupVisited?.value !== 'true' : false
// Add installed services (non-dependency services only)
props.system.services

View File

@ -1,15 +1,220 @@
import { Head } from '@inertiajs/react'
import SettingsLayout from '~/layouts/SettingsLayout'
import StyledButton from '~/components/StyledButton'
import StyledTable from '~/components/StyledTable'
import StyledSectionHeader from '~/components/StyledSectionHeader'
import ActiveDownloads from '~/components/ActiveDownloads'
import Alert from '~/components/Alert'
import { useEffect, useState } from 'react'
import { IconAlertCircle, IconArrowBigUpLines, IconCheck, IconCircleCheck, IconReload } from '@tabler/icons-react'
import { SystemUpdateStatus } from '../../../types/system'
import type { ContentUpdateCheckResult, ResourceUpdateInfo } from '../../../types/collections'
import api from '~/lib/api'
import Input from '~/components/inputs/Input'
import { useMutation } from '@tanstack/react-query'
import { useNotifications } from '~/context/NotificationContext'
function ContentUpdatesSection() {
const { addNotification } = useNotifications()
const [checkResult, setCheckResult] = useState<ContentUpdateCheckResult | null>(null)
const [isChecking, setIsChecking] = useState(false)
const [applyingIds, setApplyingIds] = useState<Set<string>>(new Set())
const [isApplyingAll, setIsApplyingAll] = useState(false)
const handleCheck = async () => {
setIsChecking(true)
try {
const result = await api.checkForContentUpdates()
if (result) {
setCheckResult(result)
}
} catch {
setCheckResult({
updates: [],
checked_at: new Date().toISOString(),
error: 'Failed to check for content updates',
})
} finally {
setIsChecking(false)
}
}
const handleApply = async (update: ResourceUpdateInfo) => {
setApplyingIds((prev) => new Set(prev).add(update.resource_id))
try {
const result = await api.applyContentUpdate(update)
if (result?.success) {
addNotification({ type: 'success', message: `Update started for ${update.resource_id}` })
// Remove from the updates list
setCheckResult((prev) =>
prev
? { ...prev, updates: prev.updates.filter((u) => u.resource_id !== update.resource_id) }
: prev
)
} else {
addNotification({ type: 'error', message: result?.error || 'Failed to start update' })
}
} catch {
addNotification({ type: 'error', message: `Failed to start update for ${update.resource_id}` })
} finally {
setApplyingIds((prev) => {
const next = new Set(prev)
next.delete(update.resource_id)
return next
})
}
}
const handleApplyAll = async () => {
if (!checkResult?.updates.length) return
setIsApplyingAll(true)
try {
const result = await api.applyAllContentUpdates(checkResult.updates)
if (result?.results) {
const succeeded = result.results.filter((r) => r.success).length
const failed = result.results.filter((r) => !r.success).length
if (succeeded > 0) {
addNotification({ type: 'success', message: `Started ${succeeded} update(s)` })
}
if (failed > 0) {
addNotification({ type: 'warning', message: `${failed} update(s) could not be started` })
}
// Remove successful updates from the list
const successIds = new Set(result.results.filter((r) => r.success).map((r) => r.resource_id))
setCheckResult((prev) =>
prev
? { ...prev, updates: prev.updates.filter((u) => !successIds.has(u.resource_id)) }
: prev
)
}
} catch {
addNotification({ type: 'error', message: 'Failed to apply updates' })
} finally {
setIsApplyingAll(false)
}
}
return (
<div className="mt-8">
<StyledSectionHeader title="Content Updates" />
<div className="bg-white rounded-lg border shadow-md overflow-hidden p-6">
<div className="flex items-center justify-between mb-4">
<p className="text-desert-stone-dark">
Check if newer versions of your installed ZIM files and maps are available.
</p>
<StyledButton
variant="primary"
icon="IconRefresh"
onClick={handleCheck}
loading={isChecking}
>
Check for Content Updates
</StyledButton>
</div>
{checkResult?.error && (
<Alert
type="warning"
title="Update Check Issue"
message={checkResult.error}
variant="bordered"
className="mb-4"
/>
)}
{checkResult && !checkResult.error && checkResult.updates.length === 0 && (
<Alert
type="success"
title="All Content Up to Date"
message="All your installed content is running the latest available version."
variant="bordered"
className="mb-4"
/>
)}
{checkResult && checkResult.updates.length > 0 && (
<div className="mt-4">
<div className="flex items-center justify-between mb-3">
<p className="text-sm text-desert-stone-dark">
{checkResult.updates.length} update(s) available
</p>
<StyledButton
variant="primary"
size="sm"
icon="IconDownload"
onClick={handleApplyAll}
loading={isApplyingAll}
>
Update All ({checkResult.updates.length})
</StyledButton>
</div>
<StyledTable
data={checkResult.updates}
columns={[
{
accessor: 'resource_id',
title: 'Title',
render: (record) => (
<span className="font-medium text-desert-green">{record.resource_id}</span>
),
},
{
accessor: 'resource_type',
title: 'Type',
render: (record) => (
<span
className={`inline-flex items-center rounded-full px-2.5 py-0.5 text-xs font-medium ${
record.resource_type === 'zim'
? 'bg-blue-100 text-blue-800'
: 'bg-emerald-100 text-emerald-800'
}`}
>
{record.resource_type === 'zim' ? 'ZIM' : 'Map'}
</span>
),
},
{
accessor: 'installed_version',
title: 'Version',
render: (record) => (
<span className="text-desert-stone-dark">
{record.installed_version} {record.latest_version}
</span>
),
},
{
accessor: 'resource_id',
title: '',
render: (record) => (
<StyledButton
variant="secondary"
size="sm"
icon="IconDownload"
onClick={() => handleApply(record)}
loading={applyingIds.has(record.resource_id)}
>
Update
</StyledButton>
),
},
]}
/>
</div>
)}
{checkResult?.checked_at && (
<p className="text-xs text-desert-stone mt-3">
Last checked: {new Date(checkResult.checked_at).toLocaleString()}
</p>
)}
</div>
<ActiveDownloads withHeader />
</div>
)
}
export default function SystemUpdatePage(props: {
system: {
updateAvailable: boolean
@ -380,6 +585,8 @@ export default function SystemUpdatePage(props: {
variant="solid"
/>
</div>
<ContentUpdatesSection />
{showLogs && (
<div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center p-4 z-50">
<div className="bg-white rounded-lg shadow-2xl max-w-4xl w-full max-h-[80vh] flex flex-col">

View File

@ -53,4 +53,11 @@ export default await Env.create(new URL('../', import.meta.url), {
*/
REDIS_HOST: Env.schema.string({ format: 'host' }),
REDIS_PORT: Env.schema.number(),
/*
|----------------------------------------------------------
| Variables for configuring Project Nomad's external API URL
|----------------------------------------------------------
*/
NOMAD_API_URL: Env.schema.string.optional(),
})

View File

@ -34,7 +34,13 @@ router.get('/easy-setup', [EasySetupController, 'index'])
router.get('/easy-setup/complete', [EasySetupController, 'complete'])
router.get('/api/easy-setup/curated-categories', [EasySetupController, 'listCuratedCategories'])
router.post('/api/manifests/refresh', [EasySetupController, 'refreshManifests'])
router.post('/api/collection-updates/check', [CollectionUpdatesController, 'checkForUpdates'])
router
.group(() => {
router.post('/check', [CollectionUpdatesController, 'checkForUpdates'])
router.post('/apply', [CollectionUpdatesController, 'applyUpdate'])
router.post('/apply-all', [CollectionUpdatesController, 'applyAllUpdates'])
})
.prefix('/api/content-updates')
router
.group(() => {

View File

@ -72,15 +72,24 @@ export type CollectionWithStatus = SpecCollection & {
total_count: number
}
export type CollectionResourceUpdateInfo = {
resource_id: string
installed_version: string
latest_version: string
latest_url: string
latest_size_mb?: number
export type ResourceUpdateCheckRequest = {
resources: Array<{
resource_id: string
resource_type: 'zim' | 'map'
installed_version: string
}>
}
export type CollectionUpdateCheckResult = {
spec_changed: boolean
resource_updates: CollectionResourceUpdateInfo[]
export type ResourceUpdateInfo = {
resource_id: string
resource_type: 'zim' | 'map'
installed_version: string
latest_version: string
download_url: string
}
export type ContentUpdateCheckResult = {
updates: ResourceUpdateInfo[]
checked_at: string
error?: string
}