project-nomad/admin/app/services/system_update_service.ts
jakeaturner 02d985db5e
feat(system): add opt-in automatic updates for the core NOMAD app
Adds a self-update path for the NOMAD admin/core image that runs without a
human in the loop, gated by layered safety checks. Recreation remains the
sidecar's job; this only decides *whether* to update now and requests it.

An hourly job (AutoUpdateJob) evaluates a side-effect-free decision pipeline
(AutoUpdateService.evaluate):
 - opt-in: disabled by default
 - in a user-configured local-time window (handles midnight wrap)
 - an eligible release: same major (major bumps stay manual), strictly newer,
   past a configurable cool-off, GA only (no drafts/prereleases), strict semver
 - pre-flight: sidecar present, no in-flight system update / downloads / app
   installs, and sufficient host disk (estimated from the registry manifest)
When all pass, it drives SystemUpdateService.requestUpdate() with a vetted tag.

Failure backoff auto-disables after 3 genuine update-request failures; transient
release-lookup errors are skips (offline-first appliances are routinely without
connectivity). Re-enabling clears the backoff state.

Settings UI exposes the toggle, window, and cool-off, with live status (eligible
target, in/out of window, last result/error). A `node ace auto-update:dry-run`
command exercises the full pipeline — and a deterministic --scenarios suite the
pure decision logic — without ever triggering an update.

New KVStore keys under autoUpdate.* hold config + last-run state.
2026-06-23 04:46:51 +00:00

115 lines
3.9 KiB
TypeScript

import logger from '@adonisjs/core/services/logger'
import { readFileSync, existsSync } from 'fs'
import { writeFile } from 'fs/promises'
import { join } from 'path'
import KVStore from '#models/kv_store'
interface UpdateStatus {
stage: 'idle' | 'starting' | 'pulling' | 'pulled' | 'recreating' | 'complete' | 'error'
progress: number
message: string
timestamp: string
}
export class SystemUpdateService {
private static SHARED_DIR = '/app/update-shared'
private static REQUEST_FILE = join(SystemUpdateService.SHARED_DIR, 'update-request')
private static STATUS_FILE = join(SystemUpdateService.SHARED_DIR, 'update-status')
private static LOG_FILE = join(SystemUpdateService.SHARED_DIR, 'update-log')
/**
* Requests a system update by creating a request file that the sidecar will detect.
*
* @param options.targetTag - Explicit Docker image tag to install (e.g. "v1.33.2").
* When omitted, falls back to the cached `system.latestVersion` (manual-update
* behavior). Auto-update passes an eligibility-vetted tag here, which may differ
* from `system.latestVersion` when the newest release is a major bump.
* @param options.requester - Identifier recorded in the request file for auditing.
*/
async requestUpdate(options?: {
targetTag?: string
requester?: string
}): Promise<{ success: boolean; message: string }> {
try {
const currentStatus = this.getUpdateStatus()
if (currentStatus && !['idle', 'complete', 'error'].includes(currentStatus.stage)) {
return {
success: false,
message: `Update already in progress (stage: ${currentStatus.stage})`,
}
}
// Determine the Docker image tag to install. Prefer an explicit caller-supplied
// tag; otherwise use the cached latest version.
let targetTag = options?.targetTag
if (!targetTag) {
const latestVersion = await KVStore.getValue('system.latestVersion')
targetTag = latestVersion ? `v${latestVersion}` : 'latest'
}
const requestData = {
requested_at: new Date().toISOString(),
requester: options?.requester ?? 'admin-api',
target_tag: targetTag,
}
await writeFile(SystemUpdateService.REQUEST_FILE, JSON.stringify(requestData, null, 2))
logger.info(`[SystemUpdateService]: System update requested (target tag: ${requestData.target_tag}) - sidecar will process shortly`)
return {
success: true,
message: 'System update initiated. The admin container will restart during the process.',
}
} catch (error) {
logger.error({ err: error }, '[SystemUpdateService] Failed to request system update')
return {
success: false,
message: 'Failed to request system update. Check server logs for details.',
}
}
}
getUpdateStatus(): UpdateStatus | null {
try {
if (!existsSync(SystemUpdateService.STATUS_FILE)) {
return {
stage: 'idle',
progress: 0,
message: 'No update in progress',
timestamp: new Date().toISOString(),
}
}
const statusContent = readFileSync(SystemUpdateService.STATUS_FILE, 'utf-8')
return JSON.parse(statusContent) as UpdateStatus
} catch (error) {
logger.error('[SystemUpdateService]: Failed to read update status:', error)
return null
}
}
getUpdateLogs(): string {
try {
if (!existsSync(SystemUpdateService.LOG_FILE)) {
return 'No update logs available'
}
return readFileSync(SystemUpdateService.LOG_FILE, 'utf-8')
} catch (error) {
logger.error('[SystemUpdateService]: Failed to read update logs:', error)
return `Error reading logs: ${error.message}`
}
}
/**
* Check if the update sidecar is reachable (i.e. shared volume is mounted)
*/
isSidecarAvailable(): boolean {
try {
return existsSync(SystemUpdateService.SHARED_DIR)
} catch (error) {
return false
}
}
}