diff --git a/.gitignore b/.gitignore index 81aedd3..4273fe3 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ server/temp # Frontend assets compiled code admin/public/assets + +# Admin specific development files +admin/storage diff --git a/README.md b/README.md index 0394c0b..2bde36b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Project N.O.M.A.D. can be installed on any Debian-based operating system (we rec *Note: sudo/root privileges are required to run the install script* ```bash -curl -fsSL https://raw.githubusercontent.com/CrosstalkSolutions/project-nomad/main/install/install/sh -o install_nomad.sh +curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/install/install_nomad.sh -o install_nomad.sh sudo bash ./install_nomad.sh ``` diff --git a/admin/app/controllers/home_controller.ts b/admin/app/controllers/home_controller.ts index e66219b..e819b18 100644 --- a/admin/app/controllers/home_controller.ts +++ b/admin/app/controllers/home_controller.ts @@ -14,7 +14,8 @@ export default class HomeController { } async home({ inertia }: HttpContext) { - const services = await this.systemService.getServices(); + const services = await this.systemService.getServices({ installedOnly: true }); + console.log(services) return inertia.render('home', { system: { services diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index 7f1aafe..c59b625 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -18,11 +18,19 @@ export default class SettingsController { } async apps({ inertia }: HttpContext) { - const services = await this.systemService.getServices(); + const services = await this.systemService.getServices({ installedOnly: false }); return inertia.render('settings/apps', { system: { services } }); } + + async zim({ inertia }: HttpContext) { + return inertia.render('settings/zim/index') + } + + async zimRemote({ inertia }: HttpContext) { + return inertia.render('settings/zim/remote-explorer'); + } } \ No newline at end of file diff --git a/admin/app/controllers/system_controller.ts b/admin/app/controllers/system_controller.ts index a805984..0307f4e 100644 --- a/admin/app/controllers/system_controller.ts +++ b/admin/app/controllers/system_controller.ts @@ -11,9 +11,8 @@ export default class SystemController { private dockerService: DockerService ) { } - async getServices({ response }: HttpContext) { - const services = await this.systemService.getServices(); - response.send(services); + async getServices({ }: HttpContext) { + return await this.systemService.getServices({ installedOnly: true }); } async installService({ request, response }: HttpContext) { diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts new file mode 100644 index 0000000..e927a33 --- /dev/null +++ b/admin/app/controllers/zim_controller.ts @@ -0,0 +1,47 @@ +import { ZimService } from '#services/zim_service'; +import { inject } from '@adonisjs/core'; +import type { HttpContext } from '@adonisjs/core/http' + +@inject() +export default class ZimController { + constructor( + private zimService: ZimService + ) { } + + async list({ }: HttpContext) { + return await this.zimService.list(); + } + + async listRemote({ request }: HttpContext) { + const { start = 0, count = 12 } = request.qs(); + return await this.zimService.listRemote({ start, count }); + } + + async downloadRemote({ request, response }: HttpContext) { + const { url } = request.body() + await this.zimService.downloadRemote(url); + + response.status(200).send({ + message: 'Download started successfully' + }); + } + + async delete({ request, response }: HttpContext) { + const { key } = request.params(); + + try { + await this.zimService.delete(key); + } catch (error) { + if (error.message === 'not_found') { + return response.status(404).send({ + message: `ZIM file with key ${key} not found` + }); + } + throw error; // Re-throw any other errors and let the global error handler catch + } + + response.status(200).send({ + message: 'ZIM file deleted successfully' + }); + } +} \ No newline at end of file diff --git a/admin/app/models/service.ts b/admin/app/models/service.ts index 8af7e95..35c08f5 100644 --- a/admin/app/models/service.ts +++ b/admin/app/models/service.ts @@ -1,10 +1,11 @@ -import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import { BaseModel, belongsTo, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import type { BelongsTo, HasMany } from '@adonisjs/lucid/types/relations' import { DateTime } from 'luxon' export default class Service extends BaseModel { static namingStrategy = new SnakeCaseNamingStrategy() - @column({ isPrimary: true }) + @column({ isPrimary: true }) declare id: number @column() @@ -14,16 +15,31 @@ export default class Service extends BaseModel { declare container_image: string @column() - declare container_command: string + declare container_command: string | null @column() declare container_config: string | null - @column() + @column({ + serialize(value) { + return Boolean(value) + }, + }) declare installed: boolean @column() - declare ui_location: string + declare depends_on: string | null + + // For services that are dependencies for other services - not intended to be installed directly by users + @column({ + serialize(value) { + return Boolean(value) + }, + }) + declare is_dependency_service: boolean + + @column() + declare ui_location: string | null @column() declare metadata: string | null @@ -33,4 +49,15 @@ export default class Service extends BaseModel { @column.dateTime({ autoCreate: true, autoUpdate: true }) declare updated_at: DateTime | null + + // Define a self-referential relationship for dependencies + @belongsTo(() => Service, { + foreignKey: 'depends_on', + }) + declare dependency: BelongsTo + + @hasMany(() => Service, { + foreignKey: 'depends_on', + }) + declare dependencies: HasMany } diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 5d3068b..49d24e3 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -15,7 +15,7 @@ export class DockerService { } async createContainerPreflight(serviceName: string): Promise<{ success: boolean; message: string }> { - const service = await Service.findBy('service_name', serviceName); + const service = await Service.query().where('service_name', serviceName).first(); if (!service) { return { success: false, @@ -33,29 +33,17 @@ export class DockerService { // Check if a service wasn't marked as installed but has an existing container // This can happen if the service was created but not properly installed // or if the container was removed manually without updating the service status. - if (await this._checkIfServiceContainerExists(serviceName)) { - const removeResult = await this._removeServiceContainer(serviceName); - if (!removeResult.success) { - return { - success: false, - message: `Failed to remove existing container for service ${serviceName}: ${removeResult.message}`, - }; - } - } - - // Attempt to parse any special container configuration - let containerConfig; - if (service.container_config) { - try { - containerConfig = JSON.parse(JSON.stringify(service.container_config)); - } catch (error) { - return { - success: false, - message: `Failed to parse container configuration for service ${service.service_name}: ${error.message}`, - }; - } - } + // if (await this._checkIfServiceContainerExists(serviceName)) { + // const removeResult = await this._removeServiceContainer(serviceName); + // if (!removeResult.success) { + // return { + // success: false, + // message: `Failed to remove existing container for service ${serviceName}: ${removeResult.message}`, + // }; + // } + // } + const containerConfig = this._parseContainerConfig(service.container_config); this._createContainer(service, containerConfig); // Don't await this method - we will use server-sent events to notify the client of progress return { @@ -72,56 +60,75 @@ export class DockerService { * @param serviceName * @returns */ - async _createContainer(service: Service, containerConfig: any): Promise { + async _createContainer(service: Service & { dependencies?: Service[] }, containerConfig: any): Promise { + try { + this._broadcastAndLog(service.service_name, 'initializing', ''); - function sendBroadcastAndLog(status: string, message: string) { - transmit.broadcast('service-installation', { - service_name: service.service_name, - timestamp: new Date().toISOString(), - status, - message, + let dependencies = []; + if (service.depends_on) { + const dependency = await Service.query().where('service_name', service.depends_on).first(); + if (dependency) { + dependencies.push(dependency); + } + } + + console.log('dependencies for service', service.service_name) + console.log(dependencies) + + // First, check if the service has any dependencies that need to be installed first + if (dependencies && dependencies.length > 0) { + this._broadcastAndLog(service.service_name, 'checking-dependencies', `Checking dependencies for service ${service.service_name}...`); + for (const dependency of dependencies) { + if (!dependency.installed) { + this._broadcastAndLog(service.service_name, 'dependency-not-installed', `Dependency service ${dependency.service_name} is not installed. Installing it first...`); + await this._createContainer(dependency, this._parseContainerConfig(dependency.container_config)); + } else { + this._broadcastAndLog(service.service_name, 'dependency-installed', `Dependency service ${dependency.service_name} is already installed.`); + } + } + } + + // Start pulling the Docker image and wait for it to complete + const pullStream = await this.docker.pull(service.container_image); + this._broadcastAndLog(service.service_name, 'pulling', `Pulling Docker image ${service.container_image}...`); + await new Promise(res => this.docker.modem.followProgress(pullStream, res)); + this._broadcastAndLog(service.service_name, 'pulled', `Docker image ${service.container_image} pulled successfully.`); + + this._broadcastAndLog(service.service_name, 'creating', `Creating Docker container for service ${service.service_name}...`); + const container = await this.docker.createContainer({ + Image: service.container_image, + name: service.service_name, + HostConfig: containerConfig?.HostConfig || undefined, + WorkingDir: containerConfig?.WorkingDir || undefined, + ExposedPorts: containerConfig?.ExposedPorts || undefined, + ...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}), + ...(service.service_name === 'open-webui' ? { Env: ['WEBUI_AUTH=False'] } : {}), // Special case for Open WebUI to disable authentication }); - logger.info(`[DockerService] [${service.service_name}] ${status}: ${message}`); + + this._broadcastAndLog(service.service_name, 'created', `Docker container for service ${service.service_name} created successfully.`); + + if (service.service_name === 'kiwix-serve') { + await this._runPreinstallActions__KiwixServe(); + this._broadcastAndLog(service.service_name, 'preinstall-complete', `Pre-install actions for Kiwix Serve completed successfully.`); + } else if (service.service_name === 'openstreetmap') { + await this._runPreinstallActions__OpenStreetMap(containerConfig); + this._broadcastAndLog(service.service_name, 'preinstall-complete', `Pre-install actions for OpenStreetMap completed successfully.`); + } + + this._broadcastAndLog(service.service_name, 'starting', `Starting Docker container for service ${service.service_name}...`); + await container.start(); + this._broadcastAndLog(service.service_name, 'started', `Docker container for service ${service.service_name} started successfully.`); + + this._broadcastAndLog(service.service_name, 'finalizing', `Finalizing installation of service ${service.service_name}...`); + + service.installed = true; + await service.save(); + + this._broadcastAndLog(service.service_name, 'completed', `Service ${service.service_name} installation completed successfully.`); + } catch (error) { + this._broadcastAndLog(service.service_name, 'error', `Error installing service ${service.service_name}: ${error.message}`); + throw new Error(`Failed to install service ${service.service_name}: ${error.message}`); } - - sendBroadcastAndLog('initializing', ''); - - // Start pulling the Docker image and wait for it to complete - const pullStream = await this.docker.pull(service.container_image); - sendBroadcastAndLog('pulling', `Pulling Docker image ${service.container_image}...`); - - await new Promise(res => this.docker.modem.followProgress(pullStream, res)); - - sendBroadcastAndLog('pulled', `Docker image ${service.container_image} pulled successfully.`); - sendBroadcastAndLog('creating', `Creating Docker container for service ${service.service_name}...`); - - const container = await this.docker.createContainer({ - Image: service.container_image, - Cmd: service.container_command.split(' '), - name: service.service_name, - HostConfig: containerConfig?.HostConfig || undefined, - WorkingDir: containerConfig?.WorkingDir || undefined, - ExposedPorts: containerConfig?.ExposedPorts || undefined, - }); - - sendBroadcastAndLog('created', `Docker container for service ${service.service_name} created successfully.`); - - if (service.service_name === 'kiwix-serve') { - sendBroadcastAndLog('preinstall', `Running pre-install actions for Kiwix Serve...`); - await this._runPreinstallActions__KiwixServe(); - sendBroadcastAndLog('preinstall-complete', `Pre-install actions for Kiwix Serve completed successfully.`); - } - - sendBroadcastAndLog('starting', `Starting Docker container for service ${service.service_name}...`); - await container.start(); - sendBroadcastAndLog('started', `Docker container for service ${service.service_name} started successfully.`); - - sendBroadcastAndLog('finalizing', `Finalizing installation of service ${service.service_name}...`); - - service.installed = true; - await service.save(); - - sendBroadcastAndLog('completed', `Service ${service.service_name} installation completed successfully.`); } async _checkIfServiceContainerExists(serviceName: string): Promise { @@ -153,13 +160,16 @@ export class DockerService { } } - async _runPreinstallActions__KiwixServe(): Promise { + private async _runPreinstallActions__KiwixServe(): Promise { /** * At least one .zim file must be available before we can start the kiwix container. * We'll download the lightweight mini Wikipedia Top 100 zim file for this purpose. **/ const WIKIPEDIA_ZIM_URL = "https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_mini_2025-06.zim" + const PATH = '/zim/wikipedia_en_100_mini_2025-06.zim'; + this._broadcastAndLog('kiwix-serve', 'preinstall', `Running pre-install actions for Kiwix Serve...`); + this._broadcastAndLog('kiwix-serve', 'preinstall', `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`); const response = await axios.get(WIKIPEDIA_ZIM_URL, { responseType: 'stream', }); @@ -171,10 +181,89 @@ export class DockerService { }); const disk = drive.use('fs'); - await disk.putStream('/zim/wikipedia_en_100_mini_2025-06.zim', stream); + await disk.putStream(PATH, stream); + this._broadcastAndLog('kiwix-serve', 'preinstall', `Downloaded Wikipedia ZIM file to ${PATH}`); + } - logger.info(`Downloaded Wikipedia ZIM file to /zim/wikipedia_en_100_mini_2025-06.zim`); + /** + * Largely follows the install instructions here: https://github.com/Overv/openstreetmap-tile-server/blob/master/README.md + */ + private async _runPreinstallActions__OpenStreetMap(containerConfig: any): Promise { + const FILE_NAME = 'us-pacific-latest.osm.pbf'; + const OSM_PBF_URL = `https://download.geofabrik.de/north-america/${FILE_NAME}`; // Download a small subregion for initial import + const PATH = `/osm/${FILE_NAME}`; + + this._broadcastAndLog('openstreetmap', 'preinstall', `Running pre-install actions for OpenStreetMap Tile Server...`); + this._broadcastAndLog('openstreetmap', 'preinstall', `Downloading OpenStreetMap PBF file from ${OSM_PBF_URL}. This may take some time...`); + const response = await axios.get(OSM_PBF_URL, { + responseType: 'stream', + }); + + const stream = response.data; + stream.on('error', (error: Error) => { + logger.error(`Error downloading OpenStreetMap PBF file: ${error.message}`); + throw error; + }); + + const disk = drive.use('fs'); + await disk.putStream(PATH, stream); + this._broadcastAndLog('openstreetmap', 'preinstall', `Downloaded OpenStreetMap PBF file to ${PATH}`); + + // Do initial import of OSM data into the tile server DB + // We'll use the same containerConfig as the actual container, just with the command set to "import" + this._broadcastAndLog('openstreetmap', 'importing', `Processing initial import of OSM data. This may take some time...`); + const data = await new Promise((resolve, reject) => { + this.docker.run(containerConfig.Image, ['import'], process.stdout, containerConfig?.HostConfig || {}, {}, + // @ts-ignore + (err: any, data: any, container: any) => { + if (err) { + logger.error(`Error running initial import for OpenStreetMap Tile Server: ${err.message}`); + return reject(err); + } + resolve(data); + }); + }); + + const [output, container] = data as [any, any]; + if (output?.StatusCode === 0) { + this._broadcastAndLog('openstreetmap', 'imported', `OpenStreetMap data imported successfully.`); + await container.remove(); + } else { + const errorMessage = `Failed to import OpenStreetMap data. Status code: ${output?.StatusCode}. Output: ${output?.Output || 'No output'}`; + this._broadcastAndLog('openstreetmap', 'error', errorMessage); + logger.error(errorMessage); + throw new Error(errorMessage); + } + } + + private _broadcastAndLog(service: string, status: string, message: string) { + transmit.broadcast('service-installation', { + service_name: service, + timestamp: new Date().toISOString(), + status, + message, + }); + logger.info(`[DockerService] [${service}] ${status}: ${message}`); + } + + private _parseContainerConfig(containerConfig: any): any { + if (!containerConfig) { + return {}; + } + + try { + // Handle the case where containerConfig is returned as an object by DB instead of a string + let toParse = containerConfig; + if (typeof containerConfig === 'object') { + toParse = JSON.stringify(containerConfig); + } + + return JSON.parse(toParse); + } catch (error) { + logger.error(`Failed to parse container configuration: ${error.message}`); + throw new Error(`Invalid container configuration: ${error.message}`); + } } async simulateSSE(): Promise { diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 11b6798..e738b8c 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -1,7 +1,15 @@ import Service from "#models/service" export class SystemService { - async getServices(): Promise<{ id: number; service_name: string; installed: boolean }[]> { - return await Service.query().orderBy('service_name', 'asc').select('id', 'service_name', 'installed', 'ui_location'); + async getServices({ + installedOnly = true, + }:{ + installedOnly?: boolean + }): Promise<{ id: number; service_name: string; installed: boolean }[]> { + const query = Service.query().orderBy('service_name', 'asc').select('id', 'service_name', 'installed', 'ui_location').where('is_dependency_service', false) + if (installedOnly) { + query.where('installed', true); + } + return await query; } } \ No newline at end of file diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts new file mode 100644 index 0000000..82abb50 --- /dev/null +++ b/admin/app/services/zim_service.ts @@ -0,0 +1,139 @@ +import drive from "@adonisjs/drive/services/main"; +import { ListRemoteZimFilesResponse, RawRemoteZimFileEntry, RemoteZimFileEntry, ZimFilesEntry } from "../../types/zim.js"; +import axios from "axios"; +import { XMLParser } from 'fast-xml-parser' +import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from "../../util/zim.js"; + +export class ZimService { + async list() { + const disk = drive.use('fs'); + const contents = await disk.listAll('/zim') + + const files: ZimFilesEntry[] = [] + for (let item of contents.objects) { + if (item.isFile) { + files.push({ + type: 'file', + key: item.key, + name: item.name + }) + } else { + files.push({ + type: 'directory', + prefix: item.prefix, + name: item.name + }) + } + } + + return { + files, + next: contents.paginationToken + } + } + + async listRemote({ start, count }: { start: number, count: number }): Promise { + const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries' + + const res = await axios.get(LIBRARY_BASE_URL, { + params: { + start: start, + count: count, + lang: 'eng' + }, + responseType: 'text' + }); + + const data = res.data; + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '', + textNodeName: '#text', + }); + const result = parser.parse(data); + + if (!isRawListRemoteZimFilesResponse(result)) { + throw new Error('Invalid response format from remote library'); + } + + const filtered = result.feed.entry.filter((entry: any) => { + return isRawRemoteZimFileEntry(entry); + }) + + const mapped: (RemoteZimFileEntry | null)[] = filtered.map((entry: RawRemoteZimFileEntry) => { + const downloadLink = entry.link.find((link: any) => { + return typeof link === 'object' && 'rel' in link && 'length' in link && 'href' in link && 'type' in link && link.type === 'application/x-zim' + }); + + if (!downloadLink) { + return null + } + + // downloadLink['href'] will end with .meta4, we need to remove that to get the actual download URL + const download_url = downloadLink['href'].substring(0, downloadLink['href'].length - 6); + const file_name = download_url.split('/').pop() || `${entry.title}.zim`; + const sizeBytes = parseInt(downloadLink['length'], 10); + + return { + id: entry.id, + title: entry.title, + updated: entry.updated, + summary: entry.summary, + size_bytes: sizeBytes || 0, + download_url: download_url, + author: entry.author.name, + file_name: file_name + } + }); + + // Filter out any null entries (those without a valid download link) + // or files that already exist in the local storage + const existing = await this.list(); + const existingKeys = new Set(existing.files.map(file => file.name)); + const withoutExisting = mapped.filter((entry): entry is RemoteZimFileEntry => entry !== null && !existingKeys.has(entry.file_name)); + + return { + items: withoutExisting, + has_more: result.feed.totalResults > start, + total_count: result.feed.totalResults, + }; + } + + async downloadRemote(url: string): Promise { + if (!url.endsWith('.zim')) { + throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`); + } + + const disk = drive.use('fs'); + const response = await axios.get(url, { + responseType: 'stream' + }); + + if (response.status !== 200) { + throw new Error(`Failed to download remote ZIM file from ${url}`); + } + + // Extract the filename from the URL + const filename = url.split('/').pop() || `downloaded-${Date.now()}.zim`; + const path = `/zim/${filename}`; + + await disk.putStream(path, response.data); + } + + async delete(key: string): Promise { + console.log('Deleting ZIM file with key:', key); + let fileName = key; + if (!fileName.endsWith('.zim')) { + fileName += '.zim'; + } + + const disk = drive.use('fs'); + const exists = await disk.exists(fileName); + + if (!exists) { + throw new Error('not_found'); + } + + await disk.delete(fileName); + } +} \ No newline at end of file diff --git a/admin/database/migrations/1751086751801_create_services_table.ts b/admin/database/migrations/1751086751801_create_services_table.ts index 62bb19b..8180092 100644 --- a/admin/database/migrations/1751086751801_create_services_table.ts +++ b/admin/database/migrations/1751086751801_create_services_table.ts @@ -6,11 +6,13 @@ export default class extends BaseSchema { async up() { this.schema.createTable(this.tableName, (table) => { table.increments('id') - table.string('service_name') - table.string('container_image') - table.string('container_command') + table.string('service_name').unique().notNullable() + table.string('container_image').notNullable() + table.string('container_command').nullable() table.json('container_config').nullable() table.boolean('installed').defaultTo(false) + table.string('depends_on').nullable().references('service_name').inTable(this.tableName).onDelete('SET NULL') + table.boolean('is_dependency_service').defaultTo(false) table.string('ui_location') table.json('metadata').nullable() table.timestamp('created_at') diff --git a/admin/database/seeders/service_seeder.ts b/admin/database/seeders/service_seeder.ts index 94695bf..d364283 100644 --- a/admin/database/seeders/service_seeder.ts +++ b/admin/database/seeders/service_seeder.ts @@ -11,15 +11,39 @@ export default class ServiceSeeder extends BaseSeeder { container_config: "{\"HostConfig\":{\"Binds\":[\"/opt/project-nomad/storage/zim:/data\"],\"PortBindings\":{\"8080/tcp\":[{\"HostPort\":\"8090\"}]}},\"ExposedPorts\":{\"8080/tcp\":{}}}", ui_location: '8090', installed: false, + is_dependency_service: false, + depends_on: null, }, { service_name: 'openstreetmap', container_image: 'overv/openstreetmap-tile-server', - container_command: 'run', + container_command: 'run --shm-size="192m"', container_config: "{\"HostConfig\":{\"Binds\":[\"/opt/project-nomad/storage/osm/db:/data/database\",\"/opt/project-nomad/storage/osm/tiles:/data/tiles\"],\"PortBindings\":{\"80/tcp\":[{\"HostPort\":\"9000\"}]}}}", ui_location: '9000', installed: false, - } + is_dependency_service: false, + depends_on: null, + }, + { + service_name: 'ollama', + container_image: 'ollama/ollama:latest', + container_command: 'serve', + container_config: "{\"HostConfig\":{\"Binds\":[\"/opt/project-nomad/storage/ollama:/root/.ollama\"],\"PortBindings\":{\"11434/tcp\":[{\"HostPort\":\"11434\"}]}}, \"ExposedPorts\":{\"11434/tcp\":{}}}", + ui_location: null, + installed: false, + is_dependency_service: true, + depends_on: null, + }, + { + service_name: 'open-webui', + container_image: 'ghcr.io/open-webui/open-webui:main', + container_command: null, + container_config: "{\"HostConfig\":{\"Env\":[\"WEBUI_AUTH=False\"],\"Binds\":[\"/opt/project-nomad/storage/open-webui:/app/backend/data\"],\"PortBindings\": {\"8080/tcp\": [{\"HostPort\": \"3000\"}]}},\"ExposedPorts\":{\"8080/tcp\": {}}}", + ui_location: '3000', + installed: false, + is_dependency_service: false, + depends_on: 'ollama', + }, ] async run() { diff --git a/admin/eslint.config.js b/admin/eslint.config.js index 9be1be3..b9e36cd 100644 --- a/admin/eslint.config.js +++ b/admin/eslint.config.js @@ -1,2 +1,3 @@ import { configApp } from '@adonisjs/eslint-config' -export default configApp() +import pluginQuery from '@tanstack/eslint-plugin-query' +export default configApp(...pluginQuery.configs['flat/recommended']) diff --git a/admin/inertia/app/app.tsx b/admin/inertia/app/app.tsx index e76722c..9f93e73 100644 --- a/admin/inertia/app/app.tsx +++ b/admin/inertia/app/app.tsx @@ -8,8 +8,10 @@ import { resolvePageComponent } from '@adonisjs/inertia/helpers' import ModalsProvider from '~/providers/ModalProvider' import { TransmitProvider } from 'react-adonis-transmit' import { generateUUID } from '~/lib/util' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' const appName = import.meta.env.VITE_APP_NAME || 'Project N.O.M.A.D.' +const queryClient = new QueryClient() // Patch the global crypto object for non-HTTPS/localhost contexts if (!window.crypto?.randomUUID) { @@ -20,7 +22,7 @@ if (!window.crypto?.randomUUID) { } createInertiaApp({ - progress: { color: '#5468FF' }, + progress: { color: '#424420' }, title: (title) => `${title} - ${appName}`, @@ -30,11 +32,13 @@ createInertiaApp({ setup({ el, App, props }) { createRoot(el).render( - - - - - + + + + + + + ) }, }) diff --git a/admin/inertia/components/StyledButton.tsx b/admin/inertia/components/StyledButton.tsx index fa84183..483f3b7 100644 --- a/admin/inertia/components/StyledButton.tsx +++ b/admin/inertia/components/StyledButton.tsx @@ -1,7 +1,7 @@ import * as Icons from '@heroicons/react/24/outline' import { useMemo } from 'react' -interface StyledButtonProps extends React.HTMLAttributes { +export interface StyledButtonProps extends React.HTMLAttributes { children: React.ReactNode // icon should be one of the HeroIcon names, e.g. ArrowTopRightOnSquareIcon icon?: keyof typeof Icons diff --git a/admin/inertia/components/StyledModal.tsx b/admin/inertia/components/StyledModal.tsx index 15b51dd..704adb6 100644 --- a/admin/inertia/components/StyledModal.tsx +++ b/admin/inertia/components/StyledModal.tsx @@ -1,17 +1,20 @@ import { Dialog, DialogBackdrop, DialogPanel, DialogTitle } from '@headlessui/react' -import StyledButton from './StyledButton' +import StyledButton, { StyledButtonProps } from './StyledButton' import React from 'react' +import classNames from '~/lib/classNames' interface StyledModalProps { onClose?: () => void title: string cancelText?: string confirmText?: string + confirmVariant?: StyledButtonProps['variant'] open: boolean onCancel?: () => void onConfirm?: () => void children: React.ReactNode icon?: React.ReactNode + large?: boolean } const StyledModal: React.FC = ({ @@ -21,9 +24,11 @@ const StyledModal: React.FC = ({ onClose, cancelText = 'Cancel', confirmText = 'Confirm', + confirmVariant = 'action', onCancel, onConfirm, icon, + large = false, }) => { return ( = ({ className="fixed inset-0 bg-gray-500/75 transition-opacity data-[closed]:opacity-0 data-[enter]:duration-300 data-[leave]:duration-200 data-[enter]:ease-out data-[leave]:ease-in" />
-
+
{icon &&
{icon}
} @@ -49,26 +62,30 @@ const StyledModal: React.FC = ({ {title} -
{children}
+
{children}
- { - if (onCancel) onCancel() - }} - > - {cancelText} - - { - if (onConfirm) onConfirm() - }} - > - {confirmText} - + {cancelText && onCancel && ( + { + if (onCancel) onCancel() + }} + > + {cancelText} + + )} + {confirmText && onConfirm && ( + { + if (onConfirm) onConfirm() + }} + > + {confirmText} + + )}
diff --git a/admin/inertia/components/StyledTable.tsx b/admin/inertia/components/StyledTable.tsx index 88964ad..33f9636 100644 --- a/admin/inertia/components/StyledTable.tsx +++ b/admin/inertia/components/StyledTable.tsx @@ -1,10 +1,14 @@ import { capitalizeFirstLetter } from '~/lib/util' import classNames from '~/lib/classNames' import LoadingSpinner from '~/components/LoadingSpinner' +import React, { RefObject } from 'react' -interface StyledTableProps> { +export type StyledTableProps = { loading?: boolean tableProps?: React.HTMLAttributes + tableRowStyle?: React.CSSProperties + tableBodyClassName?: string + tableBodyStyle?: React.CSSProperties data?: T[] noDataText?: string onRowClick?: (record: T) => void @@ -16,55 +20,82 @@ interface StyledTableProps> { }[] className?: string rowLines?: boolean + ref?: RefObject + containerProps?: React.HTMLAttributes + compact?: boolean } -function StyledTable({ +function StyledTable({ loading = false, tableProps = {}, + tableRowStyle = {}, + tableBodyClassName = '', + tableBodyStyle = {}, data = [], noDataText = 'No records found', onRowClick, columns = [], className = '', + ref, + containerProps = {}, + rowLines = true, + compact = false, }: StyledTableProps) { const { className: tableClassName, ...restTableProps } = tableProps + const leftPadding = compact ? 'pl-2' : 'pl-4 sm:pl-6' + return (
- - +
+ {columns.map((column, index) => ( ))} - + {!loading && data.length !== 0 && data.map((record, recordIdx) => ( onRowClick?.(record)} - className={onRowClick ? `cursor-pointer hover:bg-gray-100 ` : ''} + style={{ + ...tableRowStyle, + height: 'height' in record ? record.height : 'auto', + transform: + 'translateY' in record ? 'translateY(' + record.transformY + 'px)' : undefined, + }} + className={classNames( + rowLines ? 'border-b border-gray-200' : '', + onRowClick ? `cursor-pointer hover:bg-gray-100 ` : '' + )} > {columns.map((column, index) => (
{column.title ?? capitalizeFirstLetter(column.accessor.toString())}
{column.render diff --git a/admin/inertia/layouts/SettingsLayout.tsx b/admin/inertia/layouts/SettingsLayout.tsx index c94de80..94779db 100644 --- a/admin/inertia/layouts/SettingsLayout.tsx +++ b/admin/inertia/layouts/SettingsLayout.tsx @@ -1,9 +1,20 @@ -import { Cog6ToothIcon, CommandLineIcon, FolderIcon } from '@heroicons/react/24/outline' +import { + Cog6ToothIcon, + CommandLineIcon, + FolderIcon, + MagnifyingGlassIcon, +} from '@heroicons/react/24/outline' import StyledSidebar from '~/components/StyledSidebar' const navigation = [ { name: 'Apps', href: '/settings/apps', icon: CommandLineIcon, current: false }, - { name: 'ZIM Explorer', href: '/settings/zim', icon: FolderIcon, current: false }, + { name: 'ZIM Manager', href: '/settings/zim', icon: FolderIcon, current: false }, + { + name: 'Zim Remote Explorer', + href: '/settings/zim/remote-explorer', + icon: MagnifyingGlassIcon, + current: false, + }, { name: 'System', href: '/settings/system', icon: Cog6ToothIcon, current: true }, ] diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index e186a61..68d19d0 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -1,4 +1,5 @@ import axios from "axios"; +import { ListRemoteZimFilesResponse, ListZimFilesResponse, RemoteZimFileEntry } from "../../types/zim"; class API { private client; @@ -32,6 +33,39 @@ class API { throw error; } } + + async listZimFiles() { + return await this.client.get("/zim/list"); + } + + async listRemoteZimFiles({ start = 0, count = 12 }: { start?: number; count?: number }) { + return await this.client.get("/zim/list-remote", { + params: { + start, + count + } + }); + } + + async downloadRemoteZimFile(url: string) { + try { + const response = await this.client.post("/zim/download-remote", { url }); + return response.data; + } catch (error) { + console.error("Error downloading remote ZIM file:", error); + throw error; + } + } + + async deleteZimFile(key: string) { + try { + const response = await this.client.delete(`/zim/${key}`); + return response.data; + } catch (error) { + console.error("Error deleting ZIM file:", error); + throw error; + } + } } export default new API(); \ No newline at end of file diff --git a/admin/inertia/lib/navigation.ts b/admin/inertia/lib/navigation.ts index 3085320..57a77dd 100644 --- a/admin/inertia/lib/navigation.ts +++ b/admin/inertia/lib/navigation.ts @@ -14,7 +14,7 @@ export function getServiceLink(ui_location: string): string { const parsedPort = parseInt(ui_location, 10); if (!isNaN(parsedPort)) { // If it's a port number, return a link to the service on that port - return `http://${window.location.origin}:${parsedPort}`; + return `http://${window.location.hostname}:${parsedPort}`; } // Otherwise, treat it as a path return `/${ui_location}`; diff --git a/admin/inertia/lib/util.ts b/admin/inertia/lib/util.ts index b22498c..65a4359 100644 --- a/admin/inertia/lib/util.ts +++ b/admin/inertia/lib/util.ts @@ -1,32 +1,41 @@ import axios from "axios"; export function capitalizeFirstLetter(str?: string | null): string { - if (!str) return ""; - return str.charAt(0).toUpperCase() + str.slice(1); + if (!str) return ""; + return str.charAt(0).toUpperCase() + str.slice(1); +} + +export function formatBytes(bytes: number, decimals = 2): string { + if (bytes === 0) return '0 Bytes'; + const k = 1024; + const dm = decimals < 0 ? 0 : decimals; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } export async function testInternetConnection(): Promise { - try { - const response = await axios.get('https://1.1.1.1/cdn-cgi/trace', { - timeout: 5000, - headers: { - 'Cache-Control': 'no-cache', - } - }); - return response.status === 200; - } catch (error) { - console.error("Error testing internet connection:", error); - return false; - } + try { + const response = await axios.get('https://1.1.1.1/cdn-cgi/trace', { + timeout: 5000, + headers: { + 'Cache-Control': 'no-cache', + } + }); + return response.status === 200; + } catch (error) { + console.error("Error testing internet connection:", error); + return false; + } } export function generateRandomString(length: number): string { - const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - let result = ''; - for (let i = 0; i < length; i++) { - result += characters.charAt(Math.floor(Math.random() * characters.length)); - } - return result; + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + let result = ''; + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * characters.length)); + } + return result; } export function generateUUID(): string { @@ -40,10 +49,10 @@ export function generateUUID(): string { arr[i] = Math.floor(Math.random() * 256); } } - + arr[6] = (arr[6] & 0x0f) | 0x40; // Version 4 arr[8] = (arr[8] & 0x3f) | 0x80; // Variant bits - + const hex = Array.from(arr, byte => byte.toString(16).padStart(2, '0')).join(''); - return `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20)}`; + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; } \ No newline at end of file diff --git a/admin/inertia/pages/home.tsx b/admin/inertia/pages/home.tsx index b96d953..e344496 100644 --- a/admin/inertia/pages/home.tsx +++ b/admin/inertia/pages/home.tsx @@ -1,59 +1,9 @@ -import { - IconBook, - IconBrandWikipedia, - IconCalculator, - IconHelp, - IconMapRoute, - IconMessageCircleSearch, - IconPlus, - IconSettings, - IconWifiOff, -} from '@tabler/icons-react' -import { Head, Link } from '@inertiajs/react' +import { IconHelp, IconPlus, IconSettings, IconWifiOff } from '@tabler/icons-react' +import { Head } from '@inertiajs/react' import BouncingLogo from '~/components/BouncingLogo' import AppLayout from '~/layouts/AppLayout' import { getServiceLink } from '~/lib/navigation' -const NAV_ITEMS = [ - { - label: 'AI Chat', - to: '/ai-chat', - description: 'Chat with local AI models', - icon: , - }, - { - label: 'Calculators', - to: '/calculators', - description: 'Perform various calculations', - icon: , - }, - { - label: 'Ebooks', - to: '/ebooks', - description: 'Explore our collection of eBooks', - icon: , - }, - { - label: 'Kiwix (Offline Browser)', - to: '/kiwix', - description: 'Access offline content with Kiwix', - icon: , - }, - { - label: 'OpenStreetMap', - to: '/openstreetmap', - description: 'View maps and geospatial data', - icon: , - }, - - { - label: 'Wikipedia', - to: '/wikipedia', - description: 'Browse an offline Wikipedia snapshot', - icon: , - }, -] - const STATIC_ITEMS = [ { label: 'Install Apps', @@ -87,6 +37,8 @@ export default function Home(props: { } }) { const items = [] + + console.log(props.system.services) props.system.services.map((service) => { items.push({ label: service.service_name, diff --git a/admin/inertia/pages/settings/zim/index.tsx b/admin/inertia/pages/settings/zim/index.tsx new file mode 100644 index 0000000..b1ff4d9 --- /dev/null +++ b/admin/inertia/pages/settings/zim/index.tsx @@ -0,0 +1,100 @@ +import { Head, Link } from '@inertiajs/react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import StyledTable from '~/components/StyledTable' +import SettingsLayout from '~/layouts/SettingsLayout' +import { ZimFilesEntry } from '../../../../types/zim' +import api from '~/lib/api' +import StyledButton from '~/components/StyledButton' +import { useModals } from '~/context/ModalContext' +import StyledModal from '~/components/StyledModal' + +export default function ZimPage() { + const queryClient = useQueryClient() + const { openModal, closeAllModals } = useModals() + const { data, isLoading } = useQuery({ + queryKey: ['zim-files'], + queryFn: getFiles, + }) + + async function getFiles() { + const res = await api.listZimFiles() + return res.data.files + } + + async function confirmDeleteFile(file: ZimFilesEntry) { + openModal( + { + deleteFileMutation.mutateAsync(file) + closeAllModals() + }} + onCancel={closeAllModals} + open={true} + confirmText="Delete" + cancelText="Cancel" + confirmVariant="danger" + > +

+ Are you sure you want to delete {file.name}? This action cannot be undone. +

+ , + 'confirm-delete-file-modal' + ) + } + + const deleteFileMutation = useMutation({ + mutationFn: async (file: ZimFilesEntry) => api.deleteZimFile(file.name.replace('.zim', '')), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['zim-files'] }) + }, + }) + + return ( + + +
+
+
+
+

ZIM Manager

+

+ Manage your stored ZIM files and download new ones! +

+
+ + Remote Explorer + +
+ + className="font-semibold" + rowLines={true} + loading={isLoading} + compact + columns={[ + { accessor: 'name', title: 'Name' }, + { + accessor: 'actions', + title: 'Actions', + render: (record) => ( +
+ { + confirmDeleteFile(record) + }} + > + Delete + +
+ ), + }, + ]} + data={data || []} + /> +
+
+ + ) +} diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx new file mode 100644 index 0000000..47f2bf9 --- /dev/null +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -0,0 +1,173 @@ +import { keepPreviousData, useInfiniteQuery } from '@tanstack/react-query' +import api from '~/lib/api' +import { useCallback, useEffect, useMemo, useRef } from 'react' +import { useVirtualizer } from '@tanstack/react-virtual' +import StyledTable from '~/components/StyledTable' +import SettingsLayout from '~/layouts/SettingsLayout' +import { Head } from '@inertiajs/react' +import { ListRemoteZimFilesResponse, RemoteZimFileEntry } from '../../../../types/zim' +import { formatBytes } from '~/lib/util' +import StyledButton from '~/components/StyledButton' +import { useModals } from '~/context/ModalContext' +import StyledModal from '~/components/StyledModal' + +export default function ZimRemoteExplorer() { + const tableParentRef = useRef(null) + const { openModal, closeAllModals } = useModals() + const { data, fetchNextPage, isFetching, isLoading } = + useInfiniteQuery({ + queryKey: ['remote-zim-files'], + queryFn: async ({ pageParam = 0 }) => { + const pageParsed = parseInt((pageParam as number).toString(), 10) + const start = isNaN(pageParsed) ? 0 : pageParsed * 12 + const res = await api.listRemoteZimFiles({ start, count: 12 }) + return res.data + }, + initialPageParam: 0, + getNextPageParam: (_lastPage, pages) => { + if (!_lastPage.has_more) { + return undefined // No more pages to fetch + } + return pages.length + }, + refetchOnWindowFocus: false, + placeholderData: keepPreviousData, + }) + + const flatData = useMemo(() => data?.pages.flatMap((page) => page.items) || [], [data]) + const hasMore = useMemo(() => data?.pages[data.pages.length - 1]?.has_more || false, [data]) + + const fetchOnBottomReached = useCallback( + (parentRef?: HTMLDivElement | null) => { + if (parentRef) { + const { scrollHeight, scrollTop, clientHeight } = parentRef + //once the user has scrolled within 200px of the bottom of the table, fetch more data if we can + if (scrollHeight - scrollTop - clientHeight < 200 && !isFetching && hasMore) { + fetchNextPage() + } + } + }, + [fetchNextPage, isFetching, hasMore] + ) + + const virtualizer = useVirtualizer({ + count: flatData.length, + estimateSize: () => 48, // Estimate row height + getScrollElement: () => tableParentRef.current, + overscan: 5, // Number of items to render outside the visible area + }) + + //a check on mount and after a fetch to see if the table is already scrolled to the bottom and immediately needs to fetch more data + useEffect(() => { + fetchOnBottomReached(tableParentRef.current) + }, [fetchOnBottomReached]) + + async function confirmDownload(record: RemoteZimFileEntry) { + openModal( + { + downloadFile(record) + closeAllModals() + }} + onCancel={closeAllModals} + open={true} + confirmText="Download" + cancelText="Cancel" + confirmVariant="primary" + > +

+ Are you sure you want to download {record.title}? It may take some time + for it to be available depending on the file size and your internet connection. +

+ , + 'confirm-download-file-modal' + ) + } + + async function downloadFile(record: RemoteZimFileEntry) { + try { + await api.downloadRemoteZimFile(record.download_url) + } catch (error) { + console.error('Error downloading file:', error) + } + } + + return ( + + +
+
+

ZIM Remote Explorer

+

+ Browse and download remote ZIM files from the Kiwix repository! +

+ + data={flatData.map((i, idx) => { + const row = virtualizer.getVirtualItems().find((v) => v.index === idx) + return { + ...i, + height: `${row?.size || 48}px`, // Use the size from the virtualizer + translateY: row?.start || 0, + } + })} + ref={tableParentRef} + loading={isLoading} + columns={[ + { + accessor: 'title', + }, + { + accessor: 'author', + }, + { + accessor: 'summary', + }, + { + accessor: 'updated', + render(record) { + return new Intl.DateTimeFormat('en-US', { + dateStyle: 'medium', + }).format(new Date(record.updated)) + }, + }, + { + accessor: 'size_bytes', + render(record) { + return formatBytes(record.size_bytes) + }, + }, + { + accessor: 'actions', + render(record, index) { + return ( +
+ { + confirmDownload(record) + }} + > + Download + +
+ ) + }, + }, + ]} + className="relative overflow-x-auto overflow-y-auto h-[600px] w-full " + tableBodyStyle={{ + position: 'relative', + height: `${virtualizer.getTotalSize()}px`, + }} + containerProps={{ + onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement), + }} + compact + rowLines + /> +
+
+ + ) +} diff --git a/admin/package-lock.json b/admin/package-lock.json index ef614ba..95d8a33 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -27,6 +27,8 @@ "@markdoc/markdoc": "^0.5.2", "@tabler/icons-react": "^3.34.0", "@tailwindcss/vite": "^4.1.10", + "@tanstack/react-query": "^5.81.5", + "@tanstack/react-virtual": "^3.13.12", "@vinejs/vine": "^3.0.1", "@vitejs/plugin-react": "^4.6.0", "autoprefixer": "^10.4.21", @@ -34,6 +36,7 @@ "better-sqlite3": "^12.1.1", "dockerode": "^4.0.7", "edge.js": "^6.2.1", + "fast-xml-parser": "^5.2.5", "luxon": "^3.6.1", "mysql2": "^3.14.1", "pino-pretty": "^13.0.0", @@ -54,6 +57,7 @@ "@japa/plugin-adonisjs": "^4.0.0", "@japa/runner": "^4.2.0", "@swc/core": "1.11.24", + "@tanstack/eslint-plugin-query": "^5.81.2", "@types/dockerode": "^3.3.41", "@types/luxon": "^3.6.2", "@types/node": "^22.15.18", @@ -3646,6 +3650,49 @@ "vite": "^5.2.0 || ^6" } }, + "node_modules/@tanstack/eslint-plugin-query": { + "version": "5.81.2", + "resolved": "https://registry.npmjs.org/@tanstack/eslint-plugin-query/-/eslint-plugin-query-5.81.2.tgz", + "integrity": "sha512-h4k6P6fm5VhKP5NkK+0TTVpGGyKQdx6tk7NYYG7J7PkSu7ClpLgBihw7yzK8N3n5zPaF3IMyErxfoNiXWH/3/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/utils": "^8.18.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0" + } + }, + "node_modules/@tanstack/query-core": { + "version": "5.81.5", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.81.5.tgz", + "integrity": "sha512-ZJOgCy/z2qpZXWaj/oxvodDx07XcQa9BF92c0oINjHkoqUPsmm3uG08HpTaviviZ/N9eP1f9CM7mKSEkIo7O1Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/react-query": { + "version": "5.81.5", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.81.5.tgz", + "integrity": "sha512-lOf2KqRRiYWpQT86eeeftAGnjuTR35myTP8MXyvHa81VlomoAWNEd8x5vkcAfQefu0qtYCvyqLropFZqgI2EQw==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.81.5" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, "node_modules/@tanstack/react-virtual": { "version": "3.13.12", "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.12.tgz", @@ -6338,6 +6385,24 @@ "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", "license": "MIT" }, + "node_modules/fast-xml-parser": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.5.tgz", + "integrity": "sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastest-levenshtein": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", @@ -10333,6 +10398,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.1.tgz", + "integrity": "sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/strtok3": { "version": "10.3.1", "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.1.tgz", diff --git a/admin/package.json b/admin/package.json index 26bb478..ac3120f 100644 --- a/admin/package.json +++ b/admin/package.json @@ -41,6 +41,7 @@ "@japa/plugin-adonisjs": "^4.0.0", "@japa/runner": "^4.2.0", "@swc/core": "1.11.24", + "@tanstack/eslint-plugin-query": "^5.81.2", "@types/dockerode": "^3.3.41", "@types/luxon": "^3.6.2", "@types/node": "^22.15.18", @@ -72,6 +73,8 @@ "@markdoc/markdoc": "^0.5.2", "@tabler/icons-react": "^3.34.0", "@tailwindcss/vite": "^4.1.10", + "@tanstack/react-query": "^5.81.5", + "@tanstack/react-virtual": "^3.13.12", "@vinejs/vine": "^3.0.1", "@vitejs/plugin-react": "^4.6.0", "autoprefixer": "^10.4.21", @@ -79,6 +82,7 @@ "better-sqlite3": "^12.1.1", "dockerode": "^4.0.7", "edge.js": "^6.2.1", + "fast-xml-parser": "^5.2.5", "luxon": "^3.6.1", "mysql2": "^3.14.1", "pino-pretty": "^13.0.0", diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 2f9ec21..7508c5a 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -10,6 +10,7 @@ import DocsController from '#controllers/docs_controller' import HomeController from '#controllers/home_controller' import SettingsController from '#controllers/settings_controller' import SystemController from '#controllers/system_controller' +import ZimController from '#controllers/zim_controller' import router from '@adonisjs/core/services/router' import transmit from '@adonisjs/transmit/services/main' @@ -22,6 +23,8 @@ router.on('/about').renderInertia('about') router.group(() => { router.get('/system', [SettingsController, 'system']) router.get('/apps', [SettingsController, 'apps']) + router.get('/zim', [SettingsController, 'zim']) + router.get('/zim/remote-explorer', [SettingsController, 'zimRemote']) }).prefix('/settings') router.group(() => { @@ -42,4 +45,11 @@ router.group(() => { router.get('/services', [SystemController, 'getServices']) router.post('/services/install', [SystemController, 'installService']) router.post('/simulate-sse', [SystemController, 'simulateSSE']) -}).prefix('/api/system') \ No newline at end of file +}).prefix('/api/system') + +router.group(() => { + router.get('/list', [ZimController, 'list']) + router.get('/list-remote', [ZimController, 'listRemote']) + router.post('/download-remote', [ZimController, 'downloadRemote']) + router.delete('/:key', [ZimController, 'delete']) +}).prefix('/api/zim') \ No newline at end of file diff --git a/admin/types/zim.ts b/admin/types/zim.ts new file mode 100644 index 0000000..e86d181 --- /dev/null +++ b/admin/types/zim.ts @@ -0,0 +1,68 @@ +export type ZimFilesEntry = + { + type: 'file' + key: string; + name: string; + } | { + type: 'directory'; + prefix: string; + name: string; + } + +export type ListZimFilesResponse = { + files: ZimFilesEntry[] + next?: string +} + +export type ListRemoteZimFilesResponse = { + items: RemoteZimFileEntry[]; + has_more: boolean; + total_count: number; +} + +export type RawRemoteZimFileEntry = { + id: string; + title: string; + updated: string; + summary: string; + language: string; + name: string; + flavour: string; + category: string; + tags: string; + articleCount: number; + mediaCount: number; + link: Record[]; + author: { + name: string + }; + publisher: { + name: string; + }; + 'dc:issued': string; +} + +export type RawListRemoteZimFilesResponse = { + '?xml': string; + feed: { + id: string; + link: string[]; + title: string; + updated: string; + totalResults: number; + startIndex: number; + itemsPerPage: number; + entry: any[]; + } +} + +export type RemoteZimFileEntry = { + id: string; + title: string; + updated: string; + summary: string; + size_bytes: number; + download_url: string; + author: string; + file_name: string; +} \ No newline at end of file diff --git a/admin/util/zim.ts b/admin/util/zim.ts new file mode 100644 index 0000000..34b6205 --- /dev/null +++ b/admin/util/zim.ts @@ -0,0 +1,10 @@ +import { RawListRemoteZimFilesResponse, RawRemoteZimFileEntry } from "../types/zim.js"; + + +export function isRawListRemoteZimFilesResponse(obj: any): obj is RawListRemoteZimFilesResponse { + return obj && typeof obj === 'object' && 'feed' in obj && 'entry' in obj.feed && Array.isArray(obj.feed.entry); +} + +export function isRawRemoteZimFileEntry(obj: any): obj is RawRemoteZimFileEntry { + return obj && typeof obj === 'object' && 'id' in obj && 'title' in obj && 'summary' in obj; +} \ No newline at end of file diff --git a/install/install_nomad.sh b/install/install_nomad.sh index 6e1da0a..4b59bad 100644 --- a/install/install_nomad.sh +++ b/install/install_nomad.sh @@ -36,7 +36,8 @@ WAIT_FOR_IT_SCRIPT_URL="https://raw.githubusercontent.com/vishnubob/wait-for-it/ script_option_debug='true' install_actions='' nomad_dir="/opt/project-nomad" - +accepted_terms='false' +local_ip='' ################################################################################################################################################################################################### # # @@ -156,26 +157,53 @@ ensure_docker_installed() { fi } -ensure_whiptail_installed() { - if ! command -v whiptail &> /dev/null; then - header_red - echo -e "${GRAY_R}#${RESET} whiptail is not installed, attempting to install it...\\n" - if command -v apt &> /dev/null; then - apt update && apt install -y whiptail - else - echo -e "${RED}#${RESET} Unsupported package manager. Please install whiptail manually." - exit 1 - fi - fi -} - get_install_confirmation(){ - if whiptail --title "$WHIPTAIL_TITLE" --yesno "This script will install/update Project N.O.M.A.D. and its dependencies on your machine.\\n\\n Are you sure you want to continue?\\n\\nInfo:\\nVersion 1.0.0\\nAuthor: Crosstalk Solutions, LLC\\nWebsite: https://crosstalksolutions.com" 15 70; then + # Make this a regular bash prompt instead of whiptail + read -p "This script will install/update Project N.O.M.A.D. and its dependencies on your machine. Are you sure you want to continue? (y/n): " choice + case "$choice" in + y|Y ) echo -e "${GREEN}#${RESET} User chose to continue with the installation." - else + ;; + n|N ) echo -e "${RED}#${RESET} User chose not to continue with the installation." exit 0 - fi + ;; + * ) + echo "Invalid Response" + echo "User chose not to continue with the installation." + exit 0 + ;; + esac +} + +accept_terms() { + printf "\n\n" + echo "License Agreement & Terms of Use" + echo "__________________________" + printf "\n\n" + echo "Copyright 2025 Crosstalk Solutions, LLC" + printf "\n" + echo "Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:" + printf "\n" + echo "The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software." + printf "\n" + echo "THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." + echo -e "\n\n" + read -p "I have read and accept License Agreement & Terms of Use (y/n)? " choice + case "$choice" in + y|Y ) + accepted_terms='true' + ;; + n|N ) + echo "License Agreement & Terms of Use not accepted. Installation cannot continue." + exit 1 + ;; + * ) + echo "Invalid Response" + echo "License Agreement & Terms of Use not accepted. Installation cannot continue." + exit 1 + ;; + esac } get_install_directory() { @@ -255,6 +283,21 @@ start_management_containers() { echo -e "${GREEN}#${RESET} Management containers started successfully.\\n" } +get_local_ip() { + local_ip_address=$(hostname -I | awk '{print $1}') + if [[ -z "$local_ip_address" ]]; then + echo -e "${RED}#${RESET} Unable to determine local IP address. Please check your network configuration." + # Don't exit if we can't determine the local IP address, it's not critical for the installation + fi +} + +success_message() { + echo -e "${GREEN}#${RESET} Project N.O.M.A.D installation completed successfully!\\n" + echo -e "${GREEN}#${RESET} Installation files are located at /opt/project-nomad\\n\n" + echo -e "${GREEN}#${RESET} You can now access the management interface at http://localhost:8080 or http://${local_ip_address}:8080\\n" + echo -e "${GREEN}#${RESET} Thank you for supporting Project N.O.M.A.D!\\n" +} + ################################################################################################################################################################################################### # # # Main Script # @@ -266,10 +309,10 @@ check_is_debian_based check_is_bash check_has_sudo check_is_debug_mode -ensure_whiptail_installed # Main install get_install_confirmation +accept_terms ensure_docker_installed #get_install_directory create_nomad_directory @@ -277,6 +320,8 @@ download_wait_for_it_script download_entrypoint_script download_management_compose_file start_management_containers +get_local_ip +success_message # free_space_check() { # if [[ "$(df -B1 / | awk 'NR==2{print $4}')" -le '5368709120' ]]; then diff --git a/install/management_compose.yaml b/install/management_compose.yaml index 82f2bcb..cbbda6f 100644 --- a/install/management_compose.yaml +++ b/install/management_compose.yaml @@ -7,7 +7,6 @@ services: ports: - "8080:8080" volumes: - - /opt/project-nomad/data:/data - /opt/project-nomad/storage:/app/storage - /var/run/docker.sock:/var/run/docker.sock # Allows the admin service to communicate with the Host's Docker daemon - ./entrypoint.sh:/usr/local/bin/entrypoint.sh