mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-03-28 03:29:25 +01:00
feat(ZIM): improved ZIM downloading and auto-restart kiwix serve
This commit is contained in:
parent
7c2b0964dc
commit
9e216c366f
|
|
@ -35,10 +35,4 @@ export default class SystemController {
|
||||||
}
|
}
|
||||||
response.send({ success: result.success, message: result.message });
|
response.send({ success: result.success, message: result.message });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async simulateSSE({ response }: HttpContext) {
|
|
||||||
this.dockerService.simulateSSE();
|
|
||||||
response.send({ message: 'Started simulation of SSE' })
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
@ -19,10 +19,12 @@ export default class ZimController {
|
||||||
|
|
||||||
async downloadRemote({ request, response }: HttpContext) {
|
async downloadRemote({ request, response }: HttpContext) {
|
||||||
const { url } = request.body()
|
const { url } = request.body()
|
||||||
await this.zimService.downloadRemote(url);
|
const filename = await this.zimService.downloadRemote(url);
|
||||||
|
|
||||||
response.status(200).send({
|
response.status(200).send({
|
||||||
message: 'Download started successfully'
|
message: 'Download started successfully',
|
||||||
|
filename,
|
||||||
|
url
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -372,17 +372,4 @@ export class DockerService {
|
||||||
throw new Error(`Invalid container configuration: ${error.message}`);
|
throw new Error(`Invalid container configuration: ${error.message}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async simulateSSE(): Promise<void> {
|
|
||||||
// This is just a simulation of the server-sent events for testing purposes
|
|
||||||
for (let i = 0; i <= 10; i++) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
||||||
transmit.broadcast('service-installation', {
|
|
||||||
service_name: 'test-service',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
status: i === 10 ? 'completed' : 'in-progress',
|
|
||||||
message: `Test message ${i}`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
@ -1,19 +1,19 @@
|
||||||
import Service from "#models/service"
|
import Service from "#models/service"
|
||||||
import { inject } from "@adonisjs/core";
|
import { inject } from "@adonisjs/core";
|
||||||
import { DockerService } from "#services/docker_service";
|
import { DockerService } from "#services/docker_service";
|
||||||
import { ServiceStatus } from "../../types/services.js";
|
import { ServiceSlim } from "../../types/services.js";
|
||||||
|
|
||||||
@inject()
|
@inject()
|
||||||
export class SystemService {
|
export class SystemService {
|
||||||
constructor(
|
constructor(
|
||||||
private dockerService: DockerService
|
private dockerService: DockerService
|
||||||
) {}
|
) { }
|
||||||
async getServices({
|
async getServices({
|
||||||
installedOnly = true,
|
installedOnly = true,
|
||||||
}:{
|
}: {
|
||||||
installedOnly?: boolean
|
installedOnly?: boolean
|
||||||
}): Promise<{ id: number; service_name: string; installed: boolean, status: ServiceStatus }[]> {
|
}): Promise<ServiceSlim[]> {
|
||||||
const query = Service.query().orderBy('service_name', 'asc').select('id', 'service_name', 'installed', 'ui_location').where('is_dependency_service', false)
|
const query = Service.query().orderBy('service_name', 'asc').select('id', 'service_name', 'installed', 'ui_location').where('is_dependency_service', false)
|
||||||
if (installedOnly) {
|
if (installedOnly) {
|
||||||
query.where('installed', true);
|
query.where('installed', true);
|
||||||
}
|
}
|
||||||
|
|
@ -25,7 +25,7 @@ export class SystemService {
|
||||||
|
|
||||||
const statuses = await this.dockerService.getServicesStatus();
|
const statuses = await this.dockerService.getServicesStatus();
|
||||||
|
|
||||||
const toReturn = [];
|
const toReturn: ServiceSlim[] = [];
|
||||||
|
|
||||||
for (const service of services) {
|
for (const service of services) {
|
||||||
const status = statuses.find(s => s.service_name === service.service_name);
|
const status = statuses.find(s => s.service_name === service.service_name);
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,22 @@
|
||||||
import drive from "@adonisjs/drive/services/main";
|
import drive from "@adonisjs/drive/services/main";
|
||||||
import { ListRemoteZimFilesResponse, RawRemoteZimFileEntry, RemoteZimFileEntry, ZimFilesEntry } from "../../types/zim.js";
|
import { DownloadOptions, DownloadProgress, ListRemoteZimFilesResponse, RawRemoteZimFileEntry, RemoteZimFileEntry, ZimFilesEntry } from "../../types/zim.js";
|
||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
import { XMLParser } from 'fast-xml-parser'
|
import { XMLParser } from 'fast-xml-parser'
|
||||||
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from "../../util/zim.js";
|
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from "../../util/zim.js";
|
||||||
|
import transmit from "@adonisjs/transmit/services/main";
|
||||||
|
import { Transform } from "stream";
|
||||||
|
import logger from "@adonisjs/core/services/logger";
|
||||||
|
import { DockerService } from "./docker_service.js";
|
||||||
|
import { inject } from "@adonisjs/core";
|
||||||
|
|
||||||
|
@inject()
|
||||||
export class ZimService {
|
export class ZimService {
|
||||||
|
private activeDownloads = new Map<string, AbortController>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private dockerService: DockerService
|
||||||
|
) {}
|
||||||
|
|
||||||
async list() {
|
async list() {
|
||||||
const disk = drive.use('fs');
|
const disk = drive.use('fs');
|
||||||
const contents = await disk.listAll('/zim')
|
const contents = await disk.listAll('/zim')
|
||||||
|
|
@ -99,29 +111,41 @@ export class ZimService {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async downloadRemote(url: string): Promise<void> {
|
async downloadRemote(url: string, opts: DownloadOptions = {}): Promise<string> {
|
||||||
if (!url.endsWith('.zim')) {
|
if (!url.endsWith('.zim')) {
|
||||||
throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`);
|
throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const disk = drive.use('fs');
|
const existing = this.activeDownloads.get(url);
|
||||||
const response = await axios.get(url, {
|
if (existing) {
|
||||||
responseType: 'stream'
|
throw new Error(`Download already in progress for URL ${url}`);
|
||||||
});
|
|
||||||
|
|
||||||
if (response.status !== 200) {
|
|
||||||
throw new Error(`Failed to download remote ZIM file from ${url}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract the filename from the URL
|
// Extract the filename from the URL
|
||||||
const filename = url.split('/').pop() || `downloaded-${Date.now()}.zim`;
|
const filename = url.split('/').pop() || `downloaded-${Date.now()}.zim`;
|
||||||
const path = `/zim/${filename}`;
|
const path = `/zim/${filename}`;
|
||||||
|
|
||||||
await disk.putStream(path, response.data);
|
this._runDownload(url, path, opts); // Don't await - let run in background
|
||||||
|
|
||||||
|
return filename;
|
||||||
|
}
|
||||||
|
|
||||||
|
getActiveDownloads(): string[] {
|
||||||
|
return Array.from(this.activeDownloads.keys());
|
||||||
|
}
|
||||||
|
|
||||||
|
cancelDownload(url: string): boolean {
|
||||||
|
const entry = this.activeDownloads.get(url);
|
||||||
|
if (entry) {
|
||||||
|
entry.abort();
|
||||||
|
this.activeDownloads.delete(url);
|
||||||
|
transmit.broadcast(`zim-downloads`, { url, status: 'cancelled' });
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
async delete(key: string): Promise<void> {
|
async delete(key: string): Promise<void> {
|
||||||
console.log('Deleting ZIM file with key:', key);
|
|
||||||
let fileName = key;
|
let fileName = key;
|
||||||
if (!fileName.endsWith('.zim')) {
|
if (!fileName.endsWith('.zim')) {
|
||||||
fileName += '.zim';
|
fileName += '.zim';
|
||||||
|
|
@ -136,4 +160,215 @@ export class ZimService {
|
||||||
|
|
||||||
await disk.delete(fileName);
|
await disk.delete(fileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async _runDownload(url: string, path: string, opts: DownloadOptions = {}): Promise<string> {
|
||||||
|
try {
|
||||||
|
const {
|
||||||
|
max_retries = 3,
|
||||||
|
retry_delay = 2000,
|
||||||
|
timeout = 30000,
|
||||||
|
onError,
|
||||||
|
}: DownloadOptions = opts;
|
||||||
|
|
||||||
|
let attempt = 0;
|
||||||
|
while (attempt < max_retries) {
|
||||||
|
try {
|
||||||
|
const abortController = new AbortController();
|
||||||
|
this.activeDownloads.set(url, abortController);
|
||||||
|
|
||||||
|
await this._attemptDownload(
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
abortController.signal,
|
||||||
|
timeout,
|
||||||
|
);
|
||||||
|
|
||||||
|
transmit.broadcast('zim-downloads', { url, path, status: 'completed', progress: { downloaded_bytes: 0, total_bytes: 0, percentage: 100, speed: '0 B/s', time_remaining: 0 } });
|
||||||
|
await this.dockerService.affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart').catch((error) => {
|
||||||
|
logger.error(`Failed to restart KIWIX container:`, error); // Don't stop the download completion, just log the error.
|
||||||
|
});
|
||||||
|
|
||||||
|
break; // Exit loop on success
|
||||||
|
} catch (error) {
|
||||||
|
attempt++;
|
||||||
|
|
||||||
|
const isAborted = error.name === 'AbortError' || error.code === 'ABORT_ERR';
|
||||||
|
const isNetworkError = error.code === 'ECONNRESET' ||
|
||||||
|
error.code === 'ENOTFOUND' ||
|
||||||
|
error.code === 'ETIMEDOUT';
|
||||||
|
|
||||||
|
onError?.(error);
|
||||||
|
if (isAborted) {
|
||||||
|
throw new Error(`Download aborted for URL: ${url}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt < max_retries && isNetworkError) {
|
||||||
|
await this.delay(retry_delay);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(`Failed to download ${url}:`, error);
|
||||||
|
transmit.broadcast('zim-downloads', { url, error: error.message, status: 'failed' });
|
||||||
|
} finally {
|
||||||
|
this.activeDownloads.delete(url);
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _attemptDownload(
|
||||||
|
url: string,
|
||||||
|
path: string,
|
||||||
|
signal: AbortSignal,
|
||||||
|
timeout: number,
|
||||||
|
): Promise<string> {
|
||||||
|
const disk = drive.use('fs');
|
||||||
|
|
||||||
|
// Check if partial file exists for resume
|
||||||
|
let startByte = 0;
|
||||||
|
let appendMode = false;
|
||||||
|
|
||||||
|
if (await disk.exists(path)) {
|
||||||
|
const stats = await disk.getMetaData(path);
|
||||||
|
startByte = stats.contentLength;
|
||||||
|
appendMode = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get file info with HEAD request first
|
||||||
|
const headResponse = await axios.head(url, {
|
||||||
|
signal,
|
||||||
|
timeout
|
||||||
|
});
|
||||||
|
|
||||||
|
const totalBytes = parseInt(headResponse.headers['content-length'] || '0');
|
||||||
|
const supportsRangeRequests = headResponse.headers['accept-ranges'] === 'bytes';
|
||||||
|
|
||||||
|
// If file is already complete
|
||||||
|
if (startByte === totalBytes && totalBytes > 0) {
|
||||||
|
logger.info(`File ${path} is already complete`);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If server doesn't support range requests and we have a partial file, delete it
|
||||||
|
if (!supportsRangeRequests && startByte > 0) {
|
||||||
|
await disk.delete(path);
|
||||||
|
startByte = 0;
|
||||||
|
appendMode = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const headers: Record<string, string> = {};
|
||||||
|
if (supportsRangeRequests && startByte > 0) {
|
||||||
|
headers.Range = `bytes=${startByte}-`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await axios.get(url, {
|
||||||
|
responseType: 'stream',
|
||||||
|
headers,
|
||||||
|
signal,
|
||||||
|
timeout
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status !== 200 && response.status !== 206) {
|
||||||
|
throw new Error(`Failed to download: HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let downloadedBytes = startByte;
|
||||||
|
let lastProgressTime = Date.now();
|
||||||
|
let lastDownloadedBytes = startByte;
|
||||||
|
|
||||||
|
// Progress tracking stream to monitor data flow
|
||||||
|
const progressStream = new Transform({
|
||||||
|
transform(chunk: Buffer, _: any, callback: Function) {
|
||||||
|
downloadedBytes += chunk.length;
|
||||||
|
this.push(chunk);
|
||||||
|
callback();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Update progress every 500ms
|
||||||
|
const progressInterval = setInterval(() => {
|
||||||
|
this.updateProgress({
|
||||||
|
downloadedBytes,
|
||||||
|
totalBytes,
|
||||||
|
lastProgressTime,
|
||||||
|
lastDownloadedBytes,
|
||||||
|
url
|
||||||
|
});
|
||||||
|
}, 500);
|
||||||
|
|
||||||
|
// Handle errors and cleanup
|
||||||
|
const cleanup = (error?: Error) => {
|
||||||
|
clearInterval(progressInterval);
|
||||||
|
progressStream.destroy();
|
||||||
|
response.data.destroy();
|
||||||
|
if (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
response.data.on('error', cleanup);
|
||||||
|
progressStream.on('error', cleanup);
|
||||||
|
|
||||||
|
signal.addEventListener('abort', () => {
|
||||||
|
cleanup(new Error('Download aborted'));
|
||||||
|
});
|
||||||
|
|
||||||
|
// Pipe through progress stream and then to disk
|
||||||
|
const sourceStream = response.data.pipe(progressStream);
|
||||||
|
|
||||||
|
// Use disk.putStream with append mode for resumable downloads
|
||||||
|
disk.putStream(path, sourceStream, { append: appendMode })
|
||||||
|
.then(() => {
|
||||||
|
clearInterval(progressInterval);
|
||||||
|
resolve(path);
|
||||||
|
})
|
||||||
|
.catch(cleanup);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private updateProgress({
|
||||||
|
downloadedBytes,
|
||||||
|
totalBytes,
|
||||||
|
lastProgressTime,
|
||||||
|
lastDownloadedBytes,
|
||||||
|
url
|
||||||
|
}: {
|
||||||
|
downloadedBytes: number;
|
||||||
|
totalBytes: number;
|
||||||
|
lastProgressTime: number;
|
||||||
|
lastDownloadedBytes: number;
|
||||||
|
url: string;
|
||||||
|
}) {
|
||||||
|
const now = Date.now();
|
||||||
|
const timeDiff = (now - lastProgressTime) / 1000;
|
||||||
|
const bytesDiff = downloadedBytes - lastDownloadedBytes;
|
||||||
|
const rawSpeed = timeDiff > 0 ? bytesDiff / timeDiff : 0;
|
||||||
|
const timeRemaining = rawSpeed > 0 ? (totalBytes - downloadedBytes) / rawSpeed : 0;
|
||||||
|
const speed = this.formatSpeed(rawSpeed);
|
||||||
|
|
||||||
|
const progress: DownloadProgress = {
|
||||||
|
downloaded_bytes: downloadedBytes,
|
||||||
|
total_bytes: totalBytes,
|
||||||
|
percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
|
||||||
|
speed,
|
||||||
|
time_remaining: timeRemaining
|
||||||
|
};
|
||||||
|
|
||||||
|
transmit.broadcast('zim-downloads', { url, progress, status: "in_progress" });
|
||||||
|
|
||||||
|
lastProgressTime = now;
|
||||||
|
lastDownloadedBytes = downloadedBytes;
|
||||||
|
};
|
||||||
|
|
||||||
|
private delay(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
private formatSpeed(bytesPerSecond: number): string {
|
||||||
|
if (bytesPerSecond < 1024) return `${bytesPerSecond.toFixed(0)} B/s`;
|
||||||
|
if (bytesPerSecond < 1024 * 1024) return `${(bytesPerSecond / 1024).toFixed(1)} KB/s`;
|
||||||
|
return `${(bytesPerSecond / (1024 * 1024)).toFixed(1)} MB/s`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
59
admin/inertia/components/Alert.tsx
Normal file
59
admin/inertia/components/Alert.tsx
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
import { ExclamationTriangleIcon, XCircleIcon } from '@heroicons/react/24/solid'
|
||||||
|
import { IconCircleCheck } from '@tabler/icons-react'
|
||||||
|
import classNames from '~/lib/classNames'
|
||||||
|
|
||||||
|
interface AlertProps extends React.HTMLAttributes<HTMLDivElement> {
|
||||||
|
title: string
|
||||||
|
message?: string
|
||||||
|
type: 'warning' | 'error' | 'success'
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function Alert({ title, message, type, ...props }: AlertProps) {
|
||||||
|
const getIcon = () => {
|
||||||
|
const Icon =
|
||||||
|
type === 'warning'
|
||||||
|
? ExclamationTriangleIcon
|
||||||
|
: type === 'error'
|
||||||
|
? XCircleIcon
|
||||||
|
: IconCircleCheck
|
||||||
|
const color =
|
||||||
|
type === 'warning' ? 'text-yellow-400' : type === 'error' ? 'text-red-400' : 'text-green-400'
|
||||||
|
|
||||||
|
return <Icon aria-hidden="true" className={`size-5 ${color}`} />
|
||||||
|
}
|
||||||
|
|
||||||
|
const getBackground = () => {
|
||||||
|
return type === 'warning' ? 'bg-yellow-100' : type === 'error' ? 'bg-red-50' : 'bg-green-50'
|
||||||
|
}
|
||||||
|
|
||||||
|
const getTextColor = () => {
|
||||||
|
return type === 'warning'
|
||||||
|
? 'text-yellow-800'
|
||||||
|
: type === 'error'
|
||||||
|
? 'text-red-800'
|
||||||
|
: 'text-green-800'
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
{...props}
|
||||||
|
className={classNames(
|
||||||
|
getBackground(),
|
||||||
|
props.className,
|
||||||
|
'border border-gray-200 rounded-md p-3 shadow-xs'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<div className="flex">
|
||||||
|
<div className="shrink-0">{getIcon()}</div>
|
||||||
|
<div className="ml-3">
|
||||||
|
<h3 className={`text-sm font-medium ${getTextColor()}`}>{title}</h3>
|
||||||
|
{message && (
|
||||||
|
<div className={`mt-2 text-sm ${getTextColor()}`}>
|
||||||
|
<p>{message}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
27
admin/inertia/components/ProgressBar.tsx
Normal file
27
admin/inertia/components/ProgressBar.tsx
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
const ProgressBar = ({ progress, speed }: { progress: number; speed?: string }) => {
|
||||||
|
if (progress >= 100) {
|
||||||
|
return (
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<span className="text-sm text-desert-green">Download complete</span>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col">
|
||||||
|
<div className="relative w-full h-2 bg-gray-200 rounded">
|
||||||
|
<div
|
||||||
|
className="absolute top-0 left-0 h-full bg-desert-green rounded"
|
||||||
|
style={{ width: `${progress}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{speed && (
|
||||||
|
<div className="mt-1 text-sm text-gray-500">
|
||||||
|
Est. Speed: {speed}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default ProgressBar
|
||||||
|
|
@ -48,7 +48,7 @@ function StyledTable<T extends { [key: string]: any }>({
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
className={classNames(
|
className={classNames(
|
||||||
'w-full overflow-x-auto bg-white mt-10 ring-1 ring-gray-300 sm:mx-0 sm:rounded-lg p-3 shadow-md',
|
'w-full overflow-x-auto bg-white ring-1 ring-gray-300 sm:mx-0 sm:rounded-lg p-3 shadow-md',
|
||||||
className
|
className
|
||||||
)}
|
)}
|
||||||
ref={ref}
|
ref={ref}
|
||||||
|
|
|
||||||
18
admin/inertia/hooks/useServiceInstalledStatus.tsx
Normal file
18
admin/inertia/hooks/useServiceInstalledStatus.tsx
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
import { useQuery } from '@tanstack/react-query'
|
||||||
|
import { ServiceSlim } from '../../types/services'
|
||||||
|
import api from '~/lib/api'
|
||||||
|
|
||||||
|
const useServiceInstalledStatus = (serviceName: string) => {
|
||||||
|
const { data, isFetching } = useQuery<ServiceSlim[]>({
|
||||||
|
queryKey: ['installed-services'],
|
||||||
|
queryFn: () => api.listServices(),
|
||||||
|
})
|
||||||
|
|
||||||
|
const isInstalled = data?.some(
|
||||||
|
(service) => service.service_name === serviceName && service.installed
|
||||||
|
)
|
||||||
|
|
||||||
|
return { isInstalled, loading: isFetching }
|
||||||
|
}
|
||||||
|
|
||||||
|
export default useServiceInstalledStatus
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
import { ListRemoteZimFilesResponse, ListZimFilesResponse, RemoteZimFileEntry } from "../../types/zim";
|
import { ListRemoteZimFilesResponse, ListZimFilesResponse } from "../../types/zim";
|
||||||
|
import { ServiceSlim } from "../../types/services";
|
||||||
|
|
||||||
class API {
|
class API {
|
||||||
private client;
|
private client;
|
||||||
|
|
@ -23,6 +24,16 @@ class API {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async listServices() {
|
||||||
|
try {
|
||||||
|
const response = await this.client.get<Array<ServiceSlim>>("/system/services");
|
||||||
|
return response.data;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error listing services:", error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
async installService(service_name: string) {
|
async installService(service_name: string) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -57,7 +68,11 @@ class API {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async downloadRemoteZimFile(url: string) {
|
async downloadRemoteZimFile(url: string): Promise<{
|
||||||
|
message: string;
|
||||||
|
filename: string;
|
||||||
|
url: string;
|
||||||
|
}> {
|
||||||
try {
|
try {
|
||||||
const response = await this.client.post("/zim/download-remote", { url });
|
const response = await this.client.post("/zim/download-remote", { url });
|
||||||
return response.data;
|
return response.data;
|
||||||
|
|
|
||||||
|
|
@ -7,10 +7,13 @@ import api from '~/lib/api'
|
||||||
import StyledButton from '~/components/StyledButton'
|
import StyledButton from '~/components/StyledButton'
|
||||||
import { useModals } from '~/context/ModalContext'
|
import { useModals } from '~/context/ModalContext'
|
||||||
import StyledModal from '~/components/StyledModal'
|
import StyledModal from '~/components/StyledModal'
|
||||||
|
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
||||||
|
import Alert from '~/components/Alert'
|
||||||
|
|
||||||
export default function ZimPage() {
|
export default function ZimPage() {
|
||||||
const queryClient = useQueryClient()
|
const queryClient = useQueryClient()
|
||||||
const { openModal, closeAllModals } = useModals()
|
const { openModal, closeAllModals } = useModals()
|
||||||
|
const { isInstalled } = useServiceInstalledStatus('nomad_kiwix_serve')
|
||||||
const { data, isLoading } = useQuery<ZimFilesEntry[]>({
|
const { data, isLoading } = useQuery<ZimFilesEntry[]>({
|
||||||
queryKey: ['zim-files'],
|
queryKey: ['zim-files'],
|
||||||
queryFn: getFiles,
|
queryFn: getFiles,
|
||||||
|
|
@ -55,19 +58,23 @@ export default function ZimPage() {
|
||||||
<Head title="ZIM Manager | Project N.O.M.A.D." />
|
<Head title="ZIM Manager | Project N.O.M.A.D." />
|
||||||
<div className="xl:pl-72 w-full">
|
<div className="xl:pl-72 w-full">
|
||||||
<main className="px-12 py-6">
|
<main className="px-12 py-6">
|
||||||
<div className="flex items-center justify-between mb-6">
|
<div className="flex items-center justify-between">
|
||||||
<div className="flex flex-col">
|
<div className="flex flex-col">
|
||||||
<h1 className="text-4xl font-semibold mb-4">ZIM Manager</h1>
|
<h1 className="text-4xl font-semibold mb-2">ZIM Manager</h1>
|
||||||
<p className="text-gray-500 mb-4">
|
<p className="text-gray-500">
|
||||||
Manage your stored ZIM files and download new ones!
|
Manage your stored ZIM files.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<Link href="/settings/zim/remote-explorer">
|
|
||||||
<StyledButton icon={'MagnifyingGlassIcon'}>Remote Explorer</StyledButton>
|
|
||||||
</Link>
|
|
||||||
</div>
|
</div>
|
||||||
|
{!isInstalled && (
|
||||||
|
<Alert
|
||||||
|
title="The Kiwix application is not installed. Please install it to view downloaded ZIM files"
|
||||||
|
type="warning"
|
||||||
|
className="!mt-6"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
<StyledTable<ZimFilesEntry & { actions?: any }>
|
<StyledTable<ZimFilesEntry & { actions?: any }>
|
||||||
className="font-semibold"
|
className="font-semibold mt-4"
|
||||||
rowLines={true}
|
rowLines={true}
|
||||||
loading={isLoading}
|
loading={isLoading}
|
||||||
compact
|
compact
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { keepPreviousData, useInfiniteQuery } from '@tanstack/react-query'
|
import { keepPreviousData, useInfiniteQuery } from '@tanstack/react-query'
|
||||||
import api from '~/lib/api'
|
import api from '~/lib/api'
|
||||||
import { useCallback, useEffect, useMemo, useRef } from 'react'
|
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||||
import { useVirtualizer } from '@tanstack/react-virtual'
|
import { useVirtualizer } from '@tanstack/react-virtual'
|
||||||
import StyledTable from '~/components/StyledTable'
|
import StyledTable from '~/components/StyledTable'
|
||||||
import SettingsLayout from '~/layouts/SettingsLayout'
|
import SettingsLayout from '~/layouts/SettingsLayout'
|
||||||
|
|
@ -10,10 +10,24 @@ import { formatBytes } from '~/lib/util'
|
||||||
import StyledButton from '~/components/StyledButton'
|
import StyledButton from '~/components/StyledButton'
|
||||||
import { useModals } from '~/context/ModalContext'
|
import { useModals } from '~/context/ModalContext'
|
||||||
import StyledModal from '~/components/StyledModal'
|
import StyledModal from '~/components/StyledModal'
|
||||||
|
import { useTransmit } from 'react-adonis-transmit'
|
||||||
|
import ProgressBar from '~/components/ProgressBar'
|
||||||
|
import { useNotifications } from '~/context/NotificationContext'
|
||||||
|
import useInternetStatus from '~/hooks/useInternetStatus'
|
||||||
|
import Alert from '~/components/Alert'
|
||||||
|
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
||||||
|
|
||||||
export default function ZimRemoteExplorer() {
|
export default function ZimRemoteExplorer() {
|
||||||
const tableParentRef = useRef<HTMLDivElement>(null)
|
const tableParentRef = useRef<HTMLDivElement>(null)
|
||||||
|
const { subscribe } = useTransmit()
|
||||||
const { openModal, closeAllModals } = useModals()
|
const { openModal, closeAllModals } = useModals()
|
||||||
|
const { addNotification } = useNotifications()
|
||||||
|
const { isOnline } = useInternetStatus()
|
||||||
|
const { isInstalled } = useServiceInstalledStatus('nomad_kiwix_serve')
|
||||||
|
const [activeDownloads, setActiveDownloads] = useState<
|
||||||
|
Map<string, { status: string; progress: number; speed: string }>
|
||||||
|
>(new Map<string, { status: string; progress: number; speed: string }>())
|
||||||
|
|
||||||
const { data, fetchNextPage, isFetching, isLoading } =
|
const { data, fetchNextPage, isFetching, isLoading } =
|
||||||
useInfiniteQuery<ListRemoteZimFilesResponse>({
|
useInfiniteQuery<ListRemoteZimFilesResponse>({
|
||||||
queryKey: ['remote-zim-files'],
|
queryKey: ['remote-zim-files'],
|
||||||
|
|
@ -78,13 +92,38 @@ export default function ZimRemoteExplorer() {
|
||||||
>
|
>
|
||||||
<p className="text-gray-700">
|
<p className="text-gray-700">
|
||||||
Are you sure you want to download <strong>{record.title}</strong>? It may take some time
|
Are you sure you want to download <strong>{record.title}</strong>? It may take some time
|
||||||
for it to be available depending on the file size and your internet connection.
|
for it to be available depending on the file size and your internet connection. The Kiwix
|
||||||
|
application will be restarted after the download is complete.
|
||||||
</p>
|
</p>
|
||||||
</StyledModal>,
|
</StyledModal>,
|
||||||
'confirm-download-file-modal'
|
'confirm-download-file-modal'
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const unsubscribe = subscribe('zim-downloads', (data: any) => {
|
||||||
|
if (data.url && data.progress?.percentage) {
|
||||||
|
setActiveDownloads((prev) =>
|
||||||
|
new Map(prev).set(data.url, {
|
||||||
|
status: data.status,
|
||||||
|
progress: data.progress.percentage || 0,
|
||||||
|
speed: data.progress.speed || '0 KB/s',
|
||||||
|
})
|
||||||
|
)
|
||||||
|
if (data.status === 'completed') {
|
||||||
|
addNotification({
|
||||||
|
message: `The download for ${data.url} has completed successfully.`,
|
||||||
|
type: 'success',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
unsubscribe()
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
async function downloadFile(record: RemoteZimFileEntry) {
|
async function downloadFile(record: RemoteZimFileEntry) {
|
||||||
try {
|
try {
|
||||||
await api.downloadRemoteZimFile(record.download_url)
|
await api.downloadRemoteZimFile(record.download_url)
|
||||||
|
|
@ -93,15 +132,38 @@ export default function ZimRemoteExplorer() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const EntryProgressBar = useCallback(
|
||||||
|
({ url }: { url: string }) => {
|
||||||
|
const entry = activeDownloads.get(url)
|
||||||
|
return <ProgressBar progress={entry?.progress || 0} speed={entry?.speed} />
|
||||||
|
},
|
||||||
|
[activeDownloads]
|
||||||
|
)
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<SettingsLayout>
|
<SettingsLayout>
|
||||||
<Head title="ZIM Remote Explorer | Project N.O.M.A.D." />
|
<Head title="ZIM Remote Explorer | Project N.O.M.A.D." />
|
||||||
<div className="xl:pl-72 w-full">
|
<div className="xl:pl-72 w-full">
|
||||||
<main className="px-12 py-6">
|
<main className="px-12 py-6">
|
||||||
<h1 className="text-4xl font-semibold mb-4">ZIM Remote Explorer</h1>
|
<h1 className="text-4xl font-semibold mb-2">ZIM Remote Explorer</h1>
|
||||||
<p className="text-gray-500 mb-2">
|
<p className="text-gray-500">
|
||||||
Browse and download remote ZIM files from the Kiwix repository!
|
Browse and download remote ZIM files from the Kiwix repository!
|
||||||
</p>
|
</p>
|
||||||
|
{!isOnline && (
|
||||||
|
<Alert
|
||||||
|
title="No internet connection. You may not be able to download files."
|
||||||
|
message=""
|
||||||
|
type="warning"
|
||||||
|
className="!mt-6"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
{!isInstalled && (
|
||||||
|
<Alert
|
||||||
|
title="The Kiwix application is not installed. Please install it to view downloaded ZIM files"
|
||||||
|
type="warning"
|
||||||
|
className="!mt-6"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
||||||
data={flatData.map((i, idx) => {
|
data={flatData.map((i, idx) => {
|
||||||
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
||||||
|
|
@ -133,29 +195,34 @@ export default function ZimRemoteExplorer() {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
accessor: 'size_bytes',
|
accessor: 'size_bytes',
|
||||||
|
title: 'Size',
|
||||||
render(record) {
|
render(record) {
|
||||||
return formatBytes(record.size_bytes)
|
return formatBytes(record.size_bytes)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
accessor: 'actions',
|
accessor: 'actions',
|
||||||
render(record, index) {
|
render(record) {
|
||||||
|
const isDownloading = activeDownloads.has(record.download_url)
|
||||||
return (
|
return (
|
||||||
<div className="flex space-x-2">
|
<div className="flex space-x-2">
|
||||||
<StyledButton
|
{!isDownloading && (
|
||||||
icon={'ArrowDownTrayIcon'}
|
<StyledButton
|
||||||
onClick={() => {
|
icon={'ArrowDownTrayIcon'}
|
||||||
confirmDownload(record)
|
onClick={() => {
|
||||||
}}
|
confirmDownload(record)
|
||||||
>
|
}}
|
||||||
Download
|
>
|
||||||
</StyledButton>
|
Download
|
||||||
|
</StyledButton>
|
||||||
|
)}
|
||||||
|
{isDownloading && <EntryProgressBar url={record.download_url} />}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
]}
|
]}
|
||||||
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full "
|
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
|
||||||
tableBodyStyle={{
|
tableBodyStyle={{
|
||||||
position: 'relative',
|
position: 'relative',
|
||||||
height: `${virtualizer.getTotalSize()}px`,
|
height: `${virtualizer.getTotalSize()}px`,
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,6 @@ router.group(() => {
|
||||||
router.get('/services', [SystemController, 'getServices'])
|
router.get('/services', [SystemController, 'getServices'])
|
||||||
router.post('/services/affect', [SystemController, 'affectService'])
|
router.post('/services/affect', [SystemController, 'affectService'])
|
||||||
router.post('/services/install', [SystemController, 'installService'])
|
router.post('/services/install', [SystemController, 'installService'])
|
||||||
router.post('/simulate-sse', [SystemController, 'simulateSSE'])
|
|
||||||
}).prefix('/api/system')
|
}).prefix('/api/system')
|
||||||
|
|
||||||
router.group(() => {
|
router.group(() => {
|
||||||
|
|
|
||||||
|
|
@ -65,4 +65,21 @@ export type RemoteZimFileEntry = {
|
||||||
download_url: string;
|
download_url: string;
|
||||||
author: string;
|
author: string;
|
||||||
file_name: string;
|
file_name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DownloadProgress = {
|
||||||
|
downloaded_bytes: number;
|
||||||
|
total_bytes: number;
|
||||||
|
percentage: number;
|
||||||
|
speed: string;
|
||||||
|
time_remaining: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DownloadOptions = {
|
||||||
|
max_retries?: number;
|
||||||
|
retry_delay?: number;
|
||||||
|
chunk_size?: number;
|
||||||
|
timeout?: number;
|
||||||
|
onError?: (error: Error) => void;
|
||||||
|
onComplete?: (filepath: string) => void;
|
||||||
}
|
}
|
||||||
Loading…
Reference in New Issue
Block a user