diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index d1b861b..a095592 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -3,7 +3,6 @@ import { downloadCollectionValidator, filenameParamValidator, remoteDownloadWithMetadataValidator, - saveInstalledTierValidator, selectWikipediaValidator, } from '#validators/common' import { listRemoteZimValidator } from '#validators/zim' @@ -56,12 +55,6 @@ export default class ZimController { return { success } } - async saveInstalledTier({ request }: HttpContext) { - const payload = await request.validateUsing(saveInstalledTierValidator) - await this.zimService.saveInstalledTier(payload.categorySlug, payload.tierSlug) - return { success: true } - } - async delete({ request, response }: HttpContext) { const payload = await request.validateUsing(filenameParamValidator) diff --git a/admin/app/models/installed_tier.ts b/admin/app/models/installed_tier.ts deleted file mode 100644 index b9c582e..0000000 --- a/admin/app/models/installed_tier.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { DateTime } from 'luxon' -import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' - -export default class InstalledTier extends BaseModel { - static namingStrategy = new SnakeCaseNamingStrategy() - - @column({ isPrimary: true }) - declare id: number - - @column() - declare category_slug: string - - @column() - declare tier_slug: string - - @column.dateTime({ autoCreate: true }) - declare created_at: DateTime - - @column.dateTime({ autoCreate: true, autoUpdate: true }) - declare updated_at: DateTime -} diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 2b1ac61..bb787ca 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -22,7 +22,6 @@ import vine from '@vinejs/vine' import { curatedCategoriesFileSchema, curatedCollectionsFileSchema, wikipediaOptionsFileSchema } from '#validators/curated_collections' import CuratedCollection from '#models/curated_collection' import CuratedCollectionResource from '#models/curated_collection_resource' -import InstalledTier from '#models/installed_tier' import WikipediaSelection from '#models/wikipedia_selection' import ZimFileMetadata from '#models/zim_file_metadata' import { RunDownloadJob } from '#jobs/run_download_job' @@ -329,29 +328,65 @@ export class ZimService implements IZimService { data, }); - // Look up installed tiers for all categories - const installedTiers = await InstalledTier.all() - const installedTierMap = new Map( - installedTiers.map((t) => [t.category_slug, t.tier_slug]) + // Dynamically determine installed tier for each category + const categoriesWithStatus = await Promise.all( + validated.categories.map(async (category) => { + const installedTierSlug = await this.getInstalledTierForCategory(category) + return { + ...category, + installedTierSlug, + } + }) ) - // Add installedTierSlug to each category - return validated.categories.map((category) => ({ - ...category, - installedTierSlug: installedTierMap.get(category.slug), - })) + return categoriesWithStatus } catch (error) { logger.error(`[ZimService] Failed to fetch curated categories:`, error) throw new Error('Failed to fetch curated categories or invalid format was received') } } - async saveInstalledTier(categorySlug: string, tierSlug: string): Promise { - await InstalledTier.updateOrCreate( - { category_slug: categorySlug }, - { tier_slug: tierSlug } - ) - logger.info(`[ZimService] Saved installed tier: ${categorySlug} -> ${tierSlug}`) + /** + * Dynamically determines which tier is installed for a category by checking + * which tier's resources are all downloaded. Returns the highest tier that + * is fully installed (considering that higher tiers include lower tier resources) + */ + private async getInstalledTierForCategory(category: CuratedCategory): Promise { + const { files: diskFiles } = await this.list() + const diskFilenames = new Set(diskFiles.map((f) => f.name)) + + // Get all CuratedCollectionResources marked as downloaded + const downloadedResources = await CuratedCollectionResource.query() + .where('downloaded', true) + .select('url') + const downloadedUrls = new Set(downloadedResources.map((r) => r.url)) + + // Check each tier from highest to lowest (assuming tiers are ordered from low to high) + // We check in reverse to find the highest fully-installed tier + const reversedTiers = [...category.tiers].reverse() + + for (const tier of reversedTiers) { + const allResourcesInstalled = tier.resources.every((resource) => { + // Check if resource is marked as downloaded in database + if (downloadedUrls.has(resource.url)) { + return true + } + + // Fallback: check if file exists on disk (for resources not tracked in CuratedCollectionResource) + const filename = resource.url.split('/').pop() + if (filename && diskFilenames.has(filename)) { + return true + } + + return false + }) + + if (allResourcesInstalled && tier.resources.length > 0) { + return tier.slug + } + } + + return undefined } async listCuratedCollections(): Promise { @@ -372,18 +407,26 @@ export class ZimService implements IZimService { }) for (const collection of validated.collections) { - const collectionResult = await CuratedCollection.updateOrCreate( - { slug: collection.slug }, + const { resources, ...restCollection } = collection; // we'll handle resources separately + + // Upsert the collection itself + await CuratedCollection.updateOrCreate( + { slug: restCollection.slug }, { - ...collection, + ...restCollection, type: 'zim', } ) - logger.info(`[ZimService] Upserted curated collection: ${collection.slug}`) + logger.info(`[ZimService] Upserted curated collection: ${restCollection.slug}`) + + // Upsert collection's resources + const resourcesResult = await CuratedCollectionResource.updateOrCreateMany('url', resources.map((res) => ({ + ...res, + curated_collection_slug: restCollection.slug, // add the foreign key + }))) - await collectionResult.related('resources').createMany(collection.resources) logger.info( - `[ZimService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}` + `[ZimService] Upserted ${resourcesResult.length} resources for collection: ${restCollection.slug}` ) } diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index dc5dbc1..564a090 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -56,13 +56,6 @@ export const downloadCollectionValidator = vine.compile( }) ) -export const saveInstalledTierValidator = vine.compile( - vine.object({ - categorySlug: vine.string().trim().minLength(1), - tierSlug: vine.string().trim().minLength(1), - }) -) - export const selectWikipediaValidator = vine.compile( vine.object({ optionId: vine.string().trim().minLength(1), diff --git a/admin/database/migrations/1770273423670_drop_installed_tiers_table.ts b/admin/database/migrations/1770273423670_drop_installed_tiers_table.ts new file mode 100644 index 0000000..76ecf39 --- /dev/null +++ b/admin/database/migrations/1770273423670_drop_installed_tiers_table.ts @@ -0,0 +1,20 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'installed_tiers' + + async up() { + this.schema.dropTableIfExists(this.tableName) + } + + async down() { + // Recreate the table if we need to rollback + this.schema.createTable(this.tableName, (table) => { + table.increments('id') + table.string('category_slug').notNullable().unique() + table.string('tier_slug').notNullable() + table.timestamp('created_at', { useTz: true }) + table.timestamp('updated_at', { useTz: true }) + }) + } +} \ No newline at end of file diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index b4d9a37..277261e 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -354,16 +354,6 @@ class API { })() } - async saveInstalledTier(categorySlug: string, tierSlug: string) { - return catchInternal(async () => { - const response = await this.client.post<{ success: boolean }>('/zim/save-installed-tier', { - categorySlug, - tierSlug, - }) - return response.data - })() - } - async listDocs() { return catchInternal(async () => { const response = await this.client.get>('/docs/list') diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index b5bef00..59f00f4 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -403,12 +403,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim await Promise.all(downloadPromises) - // Save installed tiers for each selected category - const tierSavePromises = Array.from(selectedTiers.entries()).map( - ([categorySlug, tier]) => api.saveInstalledTier(categorySlug, tier.slug) - ) - await Promise.all(tierSavePromises) - // Select Wikipedia option if one was chosen if (selectedWikipedia && selectedWikipedia !== wikipediaState?.currentSelection?.optionId) { await api.selectWikipedia(selectedWikipedia) diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index 7ff4230..18cb38d 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -241,15 +241,12 @@ export default function ZimRemoteExplorer() { // Get all resources for this tier (including inherited ones) const resources = getAllResourcesForTier(tier, category.tiers) - // Download each resource and save the installed tier + // Download each resource try { for (const resource of resources) { await api.downloadRemoteZimFile(resource.url) } - // Save the installed tier - await api.saveInstalledTier(category.slug, tier.slug) - addNotification({ message: `Started downloading ${resources.length} files from "${category.name} - ${tier.name}"`, type: 'success', diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 0a552d2..c104b44 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -149,7 +149,7 @@ router router.post('/fetch-latest-collections', [ZimController, 'fetchLatestCollections']) router.post('/download-remote', [ZimController, 'downloadRemote']) router.post('/download-collection', [ZimController, 'downloadCollection']) - router.post('/save-installed-tier', [ZimController, 'saveInstalledTier']) + router.get('/wikipedia', [ZimController, 'getWikipediaState']) router.post('/wikipedia/select', [ZimController, 'selectWikipedia']) router.delete('/:filename', [ZimController, 'delete'])