diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c index 81df4cb5606b..3c067aadc52d 100644 --- a/drivers/gpu/drm/ttm/ttm_backup.c +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -6,9 +6,10 @@ #include #include -#include #include +#include "ttm_pool_internal.h" + /* * Need to map shmem indices to handle since a handle value * of 0 means error, following the swp_entry_t convention. @@ -68,17 +69,23 @@ int ttm_backup_copy_page(struct file *backup, struct page *dst, } /** - * ttm_backup_backup_page() - Backup a page + * ttm_backup_backup_folio() - Backup a folio * @backup: The struct backup pointer to use. - * @page: The page to back up. - * @writeback: Whether to perform immediate writeback of the page. + * @folio: The folio to back up. + * @order: The allocation order of @folio. Since TTM allocates higher-order + * pages without __GFP_COMP, folio_nr_pages(@folio) would always + * return 1; the caller must pass the true order explicitly. + * @writeback: Whether to perform immediate writeback of the folio's pages. * This may have performance implications. - * @idx: A unique integer for each page and each struct backup. + * @idx: A unique integer for the first page of the folio and each struct backup. * This allows the backup implementation to avoid managing * its address space separately. - * @page_gfp: The gfp value used when the page was allocated. - * This is used for accounting purposes. + * @folio_gfp: The gfp value used when the folio was allocated. + * Currently unused. * @alloc_gfp: The gfp to be used when allocating memory. + * @nr_pages_backed: Output. On a successful return, set to the number of + * pages actually backed up, which may be less than (1 << @order) + * if an -ENOMEM was encountered mid-folio. * * Context: If called from reclaim context, the caller needs to * assert that the shrinker gfp has __GFP_FS set, to avoid @@ -87,53 +94,87 @@ int ttm_backup_copy_page(struct file *backup, struct page *dst, * that the shrinker gfp has __GFP_IO set, since without it, * we're not allowed to start backup IO. * - * Return: A handle on success. Negative error code on failure. - * - * Note: This function could be extended to back up a folio and - * implementations would then split the folio internally if needed. - * Drawback is that the caller would then have to keep track of - * the folio size- and usage. + * Return: A handle for the first backed-up page on success (handles for + * subsequent pages follow sequentially). -ENOMEM if no pages could be backed + * up. Any other negative error code if a non-ENOMEM failure occurred; in that + * case any pages backed up so far are truncated before returning. */ s64 -ttm_backup_backup_page(struct file *backup, struct page *page, - bool writeback, pgoff_t idx, gfp_t page_gfp, - gfp_t alloc_gfp) +ttm_backup_backup_folio(struct file *backup, struct folio *folio, + unsigned int order, bool writeback, pgoff_t idx, + gfp_t folio_gfp, gfp_t alloc_gfp, + pgoff_t *nr_pages_backed) { struct address_space *mapping = backup->f_mapping; - unsigned long handle = 0; + int nr_pages = 1 << order; struct folio *to_folio; - int ret; + int ret, i; - to_folio = shmem_read_folio_gfp(mapping, idx, alloc_gfp); - if (IS_ERR(to_folio)) - return PTR_ERR(to_folio); + *nr_pages_backed = 0; - folio_mark_accessed(to_folio); - folio_lock(to_folio); - folio_mark_dirty(to_folio); - copy_highpage(folio_file_page(to_folio, idx), page); - handle = ttm_backup_shmem_idx_to_handle(idx); + for (i = 0; i < nr_pages; ) { + int to_nr, j; - if (writeback && !folio_mapped(to_folio) && - folio_clear_dirty_for_io(to_folio)) { - folio_set_reclaim(to_folio); - ret = shmem_writeout(to_folio, NULL, NULL); - if (!folio_test_writeback(to_folio)) - folio_clear_reclaim(to_folio); /* - * If writeout succeeds, it unlocks the folio. errors - * are otherwise dropped, since writeout is only best - * effort here. + * Only inject past the first subpage so *nr_pages_backed is + * always > 0 here, matching a genuine mid-compound -ENOMEM + * and driving the caller's reactive split fallback instead + * of an early, no-progress failure. */ - if (ret) + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && i && + ttm_backup_fault_inject_folio()) + to_folio = ERR_PTR(-ENOMEM); + else + to_folio = shmem_read_folio_gfp(mapping, idx + i, alloc_gfp); + if (IS_ERR(to_folio)) { + int err = PTR_ERR(to_folio); + + if (err == -ENOMEM && *nr_pages_backed) + return ttm_backup_shmem_idx_to_handle(idx); + + if (*nr_pages_backed) { + shmem_truncate_range(file_inode(backup), + (loff_t)idx << PAGE_SHIFT, + ((loff_t)(idx + i) << PAGE_SHIFT) - 1); + /* + * The pages just truncated are no longer + * backed up; don't let the caller mistake + * them for valid handles. + */ + *nr_pages_backed = 0; + } + return err; + } + + to_nr = min_t(int, nr_pages - i, + folio_next_index(to_folio) - (idx + i)); + + folio_mark_accessed(to_folio); + folio_lock(to_folio); + folio_mark_dirty(to_folio); + + for (j = 0; j < to_nr; j++) + copy_highpage(folio_file_page(to_folio, idx + i + j), + folio_page(folio, i + j)); + + if (writeback && !folio_mapped(to_folio) && + folio_clear_dirty_for_io(to_folio)) { + folio_set_reclaim(to_folio); + ret = shmem_writeout(to_folio, NULL, NULL); + if (!folio_test_writeback(to_folio)) + folio_clear_reclaim(to_folio); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(to_folio); + } else { folio_unlock(to_folio); - } else { - folio_unlock(to_folio); + } + + folio_put(to_folio); + i += to_nr; + *nr_pages_backed = i; } - folio_put(to_folio); - - return handle; + return ttm_backup_shmem_idx_to_handle(idx); } /** diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index 46983e7de7a3..e7648ad3e615 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -53,8 +53,23 @@ #ifdef CONFIG_FAULT_INJECTION #include static DECLARE_FAULT_ATTR(backup_fault_inject); + +/* + * Exposed to ttm_backup.c so a mid-compound subpage can be made to fail + * with -ENOMEM, exercising the reactive split-and-retry fallback in + * ttm_pool_backup() for high-order backups. + */ +bool ttm_backup_fault_inject_folio(void) +{ + return should_fail(&backup_fault_inject, 1); +} #else #define should_fail(...) false + +bool ttm_backup_fault_inject_folio(void) +{ + return false; +} #endif /** @@ -487,7 +502,7 @@ static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p) /** * DOC: Partial backup and restoration of a struct ttm_tt. * - * Swapout using ttm_backup_backup_page() and swapin using + * Swapout using ttm_backup_backup_folio() and swapin using * ttm_backup_copy_page() may fail. * The former most likely due to lack of swap-space or memory, the latter due * to lack of memory or because of signal interruption during waits. @@ -1036,12 +1051,12 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, { struct file *backup = tt->backup; struct page *page; - unsigned long handle; gfp_t alloc_gfp; gfp_t gfp; int ret = 0; pgoff_t shrunken = 0; - pgoff_t i, num_pages; + pgoff_t i, j, num_pages, npages; + pgoff_t nr_backed; if (WARN_ON(ttm_tt_is_backed_up(tt))) return -EINVAL; @@ -1119,9 +1134,11 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, if (IS_ENABLED(CONFIG_FAULT_INJECTION) && should_fail(&backup_fault_inject, 1)) num_pages = DIV_ROUND_UP(num_pages, 2); - for (i = 0; i < num_pages; ++i) { - s64 shandle; + for (i = 0; i < num_pages; i += npages) { + unsigned int order; + s64 handle; + npages = 1; page = tt->pages[i]; if (unlikely(!page)) continue; @@ -1130,19 +1147,61 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, if (unlikely(ttm_backup_page_ptr_is_handle(page))) continue; - ttm_pool_split_for_swap(pool, page); + order = ttm_pool_page_order(pool, page); + npages = 1UL << order; - shandle = ttm_backup_backup_page(backup, page, flags->writeback, i, - gfp, alloc_gfp); - if (shandle < 0) { - /* We allow partially shrunken tts */ - ret = shandle; + /* + * We don't allow dipping kernel reserves for high order backup + */ + if (order) + alloc_gfp |= __GFP_NOMEMALLOC; + else + alloc_gfp &= ~__GFP_NOMEMALLOC; + + /* + * Back up the compound atomically at its native order. If + * fault injection truncated num_pages mid-compound, skip + * the partial tail rather than splitting. + */ + if (unlikely(i + npages > num_pages)) + break; + + handle = ttm_backup_backup_folio(backup, page_folio(page), + order, flags->writeback, i, + gfp, alloc_gfp, + &nr_backed); + /* + * Zero progress on this compound (whether order 0 or a + * high-order compound that failed before backing up even + * its first subpage) is unrecoverable: bail out rather than + * looping forever with npages == nr_backed == 0 below. + */ + if (unlikely(handle < 0 && !nr_backed)) { + ret = handle; break; } - handle = shandle; - tt->pages[i] = ttm_backup_handle_to_page_ptr(handle); - __free_pages_gpu_account(page, 0, false); - shrunken++; + + for (j = 0; j < nr_backed; j++) + tt->pages[i + j] = ttm_backup_handle_to_page_ptr(handle + j); + + shrunken += nr_backed; + + if (unlikely(nr_backed < npages)) { + /* + * Partial OOM backup: split the compound and free the + * subpages whose content is now in shmem. Continue the + * loop from the first un-backed order-0 page. + */ + ttm_pool_split_for_swap(pool, page); + for (j = 0; j < nr_backed; j++) + __free_pages_gpu_account(page + j, 0, false); + npages = nr_backed; + continue; + } + + /* Fully backed up: free at native order. */ + page->private = 0; + __free_pages_gpu_account(page, order, false); } return shrunken ? shrunken : ret; diff --git a/drivers/gpu/drm/ttm/ttm_pool_internal.h b/drivers/gpu/drm/ttm/ttm_pool_internal.h index 24c179fd69d1..cbb17a2129fe 100644 --- a/drivers/gpu/drm/ttm/ttm_pool_internal.h +++ b/drivers/gpu/drm/ttm/ttm_pool_internal.h @@ -22,4 +22,12 @@ static inline unsigned int ttm_pool_beneficial_order(struct ttm_pool *pool) return pool->alloc_flags & 0xff; } +/* + * Implemented in ttm_pool.c, used by ttm_backup.c. Returns true if a fault + * should be injected mid-compound to test the reactive split-and-retry + * fallback in ttm_pool_backup(). Always returns false when + * CONFIG_FAULT_INJECTION is disabled. + */ +bool ttm_backup_fault_inject_folio(void); + #endif diff --git a/include/drm/ttm/ttm_backup.h b/include/drm/ttm/ttm_backup.h index 29b9c855af77..49efa713e87c 100644 --- a/include/drm/ttm/ttm_backup.h +++ b/include/drm/ttm/ttm_backup.h @@ -13,9 +13,8 @@ * ttm_backup_handle_to_page_ptr() - Convert handle to struct page pointer * @handle: The handle to convert. * - * Converts an opaque handle received from the - * ttm_backup_backup_page() function to an (invalid) - * struct page pointer suitable for a struct page array. + * Converts an opaque handle received from a ttm_backup_backup_*() + * function to an (invalid) struct page pointer suitable for a struct page array. * * Return: An (invalid) struct page pointer. */ @@ -59,9 +58,10 @@ int ttm_backup_copy_page(struct file *backup, struct page *dst, pgoff_t handle, bool intr, gfp_t additional_gfp); s64 -ttm_backup_backup_page(struct file *backup, struct page *page, - bool writeback, pgoff_t idx, gfp_t page_gfp, - gfp_t alloc_gfp); +ttm_backup_backup_folio(struct file *backup, struct folio *folio, + unsigned int order, bool writeback, pgoff_t idx, + gfp_t folio_gfp, gfp_t alloc_gfp, + pgoff_t *nr_pages_backed); void ttm_backup_fini(struct file *backup);