diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index c87210a06c31..f2c92902e4a3 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -12,9 +12,16 @@ #include +enum drm_buddy_free_tree { + DRM_BUDDY_CLEAR_TREE = 0, + DRM_BUDDY_DIRTY_TREE, + DRM_BUDDY_MAX_FREE_TREES, +}; + static struct kmem_cache *slab_blocks; -#define rbtree_get_free_block(node) rb_entry((node), struct drm_buddy_block, rb) +#define for_each_free_tree(tree) \ + for ((tree) = 0; (tree) < DRM_BUDDY_MAX_FREE_TREES; (tree)++) static struct drm_buddy_block *drm_block_alloc(struct drm_buddy *mm, struct drm_buddy_block *parent, @@ -45,6 +52,30 @@ static void drm_block_free(struct drm_buddy *mm, kmem_cache_free(slab_blocks, block); } +static enum drm_buddy_free_tree +get_block_tree(struct drm_buddy_block *block) +{ + return drm_buddy_block_is_clear(block) ? + DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; +} + +static struct drm_buddy_block * +rbtree_get_free_block(const struct rb_node *node) +{ + return node ? rb_entry(node, struct drm_buddy_block, rb) : NULL; +} + +static struct drm_buddy_block * +rbtree_last_free_block(struct rb_root *root) +{ + return rbtree_get_free_block(rb_last(root)); +} + +static bool rbtree_is_empty(struct rb_root *root) +{ + return RB_EMPTY_ROOT(root); +} + static bool drm_buddy_block_offset_less(const struct drm_buddy_block *block, const struct drm_buddy_block *node) { @@ -59,37 +90,28 @@ static bool rbtree_block_offset_less(struct rb_node *block, } static void rbtree_insert(struct drm_buddy *mm, - struct drm_buddy_block *block) + struct drm_buddy_block *block, + enum drm_buddy_free_tree tree) { rb_add(&block->rb, - &mm->free_tree[drm_buddy_block_order(block)], + &mm->free_trees[tree][drm_buddy_block_order(block)], rbtree_block_offset_less); } static void rbtree_remove(struct drm_buddy *mm, struct drm_buddy_block *block) { + unsigned int order = drm_buddy_block_order(block); + enum drm_buddy_free_tree tree; struct rb_root *root; - root = &mm->free_tree[drm_buddy_block_order(block)]; + tree = get_block_tree(block); + root = &mm->free_trees[tree][order]; + rb_erase(&block->rb, root); - RB_CLEAR_NODE(&block->rb); } -static struct drm_buddy_block * -rbtree_last_entry(struct drm_buddy *mm, unsigned int order) -{ - struct rb_node *node = rb_last(&mm->free_tree[order]); - - return node ? rb_entry(node, struct drm_buddy_block, rb) : NULL; -} - -static bool rbtree_is_empty(struct drm_buddy *mm, unsigned int order) -{ - return RB_EMPTY_ROOT(&mm->free_tree[order]); -} - static void clear_reset(struct drm_buddy_block *block) { block->header &= ~DRM_BUDDY_HEADER_CLEAR; @@ -112,10 +134,13 @@ static void mark_allocated(struct drm_buddy *mm, static void mark_free(struct drm_buddy *mm, struct drm_buddy_block *block) { + enum drm_buddy_free_tree tree; + block->header &= ~DRM_BUDDY_HEADER_STATE; block->header |= DRM_BUDDY_FREE; - rbtree_insert(mm, block); + tree = get_block_tree(block); + rbtree_insert(mm, block, tree); } static void mark_split(struct drm_buddy *mm, @@ -201,7 +226,7 @@ static int __force_merge(struct drm_buddy *mm, u64 end, unsigned int min_order) { - unsigned int order; + unsigned int tree, order; int i; if (!min_order) @@ -210,45 +235,48 @@ static int __force_merge(struct drm_buddy *mm, if (min_order > mm->max_order) return -EINVAL; - for (i = min_order - 1; i >= 0; i--) { - struct rb_root *root = &mm->free_tree[i]; - struct rb_node *iter; + for_each_free_tree(tree) { + for (i = min_order - 1; i >= 0; i--) { + struct rb_node *iter = rb_last(&mm->free_trees[tree][i]); - iter = rb_last(root); + while (iter) { + struct drm_buddy_block *block, *buddy; + u64 block_start, block_end; - while (iter) { - struct drm_buddy_block *block, *buddy; - u64 block_start, block_end; - - block = rbtree_get_free_block(iter); - iter = rb_prev(iter); - - if (!block || !block->parent) - continue; - - block_start = drm_buddy_block_offset(block); - block_end = block_start + drm_buddy_block_size(mm, block) - 1; - - if (!contains(start, end, block_start, block_end)) - continue; - - buddy = __get_buddy(block); - if (!drm_buddy_block_is_free(buddy)) - continue; - - WARN_ON(drm_buddy_block_is_clear(block) == - drm_buddy_block_is_clear(buddy)); - - if (iter == &buddy->rb) + block = rbtree_get_free_block(iter); iter = rb_prev(iter); - rbtree_remove(mm, block); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail -= drm_buddy_block_size(mm, block); + if (!block || !block->parent) + continue; - order = __drm_buddy_free(mm, block, true); - if (order >= min_order) - return 0; + block_start = drm_buddy_block_offset(block); + block_end = block_start + drm_buddy_block_size(mm, block) - 1; + + if (!contains(start, end, block_start, block_end)) + continue; + + buddy = __get_buddy(block); + if (!drm_buddy_block_is_free(buddy)) + continue; + + WARN_ON(drm_buddy_block_is_clear(block) == + drm_buddy_block_is_clear(buddy)); + + /* + * Advance to the next node when the current node is the buddy, + * as freeing the block will also remove its buddy from the tree. + */ + if (iter == &buddy->rb) + iter = rb_prev(iter); + + rbtree_remove(mm, block); + if (drm_buddy_block_is_clear(block)) + mm->clear_avail -= drm_buddy_block_size(mm, block); + + order = __drm_buddy_free(mm, block, true); + if (order >= min_order) + return 0; + } } } @@ -269,7 +297,7 @@ static int __force_merge(struct drm_buddy *mm, */ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) { - unsigned int i; + unsigned int i, j, root_count = 0; u64 offset = 0; if (size < chunk_size) @@ -291,14 +319,22 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) BUG_ON(mm->max_order > DRM_BUDDY_MAX_ORDER); - mm->free_tree = kmalloc_array(mm->max_order + 1, - sizeof(struct rb_root), - GFP_KERNEL); - if (!mm->free_tree) + mm->free_trees = kmalloc_array(DRM_BUDDY_MAX_FREE_TREES, + sizeof(*mm->free_trees), + GFP_KERNEL); + if (!mm->free_trees) return -ENOMEM; - for (i = 0; i <= mm->max_order; ++i) - mm->free_tree[i] = RB_ROOT; + for_each_free_tree(i) { + mm->free_trees[i] = kmalloc_array(mm->max_order + 1, + sizeof(struct rb_root), + GFP_KERNEL); + if (!mm->free_trees[i]) + goto out_free_tree; + + for (j = 0; j <= mm->max_order; ++j) + mm->free_trees[i][j] = RB_ROOT; + } mm->n_roots = hweight64(size); @@ -308,8 +344,6 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) if (!mm->roots) goto out_free_tree; - i = 0; - /* * Split into power-of-two blocks, in case we are given a size that is * not itself a power-of-two. @@ -328,24 +362,26 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) mark_free(mm, root); - BUG_ON(i > mm->max_order); + BUG_ON(root_count > mm->max_order); BUG_ON(drm_buddy_block_size(mm, root) < chunk_size); - mm->roots[i] = root; + mm->roots[root_count] = root; offset += root_size; size -= root_size; - i++; + root_count++; } while (size); return 0; out_free_roots: - while (i--) - drm_block_free(mm, mm->roots[i]); + while (root_count--) + drm_block_free(mm, mm->roots[root_count]); kfree(mm->roots); out_free_tree: - kfree(mm->free_tree); + while (i--) + kfree(mm->free_trees[i]); + kfree(mm->free_trees); return -ENOMEM; } EXPORT_SYMBOL(drm_buddy_init); @@ -381,8 +417,9 @@ void drm_buddy_fini(struct drm_buddy *mm) WARN_ON(mm->avail != mm->size); + for_each_free_tree(i) + kfree(mm->free_trees[i]); kfree(mm->roots); - kfree(mm->free_tree); } EXPORT_SYMBOL(drm_buddy_fini); @@ -406,8 +443,7 @@ static int split_block(struct drm_buddy *mm, return -ENOMEM; } - mark_free(mm, block->left); - mark_free(mm, block->right); + mark_split(mm, block); if (drm_buddy_block_is_clear(block)) { mark_cleared(block->left); @@ -415,7 +451,8 @@ static int split_block(struct drm_buddy *mm, clear_reset(block); } - mark_split(mm, block); + mark_free(mm, block->left); + mark_free(mm, block->right); return 0; } @@ -448,6 +485,7 @@ EXPORT_SYMBOL(drm_get_buddy); */ void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear) { + enum drm_buddy_free_tree src_tree, dst_tree; u64 root_size, size, start; unsigned int order; int i; @@ -462,19 +500,24 @@ void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear) size -= root_size; } + src_tree = is_clear ? DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE; + dst_tree = is_clear ? DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; + for (i = 0; i <= mm->max_order; ++i) { + struct rb_root *root = &mm->free_trees[src_tree][i]; struct drm_buddy_block *block, *tmp; - rbtree_postorder_for_each_entry_safe(block, tmp, &mm->free_tree[i], rb) { - if (is_clear != drm_buddy_block_is_clear(block)) { - if (is_clear) { - mark_cleared(block); - mm->clear_avail += drm_buddy_block_size(mm, block); - } else { - clear_reset(block); - mm->clear_avail -= drm_buddy_block_size(mm, block); - } + rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { + rbtree_remove(mm, block); + if (is_clear) { + mark_cleared(block); + mm->clear_avail += drm_buddy_block_size(mm, block); + } else { + clear_reset(block); + mm->clear_avail -= drm_buddy_block_size(mm, block); } + + rbtree_insert(mm, block, dst_tree); } } } @@ -664,27 +707,17 @@ __drm_buddy_alloc_range_bias(struct drm_buddy *mm, } static struct drm_buddy_block * -get_maxblock(struct drm_buddy *mm, unsigned int order, - unsigned long flags) +get_maxblock(struct drm_buddy *mm, + unsigned int order, + enum drm_buddy_free_tree tree) { struct drm_buddy_block *max_block = NULL, *block = NULL; + struct rb_root *root; unsigned int i; for (i = order; i <= mm->max_order; ++i) { - struct rb_node *iter = rb_last(&mm->free_tree[i]); - struct drm_buddy_block *tmp_block; - - while (iter) { - tmp_block = rbtree_get_free_block(iter); - - if (!block_incompatible(tmp_block, flags)) { - block = tmp_block; - break; - } - - iter = rb_prev(iter); - } - + root = &mm->free_trees[tree][i]; + block = rbtree_last_free_block(root); if (!block) continue; @@ -708,39 +741,37 @@ alloc_from_freetree(struct drm_buddy *mm, unsigned long flags) { struct drm_buddy_block *block = NULL; + struct rb_root *root; + enum drm_buddy_free_tree tree; unsigned int tmp; int err; + tree = (flags & DRM_BUDDY_CLEAR_ALLOCATION) ? + DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; + if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) { - block = get_maxblock(mm, order, flags); + block = get_maxblock(mm, order, tree); if (block) /* Store the obtained block order */ tmp = drm_buddy_block_order(block); } else { for (tmp = order; tmp <= mm->max_order; ++tmp) { - struct rb_node *iter = rb_last(&mm->free_tree[tmp]); - struct drm_buddy_block *tmp_block; - - while (iter) { - tmp_block = rbtree_get_free_block(iter); - - if (!block_incompatible(tmp_block, flags)) { - block = tmp_block; - break; - } - - iter = rb_prev(iter); - } - + /* Get RB tree root for this order and tree */ + root = &mm->free_trees[tree][tmp]; + block = rbtree_last_free_block(root); if (block) break; } } if (!block) { - /* Fallback method */ + /* Try allocating from the other tree */ + tree = (tree == DRM_BUDDY_CLEAR_TREE) ? + DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE; + for (tmp = order; tmp <= mm->max_order; ++tmp) { - block = rbtree_last_entry(mm, tmp); + root = &mm->free_trees[tree][tmp]; + block = rbtree_last_free_block(root); if (block) break; } @@ -885,10 +916,9 @@ static int __alloc_contig_try_harder(struct drm_buddy *mm, { u64 rhs_offset, lhs_offset, lhs_size, filled; struct drm_buddy_block *block; + unsigned int tree, order; LIST_HEAD(blocks_lhs); - struct rb_node *iter; unsigned long pages; - unsigned int order; u64 modify_size; int err; @@ -898,40 +928,45 @@ static int __alloc_contig_try_harder(struct drm_buddy *mm, if (order == 0) return -ENOSPC; - if (rbtree_is_empty(mm, order)) - return -ENOSPC; + for_each_free_tree(tree) { + struct rb_root *root; + struct rb_node *iter; - iter = rb_last(&mm->free_tree[order]); + root = &mm->free_trees[tree][order]; + if (rbtree_is_empty(root)) + continue; - while (iter) { - block = rbtree_get_free_block(iter); + iter = rb_last(root); + while (iter) { + block = rbtree_get_free_block(iter); - /* Allocate blocks traversing RHS */ - rhs_offset = drm_buddy_block_offset(block); - err = __drm_buddy_alloc_range(mm, rhs_offset, size, - &filled, blocks); - if (!err || err != -ENOSPC) - return err; + /* Allocate blocks traversing RHS */ + rhs_offset = drm_buddy_block_offset(block); + err = __drm_buddy_alloc_range(mm, rhs_offset, size, + &filled, blocks); + if (!err || err != -ENOSPC) + return err; - lhs_size = max((size - filled), min_block_size); - if (!IS_ALIGNED(lhs_size, min_block_size)) - lhs_size = round_up(lhs_size, min_block_size); + lhs_size = max((size - filled), min_block_size); + if (!IS_ALIGNED(lhs_size, min_block_size)) + lhs_size = round_up(lhs_size, min_block_size); - /* Allocate blocks traversing LHS */ - lhs_offset = drm_buddy_block_offset(block) - lhs_size; - err = __drm_buddy_alloc_range(mm, lhs_offset, lhs_size, - NULL, &blocks_lhs); - if (!err) { - list_splice(&blocks_lhs, blocks); - return 0; - } else if (err != -ENOSPC) { + /* Allocate blocks traversing LHS */ + lhs_offset = drm_buddy_block_offset(block) - lhs_size; + err = __drm_buddy_alloc_range(mm, lhs_offset, lhs_size, + NULL, &blocks_lhs); + if (!err) { + list_splice(&blocks_lhs, blocks); + return 0; + } else if (err != -ENOSPC) { + drm_buddy_free_list_internal(mm, blocks); + return err; + } + /* Free blocks for the next iteration */ drm_buddy_free_list_internal(mm, blocks); - return err; - } - /* Free blocks for the next iteration */ - drm_buddy_free_list_internal(mm, blocks); - iter = rb_prev(iter); + iter = rb_prev(iter); + } } return -ENOSPC; @@ -1243,11 +1278,17 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p) for (order = mm->max_order; order >= 0; order--) { struct drm_buddy_block *block, *tmp; + struct rb_root *root; u64 count = 0, free; + unsigned int tree; - rbtree_postorder_for_each_entry_safe(block, tmp, &mm->free_tree[order], rb) { - BUG_ON(!drm_buddy_block_is_free(block)); - count++; + for_each_free_tree(tree) { + root = &mm->free_trees[tree][order]; + + rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { + BUG_ON(!drm_buddy_block_is_free(block)); + count++; + } } drm_printf(p, "order-%2d ", order); diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index 9ee105d4309f..d7891d08f67a 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -73,7 +73,7 @@ struct drm_buddy_block { */ struct drm_buddy { /* Maintain a free list for each order. */ - struct rb_root *free_tree; + struct rb_root **free_trees; /* * Maintain explicit binary tree(s) to track the allocation of the