diff --git a/mm/vmalloc.c b/mm/vmalloc.c index e3c412430a6f..81cad5095889 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2503,67 +2503,94 @@ static void free_unmap_vmap_area(struct vmap_area *va) free_vmap_area_noflush(va); } -struct vmap_area *find_vmap_area(unsigned long addr) +static inline int next_vmap_node_id(int i) +{ + return (i + nr_vmap_nodes - 1) % nr_vmap_nodes; +} + +enum vmap_lock_mode { + VMAP_LOCK, + VMAP_TRYLOCK, +}; + +/* + * Search for a vmap_area at @addr across all vmap nodes. An + * addr_to_node_id(addr) converts an address to a node index where + * a VA is located. If VA spans several zones and passed addr is not + * the same as va->va_start, what is not common, we may need to scan + * extra nodes. See an example: + * + * <----va----> + * -|-----|-----|-----|-----|- + * 1 2 0 1 + * + * VA resides in node 1 whereas it spans 1, 2 an 0. If passed addr + * is within 2 or 0 nodes we should do extra work. + * + * Returns the VA with @locked_vn->busy.lock held; the caller must + * release it. If @mode is VMAP_TRYLOCK, nodes that cannot be locked + * are skipped. + */ +static struct vmap_area * +find_vmap_area_lock(unsigned long addr, struct vmap_node **locked_vn, + enum vmap_lock_mode mode) { struct vmap_node *vn; struct vmap_area *va; int i, j; + *locked_vn = NULL; + if (unlikely(!vmap_initialized)) return NULL; - /* - * An addr_to_node_id(addr) converts an address to a node index - * where a VA is located. If VA spans several zones and passed - * addr is not the same as va->va_start, what is not common, we - * may need to scan extra nodes. See an example: - * - * <----va----> - * -|-----|-----|-----|-----|- - * 1 2 0 1 - * - * VA resides in node 1 whereas it spans 1, 2 an 0. If passed - * addr is within 2 or 0 nodes we should do extra work. - */ i = j = addr_to_node_id(addr); do { vn = &vmap_nodes[i]; - spin_lock(&vn->busy.lock); - va = __find_vmap_area(addr, &vn->busy.root); - spin_unlock(&vn->busy.lock); + if (mode == VMAP_LOCK) { + spin_lock(&vn->busy.lock); + } else { + if (!spin_trylock(&vn->busy.lock)) + continue; + } - if (va) + va = __find_vmap_area(addr, &vn->busy.root); + if (va) { + *locked_vn = vn; return va; - } while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j); + } + + spin_unlock(&vn->busy.lock); + } while ((i = next_vmap_node_id(i)) != j); return NULL; } +struct vmap_area *find_vmap_area(unsigned long addr) +{ + struct vmap_node *vn; + struct vmap_area *va; + + va = find_vmap_area_lock(addr, &vn, VMAP_LOCK); + if (va) + spin_unlock(&vn->busy.lock); + + return va; +} + static struct vmap_area *find_unlink_vmap_area(unsigned long addr) { struct vmap_node *vn; struct vmap_area *va; - int i, j; - /* - * Check the comment in the find_vmap_area() about the loop. - */ - i = j = addr_to_node_id(addr); - do { - vn = &vmap_nodes[i]; - - spin_lock(&vn->busy.lock); - va = __find_vmap_area(addr, &vn->busy.root); - if (va) - unlink_va(va, &vn->busy.root); + va = find_vmap_area_lock(addr, &vn, VMAP_LOCK); + if (va) { + unlink_va(va, &vn->busy.root); spin_unlock(&vn->busy.lock); + } - if (va) - return va; - } while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j); - - return NULL; + return va; } /*** Per cpu kva allocator ***/ @@ -5269,14 +5296,11 @@ bool vmalloc_dump_obj(void *object) unsigned long nr_pages; addr = PAGE_ALIGN_DOWN((unsigned long) object); - vn = addr_to_node(addr); - if (!spin_trylock(&vn->busy.lock)) - return false; - - va = __find_vmap_area(addr, &vn->busy.root); + va = find_vmap_area_lock(addr, &vn, VMAP_TRYLOCK); if (!va || !va->vm) { - spin_unlock(&vn->busy.lock); + if (va) + spin_unlock(&vn->busy.lock); return false; }