diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 4bae3796c42f..eed7f8f1e417 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2902,8 +2902,10 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **page_array); +struct page *bpf_alloc_page(int nid, bool sleepable); +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable); +void bpf_free_pages(struct llist_head *pages); #ifdef CONFIG_MEMCG void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg, struct mem_cgroup **new_memcg); @@ -3299,7 +3301,7 @@ const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr); + u16 desc_idx, u8 **func_addr); struct bpf_core_ctx { struct bpf_verifier_log *log; @@ -3642,7 +3644,7 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, static inline int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { return -ENOTSUPP; } diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 811342e3c041..79c4be5c33f4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1788,24 +1788,29 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 +/* Each kfunc can have its canonical and one specialized call target. */ +#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2) +static_assert(MAX_KFUNC_CALL_DESCS <= S16_MAX + 1); struct bpf_kfunc_desc { struct btf_func_model func_model; struct bpf_func_proto proto; u32 func_id; - s32 imm; u16 offset; unsigned long addr; }; struct bpf_kfunc_desc_tab { u32 nr_descs; + u32 nr_base_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during - * verification. JITs do lookups by bpf_insn, where func_id may not be - * available, therefore at the end of verification do_misc_fixups() - * sorts this by imm and offset. + * verification. JITs use the descriptor index stored in the finalized + * call's off field. The first nr_base_descs entries are the canonical + * descriptors used for verifier lookups. Call specialization may append + * immutable descriptors for additional targets. * - * Grown one entry at a time by bpf_add_kfunc_call(). + * Grown one entry at a time by bpf_add_kfunc_call() and during + * call specialization. */ struct bpf_kfunc_desc descs[]; }; diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index c6369ea5e208..0ff707707da0 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -146,7 +146,7 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) struct apply_range_data { struct bpf_arena *arena; - struct page **pages; + struct llist_head *pages; int i; }; @@ -158,13 +158,17 @@ struct clear_range_data { static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; + struct llist_node *node; struct page *page; pte_t pteval; if (!data) return 0; - page = d->pages[d->i]; + node = READ_ONCE(d->pages->first); + if (WARN_ON_ONCE(!node)) + return -EINVAL; + page = llist_entry(node, struct page, pcp_llist); /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; @@ -197,6 +201,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) return -EBUSY; set_pte_at(&init_mm, addr, pte, pteval); #endif + WARN_ON_ONCE(llist_del_first(d->pages) != node); d->i++; WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; @@ -312,8 +317,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); - err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); - if (err) + arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE, true); + if (!arena->scratch_page) goto err_free_arena; range_tree_init(&arena->rt); @@ -481,7 +486,9 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; + LLIST_HEAD(pages); struct page *page, *new_page = NULL; + struct apply_range_data data; vm_fault_t fault_ret; long kbase, kaddr; unsigned long flags; @@ -543,8 +550,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) * The probed page was freed meanwhile or preallocation failed; * try the non-blocking allocator, we cannot sleep here. */ - ret = bpf_map_alloc_pages(map, map->numa_node, 1, &new_page); - if (ret) { + new_page = bpf_alloc_page(map->numa_node, false); + if (!new_page) { fault_ret = VM_FAULT_SIGBUS; goto out_err_locked_memcg; } @@ -555,12 +562,14 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) fault_ret = VM_FAULT_SIGBUS; goto out_err_locked_memcg; } - struct apply_range_data data = { - .arena = arena, .pages = &new_page, .i = 0 - }; + llist_add(&new_page->pcp_llist, &pages); + data.arena = arena; + data.pages = &pages; + data.i = 0; ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data); if (ret) { + llist_del_first(&pages); range_tree_set(&arena->rt, vmf->pgoff, 1); fault_ret = VM_FAULT_SIGBUS; goto out_err_locked_memcg; @@ -704,6 +713,27 @@ static u64 clear_lo32(u64 val) return val & ~(u64)~0U; } +static int arena_adjust_tree(struct bpf_arena *arena, long uaddr, long page_cnt, long *pgoff) +{ + int ret; + + /* Special case where user is requesting specific range. */ + if (uaddr) { + ret = is_range_tree_set(&arena->rt, *pgoff, page_cnt); + if (ret) + return ret; + return range_tree_clear(&arena->rt, *pgoff, page_cnt); + } + + ret = range_tree_find(&arena->rt, page_cnt); + if (ret < 0) + return ret; + + *pgoff = ret; + + return range_tree_clear(&arena->rt, *pgoff, page_cnt); +} + /* * Allocate pages and vmap them into kernel vmalloc area. * Later the pages will be mmaped into user space vma. @@ -716,13 +746,13 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena); struct mem_cgroup *new_memcg, *old_memcg; struct apply_range_data data; - struct page **pages = NULL; - long remaining, mapped = 0; - long alloc_pages; + LLIST_HEAD(pages); + long mapped = 0; unsigned long flags; long pgoff = 0; u32 uaddr32; - int ret, i; + long addr = 0; + int ret; if (node_id != NUMA_NO_NODE && ((unsigned int)node_id >= nr_node_ids || !node_online(node_id))) @@ -739,89 +769,63 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt /* requested address will be outside of user VMA */ return 0; } - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - /* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */ - alloc_pages = min(page_cnt, KMALLOC_MAX_CACHE_SIZE / sizeof(struct page *)); - pages = kmalloc_nolock(alloc_pages * sizeof(struct page *), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!pages) { - bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; - } + + ret = bpf_alloc_pages(node_id, page_cnt, &pages, sleepable); + if (ret) + goto out_memcg; + data.arena = arena; - data.pages = pages; + data.pages = &pages; + data.i = 0; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; - if (uaddr) { - ret = is_range_tree_set(&arena->rt, pgoff, page_cnt); - if (ret) - goto out_unlock_free_pages; - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); - } else { - ret = pgoff = range_tree_find(&arena->rt, page_cnt); - if (pgoff >= 0) - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); + ret = arena_adjust_tree(arena, uaddr, page_cnt, &pgoff); + if (ret) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + goto out_free_pages; } - if (ret) - goto out_unlock_free_pages; - remaining = page_cnt; uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); - while (remaining) { - long this_batch = min(remaining, alloc_pages); + /* + * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 + * will not overflow 32-bit. Lower 32-bit need to represent + * contiguous user address range. + * Map these pages at kern_vm_start base. + * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow + * lower 32-bit and it's ok. + */ + ret = apply_to_page_range(&init_mm, kern_vm_start + uaddr32, + page_cnt << PAGE_SHIFT, apply_range_set_cb, &data); + mapped = data.i; + if (ret) + goto out_unmap; - /* zeroing is needed, since alloc_pages_bulk() only fills in non-zero entries */ - memset(pages, 0, this_batch * sizeof(struct page *)); - - ret = bpf_map_alloc_pages(&arena->map, node_id, this_batch, pages); - if (ret) - goto out; - - /* - * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 - * will not overflow 32-bit. Lower 32-bit need to represent - * contiguous user address range. - * Map these pages at kern_vm_start base. - * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow - * lower 32-bit and it's ok. - */ - data.i = 0; - ret = apply_to_page_range(&init_mm, - kern_vm_start + uaddr32 + (mapped << PAGE_SHIFT), - this_batch << PAGE_SHIFT, apply_range_set_cb, &data); - if (ret) { - /* data.i pages were mapped, account them and free the remaining */ - mapped += data.i; - for (i = data.i; i < this_batch; i++) - free_pages_nolock(pages[i], 0); - goto out; - } - - mapped += this_batch; - remaining -= this_batch; - } flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - kfree_nolock(pages); - bpf_map_memcg_exit(old_memcg, new_memcg); - return clear_lo32(arena->user_vm_start) + uaddr32; -out: + + addr = clear_lo32(arena->user_vm_start) + uaddr32; + goto out_memcg; + +out_unmap: + /* Error handling: Undo partial mappings. */ + flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - if (mapped) { - flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); + if (mapped) arena_free_pages(arena, uaddr32, mapped, sleepable); - } - goto out_free_pages; -out_unlock_free_pages: - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + out_free_pages: - kfree_nolock(pages); + /* Error handling: Free back any unmapped pages. */ + bpf_free_pages(&pages); + +out_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; + + return addr; } /* diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 37cf130ebb57..cb7219ff68bd 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include #include #include @@ -117,73 +115,26 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) return dst_reg; } -static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) -{ - const struct bpf_kfunc_desc *d0 = a; - const struct bpf_kfunc_desc *d1 = b; - - if (d0->imm != d1->imm) - return d0->imm < d1->imm ? -1 : 1; - if (d0->offset != d1->offset) - return d0->offset < d1->offset ? -1 : 1; - return 0; -} - const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) { - const struct bpf_kfunc_desc desc = { - .imm = insn->imm, - .offset = insn->off, - }; const struct bpf_kfunc_desc *res; struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - res = bsearch(&desc, tab->descs, tab->nr_descs, - sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); - - return res ? &res->func_model : NULL; -} - -static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) -{ - unsigned long call_imm; + if (insn->off < 0 || insn->off >= tab->nr_descs) + return NULL; + res = &tab->descs[insn->off]; if (bpf_jit_supports_far_kfunc_call()) { - call_imm = desc->func_id; - } else { - call_imm = BPF_CALL_IMM(desc->addr); - /* Check whether the relative offset overflows desc->imm */ - if ((unsigned long)(s32)call_imm != call_imm) { - verbose(env, "address of kernel func_id %u is out of range\n", - desc->func_id); - return -EINVAL; - } - } - desc->imm = call_imm; - return 0; -} - -static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) -{ - struct bpf_kfunc_desc_tab *tab; - int i, err; - - tab = env->prog->aux->kfunc_tab; - if (!tab) - return 0; - - for (i = 0; i < tab->nr_descs; i++) { - err = set_kfunc_desc_imm(env, &tab->descs[i]); - if (err) - return err; + if (res->func_id != insn->imm) + return NULL; + } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { + return NULL; } - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), - kfunc_desc_cmp_by_imm_off, NULL); - return 0; + return &res->func_model; } static int add_kfunc_in_insns(struct bpf_verifier_env *env, @@ -2720,10 +2671,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } } - ret = sort_kfunc_descs_by_imm_off(env); - if (ret) - return ret; - return 0; } @@ -2897,4 +2844,3 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) return 0; } - diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ac52f4ae414c..ef8fb2f6e6e3 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -602,38 +602,45 @@ static bool can_alloc_pages(void) !IS_ENABLED(CONFIG_PREEMPT_RT); } -static struct page *__bpf_alloc_page(int nid) +struct page *bpf_alloc_page(int nid, bool sleepable) { - if (!can_alloc_pages()) + if (!sleepable || !can_alloc_pages()) return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0); return alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT - | __GFP_NOWARN, + | __GFP_NOWARN | __GFP_RETRY_MAYFAIL, 0); } -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **pages) +void bpf_free_pages(struct llist_head *pages) { - unsigned long i, j; + struct llist_node *node; + struct page *page, *tmp; + + node = llist_del_all(pages); + llist_for_each_entry_safe(page, tmp, node, pcp_llist) + free_pages_nolock(page, 0); +} + +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable) +{ + unsigned long i; struct page *pg; - int ret = 0; for (i = 0; i < nr_pages; i++) { - pg = __bpf_alloc_page(nid); - - if (pg) { - pages[i] = pg; - continue; - } - for (j = 0; j < i; j++) - free_pages_nolock(pages[j], 0); - ret = -ENOMEM; - break; + pg = bpf_alloc_page(nid, sleepable); + if (!pg) + goto free_pages; + llist_add(&pg->pcp_llist, pages); } - return ret; + return 0; + +free_pages: + bpf_free_pages(pages); + return -ENOMEM; } static int btf_field_cmp(const void *a, const void *b) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b840b3eb9b22..ba0dfefe7209 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2579,17 +2579,21 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - return bsearch(&desc, tab->descs, tab->nr_descs, + return bsearch(&desc, tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); } int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { + struct bpf_kfunc_desc_tab *tab; const struct bpf_kfunc_desc *desc; - desc = find_kfunc_desc(prog, func_id, btf_fd_idx); - if (!desc) + tab = prog->aux->kfunc_tab; + if (desc_idx >= tab->nr_descs) + return -EFAULT; + desc = &tab->descs[desc_idx]; + if (desc->func_id != func_id) return -EFAULT; *func_addr = (u8 *)desc->addr; @@ -2929,10 +2933,15 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (find_kfunc_desc(env->prog, func_id, offset)) return 0; - if (tab->nr_descs == MAX_KFUNC_DESCS) { + if (tab->nr_base_descs == MAX_KFUNC_DESCS) { verbose(env, "too many different kernel function calls\n"); return -E2BIG; } + if (tab->nr_descs != tab->nr_base_descs) { + verifier_bug(env, "unexpected specialized func desc found (%d descs, %d base descs)", + tab->nr_descs, tab->nr_base_descs); + return -EFAULT; + } err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) @@ -2990,7 +2999,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc->addr = addr; desc->func_model = func_model; tab->nr_descs++; - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), + tab->nr_base_descs++; + sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } @@ -22017,6 +22027,66 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc return 0; } +static int add_kfunc_desc_target(struct bpf_verifier_env *env, + const struct bpf_kfunc_desc *target_desc, + u16 base_desc_idx, u16 *desc_idx) +{ + struct bpf_kfunc_desc desc = *target_desc; + struct bpf_kfunc_desc_tab *new_tab; + struct bpf_kfunc_desc_tab *tab; + struct bpf_prog_aux *prog_aux; + u32 i; + + prog_aux = env->prog->aux; + tab = prog_aux->kfunc_tab; + + if (base_desc_idx >= tab->nr_base_descs) { + verifier_bug(env, "kfunc desc_idx %d out of bounds (%d descriptors)", + base_desc_idx, tab->nr_base_descs); + return -EFAULT; + } + if (tab->descs[base_desc_idx].func_id != desc.func_id) { + verifier_bug(env, "kfunc desc %d has invalid id (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].func_id, desc.func_id); + return -EFAULT; + } + if (tab->descs[base_desc_idx].offset != desc.offset) { + verifier_bug(env, "kfunc desc %d has invalid offset (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].offset, desc.offset); + return -EFAULT; + } + + if (tab->descs[base_desc_idx].addr == desc.addr) { + *desc_idx = base_desc_idx; + return 0; + } + + for (i = tab->nr_base_descs; i < tab->nr_descs; i++) { + if (tab->descs[i].func_id == desc.func_id && + tab->descs[i].offset == desc.offset && + tab->descs[i].addr == desc.addr) { + *desc_idx = i; + return 0; + } + } + + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) { + verbose(env, "too many different kernel function call targets\n"); + return -E2BIG; + } + + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), + GFP_KERNEL_ACCOUNT); + if (!new_tab) + return -ENOMEM; + tab = new_tab; + prog_aux->kfunc_tab = tab; + + *desc_idx = tab->nr_descs; + tab->descs[tab->nr_descs++] = desc; + return 0; +} + static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, u16 struct_meta_reg, u16 node_offset_reg, @@ -22037,7 +22107,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { + struct bpf_kfunc_desc desc_copy; struct bpf_kfunc_desc *desc; + unsigned long call_imm; + u16 base_desc_idx, desc_idx; + bool near_call; int err; if (!insn->imm) { @@ -22057,13 +22131,30 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn->imm); return -EFAULT; } + base_desc_idx = desc - env->prog->aux->kfunc_tab->descs; + + near_call = !bpf_jit_supports_far_kfunc_call(); + desc_copy = *desc; + desc = &desc_copy; err = specialize_kfunc(env, desc, insn_idx); if (err) return err; - if (!bpf_jit_supports_far_kfunc_call()) - insn->imm = BPF_CALL_IMM(desc->addr); + if (near_call) { + call_imm = BPF_CALL_IMM(desc->addr); + if ((unsigned long)(s32)call_imm != call_imm) { + verbose(env, "address of kernel func_id %u is out of range\n", + desc->func_id); + return -EINVAL; + } + insn->imm = call_imm; + } + + err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx); + if (err) + return err; + insn->off = desc_idx; if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; diff --git a/tools/testing/selftests/bpf/prog_tests/file_reader.c b/tools/testing/selftests/bpf/prog_tests/file_reader.c index 48aae7ea0e4b..e59c6c87e9d5 100644 --- a/tools/testing/selftests/bpf/prog_tests/file_reader.c +++ b/tools/testing/selftests/bpf/prog_tests/file_reader.c @@ -7,10 +7,12 @@ #include "file_reader_fail.skel.h" #include #include +#include const char *user_ptr = "hello world"; char file_contents[256000]; void *addr; +__u64 beyond_eof_offset; void *get_executable_base_addr(void) { @@ -26,12 +28,18 @@ void *get_executable_base_addr(void) static int initialize_file_contents(void) { + struct stat st; int fd, page_sz = sysconf(_SC_PAGESIZE); ssize_t n = 0, cur; fd = open("/proc/self/exe", O_RDONLY); if (!ASSERT_OK_FD(fd, "Open /proc/self/exe\n")) return 1; + if (!ASSERT_OK(fstat(fd, &st), "fstat /proc/self/exe")) { + close(fd); + return 1; + } + beyond_eof_offset = st.st_size + (1ULL << 30); do { cur = read(fd, file_contents + n, sizeof(file_contents) - n); @@ -75,6 +83,7 @@ static void run_test(const char *prog_name) memcpy(skel->bss->user_buf, file_contents, sizeof(file_contents)); skel->bss->pid = getpid(); + skel->bss->beyond_eof_offset = beyond_eof_offset; err = file_reader__load(skel); if (!ASSERT_OK(err, "file_reader__load")) @@ -110,6 +119,12 @@ void test_file_reader(void) if (test__start_subtest("on_open_validate_file_read")) run_test("on_open_validate_file_read"); + if (test__start_subtest("on_open_non_sleepable_first")) + run_test("on_open_non_sleepable_first"); + + if (test__start_subtest("on_open_sleepable_first")) + run_test("on_open_sleepable_first"); + if (test__start_subtest("negative")) RUN_TESTS(file_reader_fail); } diff --git a/tools/testing/selftests/bpf/progs/file_reader.c b/tools/testing/selftests/bpf/progs/file_reader.c index aa2c05cce2b3..8b972fd26d73 100644 --- a/tools/testing/selftests/bpf/progs/file_reader.c +++ b/tools/testing/selftests/bpf/progs/file_reader.c @@ -27,9 +27,14 @@ char tmp_buf[256000]; int pid = 0; int err, run_success = 0; +__u64 beyond_eof_offset; static int validate_file_read(struct file *file); static int task_work_callback(struct bpf_map *map, void *key, void *value); +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value); + +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; SEC("lsm/file_open") int on_open_expect_fault(void *c) @@ -81,6 +86,101 @@ int on_open_validate_file_read(void *c) return 0; } +/* + * Exercise bpf_dynptr_from_file() first from a non-sleepable LSM program and + * then from its sleepable task-work callback. Reading beyond EOF makes the two + * backing implementations return different errors. + */ +SEC("lsm/file_open") +int on_open_non_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct elem *work; + struct file *file; + int key = 0; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 1; + return 0; + } + + /* The non-sleepable reader cannot fault in an uncached folio. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EFAULT) { + err = 2; + return 0; + } + + work = bpf_map_lookup_elem(&arrmap, &key); + if (!work) { + err = 3; + return 0; + } + + ret = bpf_task_work_schedule_signal(task, &work->tw, &arrmap, + sleepable_second_callback); + if (ret) + err = 4; + return 0; +} + +/* + * Exercise the opposite fixup order: the first call is made from a sleepable + * LSM program, while the RCU read-side section makes the second non-sleepable. + */ +SEC("lsm.s/file_open") +int on_open_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 7; + return 0; + } + + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EIO) { + err = 8; + goto out; + } + + bpf_rcu_read_lock(); + ret = bpf_dynptr_from_file(file, 0, &dynptr); + bpf_rcu_read_unlock(); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EFAULT) { + err = 9; + goto out; + } + + run_success = 1; +out: + bpf_put_file(file); + return 0; +} + /* Called in a sleepable context, read 256K bytes, cross check with user space read data */ static int task_work_callback(struct bpf_map *map, void *key, void *value) { @@ -97,6 +197,35 @@ static int task_work_callback(struct bpf_map *map, void *key, void *value) return 0; } +/* Task-work callbacks are verified as sleepable. */ +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 5; + return 0; + } + + /* freader_fetch() converts __kernel_read()'s short read at EOF to -EIO. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EIO) { + err = 6; + return 0; + } + + run_success = 1; + return 0; +} + static int verify_dynptr_read(struct bpf_dynptr *ptr, u32 off, char *user_buf, u32 len) { int i; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c index 6ab8730d4878..dbdea14ca76f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c @@ -10,6 +10,9 @@ #include #define ARENA_SIZE (1ull << 32) +#define LARGE_PAGE_CNT 1025 + +volatile int zero = 0; struct { __uint(type, BPF_MAP_TYPE_ARENA); @@ -284,6 +287,7 @@ int big_alloc2(void *ctx) return 0; } +/* Nonsleepable because it binds to a socket program. */ SEC("socket") __success __retval(0) int big_alloc3(void *ctx) @@ -291,24 +295,56 @@ int big_alloc3(void *ctx) #if defined(__BPF_FEATURE_ADDR_SPACE_CAST) char __arena *pages; u64 i; + int err = 0; - /* - * Allocate 2051 pages in one go to check how kmalloc_nolock() handles large requests. - * Since kmalloc_nolock() can allocate up to 1024 struct page * at a time, this call should - * result in three batches: two batches of 1024 pages each, followed by a final batch of 3 - * pages. - */ - pages = bpf_arena_alloc_pages(&arena, NULL, 2051, NUMA_NO_NODE, 0); + /* Verify that a nonsleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); if (!pages) - return 0; + return 1; - bpf_for(i, 0, 2051) - pages[i * PAGE_SIZE] = 123; - bpf_for(i, 0, 2051) - if (pages[i * PAGE_SIZE] != 123) - return i; + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; - bpf_arena_free_pages(&arena, pages, 2051); + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } + + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; +#endif + return 0; +} + +/* SYSCALL programs are always sleepable. */ +SEC("syscall") +__success __retval(0) +int big_alloc4(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + char __arena *pages; + u64 i; + int err = 0; + + /* Verify that a sleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); + if (!pages) + return 1; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } + + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; #endif return 0; }