diff --git a/include/linux/bpf.h b/include/linux/bpf.h index f4e8d372253a..04cadd987169 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -145,6 +145,7 @@ struct bpf_map_ops { int (*map_direct_value_meta)(const struct bpf_map *map, u64 imm, u32 *off); int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); + vm_fault_t (*map_mmap_fault)(struct bpf_map *map, struct vm_fault *vmf); __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, struct poll_table_struct *pts); unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr, diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 34865701f7f7..ef315b168b29 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -608,17 +608,42 @@ static int array_map_check_btf(struct bpf_map *map, static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) { struct bpf_array *array = container_of(map, struct bpf_array, map); - pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT; if (!(map->map_flags & BPF_F_MMAPABLE)) return -EINVAL; - if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > + /* use u64 math so the offset cannot overflow on 32-bit archs */ + if ((u64)vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) return -EINVAL; - return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), - vma->vm_pgoff + pgoff); + /* + * Pages are faulted in on demand by array_map_mmap_fault(). Set the + * same flags that the eager remap_vmalloc_range() path used to set + * via vm_insert_page(), so that e.g. NUMA balancing keeps skipping + * these VMAs. + */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); + + return 0; +} + +static vm_fault_t array_map_mmap_fault(struct bpf_map *map, + struct vm_fault *vmf) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + struct page *page; + + page = vmalloc_to_page(array->value + ((u64)vmf->pgoff << PAGE_SHIFT)); + if (!page) + return VM_FAULT_SIGBUS; + + /* the eager remap_vmalloc_range() flushed via vm_insert_page() */ + flush_dcache_folio(page_folio(page)); + get_page(page); + vmf->page = page; + + return 0; } static bool array_map_meta_equal(const struct bpf_map *meta0, @@ -844,6 +869,7 @@ const struct bpf_map_ops array_map_ops = { .map_direct_value_addr = array_map_direct_value_addr, .map_direct_value_meta = array_map_direct_value_meta, .map_mmap = array_map_mmap, + .map_mmap_fault = array_map_mmap_fault, .map_seq_show_elem = array_map_seq_show_elem, .map_check_btf = array_map_check_btf, .map_lookup_batch = generic_map_lookup_batch, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 7d8c3e8e6d62..6874ba1424af 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1076,11 +1076,24 @@ static void bpf_map_mmap_close(struct vm_area_struct *vma) bpf_map_write_active_dec(map); } +static vm_fault_t bpf_map_mmap_fault(struct vm_fault *vmf) +{ + struct bpf_map *map = vmf->vma->vm_private_data; + + return map->ops->map_mmap_fault(map, vmf); +} + static const struct vm_operations_struct bpf_map_default_vmops = { .open = bpf_map_mmap_open, .close = bpf_map_mmap_close, }; +static const struct vm_operations_struct bpf_map_lazy_vmops = { + .open = bpf_map_mmap_open, + .close = bpf_map_mmap_close, + .fault = bpf_map_mmap_fault, +}; + static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) { struct bpf_map *map = filp->private_data; @@ -1116,7 +1129,7 @@ static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) return err; /* set default open/close callbacks */ - vma->vm_ops = &bpf_map_default_vmops; + vma->vm_ops = map->ops->map_mmap_fault ? &bpf_map_lazy_vmops : &bpf_map_default_vmops; vma->vm_private_data = map; vm_flags_clear(vma, VM_MAYEXEC); /* If mapping is read-only, then disallow potentially re-mapping with