github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-12 16:18:45 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

vdso/datastore: Allocate data pages dynamically

Browse Source 
Allocating the data pages as part of the kernel image does not work on
SPARC. The MMU will raise a fault when userspace tries to access them.

Allocate the data pages through the page allocator instead.

Unused pages in the vDSO VMA are still allocated to keep the virtual
addresses aligned. Switch the mapping from PFNs to 'struct page' as that is
required for dynamically allocated pages.  This also aligns the allocation
of the datapages with the code pages and is a prerequisite for mlockall()
support.

VM_MIXEDMAP is necessary for the call to vmf_insert_page() in the timens
prefault path to work.

The data pages need to be order-0, non-compound pages so that the mapping
to userspace and the different orderings work.

These pages are also used by the timekeeping, random pool and architecture
initialization code. Some of these are running before the page allocator is
available. To keep these subsytems working without changes, introduce
early, statically data storage which will then replaced by the real one as
soon as that is available.

Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Reviewed-by: Christophe Leroy (CS GROUP) <chleroy@kernel.org>
Link: https://patch.msgid.link/20260304-vdso-sparc64-generic-2-v6-3-d8eb3b0e1410@linutronix.de
This commit is contained in:
Thomas Weißschuh 2026-03-04 08:49:00 +01:00 committed by Thomas Gleixner
parent c0c9439ba3
commit 05988dba11
3 changed files with 68 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/linux/vdso_datastore.h
View File

@ -2,9 +2,15 @@
#ifndef _LINUX_VDSO_DATASTORE_H
#define _LINUX_VDSO_DATASTORE_H
#ifdef CONFIG_HAVE_GENERIC_VDSO
#include <linux/mm_types.h>
extern const struct vm_special_mapping vdso_vvar_mapping;
struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
void __init vdso_setup_data_pages(void);
#else /* !CONFIG_HAVE_GENERIC_VDSO */
static inline void vdso_setup_data_pages(void) { }
#endif /* CONFIG_HAVE_GENERIC_VDSO */
#endif /* _LINUX_VDSO_DATASTORE_H */

2
init/main.c
View File

@ -105,6 +105,7 @@
#include <linux/ptdump.h>
#include <linux/time_namespace.h>
#include <linux/unaligned.h>
#include <linux/vdso_datastore.h>
#include <net/net_namespace.h>
#include <asm/io.h>
@ -1119,6 +1120,7 @@ void start_kernel(void)
srcu_init();
hrtimers_init();
softirq_init();
vdso_setup_data_pages();
timekeeping_init();
time_init();

92
lib/vdso/datastore.c
View File

@ -1,52 +1,79 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/linkage.h>
#include <linux/gfp.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/time_namespace.h>
#include <linux/types.h>
#include <linux/vdso_datastore.h>
#include <vdso/datapage.h>
/*
* The vDSO data page.
*/
static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
#ifdef CONFIG_GENERIC_GETTIMEOFDAY
static union {
struct vdso_time_data data;
u8 page[PAGE_SIZE];
} vdso_time_data_store __page_aligned_data;
struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data;
static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
struct vdso_time_data *vdso_k_time_data __refdata =
(void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
#ifdef CONFIG_VDSO_GETRANDOM
static union {
struct vdso_rng_data data;
u8 page[PAGE_SIZE];
} vdso_rng_data_store __page_aligned_data;
struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data;
static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE);
struct vdso_rng_data *vdso_k_rng_data __refdata =
(void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
#endif /* CONFIG_VDSO_GETRANDOM */
#ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
static union {
struct vdso_arch_data data;
u8 page[VDSO_ARCH_DATA_SIZE];
} vdso_arch_data_store __page_aligned_data;
struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data;
struct vdso_arch_data *vdso_k_arch_data __refdata =
(void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
#endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
void __init vdso_setup_data_pages(void)
{
unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
struct page *pages;
/*
* Allocate the data pages dynamically. SPARC does not support mapping
* static pages to be mapped into userspace.
* It is also a requirement for mlockall() support.
*
* Do not use folios. In time namespaces the pages are mapped in a different order
* to userspace, which is not handled by the folio optimizations in finish_fault().
*/
pages = alloc_pages(GFP_KERNEL, order);
if (!pages)
panic("Unable to allocate VDSO storage pages");
/* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
split_page(pages, order);
/* Move the data already written by other subsystems to the new pages */
memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET);
if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET);
if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START);
}
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct page *timens_page = find_timens_vvar_page(vma);
unsigned long pfn;
struct page *page, *timens_page;
timens_page = find_timens_vvar_page(vma);
switch (vmf->pgoff) {
case VDSO_TIME_PAGE_OFFSET:
if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
page = virt_to_page(vdso_k_time_data);
if (timens_page) {
/*
* Fault in VVAR page too, since it will be accessed
@ -56,10 +83,10 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
vm_fault_t err;
addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
err = vmf_insert_pfn(vma, addr, pfn);
err = vmf_insert_page(vma, addr, page);
if (unlikely(err & VM_FAULT_ERROR))
return err;
pfn = page_to_pfn(timens_page);
page = timens_page;
}
break;
case VDSO_TIMENS_PAGE_OFFSET:
@ -72,24 +99,25 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
*/
if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
page = virt_to_page(vdso_k_time_data);
break;
case VDSO_RNG_PAGE_OFFSET:
if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data));
page = virt_to_page(vdso_k_rng_data);
break;
case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) +
vmf->pgoff - VDSO_ARCH_PAGES_START;
page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START;
break;
default:
return VM_FAULT_SIGBUS;
}
return vmf_insert_pfn(vma, vmf->address, pfn);
get_page(page);
vmf->page = page;
return 0;
}
const struct vm_special_mapping vdso_vvar_mapping = {
@ -101,7 +129,7 @@ struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned
{
return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP |
VM_PFNMAP | VM_SEALED_SYSMAP,
VM_MIXEDMAP | VM_SEALED_SYSMAP,
&vdso_vvar_mapping);
}