diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..ef5bb6981133 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1098,19 +1098,21 @@ static inline void rcu_read_unlock_migrate(void) /* * In mm/slab_common.c, no suitable header to include here. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr); +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the * comment of kfree_rcu() for details. */ -#define kvfree_rcu_arg_2(ptr, rhf) \ +#define kvfree_rcu_arg_2(ptr, kvrhf) \ do { \ typeof (ptr) ___p = (ptr); \ + struct kvfree_rcu_head *___head; \ \ if (___p) { \ - BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096); \ - kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + ___head = (struct kvfree_rcu_head *) &(___p)->kvrhf; \ + kvfree_call_rcu(___head, (void *) (___p)); \ } \ } while (0) diff --git a/include/linux/types.h b/include/linux/types.h index 93166b0b0617..7d1d305a763e 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -255,6 +255,16 @@ struct callback_head { } __attribute__((aligned(sizeof(void *)))); #define rcu_head callback_head +#ifdef CONFIG_KVFREE_RCU_BATCHED +struct kvfree_rcu_head { + struct kvfree_rcu_head *next; +}; +#else +struct kvfree_rcu_head { + struct rcu_head head; +}; +#endif + typedef void (*rcu_callback_t)(struct rcu_head *head); typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 5fbdabe3faea..a74027126a2f 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -625,7 +625,7 @@ TRACE_EVENT_RCU(rcu_invoke_callback, */ TRACE_EVENT_RCU(rcu_invoke_kvfree_callback, - TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset), + TP_PROTO(const char *rcuname, struct kvfree_rcu_head *rhp, unsigned long offset), TP_ARGS(rcuname, rhp, offset), diff --git a/mm/slab.h b/mm/slab.h index ddcf59230d2f..1dea45402794 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -352,6 +352,37 @@ static inline int objs_per_slab(const struct kmem_cache *cache, return slab->objects; } +/* + * kvfree_rcu_head offset can be only less than page size. + * Calculate the start address while preserving the KASAN tag. + */ +static inline void *kvmalloc_obj_start_addr(void *head) +{ + unsigned long offset; + + if (unlikely(is_vmalloc_addr(head))) { + offset = offset_in_page(head); + } else { + struct slab *slab = virt_to_slab(head); + + if (!slab) { + offset = offset_in_page(head); + } else if (is_kfence_address(head)) { + offset = head - kfence_object_start(head); + } else { + struct kmem_cache *s = slab->slab_cache; + unsigned int idx = __obj_to_index(s, slab_address(slab), head); + void *obj = slab_address(slab) + s->size * idx; + + obj = fixup_red_left(s, obj); + obj = kasan_reset_tag(obj); + offset = kasan_reset_tag(head) - obj; + } + } + + return head - offset; +} + /* * State of the slab allocator. * @@ -787,6 +818,7 @@ void __check_heap_object(const void *ptr, unsigned long n, const struct slab *slab, bool to_user); void deferred_work_barrier(void); +void defer_kfree_rcu(struct kvfree_rcu_head *head); static inline bool slub_debug_orig_size(struct kmem_cache *s) { diff --git a/mm/slab_common.c b/mm/slab_common.c index e776fc4516c8..661f1fc24c2d 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1282,11 +1282,11 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); #ifndef CONFIG_KVFREE_RCU_BATCHED -void kvfree_call_rcu(struct rcu_head *head, void *ptr) +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) { if (head) { kasan_record_aux_stack(ptr); - call_rcu(head, kvfree_rcu_cb); + call_rcu(&head->head, kvfree_rcu_cb); return; } @@ -1363,7 +1363,7 @@ struct kvfree_rcu_bulk_data { struct kfree_rcu_cpu_work { struct rcu_work rcu_work; - struct rcu_head *head_free; + struct kvfree_rcu_head *head_free; struct rcu_gp_oldstate head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; @@ -1399,7 +1399,7 @@ struct kfree_rcu_cpu_work { struct kfree_rcu_cpu { // Objects queued on a linked list // through their rcu_head structures. - struct rcu_head *head; + struct kvfree_rcu_head *head; unsigned long head_gp_snap; atomic_t head_count; @@ -1540,12 +1540,12 @@ kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp, } static void -kvfree_rcu_list(struct rcu_head *head) +kvfree_rcu_list(struct kvfree_rcu_head *head) { - struct rcu_head *next; + struct kvfree_rcu_head *next; for (; head; head = next) { - void *ptr = (void *) head->func; + void *ptr = kvmalloc_obj_start_addr(head); unsigned long offset = (void *) head - ptr; next = head->next; @@ -1569,7 +1569,7 @@ static void kfree_rcu_work(struct work_struct *work) unsigned long flags; struct kvfree_rcu_bulk_data *bnode, *n; struct list_head bulk_head[FREE_N_CHANNELS]; - struct rcu_head *head; + struct kvfree_rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; struct rcu_gp_oldstate head_gp_snap; @@ -1700,7 +1700,7 @@ kvfree_rcu_drain_ready(struct kfree_rcu_cpu *krcp) { struct list_head bulk_ready[FREE_N_CHANNELS]; struct kvfree_rcu_bulk_data *bnode, *n; - struct rcu_head *head_ready = NULL; + struct kvfree_rcu_head *head_ready = NULL; unsigned long flags; int i; @@ -1963,7 +1963,7 @@ void __init kfree_rcu_scheduler_running(void) * be free'd in workqueue context. This allows us to: batch requests together to * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr) +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) { unsigned long flags; struct kfree_rcu_cpu *krcp; @@ -2001,7 +2001,6 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) // Inline if kvfree_rcu(one_arg) call. goto unlock_return; - head->func = ptr; head->next = krcp->head; WRITE_ONCE(krcp->head, head); atomic_inc(&krcp->head_count); diff --git a/mm/slub.c b/mm/slub.c index 4a07ba92b64f..0a7024c8c090 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6681,43 +6681,21 @@ static void free_large_kmalloc(struct page *page, void *object) */ void kvfree_rcu_cb(struct rcu_head *head) { - void *obj = head; - struct page *page; - struct slab *slab; - struct kmem_cache *s; - void *slab_addr; + void *obj; + + obj = kvmalloc_obj_start_addr(head); if (is_vmalloc_addr(obj)) { - obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); vfree(obj); - return; - } - - page = virt_to_page(obj); - slab = page_slab(page); - if (!slab) { - /* - * rcu_head offset can be only less than page size so no need to - * consider allocation order - */ - obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); - free_large_kmalloc(page, obj); - return; - } - - s = slab->slab_cache; - slab_addr = slab_address(slab); - - if (is_kfence_address(obj)) { - obj = kfence_object_start(obj); } else { - unsigned int idx = __obj_to_index(s, slab_addr, obj); + struct page *page = virt_to_page(obj); + struct slab *slab = page_slab(page); - obj = slab_addr + s->size * idx; - obj = fixup_red_left(s, obj); + if (slab) + slab_free(slab->slab_cache, slab, obj, _RET_IP_); + else + free_large_kmalloc(page, obj); } - - slab_free(s, slab, obj, _RET_IP_); } /**