diff --git a/MAINTAINERS b/MAINTAINERS index efa1e8e6a010..4929b0a956df 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -27984,6 +27984,22 @@ F: kernel/trace/trace_irqsoff.c F: kernel/trace/trace_osnoise.c F: kernel/trace/trace_sched_wakeup.c +TRACING RING BUFFER +M: Steven Rostedt +M: Vincent Donnefort +M: Masami Hiramatsu +L: linux-kernel@vger.kernel.org +L: linux-trace-kernel@vger.kernel.org +S: Maintained +Q: https://patchwork.kernel.org/project/linux-trace-kernel/list/ +T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git +F: Documentation/trace/ring-buffer* +F: Documentation/trace/remotes.rst +F: include/linux/ring-buffer*.h +F: kernel/trace/*ring_buffer*.c +F: kernel/trace/remote*.c +F: tools/testing/selftests/ring-buffer/ + TRADITIONAL CHINESE DOCUMENTATION M: Chen-Yu Yeh M: Weijie Yuan diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index eac3e9080c3c..e4370064273e 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -98,6 +98,13 @@ struct trace_buffer *__ring_buffer_alloc_range(unsigned long size, unsigned flag void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size); +/* + * A minimum of 2 pages ensures we can always swap a page out of the ring + * buffer. This does not account for the reader page or any other page + * containing meta data used by static ring buffers. + */ +#define RING_BUFFER_MIN_NR_PAGES 2UL + /* * Because the ring buffer is generic, if other users of the ring buffer get * traced by ftrace, it can produce lockdep warnings. We need to keep each @@ -287,7 +294,7 @@ static inline struct ring_buffer_desc *__first_ring_buffer_desc(struct trace_buf static inline unsigned long __calc_nr_pages_ring_buffer_desc(size_t size) { /* Takes into account the reader page */ - return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), 2UL) + 1; + return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), RING_BUFFER_MIN_NR_PAGES) + 1; } static inline size_t trace_buffer_desc_size(size_t buffer_size, unsigned int nr_cpus) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9c4e40847e43..b52f51db9045 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -642,6 +642,11 @@ static __always_inline unsigned int rb_subbuf_max_data_size(struct trace_buffer return rb_subbuf_capacity(buffer) - RB_EVNT_HDR_SIZE - sizeof(event->array[0]); } +static unsigned long rb_calculate_nr_pages(struct trace_buffer *buffer, unsigned long size) +{ + return max(DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)), RING_BUFFER_MIN_NR_PAGES); +} + /** * rb_subbuf_start - Get the start address of a subbuffer * @buffer: A trace buffer @@ -2868,7 +2873,7 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, goto fail_free_buffers; /* Need at least two pages plus the reader page */ - if (nr_pages < 3) + if (nr_pages < RING_BUFFER_MIN_NR_PAGES + 1) goto fail_free_buffers; again: @@ -2880,7 +2885,7 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, ptr += subbuf_size * nr_pages; } if (ptr > end) { - if (nr_pages <= 3) + if (nr_pages <= RING_BUFFER_MIN_NR_PAGES + 1) goto fail_free_buffers; nr_pages--; goto again; @@ -2903,13 +2908,10 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, if (nr_pages > rb_static_max_pages()) goto fail_free_buffers; - if (nr_pages < 2) + if (nr_pages < RING_BUFFER_MIN_NR_PAGES) goto fail_free_buffers; } else { - /* need at least two pages */ - nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); - if (nr_pages < 2) - nr_pages = 2; + nr_pages = rb_calculate_nr_pages(buffer, size); } cpumask_set_cpu(cpu, buffer->cpumask); @@ -3284,6 +3286,21 @@ static void update_pages_handler(struct work_struct *work) complete(&cpu_buffer->update_done); } +static bool rb_resize_disabled(struct trace_buffer *buffer, int cpu) +{ + lockdep_assert_held(&buffer->mutex); + + if (cpu != RING_BUFFER_ALL_CPUS) + return atomic_read(&buffer->buffers[cpu]->resize_disabled); + + for_each_buffer_cpu(buffer, cpu) { + if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) + return true; + } + + return false; +} + /** * ring_buffer_resize - resize the ring buffer * @buffer: the buffer to resize. @@ -3322,26 +3339,19 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, mutex_lock(&buffer->mutex); atomic_inc(&buffer->resizing); - nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); + nr_pages = rb_calculate_nr_pages(buffer, size); - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; + /* + * Don't succeed if resizing is disabled, as a reader might be + * manipulating the ring buffer and is expecting a sane state while + * this is true. + */ + if (rb_resize_disabled(buffer, cpu_id)) { + err = -EBUSY; + goto out_err_unlock; + } if (cpu_id == RING_BUFFER_ALL_CPUS) { - /* - * Don't succeed if resizing is disabled, as a reader might be - * manipulating the ring buffer and is expecting a sane state while - * this is true. - */ - for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto out_err_unlock; - } - } - /* calculate the pages to update */ for_each_buffer_cpu(buffer, cpu) { cpu_buffer = buffer->buffers[cpu]; @@ -3413,16 +3423,6 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, if (nr_pages == cpu_buffer->nr_pages) goto out; - /* - * Don't succeed if resizing is disabled, as a reader might be - * manipulating the ring buffer and is expecting a sane state while - * this is true. - */ - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto out_err_unlock; - } - cpu_buffer->nr_pages_to_update = nr_pages - cpu_buffer->nr_pages; @@ -7466,6 +7466,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (psize > RB_WRITE_MASK + 1) return -EINVAL; + guard(cpus_read_lock)(); + /* prevent another thread from changing buffer sizes */ guard(mutex)(&buffer->mutex); @@ -7475,13 +7477,9 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) old_capacity = rb_subbuf_capacity(buffer); - /* The mmap fast path reads subbuf_order without buffer->mutex. */ - for_each_buffer_cpu(buffer, cpu) { - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) - return -EBUSY; - } + /* Check it is resizable before we touch subbuf_order */ + if (rb_resize_disabled(buffer, RING_BUFFER_ALL_CPUS)) + return -EBUSY; atomic_inc(&buffer->record_disabled); @@ -7492,24 +7490,11 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { - - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - cpu_buffer = buffer->buffers[cpu]; - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto error; - } - /* Update the number of pages to match the new size */ - nr_pages = old_capacity * buffer->buffers[cpu]->nr_pages; - nr_pages = DIV_ROUND_UP(nr_pages, rb_subbuf_capacity(buffer)); - - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; + nr_pages = rb_calculate_nr_pages(buffer, + old_capacity * buffer->buffers[cpu]->nr_pages); cpu_buffer->nr_pages_to_update = nr_pages; @@ -7531,9 +7516,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) struct list_head old_pages; unsigned long flags; - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - cpu_buffer = buffer->buffers[cpu]; raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); @@ -8100,7 +8082,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node) } /* allocate minimum pages, user can later expand it */ if (!nr_pages_same) - nr_pages = 2; + nr_pages = RING_BUFFER_MIN_NR_PAGES; buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); if (!buffer->buffers[cpu]) { diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index 49913bb0057a..abfaf9bee12e 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -383,8 +383,8 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, void *page; int i; - /* At least 1 reader page and two pages in the ring-buffer */ - if (desc->nr_page_va < 3) + /* At least RING_BUFFER_MIN_NR_PAGES and 1 reader page */ + if (desc->nr_page_va < RING_BUFFER_MIN_NR_PAGES + 1) return -EINVAL; memset(cpu_buffer, 0, sizeof(*cpu_buffer));