From f909f0308267dc49fbf122f60e1ec7ddcd1b92c7 Mon Sep 17 00:00:00 2001 From: Carlos Llamas Date: Tue, 10 Dec 2024 14:31:00 +0000 Subject: binder: store shrinker metadata under page->private Instead of pre-allocating an entire array of struct binder_lru_page in alloc->pages, install the shrinker metadata under page->private. This ensures the memory is allocated and released as needed alongside pages. By converting the alloc->pages[] into an array of struct page pointers, we can access these pages directly and only reference the shrinker metadata where it's being used (e.g. inside the shrinker's callback). Rename struct binder_lru_page to struct binder_shrinker_mdata to better reflect its purpose. Add convenience functions that wrap the allocation and freeing of pages along with their shrinker metadata. Note I've reworked this patch to avoid using page->lru and page->index directly, as Matthew pointed out that these are being removed [1]. Link: https://lore.kernel.org/all/ZzziucEm3np6e7a0@casper.infradead.org/ [1] Cc: Matthew Wilcox Cc: Liam R. Howlett Reviewed-by: Suren Baghdasaryan Signed-off-by: Carlos Llamas Link: https://lore.kernel.org/r/20241210143114.661252-5-cmllamas@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/android/binder_alloc_selftest.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'drivers/android/binder_alloc_selftest.c') diff --git a/drivers/android/binder_alloc_selftest.c b/drivers/android/binder_alloc_selftest.c index 81442fe20a69..a4c650843bee 100644 --- a/drivers/android/binder_alloc_selftest.c +++ b/drivers/android/binder_alloc_selftest.c @@ -105,10 +105,10 @@ static bool check_buffer_pages_allocated(struct binder_alloc *alloc, page_addr = buffer->user_data; for (; page_addr < end; page_addr += PAGE_SIZE) { page_index = (page_addr - alloc->buffer) / PAGE_SIZE; - if (!alloc->pages[page_index].page_ptr || - !list_empty(&alloc->pages[page_index].lru)) { + if (!alloc->pages[page_index] || + !list_empty(page_to_lru(alloc->pages[page_index]))) { pr_err("expect alloc but is %s at page index %d\n", - alloc->pages[page_index].page_ptr ? + alloc->pages[page_index] ? "lru" : "free", page_index); return false; } @@ -148,10 +148,10 @@ static void binder_selftest_free_buf(struct binder_alloc *alloc, * if binder shrinker ran during binder_alloc_free_buf * calls above. */ - if (list_empty(&alloc->pages[i].lru)) { + if (list_empty(page_to_lru(alloc->pages[i]))) { pr_err_size_seq(sizes, seq); pr_err("expect lru but is %s at page index %d\n", - alloc->pages[i].page_ptr ? "alloc" : "free", i); + alloc->pages[i] ? "alloc" : "free", i); binder_selftest_failures++; } } @@ -168,9 +168,9 @@ static void binder_selftest_free_page(struct binder_alloc *alloc) } for (i = 0; i < (alloc->buffer_size / PAGE_SIZE); i++) { - if (alloc->pages[i].page_ptr) { + if (alloc->pages[i]) { pr_err("expect free but is %s at page index %d\n", - list_empty(&alloc->pages[i].lru) ? + list_empty(page_to_lru(alloc->pages[i])) ? "alloc" : "lru", i); binder_selftest_failures++; } -- cgit v1.2.3 From 072010abc3ad98bc20198dbe60ef13233a0a357c Mon Sep 17 00:00:00 2001 From: Carlos Llamas Date: Tue, 10 Dec 2024 14:31:01 +0000 Subject: binder: replace alloc->vma with alloc->mapped It is unsafe to use alloc->vma outside of the mmap_sem. Instead, add a new boolean alloc->mapped to save the vma state (mapped or unmmaped) and use this as a replacement for alloc->vma to validate several paths. Using the alloc->vma caused several performance and security issues in the past. Now that it has been replaced with either vm_lookup() or the alloc->mapped state, we can finally remove it. Cc: Minchan Kim Cc: Liam R. Howlett Cc: Matthew Wilcox Cc: Suren Baghdasaryan Reviewed-by: Suren Baghdasaryan Signed-off-by: Carlos Llamas Link: https://lore.kernel.org/r/20241210143114.661252-6-cmllamas@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/android/binder_alloc.c | 48 ++++++++++++++++++--------------- drivers/android/binder_alloc.h | 6 ++--- drivers/android/binder_alloc_selftest.c | 2 +- 3 files changed, 30 insertions(+), 26 deletions(-) (limited to 'drivers/android/binder_alloc_selftest.c') diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 3e30ac5b4861..ed79d7c146c8 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -220,6 +220,19 @@ static void binder_lru_freelist_add(struct binder_alloc *alloc, } } +static inline +void binder_alloc_set_mapped(struct binder_alloc *alloc, bool state) +{ + /* pairs with smp_load_acquire in binder_alloc_is_mapped() */ + smp_store_release(&alloc->mapped, state); +} + +static inline bool binder_alloc_is_mapped(struct binder_alloc *alloc) +{ + /* pairs with smp_store_release in binder_alloc_set_mapped() */ + return smp_load_acquire(&alloc->mapped); +} + static struct page *binder_page_alloc(struct binder_alloc *alloc, unsigned long index) { @@ -271,7 +284,7 @@ static int binder_install_single_page(struct binder_alloc *alloc, mmap_read_lock(alloc->mm); vma = vma_lookup(alloc->mm, addr); - if (!vma || vma != alloc->vma) { + if (!vma || !binder_alloc_is_mapped(alloc)) { binder_free_page(page); pr_err("%d: %s failed, no vma\n", alloc->pid, __func__); ret = -ESRCH; @@ -379,20 +392,6 @@ static void binder_lru_freelist_del(struct binder_alloc *alloc, } } -static inline void binder_alloc_set_vma(struct binder_alloc *alloc, - struct vm_area_struct *vma) -{ - /* pairs with smp_load_acquire in binder_alloc_get_vma() */ - smp_store_release(&alloc->vma, vma); -} - -static inline struct vm_area_struct *binder_alloc_get_vma( - struct binder_alloc *alloc) -{ - /* pairs with smp_store_release in binder_alloc_set_vma() */ - return smp_load_acquire(&alloc->vma); -} - static void debug_no_space_locked(struct binder_alloc *alloc) { size_t largest_alloc_size = 0; @@ -626,7 +625,7 @@ struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc, int ret; /* Check binder_alloc is fully initialized */ - if (!binder_alloc_get_vma(alloc)) { + if (!binder_alloc_is_mapped(alloc)) { binder_alloc_debug(BINDER_DEBUG_USER_ERROR, "%d: binder_alloc_buf, no vma\n", alloc->pid); @@ -908,7 +907,7 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, alloc->free_async_space = alloc->buffer_size / 2; /* Signal binder_alloc is fully initialized */ - binder_alloc_set_vma(alloc, vma); + binder_alloc_set_mapped(alloc, true); return 0; @@ -938,7 +937,7 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) buffers = 0; mutex_lock(&alloc->mutex); - BUG_ON(alloc->vma); + BUG_ON(alloc->mapped); while ((n = rb_first(&alloc->allocated_buffers))) { buffer = rb_entry(n, struct binder_buffer, rb_node); @@ -1044,7 +1043,7 @@ void binder_alloc_print_pages(struct seq_file *m, * Make sure the binder_alloc is fully initialized, otherwise we might * read inconsistent state. */ - if (binder_alloc_get_vma(alloc) != NULL) { + if (binder_alloc_is_mapped(alloc)) { for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { page = binder_get_installed_page(alloc, i); if (!page) @@ -1084,12 +1083,12 @@ int binder_alloc_get_allocated_count(struct binder_alloc *alloc) * @alloc: binder_alloc for this proc * * Called from binder_vma_close() when releasing address space. - * Clears alloc->vma to prevent new incoming transactions from + * Clears alloc->mapped to prevent new incoming transactions from * allocating more buffers. */ void binder_alloc_vma_close(struct binder_alloc *alloc) { - binder_alloc_set_vma(alloc, NULL); + binder_alloc_set_mapped(alloc, false); } /** @@ -1125,7 +1124,12 @@ enum lru_status binder_alloc_free_page(struct list_head *item, page_addr = alloc->buffer + index * PAGE_SIZE; vma = vma_lookup(mm, page_addr); - if (vma && vma != binder_alloc_get_vma(alloc)) + /* + * Since a binder_alloc can only be mapped once, we ensure + * the vma corresponds to this mapping by checking whether + * the binder_alloc is still mapped. + */ + if (vma && !binder_alloc_is_mapped(alloc)) goto err_invalid_vma; trace_binder_unmap_kernel_start(alloc, index); diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h index d71f99189ef5..3ebb12afd4de 100644 --- a/drivers/android/binder_alloc.h +++ b/drivers/android/binder_alloc.h @@ -82,8 +82,6 @@ static inline struct list_head *page_to_lru(struct page *p) /** * struct binder_alloc - per-binder proc state for binder allocator * @mutex: protects binder_alloc fields - * @vma: vm_area_struct passed to mmap_handler - * (invariant after mmap) * @mm: copy of task->mm (invariant after open) * @buffer: base of per-proc address space mapped via mmap * @buffers: list of all buffers for this proc @@ -96,6 +94,8 @@ static inline struct list_head *page_to_lru(struct page *p) * @buffer_size: size of address space specified via mmap * @pid: pid for associated binder_proc (invariant after init) * @pages_high: high watermark of offset in @pages + * @mapped: whether the vm area is mapped, each binder instance is + * allowed a single mapping throughout its lifetime * @oneway_spam_detected: %true if oneway spam detection fired, clear that * flag once the async buffer has returned to a healthy state * @@ -106,7 +106,6 @@ static inline struct list_head *page_to_lru(struct page *p) */ struct binder_alloc { struct mutex mutex; - struct vm_area_struct *vma; struct mm_struct *mm; unsigned long buffer; struct list_head buffers; @@ -117,6 +116,7 @@ struct binder_alloc { size_t buffer_size; int pid; size_t pages_high; + bool mapped; bool oneway_spam_detected; }; diff --git a/drivers/android/binder_alloc_selftest.c b/drivers/android/binder_alloc_selftest.c index a4c650843bee..6a64847a8555 100644 --- a/drivers/android/binder_alloc_selftest.c +++ b/drivers/android/binder_alloc_selftest.c @@ -291,7 +291,7 @@ void binder_selftest_alloc(struct binder_alloc *alloc) if (!binder_selftest_run) return; mutex_lock(&binder_selftest_lock); - if (!binder_selftest_run || !alloc->vma) + if (!binder_selftest_run || !alloc->mapped) goto done; pr_info("STARTED\n"); binder_selftest_alloc_offset(alloc, end_offset, 0); -- cgit v1.2.3 From 0a7bf6866d416e4f8f452419410359b6a82639d1 Mon Sep 17 00:00:00 2001 From: Carlos Llamas Date: Tue, 10 Dec 2024 14:31:02 +0000 Subject: binder: rename alloc->buffer to vm_start The alloc->buffer field in struct binder_alloc stores the starting address of the mapped vma, rename this field to alloc->vm_start to better reflect its purpose. It also avoids confusion with the binder buffer concept, e.g. transaction->buffer. No functional changes in this patch. Reviewed-by: Suren Baghdasaryan Signed-off-by: Carlos Llamas Link: https://lore.kernel.org/r/20241210143114.661252-7-cmllamas@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/android/binder.c | 2 +- drivers/android/binder_alloc.c | 28 ++++++++++++++-------------- drivers/android/binder_alloc.h | 4 ++-- drivers/android/binder_alloc_selftest.c | 2 +- drivers/android/binder_trace.h | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) (limited to 'drivers/android/binder_alloc_selftest.c') diff --git a/drivers/android/binder.c b/drivers/android/binder.c index ef353ca13c35..9962c606cabd 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -6374,7 +6374,7 @@ static void print_binder_transaction_ilocked(struct seq_file *m, seq_printf(m, " node %d", buffer->target_node->debug_id); seq_printf(m, " size %zd:%zd offset %lx\n", buffer->data_size, buffer->offsets_size, - proc->alloc.buffer - buffer->user_data); + proc->alloc.vm_start - buffer->user_data); } static void print_binder_work_ilocked(struct seq_file *m, diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index ed79d7c146c8..9cb47e1bc6be 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -61,7 +61,7 @@ static size_t binder_alloc_buffer_size(struct binder_alloc *alloc, struct binder_buffer *buffer) { if (list_is_last(&buffer->entry, &alloc->buffers)) - return alloc->buffer + alloc->buffer_size - buffer->user_data; + return alloc->vm_start + alloc->buffer_size - buffer->user_data; return binder_buffer_next(buffer)->user_data - buffer->user_data; } @@ -203,7 +203,7 @@ static void binder_lru_freelist_add(struct binder_alloc *alloc, size_t index; int ret; - index = (page_addr - alloc->buffer) / PAGE_SIZE; + index = (page_addr - alloc->vm_start) / PAGE_SIZE; page = binder_get_installed_page(alloc, index); if (!page) continue; @@ -305,7 +305,7 @@ static int binder_install_single_page(struct binder_alloc *alloc, FOLL_NOFAULT, &page, NULL); if (npages <= 0) { pr_err("%d: failed to find page at offset %lx\n", - alloc->pid, addr - alloc->buffer); + alloc->pid, addr - alloc->vm_start); ret = -ESRCH; break; } @@ -317,7 +317,7 @@ static int binder_install_single_page(struct binder_alloc *alloc, default: binder_free_page(page); pr_err("%d: %s failed to insert page at offset %lx with %d\n", - alloc->pid, __func__, addr - alloc->buffer, ret); + alloc->pid, __func__, addr - alloc->vm_start, ret); ret = -ENOMEM; break; } @@ -342,7 +342,7 @@ static int binder_install_buffer_pages(struct binder_alloc *alloc, unsigned long index; int ret; - index = (page_addr - alloc->buffer) / PAGE_SIZE; + index = (page_addr - alloc->vm_start) / PAGE_SIZE; if (binder_get_installed_page(alloc, index)) continue; @@ -371,7 +371,7 @@ static void binder_lru_freelist_del(struct binder_alloc *alloc, unsigned long index; bool on_lru; - index = (page_addr - alloc->buffer) / PAGE_SIZE; + index = (page_addr - alloc->vm_start) / PAGE_SIZE; page = binder_get_installed_page(alloc, index); if (page) { @@ -723,8 +723,8 @@ static void binder_free_buf_locked(struct binder_alloc *alloc, BUG_ON(buffer->free); BUG_ON(size > buffer_size); BUG_ON(buffer->transaction != NULL); - BUG_ON(buffer->user_data < alloc->buffer); - BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size); + BUG_ON(buffer->user_data < alloc->vm_start); + BUG_ON(buffer->user_data > alloc->vm_start + alloc->buffer_size); if (buffer->async_transaction) { alloc->free_async_space += buffer_size; @@ -783,7 +783,7 @@ static struct page *binder_alloc_get_page(struct binder_alloc *alloc, pgoff_t *pgoffp) { binder_size_t buffer_space_offset = buffer_offset + - (buffer->user_data - alloc->buffer); + (buffer->user_data - alloc->vm_start); pgoff_t pgoff = buffer_space_offset & ~PAGE_MASK; size_t index = buffer_space_offset >> PAGE_SHIFT; @@ -882,7 +882,7 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, SZ_4M); mutex_unlock(&binder_alloc_mmap_lock); - alloc->buffer = vma->vm_start; + alloc->vm_start = vma->vm_start; alloc->pages = kvcalloc(alloc->buffer_size / PAGE_SIZE, sizeof(alloc->pages[0]), @@ -900,7 +900,7 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, goto err_alloc_buf_struct_failed; } - buffer->user_data = alloc->buffer; + buffer->user_data = alloc->vm_start; list_add(&buffer->entry, &alloc->buffers); buffer->free = 1; binder_insert_free_buffer(alloc, buffer); @@ -915,7 +915,7 @@ err_alloc_buf_struct_failed: kvfree(alloc->pages); alloc->pages = NULL; err_alloc_pages_failed: - alloc->buffer = 0; + alloc->vm_start = 0; mutex_lock(&binder_alloc_mmap_lock); alloc->buffer_size = 0; err_already_mapped: @@ -1016,7 +1016,7 @@ void binder_alloc_print_allocated(struct seq_file *m, buffer = rb_entry(n, struct binder_buffer, rb_node); seq_printf(m, " buffer %d: %lx size %zd:%zd:%zd %s\n", buffer->debug_id, - buffer->user_data - alloc->buffer, + buffer->user_data - alloc->vm_start, buffer->data_size, buffer->offsets_size, buffer->extra_buffers_size, buffer->transaction ? "active" : "delivered"); @@ -1121,7 +1121,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, goto err_get_alloc_mutex_failed; index = mdata->page_index; - page_addr = alloc->buffer + index * PAGE_SIZE; + page_addr = alloc->vm_start + index * PAGE_SIZE; vma = vma_lookup(mm, page_addr); /* diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h index 3ebb12afd4de..feecd7414241 100644 --- a/drivers/android/binder_alloc.h +++ b/drivers/android/binder_alloc.h @@ -83,7 +83,7 @@ static inline struct list_head *page_to_lru(struct page *p) * struct binder_alloc - per-binder proc state for binder allocator * @mutex: protects binder_alloc fields * @mm: copy of task->mm (invariant after open) - * @buffer: base of per-proc address space mapped via mmap + * @vm_start: base of per-proc address space mapped via mmap * @buffers: list of all buffers for this proc * @free_buffers: rb tree of buffers available for allocation * sorted by size @@ -107,7 +107,7 @@ static inline struct list_head *page_to_lru(struct page *p) struct binder_alloc { struct mutex mutex; struct mm_struct *mm; - unsigned long buffer; + unsigned long vm_start; struct list_head buffers; struct rb_root free_buffers; struct rb_root allocated_buffers; diff --git a/drivers/android/binder_alloc_selftest.c b/drivers/android/binder_alloc_selftest.c index 6a64847a8555..c88735c54848 100644 --- a/drivers/android/binder_alloc_selftest.c +++ b/drivers/android/binder_alloc_selftest.c @@ -104,7 +104,7 @@ static bool check_buffer_pages_allocated(struct binder_alloc *alloc, end = PAGE_ALIGN(buffer->user_data + size); page_addr = buffer->user_data; for (; page_addr < end; page_addr += PAGE_SIZE) { - page_index = (page_addr - alloc->buffer) / PAGE_SIZE; + page_index = (page_addr - alloc->vm_start) / PAGE_SIZE; if (!alloc->pages[page_index] || !list_empty(page_to_lru(alloc->pages[page_index]))) { pr_err("expect alloc but is %s at page index %d\n", diff --git a/drivers/android/binder_trace.h b/drivers/android/binder_trace.h index fe38c6fc65d0..16de1b9e72f7 100644 --- a/drivers/android/binder_trace.h +++ b/drivers/android/binder_trace.h @@ -328,7 +328,7 @@ TRACE_EVENT(binder_update_page_range, TP_fast_assign( __entry->proc = alloc->pid; __entry->allocate = allocate; - __entry->offset = start - alloc->buffer; + __entry->offset = start - alloc->vm_start; __entry->size = end - start; ), TP_printk("proc=%d allocate=%d offset=%zu size=%zu", -- cgit v1.2.3