From 3367aa7d74d240261de2543ddb35531ccad9d884 Mon Sep 17 00:00:00 2001 From: Javier Martinez Canillas Date: Wed, 11 May 2022 13:30:39 +0200 Subject: fbdev: Restart conflicting fb removal loop when unregistering devices Drivers that want to remove registered conflicting framebuffers prior to register their own framebuffer, call to remove_conflicting_framebuffers(). This function takes the registration_lock mutex, to prevent a race when drivers register framebuffer devices. But if a conflicting framebuffer device is found, the underlaying platform device is unregistered and this will lead to the platform driver .remove callback to be called. Which in turn will call to unregister_framebuffer() that takes the same lock. To prevent this, a struct fb_info.forced_out field was used as indication to unregister_framebuffer() whether the mutex has to be grabbed or not. But this could be unsafe, since the fbdev core is making assumptions about what drivers may or may not do in their .remove callbacks. Allowing to run these callbacks with the registration_lock held can cause deadlocks, since the fbdev core has no control over what drivers do in their removal path. A better solution is to drop the lock before platform_device_unregister(), so unregister_framebuffer() can take it when called from the fbdev driver. The lock is acquired again after the device has been unregistered and at this point the removal loop can be restarted. Since the conflicting framebuffer device has already been removed, the loop would just finish when no more conflicting framebuffers are found. Suggested-by: Daniel Vetter Signed-off-by: Javier Martinez Canillas Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20220511113039.1252432-1-javierm@redhat.com --- include/linux/fb.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 69c67c70fa78..bbe1e4571899 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -511,7 +511,6 @@ struct fb_info { } *apertures; bool skip_vt_switch; /* no VT switch on suspend/resume required */ - bool forced_out; /* set when being removed by another driver */ }; static inline struct apertures_struct *alloc_apertures(unsigned int max_num) { -- cgit v1.2.3 From 93984f19e7bce4c18084a6ef3dacafb155b806ed Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 29 Apr 2022 21:00:23 +0000 Subject: KVM: Fully serialize gfn=>pfn cache refresh via mutex Protect gfn=>pfn cache refresh with a mutex to fully serialize refreshes. The refresh logic doesn't protect against - concurrent unmaps, or refreshes with different GPAs (which may or may not happen in practice, for example if a cache is only used under vcpu->mutex; but it's allowed in the code) - a false negative on the memslot generation. If the first refresh sees a stale memslot generation, it will refresh the hva and generation before moving on to the hva=>pfn translation. If it then drops gpc->lock, a different user of the cache can come along, acquire gpc->lock, see that the memslot generation is fresh, and skip the hva=>pfn update due to the userspace address also matching (because it too was updated). The refresh path can already sleep during hva=>pfn resolution, so wrap the refresh with a mutex to ensure that any given refresh runs to completion before other callers can start their refresh. Cc: stable@vger.kernel.org Cc: Lai Jiangshan Signed-off-by: Sean Christopherson Message-Id: <20220429210025.3293691-7-seanjc@google.com> Signed-off-by: Paolo Bonzini --- include/linux/kvm_types.h | 2 ++ virt/kvm/pfncache.c | 12 ++++++++++++ 2 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index ac1ebb37a0ff..f328a01db4fe 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -19,6 +19,7 @@ struct kvm_memslots; enum kvm_mr_change; #include +#include #include #include @@ -69,6 +70,7 @@ struct gfn_to_pfn_cache { struct kvm_vcpu *vcpu; struct list_head list; rwlock_t lock; + struct mutex refresh_lock; void *khva; kvm_pfn_t pfn; enum pfn_cache_usage usage; diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 05cb0bcbf662..f610d3945b69 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -157,6 +157,13 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, if (page_offset + len > PAGE_SIZE) return -EINVAL; + /* + * If another task is refreshing the cache, wait for it to complete. + * There is no guarantee that concurrent refreshes will see the same + * gpa, memslots generation, etc..., so they must be fully serialized. + */ + mutex_lock(&gpc->refresh_lock); + write_lock_irq(&gpc->lock); old_pfn = gpc->pfn; @@ -248,6 +255,8 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, out: write_unlock_irq(&gpc->lock); + mutex_unlock(&gpc->refresh_lock); + gpc_release_pfn_and_khva(kvm, old_pfn, old_khva); return ret; @@ -259,6 +268,7 @@ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) void *old_khva; kvm_pfn_t old_pfn; + mutex_lock(&gpc->refresh_lock); write_lock_irq(&gpc->lock); gpc->valid = false; @@ -274,6 +284,7 @@ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) gpc->pfn = KVM_PFN_ERR_FAULT; write_unlock_irq(&gpc->lock); + mutex_unlock(&gpc->refresh_lock); gpc_release_pfn_and_khva(kvm, old_pfn, old_khva); } @@ -288,6 +299,7 @@ int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, if (!gpc->active) { rwlock_init(&gpc->lock); + mutex_init(&gpc->refresh_lock); gpc->khva = NULL; gpc->pfn = KVM_PFN_ERR_FAULT; -- cgit v1.2.3 From 01357a5a45ed8eb9543183f5c9c6713ae60fc1f3 Mon Sep 17 00:00:00 2001 From: Christian König Date: Sun, 24 Apr 2022 16:55:14 +0200 Subject: dma-buf: cleanup dma_fence_unwrap implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the code from the inline functions into exported functions. Signed-off-by: Christian König Acked-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20220518135844.3338-3-christian.koenig@amd.com --- drivers/dma-buf/Makefile | 2 +- drivers/dma-buf/dma-fence-unwrap.c | 59 ++++++++++++++++++++++++++++++++++++++ include/linux/dma-fence-unwrap.h | 52 +++------------------------------ 3 files changed, 64 insertions(+), 49 deletions(-) create mode 100644 drivers/dma-buf/dma-fence-unwrap.c (limited to 'include/linux') diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile index 4c9eb53ba3f8..70ec901edf2c 100644 --- a/drivers/dma-buf/Makefile +++ b/drivers/dma-buf/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y := dma-buf.o dma-fence.o dma-fence-array.o dma-fence-chain.o \ - dma-resv.o + dma-fence-unwrap.o dma-resv.o obj-$(CONFIG_DMABUF_HEAPS) += dma-heap.o obj-$(CONFIG_DMABUF_HEAPS) += heaps/ obj-$(CONFIG_SYNC_FILE) += sync_file.o diff --git a/drivers/dma-buf/dma-fence-unwrap.c b/drivers/dma-buf/dma-fence-unwrap.c new file mode 100644 index 000000000000..711be125428c --- /dev/null +++ b/drivers/dma-buf/dma-fence-unwrap.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * dma-fence-util: misc functions for dma_fence objects + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. + * Authors: + * Christian König + */ + +#include +#include +#include +#include + +/* Internal helper to start new array iteration, don't use directly */ +static struct dma_fence * +__dma_fence_unwrap_array(struct dma_fence_unwrap *cursor) +{ + cursor->array = dma_fence_chain_contained(cursor->chain); + cursor->index = 0; + return dma_fence_array_first(cursor->array); +} + +/** + * dma_fence_unwrap_first - return the first fence from fence containers + * @head: the entrypoint into the containers + * @cursor: current position inside the containers + * + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the + * first fence. + */ +struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head, + struct dma_fence_unwrap *cursor) +{ + cursor->chain = dma_fence_get(head); + return __dma_fence_unwrap_array(cursor); +} +EXPORT_SYMBOL_GPL(dma_fence_unwrap_first); + +/** + * dma_fence_unwrap_next - return the next fence from a fence containers + * @cursor: current position inside the containers + * + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return + * the next fence from them. + */ +struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) +{ + struct dma_fence *tmp; + + ++cursor->index; + tmp = dma_fence_array_next(cursor->array, cursor->index); + if (tmp) + return tmp; + + cursor->chain = dma_fence_chain_walk(cursor->chain); + return __dma_fence_unwrap_array(cursor); +} +EXPORT_SYMBOL_GPL(dma_fence_unwrap_next); diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h index 77e335a1bcac..e7c219da4ed7 100644 --- a/include/linux/dma-fence-unwrap.h +++ b/include/linux/dma-fence-unwrap.h @@ -1,7 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * fence-chain: chain fences together in a timeline - * * Copyright (C) 2022 Advanced Micro Devices, Inc. * Authors: * Christian König @@ -10,8 +8,7 @@ #ifndef __LINUX_DMA_FENCE_UNWRAP_H #define __LINUX_DMA_FENCE_UNWRAP_H -#include -#include +struct dma_fence; /** * struct dma_fence_unwrap - cursor into the container structure @@ -33,50 +30,9 @@ struct dma_fence_unwrap { unsigned int index; }; -/* Internal helper to start new array iteration, don't use directly */ -static inline struct dma_fence * -__dma_fence_unwrap_array(struct dma_fence_unwrap * cursor) -{ - cursor->array = dma_fence_chain_contained(cursor->chain); - cursor->index = 0; - return dma_fence_array_first(cursor->array); -} - -/** - * dma_fence_unwrap_first - return the first fence from fence containers - * @head: the entrypoint into the containers - * @cursor: current position inside the containers - * - * Unwraps potential dma_fence_chain/dma_fence_array containers and return the - * first fence. - */ -static inline struct dma_fence * -dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor) -{ - cursor->chain = dma_fence_get(head); - return __dma_fence_unwrap_array(cursor); -} - -/** - * dma_fence_unwrap_next - return the next fence from a fence containers - * @cursor: current position inside the containers - * - * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return - * the next fence from them. - */ -static inline struct dma_fence * -dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) -{ - struct dma_fence *tmp; - - ++cursor->index; - tmp = dma_fence_array_next(cursor->array, cursor->index); - if (tmp) - return tmp; - - cursor->chain = dma_fence_chain_walk(cursor->chain); - return __dma_fence_unwrap_array(cursor); -} +struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head, + struct dma_fence_unwrap *cursor); +struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor); /** * dma_fence_unwrap_for_each - iterate over all fences in containers -- cgit v1.2.3 From 8f61973718485f3e89bc4f408f929048b7b47c83 Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 4 May 2022 13:01:29 +0200 Subject: dma-buf: return only unsignaled fences in dma_fence_unwrap_for_each v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dma_fence_chain containers cleanup signaled fences automatically, so filter those out from arrays as well. v2: fix missing walk over the array v3: massively simplify the patch and actually update the description. Signed-off-by: Christian König Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20220518135844.3338-4-christian.koenig@amd.com --- include/linux/dma-fence-unwrap.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h index e7c219da4ed7..a4d342fef8e0 100644 --- a/include/linux/dma-fence-unwrap.h +++ b/include/linux/dma-fence-unwrap.h @@ -43,9 +43,13 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor); * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all * potential fences in them. If @head is just a normal fence only that one is * returned. + * + * Note that signalled fences are opportunistically filtered out, which + * means the iteration is potentially over no fence at all. */ #define dma_fence_unwrap_for_each(fence, cursor, head) \ for (fence = dma_fence_unwrap_first(head, cursor); fence; \ - fence = dma_fence_unwrap_next(cursor)) + fence = dma_fence_unwrap_next(cursor)) \ + if (!dma_fence_is_signaled(fence)) #endif -- cgit v1.2.3 From 245a4a7b531cffb41233a716497c25b06835cf4b Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 25 Apr 2022 14:22:12 +0200 Subject: dma-buf: generalize dma_fence unwrap & merging v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a dma_fence_unwrap_merge() macro which allows to unwrap fences which potentially can be containers as well and then merge them back together into a flat dma_fence_array. v2: rename the function, add some more comments about how the wrapper is used, move filtering of signaled fences into the unwrap iterator, add complex selftest which covers more cases. v3: fix signaled fence filtering once more Signed-off-by: Christian König Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20220518135844.3338-5-christian.koenig@amd.com --- drivers/dma-buf/dma-fence-unwrap.c | 103 +++++++++++++++++++++++++++++ drivers/dma-buf/st-dma-fence-unwrap.c | 109 +++++++++++++++++++++++++++++++ drivers/dma-buf/sync_file.c | 119 ++-------------------------------- include/linux/dma-fence-unwrap.h | 24 +++++++ 4 files changed, 242 insertions(+), 113 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma-buf/dma-fence-unwrap.c b/drivers/dma-buf/dma-fence-unwrap.c index 711be125428c..502a65ea6d44 100644 --- a/drivers/dma-buf/dma-fence-unwrap.c +++ b/drivers/dma-buf/dma-fence-unwrap.c @@ -11,6 +11,7 @@ #include #include #include +#include /* Internal helper to start new array iteration, don't use directly */ static struct dma_fence * @@ -57,3 +58,105 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) return __dma_fence_unwrap_array(cursor); } EXPORT_SYMBOL_GPL(dma_fence_unwrap_next); + +/* Implementation for the dma_fence_merge() marco, don't use directly */ +struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, + struct dma_fence **fences, + struct dma_fence_unwrap *iter) +{ + struct dma_fence_array *result; + struct dma_fence *tmp, **array; + unsigned int i; + size_t count; + + count = 0; + for (i = 0; i < num_fences; ++i) { + dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) + ++count; + } + + if (count == 0) + return dma_fence_get_stub(); + + array = kmalloc_array(count, sizeof(*array), GFP_KERNEL); + if (!array) + return NULL; + + /* + * This trashes the input fence array and uses it as position for the + * following merge loop. This works because the dma_fence_merge() + * wrapper macro is creating this temporary array on the stack together + * with the iterators. + */ + for (i = 0; i < num_fences; ++i) + fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]); + + count = 0; + do { + unsigned int sel; + +restart: + tmp = NULL; + for (i = 0; i < num_fences; ++i) { + struct dma_fence *next; + + while (fences[i] && dma_fence_is_signaled(fences[i])) + fences[i] = dma_fence_unwrap_next(&iter[i]); + + next = fences[i]; + if (!next) + continue; + + /* + * We can't guarantee that inpute fences are ordered by + * context, but it is still quite likely when this + * function is used multiple times. So attempt to order + * the fences by context as we pass over them and merge + * fences with the same context. + */ + if (!tmp || tmp->context > next->context) { + tmp = next; + sel = i; + + } else if (tmp->context < next->context) { + continue; + + } else if (dma_fence_is_later(tmp, next)) { + fences[i] = dma_fence_unwrap_next(&iter[i]); + goto restart; + } else { + fences[sel] = dma_fence_unwrap_next(&iter[sel]); + goto restart; + } + } + + if (tmp) { + array[count++] = dma_fence_get(tmp); + fences[sel] = dma_fence_unwrap_next(&iter[sel]); + } + } while (tmp); + + if (count == 0) { + tmp = dma_fence_get_stub(); + goto return_tmp; + } + + if (count == 1) { + tmp = array[0]; + goto return_tmp; + } + + result = dma_fence_array_create(count, array, + dma_fence_context_alloc(1), + 1, false); + if (!result) { + tmp = NULL; + goto return_tmp; + } + return &result->base; + +return_tmp: + kfree(array); + return tmp; +} +EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge); diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c index e20c5a7dcfe4..4105d5ea8dde 100644 --- a/drivers/dma-buf/st-dma-fence-unwrap.c +++ b/drivers/dma-buf/st-dma-fence-unwrap.c @@ -238,6 +238,113 @@ static int unwrap_chain_array(void *arg) return err; } +static int unwrap_merge(void *arg) +{ + struct dma_fence *fence, *f1, *f2, *f3; + struct dma_fence_unwrap iter; + int err = 0; + + f1 = mock_fence(); + if (!f1) + return -ENOMEM; + + f2 = mock_fence(); + if (!f2) { + err = -ENOMEM; + goto error_put_f1; + } + + f3 = dma_fence_unwrap_merge(f1, f2); + if (!f3) { + err = -ENOMEM; + goto error_put_f2; + } + + dma_fence_unwrap_for_each(fence, &iter, f3) { + if (fence == f1) { + dma_fence_put(f1); + f1 = NULL; + } else if (fence == f2) { + dma_fence_put(f2); + f2 = NULL; + } else { + pr_err("Unexpected fence!\n"); + err = -EINVAL; + } + } + + if (f1 || f2) { + pr_err("Not all fences seen!\n"); + err = -EINVAL; + } + + dma_fence_put(f3); +error_put_f2: + dma_fence_put(f2); +error_put_f1: + dma_fence_put(f1); + return err; +} + +static int unwrap_merge_complex(void *arg) +{ + struct dma_fence *fence, *f1, *f2, *f3, *f4, *f5; + struct dma_fence_unwrap iter; + int err = -ENOMEM; + + f1 = mock_fence(); + if (!f1) + return -ENOMEM; + + f2 = mock_fence(); + if (!f2) + goto error_put_f1; + + f3 = dma_fence_unwrap_merge(f1, f2); + if (!f3) + goto error_put_f2; + + /* The resulting array has the fences in reverse */ + f4 = dma_fence_unwrap_merge(f2, f1); + if (!f4) + goto error_put_f3; + + /* Signaled fences should be filtered, the two arrays merged. */ + f5 = dma_fence_unwrap_merge(f3, f4, dma_fence_get_stub()); + if (!f5) + goto error_put_f4; + + err = 0; + dma_fence_unwrap_for_each(fence, &iter, f5) { + if (fence == f1) { + dma_fence_put(f1); + f1 = NULL; + } else if (fence == f2) { + dma_fence_put(f2); + f2 = NULL; + } else { + pr_err("Unexpected fence!\n"); + err = -EINVAL; + } + } + + if (f1 || f2) { + pr_err("Not all fences seen!\n"); + err = -EINVAL; + } + + dma_fence_put(f5); +error_put_f4: + dma_fence_put(f4); +error_put_f3: + dma_fence_put(f3); +error_put_f2: + dma_fence_put(f2); +error_put_f1: + dma_fence_put(f1); + return err; +} + int dma_fence_unwrap(void) { static const struct subtest tests[] = { @@ -245,6 +352,8 @@ int dma_fence_unwrap(void) SUBTEST(unwrap_array), SUBTEST(unwrap_chain), SUBTEST(unwrap_chain_array), + SUBTEST(unwrap_merge), + SUBTEST(unwrap_merge_complex), }; return subtests(tests, NULL); diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c index 0fe564539166..3ebec19a8e02 100644 --- a/drivers/dma-buf/sync_file.c +++ b/drivers/dma-buf/sync_file.c @@ -146,50 +146,6 @@ char *sync_file_get_name(struct sync_file *sync_file, char *buf, int len) return buf; } -static int sync_file_set_fence(struct sync_file *sync_file, - struct dma_fence **fences, int num_fences) -{ - struct dma_fence_array *array; - - /* - * The reference for the fences in the new sync_file and held - * in add_fence() during the merge procedure, so for num_fences == 1 - * we already own a new reference to the fence. For num_fence > 1 - * we own the reference of the dma_fence_array creation. - */ - - if (num_fences == 0) { - sync_file->fence = dma_fence_get_stub(); - kfree(fences); - - } else if (num_fences == 1) { - sync_file->fence = fences[0]; - kfree(fences); - - } else { - array = dma_fence_array_create(num_fences, fences, - dma_fence_context_alloc(1), - 1, false); - if (!array) - return -ENOMEM; - - sync_file->fence = &array->base; - } - - return 0; -} - -static void add_fence(struct dma_fence **fences, - int *i, struct dma_fence *fence) -{ - fences[*i] = fence; - - if (!dma_fence_is_signaled(fence)) { - dma_fence_get(fence); - (*i)++; - } -} - /** * sync_file_merge() - merge two sync_files * @name: name of new fence @@ -203,84 +159,21 @@ static void add_fence(struct dma_fence **fences, static struct sync_file *sync_file_merge(const char *name, struct sync_file *a, struct sync_file *b) { - struct dma_fence *a_fence, *b_fence, **fences; - struct dma_fence_unwrap a_iter, b_iter; - unsigned int index, num_fences; struct sync_file *sync_file; + struct dma_fence *fence; sync_file = sync_file_alloc(); if (!sync_file) return NULL; - num_fences = 0; - dma_fence_unwrap_for_each(a_fence, &a_iter, a->fence) - ++num_fences; - dma_fence_unwrap_for_each(b_fence, &b_iter, b->fence) - ++num_fences; - - if (num_fences > INT_MAX) - goto err_free_sync_file; - - fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL); - if (!fences) - goto err_free_sync_file; - - /* - * We can't guarantee that fences in both a and b are ordered, but it is - * still quite likely. - * - * So attempt to order the fences as we pass over them and merge fences - * with the same context. - */ - - index = 0; - for (a_fence = dma_fence_unwrap_first(a->fence, &a_iter), - b_fence = dma_fence_unwrap_first(b->fence, &b_iter); - a_fence || b_fence; ) { - - if (!b_fence) { - add_fence(fences, &index, a_fence); - a_fence = dma_fence_unwrap_next(&a_iter); - - } else if (!a_fence) { - add_fence(fences, &index, b_fence); - b_fence = dma_fence_unwrap_next(&b_iter); - - } else if (a_fence->context < b_fence->context) { - add_fence(fences, &index, a_fence); - a_fence = dma_fence_unwrap_next(&a_iter); - - } else if (b_fence->context < a_fence->context) { - add_fence(fences, &index, b_fence); - b_fence = dma_fence_unwrap_next(&b_iter); - - } else if (__dma_fence_is_later(a_fence->seqno, b_fence->seqno, - a_fence->ops)) { - add_fence(fences, &index, a_fence); - a_fence = dma_fence_unwrap_next(&a_iter); - b_fence = dma_fence_unwrap_next(&b_iter); - - } else { - add_fence(fences, &index, b_fence); - a_fence = dma_fence_unwrap_next(&a_iter); - b_fence = dma_fence_unwrap_next(&b_iter); - } + fence = dma_fence_unwrap_merge(a->fence, b->fence); + if (!fence) { + fput(sync_file->file); + return NULL; } - - if (sync_file_set_fence(sync_file, fences, index) < 0) - goto err_put_fences; - + sync_file->fence = fence; strlcpy(sync_file->user_name, name, sizeof(sync_file->user_name)); return sync_file; - -err_put_fences: - while (index) - dma_fence_put(fences[--index]); - kfree(fences); - -err_free_sync_file: - fput(sync_file->file); - return NULL; } static int sync_file_release(struct inode *inode, struct file *file) diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h index a4d342fef8e0..390de1ee9d35 100644 --- a/include/linux/dma-fence-unwrap.h +++ b/include/linux/dma-fence-unwrap.h @@ -52,4 +52,28 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor); fence = dma_fence_unwrap_next(cursor)) \ if (!dma_fence_is_signaled(fence)) +struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, + struct dma_fence **fences, + struct dma_fence_unwrap *cursors); + +/** + * dma_fence_unwrap_merge - unwrap and merge fences + * + * All fences given as parameters are unwrapped and merged back together as flat + * dma_fence_array. Useful if multiple containers need to be merged together. + * + * Implemented as a macro to allocate the necessary arrays on the stack and + * account the stack frame size to the caller. + * + * Returns NULL on memory allocation failure, a dma_fence object representing + * all the given fences otherwise. + */ +#define dma_fence_unwrap_merge(...) \ + ({ \ + struct dma_fence *__f[] = { __VA_ARGS__ }; \ + struct dma_fence_unwrap __c[ARRAY_SIZE(__f)]; \ + \ + __dma_fence_unwrap_merge(ARRAY_SIZE(__f), __f, __c); \ + }) + #endif -- cgit v1.2.3 From d8616ee2affcff37c5d315310da557a694a3303d Mon Sep 17 00:00:00 2001 From: Wang Yufen Date: Tue, 24 May 2022 15:53:11 +0800 Subject: bpf, sockmap: Fix sk->sk_forward_alloc warn_on in sk_stream_kill_queues During TCP sockmap redirect pressure test, the following warning is triggered: WARNING: CPU: 3 PID: 2145 at net/core/stream.c:205 sk_stream_kill_queues+0xbc/0xd0 CPU: 3 PID: 2145 Comm: iperf Kdump: loaded Tainted: G W 5.10.0+ #9 Call Trace: inet_csk_destroy_sock+0x55/0x110 inet_csk_listen_stop+0xbb/0x380 tcp_close+0x41b/0x480 inet_release+0x42/0x80 __sock_release+0x3d/0xa0 sock_close+0x11/0x20 __fput+0x9d/0x240 task_work_run+0x62/0x90 exit_to_user_mode_prepare+0x110/0x120 syscall_exit_to_user_mode+0x27/0x190 entry_SYSCALL_64_after_hwframe+0x44/0xa9 The reason we observed is that: When the listener is closing, a connection may have completed the three-way handshake but not accepted, and the client has sent some packets. The child sks in accept queue release by inet_child_forget()->inet_csk_destroy_sock(), but psocks of child sks have not released. To fix, add sock_map_destroy to release psocks. Signed-off-by: Wang Yufen Signed-off-by: Daniel Borkmann Signed-off-by: Andrii Nakryiko Acked-by: Jakub Sitnicki Acked-by: John Fastabend Link: https://lore.kernel.org/bpf/20220524075311.649153-1-wangyufen@huawei.com --- include/linux/bpf.h | 1 + include/linux/skmsg.h | 1 + net/core/skmsg.c | 1 + net/core/sock_map.c | 23 +++++++++++++++++++++++ net/ipv4/tcp_bpf.c | 1 + 5 files changed, 27 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2b914a56a2c5..8e6092d0ea95 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2104,6 +2104,7 @@ int sock_map_bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr); void sock_map_unhash(struct sock *sk); +void sock_map_destroy(struct sock *sk); void sock_map_close(struct sock *sk, long timeout); #else static inline int bpf_prog_offload_init(struct bpf_prog *prog, diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index c5a2d6f50f25..153b6dec9b6a 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -95,6 +95,7 @@ struct sk_psock { spinlock_t link_lock; refcount_t refcnt; void (*saved_unhash)(struct sock *sk); + void (*saved_destroy)(struct sock *sk); void (*saved_close)(struct sock *sk, long timeout); void (*saved_write_space)(struct sock *sk); void (*saved_data_ready)(struct sock *sk); diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 22b983ade0e7..7e03f96e441b 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -715,6 +715,7 @@ struct sk_psock *sk_psock_init(struct sock *sk, int node) psock->eval = __SK_NONE; psock->sk_proto = prot; psock->saved_unhash = prot->unhash; + psock->saved_destroy = prot->destroy; psock->saved_close = prot->close; psock->saved_write_space = sk->sk_write_space; diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 81d4b4756a02..9f08ccfaf6da 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1561,6 +1561,29 @@ void sock_map_unhash(struct sock *sk) } EXPORT_SYMBOL_GPL(sock_map_unhash); +void sock_map_destroy(struct sock *sk) +{ + void (*saved_destroy)(struct sock *sk); + struct sk_psock *psock; + + rcu_read_lock(); + psock = sk_psock_get(sk); + if (unlikely(!psock)) { + rcu_read_unlock(); + if (sk->sk_prot->destroy) + sk->sk_prot->destroy(sk); + return; + } + + saved_destroy = psock->saved_destroy; + sock_map_remove_links(sk, psock); + rcu_read_unlock(); + sk_psock_stop(psock, true); + sk_psock_put(sk, psock); + saved_destroy(sk); +} +EXPORT_SYMBOL_GPL(sock_map_destroy); + void sock_map_close(struct sock *sk, long timeout) { void (*saved_close)(struct sock *sk, long timeout); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index be3947e70fec..38550bb1b90b 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -540,6 +540,7 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], struct proto *base) { prot[TCP_BPF_BASE] = *base; + prot[TCP_BPF_BASE].destroy = sock_map_destroy; prot[TCP_BPF_BASE].close = sock_map_close; prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; -- cgit v1.2.3 From 8d5976089c97a4beeeda4de59e2fba9862946893 Mon Sep 17 00:00:00 2001 From: Xiang wangx Date: Mon, 6 Jun 2022 10:23:13 +0800 Subject: platform/chrome: cros_ec_commands: Fix syntax errors in comments Delete the redundant word 'using'. Signed-off-by: Xiang wangx Reviewed-by: Tzung-Bi Shih Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220606022313.22912-1-wangxiang@cdjrlc.com --- include/linux/platform_data/cros_ec_commands.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 8cfa8cfca77e..e59f51c41a1c 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -787,7 +787,7 @@ struct ec_host_response { * * Packets always start with a request or response header. They are followed * by data_len bytes of data. If the data_crc_present flag is set, the data - * bytes are followed by a CRC-8 of that data, using using x^8 + x^2 + x + 1 + * bytes are followed by a CRC-8 of that data, using x^8 + x^2 + x + 1 * polynomial. * * Host algorithm when sending a request q: -- cgit v1.2.3 From 657f8bd88cb5a968d907fd1c891cee52dc156caa Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sat, 21 May 2022 13:10:23 +0200 Subject: spi: fix typo in comment Spelling mistake (triple letters) in comment. Detected with the help of Coccinelle. Signed-off-by: Julia Lawall Link: https://lore.kernel.org/r/20220521111145.81697-13-Julia.Lawall@inria.fr Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index d361ba26203b..eadfd3ba3cbc 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -1127,7 +1127,7 @@ spi_max_transfer_size(struct spi_device *spi) if (ctlr->max_transfer_size) tr_max = ctlr->max_transfer_size(spi); - /* transfer size limit must not be greater than messsage size limit */ + /* transfer size limit must not be greater than message size limit */ return min(tr_max, msg_max); } -- cgit v1.2.3 From 6598b91b5ac32bc756d7c3000a31f775d4ead1c4 Mon Sep 17 00:00:00 2001 From: David Jander Date: Tue, 24 May 2022 11:18:08 +0200 Subject: spi: spi.c: Convert statistics to per-cpu u64_stats_t This change gives a dramatic performance improvement in the hot path, since many costly spin_lock_irqsave() calls can be avoided. On an i.MX8MM system with a MCP2518FD CAN controller connected via SPI, the time the driver takes to handle interrupts, or in other words the time the IRQ line of the CAN controller stays low is mainly dominated by the time it takes to do 3 relatively short sync SPI transfers. The effect of this patch is a reduction of this time from 136us down to only 98us. Suggested-by: Andrew Lunn Signed-off-by: David Jander Link: https://lore.kernel.org/r/20220524091808.2269898-1-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 143 +++++++++++++++++++++++++++++++++--------------- include/linux/spi/spi.h | 52 ++++++++++-------- 2 files changed, 127 insertions(+), 68 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index ea09d1b42bf6..d94822bf3cec 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -33,6 +33,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -49,6 +50,7 @@ static void spidev_release(struct device *dev) spi_controller_put(spi->controller); kfree(spi->driver_override); + free_percpu(spi->pcpu_statistics); kfree(spi); } @@ -93,6 +95,47 @@ static ssize_t driver_override_show(struct device *dev, } static DEVICE_ATTR_RW(driver_override); +static struct spi_statistics *spi_alloc_pcpu_stats(struct device *dev) +{ + struct spi_statistics __percpu *pcpu_stats; + + if (dev) + pcpu_stats = devm_alloc_percpu(dev, struct spi_statistics); + else + pcpu_stats = alloc_percpu_gfp(struct spi_statistics, GFP_KERNEL); + + if (pcpu_stats) { + int cpu; + + for_each_possible_cpu(cpu) { + struct spi_statistics *stat; + + stat = per_cpu_ptr(pcpu_stats, cpu); + u64_stats_init(&stat->syncp); + } + } + return pcpu_stats; +} + +#define spi_pcpu_stats_totalize(ret, in, field) \ +do { \ + int i; \ + ret = 0; \ + for_each_possible_cpu(i) { \ + const struct spi_statistics *pcpu_stats; \ + u64 inc; \ + unsigned int start; \ + pcpu_stats = per_cpu_ptr(in, i); \ + do { \ + start = u64_stats_fetch_begin_irq( \ + &pcpu_stats->syncp); \ + inc = u64_stats_read(&pcpu_stats->field); \ + } while (u64_stats_fetch_retry_irq( \ + &pcpu_stats->syncp, start)); \ + ret += inc; \ + } \ +} while (0) + #define SPI_STATISTICS_ATTRS(field, file) \ static ssize_t spi_controller_##field##_show(struct device *dev, \ struct device_attribute *attr, \ @@ -100,7 +143,7 @@ static ssize_t spi_controller_##field##_show(struct device *dev, \ { \ struct spi_controller *ctlr = container_of(dev, \ struct spi_controller, dev); \ - return spi_statistics_##field##_show(&ctlr->statistics, buf); \ + return spi_statistics_##field##_show(ctlr->pcpu_statistics, buf); \ } \ static struct device_attribute dev_attr_spi_controller_##field = { \ .attr = { .name = file, .mode = 0444 }, \ @@ -111,47 +154,46 @@ static ssize_t spi_device_##field##_show(struct device *dev, \ char *buf) \ { \ struct spi_device *spi = to_spi_device(dev); \ - return spi_statistics_##field##_show(&spi->statistics, buf); \ + return spi_statistics_##field##_show(spi->pcpu_statistics, buf); \ } \ static struct device_attribute dev_attr_spi_device_##field = { \ .attr = { .name = file, .mode = 0444 }, \ .show = spi_device_##field##_show, \ } -#define SPI_STATISTICS_SHOW_NAME(name, file, field, format_string) \ +#define SPI_STATISTICS_SHOW_NAME(name, file, field) \ static ssize_t spi_statistics_##name##_show(struct spi_statistics *stat, \ char *buf) \ { \ - unsigned long flags; \ ssize_t len; \ - spin_lock_irqsave(&stat->lock, flags); \ - len = sysfs_emit(buf, format_string "\n", stat->field); \ - spin_unlock_irqrestore(&stat->lock, flags); \ + u64 val; \ + spi_pcpu_stats_totalize(val, stat, field); \ + len = sysfs_emit(buf, "%llu\n", val); \ return len; \ } \ SPI_STATISTICS_ATTRS(name, file) -#define SPI_STATISTICS_SHOW(field, format_string) \ +#define SPI_STATISTICS_SHOW(field) \ SPI_STATISTICS_SHOW_NAME(field, __stringify(field), \ - field, format_string) + field) -SPI_STATISTICS_SHOW(messages, "%lu"); -SPI_STATISTICS_SHOW(transfers, "%lu"); -SPI_STATISTICS_SHOW(errors, "%lu"); -SPI_STATISTICS_SHOW(timedout, "%lu"); +SPI_STATISTICS_SHOW(messages); +SPI_STATISTICS_SHOW(transfers); +SPI_STATISTICS_SHOW(errors); +SPI_STATISTICS_SHOW(timedout); -SPI_STATISTICS_SHOW(spi_sync, "%lu"); -SPI_STATISTICS_SHOW(spi_sync_immediate, "%lu"); -SPI_STATISTICS_SHOW(spi_async, "%lu"); +SPI_STATISTICS_SHOW(spi_sync); +SPI_STATISTICS_SHOW(spi_sync_immediate); +SPI_STATISTICS_SHOW(spi_async); -SPI_STATISTICS_SHOW(bytes, "%llu"); -SPI_STATISTICS_SHOW(bytes_rx, "%llu"); -SPI_STATISTICS_SHOW(bytes_tx, "%llu"); +SPI_STATISTICS_SHOW(bytes); +SPI_STATISTICS_SHOW(bytes_rx); +SPI_STATISTICS_SHOW(bytes_tx); #define SPI_STATISTICS_TRANSFER_BYTES_HISTO(index, number) \ SPI_STATISTICS_SHOW_NAME(transfer_bytes_histo##index, \ "transfer_bytes_histo_" number, \ - transfer_bytes_histo[index], "%lu") + transfer_bytes_histo[index]) SPI_STATISTICS_TRANSFER_BYTES_HISTO(0, "0-1"); SPI_STATISTICS_TRANSFER_BYTES_HISTO(1, "2-3"); SPI_STATISTICS_TRANSFER_BYTES_HISTO(2, "4-7"); @@ -170,7 +212,7 @@ SPI_STATISTICS_TRANSFER_BYTES_HISTO(14, "16384-32767"); SPI_STATISTICS_TRANSFER_BYTES_HISTO(15, "32768-65535"); SPI_STATISTICS_TRANSFER_BYTES_HISTO(16, "65536+"); -SPI_STATISTICS_SHOW(transfers_split_maxsize, "%lu"); +SPI_STATISTICS_SHOW(transfers_split_maxsize); static struct attribute *spi_dev_attrs[] = { &dev_attr_modalias.attr, @@ -267,30 +309,30 @@ static const struct attribute_group *spi_master_groups[] = { NULL, }; -static void spi_statistics_add_transfer_stats(struct spi_statistics *stats, +static void spi_statistics_add_transfer_stats(struct spi_statistics *pcpu_stats, struct spi_transfer *xfer, struct spi_controller *ctlr) { - unsigned long flags; int l2len = min(fls(xfer->len), SPI_STATISTICS_HISTO_SIZE) - 1; + struct spi_statistics *stats = this_cpu_ptr(pcpu_stats); if (l2len < 0) l2len = 0; - spin_lock_irqsave(&stats->lock, flags); + u64_stats_update_begin(&stats->syncp); - stats->transfers++; - stats->transfer_bytes_histo[l2len]++; + u64_stats_inc(&stats->transfers); + u64_stats_inc(&stats->transfer_bytes_histo[l2len]); - stats->bytes += xfer->len; + u64_stats_add(&stats->bytes, xfer->len); if ((xfer->tx_buf) && (xfer->tx_buf != ctlr->dummy_tx)) - stats->bytes_tx += xfer->len; + u64_stats_add(&stats->bytes_tx, xfer->len); if ((xfer->rx_buf) && (xfer->rx_buf != ctlr->dummy_rx)) - stats->bytes_rx += xfer->len; + u64_stats_add(&stats->bytes_rx, xfer->len); - spin_unlock_irqrestore(&stats->lock, flags); + u64_stats_update_end(&stats->syncp); } /* @@ -519,14 +561,19 @@ struct spi_device *spi_alloc_device(struct spi_controller *ctlr) return NULL; } + spi->pcpu_statistics = spi_alloc_pcpu_stats(NULL); + if (!spi->pcpu_statistics) { + kfree(spi); + spi_controller_put(ctlr); + return NULL; + } + spi->master = spi->controller = ctlr; spi->dev.parent = &ctlr->dev; spi->dev.bus = &spi_bus_type; spi->dev.release = spidev_release; spi->mode = ctlr->buswidth_override_bits; - spin_lock_init(&spi->statistics.lock); - device_initialize(&spi->dev); return spi; } @@ -1225,8 +1272,8 @@ static int spi_transfer_wait(struct spi_controller *ctlr, struct spi_message *msg, struct spi_transfer *xfer) { - struct spi_statistics *statm = &ctlr->statistics; - struct spi_statistics *stats = &msg->spi->statistics; + struct spi_statistics *statm = ctlr->pcpu_statistics; + struct spi_statistics *stats = msg->spi->pcpu_statistics; u32 speed_hz = xfer->speed_hz; unsigned long long ms; @@ -1382,8 +1429,8 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, struct spi_transfer *xfer; bool keep_cs = false; int ret = 0; - struct spi_statistics *statm = &ctlr->statistics; - struct spi_statistics *stats = &msg->spi->statistics; + struct spi_statistics *statm = ctlr->pcpu_statistics; + struct spi_statistics *stats = msg->spi->pcpu_statistics; spi_set_cs(msg->spi, true, false); @@ -3029,7 +3076,11 @@ int spi_register_controller(struct spi_controller *ctlr) } } /* add statistics */ - spin_lock_init(&ctlr->statistics.lock); + ctlr->pcpu_statistics = spi_alloc_pcpu_stats(dev); + if (!ctlr->pcpu_statistics) { + dev_err(dev, "Error allocating per-cpu statistics\n"); + goto destroy_queue; + } mutex_lock(&board_lock); list_add_tail(&ctlr->list, &spi_controller_list); @@ -3042,6 +3093,8 @@ int spi_register_controller(struct spi_controller *ctlr) acpi_register_spi_devices(ctlr); return status; +destroy_queue: + spi_destroy_queue(ctlr); free_bus_id: mutex_lock(&board_lock); idr_remove(&spi_master_idr, ctlr->bus_num); @@ -3367,9 +3420,9 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, *xferp = &xfers[count - 1]; /* increment statistics counters */ - SPI_STATISTICS_INCREMENT_FIELD(&ctlr->statistics, + SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, transfers_split_maxsize); - SPI_STATISTICS_INCREMENT_FIELD(&msg->spi->statistics, + SPI_STATISTICS_INCREMENT_FIELD(msg->spi->pcpu_statistics, transfers_split_maxsize); return 0; @@ -3760,8 +3813,8 @@ static int __spi_async(struct spi_device *spi, struct spi_message *message) message->spi = spi; - SPI_STATISTICS_INCREMENT_FIELD(&ctlr->statistics, spi_async); - SPI_STATISTICS_INCREMENT_FIELD(&spi->statistics, spi_async); + SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_async); + SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_async); trace_spi_message_submit(message); @@ -3908,8 +3961,8 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) message->context = &done; message->spi = spi; - SPI_STATISTICS_INCREMENT_FIELD(&ctlr->statistics, spi_sync); - SPI_STATISTICS_INCREMENT_FIELD(&spi->statistics, spi_sync); + SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync); + SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync); /* * If we're not using the legacy transfer method then we will @@ -3932,9 +3985,9 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) if (status == 0) { /* Push out the messages in the calling context if we can */ if (ctlr->transfer == spi_queued_transfer) { - SPI_STATISTICS_INCREMENT_FIELD(&ctlr->statistics, + SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync_immediate); - SPI_STATISTICS_INCREMENT_FIELD(&spi->statistics, + SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync_immediate); __spi_pump_messages(ctlr, false); } diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index eadfd3ba3cbc..eac8d3caf954 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -17,6 +17,7 @@ #include #include +#include struct dma_chan; struct software_node; @@ -59,37 +60,42 @@ extern struct bus_type spi_bus_type; * maxsize limit */ struct spi_statistics { - spinlock_t lock; /* lock for the whole structure */ + struct u64_stats_sync syncp; - unsigned long messages; - unsigned long transfers; - unsigned long errors; - unsigned long timedout; + u64_stats_t messages; + u64_stats_t transfers; + u64_stats_t errors; + u64_stats_t timedout; - unsigned long spi_sync; - unsigned long spi_sync_immediate; - unsigned long spi_async; + u64_stats_t spi_sync; + u64_stats_t spi_sync_immediate; + u64_stats_t spi_async; - unsigned long long bytes; - unsigned long long bytes_rx; - unsigned long long bytes_tx; + u64_stats_t bytes; + u64_stats_t bytes_rx; + u64_stats_t bytes_tx; #define SPI_STATISTICS_HISTO_SIZE 17 - unsigned long transfer_bytes_histo[SPI_STATISTICS_HISTO_SIZE]; + u64_stats_t transfer_bytes_histo[SPI_STATISTICS_HISTO_SIZE]; - unsigned long transfers_split_maxsize; + u64_stats_t transfers_split_maxsize; }; -#define SPI_STATISTICS_ADD_TO_FIELD(stats, field, count) \ - do { \ - unsigned long flags; \ - spin_lock_irqsave(&(stats)->lock, flags); \ - (stats)->field += count; \ - spin_unlock_irqrestore(&(stats)->lock, flags); \ +#define SPI_STATISTICS_ADD_TO_FIELD(pcpu_stats, field, count) \ + do { \ + struct spi_statistics *__lstats = this_cpu_ptr(pcpu_stats); \ + u64_stats_update_begin(&__lstats->syncp); \ + u64_stats_add(&__lstats->field, count); \ + u64_stats_update_end(&__lstats->syncp); \ } while (0) -#define SPI_STATISTICS_INCREMENT_FIELD(stats, field) \ - SPI_STATISTICS_ADD_TO_FIELD(stats, field, 1) +#define SPI_STATISTICS_INCREMENT_FIELD(pcpu_stats, field) \ + do { \ + struct spi_statistics *__lstats = this_cpu_ptr(pcpu_stats); \ + u64_stats_update_begin(&__lstats->syncp); \ + u64_stats_inc(&__lstats->field); \ + u64_stats_update_end(&__lstats->syncp); \ + } while (0) /** * struct spi_delay - SPI delay information @@ -194,7 +200,7 @@ struct spi_device { struct spi_delay cs_inactive; /* the statistics */ - struct spi_statistics statistics; + struct spi_statistics __percpu *pcpu_statistics; /* * likely need more hooks for more protocol options affecting how @@ -647,7 +653,7 @@ struct spi_controller { s8 max_native_cs; /* statistics */ - struct spi_statistics statistics; + struct spi_statistics __percpu *pcpu_statistics; /* DMA channels for use with core dmaengine helpers */ struct dma_chan *dma_tx; -- cgit v1.2.3 From fc602b4f692cb83c937b5f79628bca32b60c4402 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Sat, 4 Jun 2022 12:32:50 +0100 Subject: mtd: spinand: Add support for ATO25D1GA Add support for the ATO25D1GA SPI NAND flash. Datasheet: - https://atta.szlcsc.com/upload/public/pdf/source/20191212/C469320_04599D67B03B078044EB65FF5AEDDDE9.pdf Signed-off-by: Aidan MacDonald Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20220604113250.4745-1-aidanmacdonald.0x0@gmail.com --- drivers/mtd/nand/spi/Makefile | 2 +- drivers/mtd/nand/spi/ato.c | 86 +++++++++++++++++++++++++++++++++++++++++++ drivers/mtd/nand/spi/core.c | 1 + include/linux/mtd/spinand.h | 1 + 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 drivers/mtd/nand/spi/ato.c (limited to 'include/linux') diff --git a/drivers/mtd/nand/spi/Makefile b/drivers/mtd/nand/spi/Makefile index 80dabe6ff0f3..b520fe634041 100644 --- a/drivers/mtd/nand/spi/Makefile +++ b/drivers/mtd/nand/spi/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -spinand-objs := core.o gigadevice.o macronix.o micron.o paragon.o toshiba.o winbond.o xtx.o +spinand-objs := core.o ato.o gigadevice.o macronix.o micron.o paragon.o toshiba.o winbond.o xtx.o obj-$(CONFIG_MTD_SPI_NAND) += spinand.o diff --git a/drivers/mtd/nand/spi/ato.c b/drivers/mtd/nand/spi/ato.c new file mode 100644 index 000000000000..82b377c06812 --- /dev/null +++ b/drivers/mtd/nand/spi/ato.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Aidan MacDonald + * + * Author: Aidan MacDonald + */ + +#include +#include +#include + + +#define SPINAND_MFR_ATO 0x9b + + +static SPINAND_OP_VARIANTS(read_cache_variants, + SPINAND_PAGE_READ_FROM_CACHE_X4_OP(0, 1, NULL, 0), + SPINAND_PAGE_READ_FROM_CACHE_OP(true, 0, 1, NULL, 0), + SPINAND_PAGE_READ_FROM_CACHE_OP(false, 0, 1, NULL, 0)); + +static SPINAND_OP_VARIANTS(write_cache_variants, + SPINAND_PROG_LOAD_X4(true, 0, NULL, 0), + SPINAND_PROG_LOAD(true, 0, NULL, 0)); + +static SPINAND_OP_VARIANTS(update_cache_variants, + SPINAND_PROG_LOAD_X4(false, 0, NULL, 0), + SPINAND_PROG_LOAD(false, 0, NULL, 0)); + + +static int ato25d1ga_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + if (section > 3) + return -ERANGE; + + region->offset = (16 * section) + 8; + region->length = 8; + return 0; +} + +static int ato25d1ga_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + if (section > 3) + return -ERANGE; + + if (section) { + region->offset = (16 * section); + region->length = 8; + } else { + /* first byte of section 0 is reserved for the BBM */ + region->offset = 1; + region->length = 7; + } + + return 0; +} + +static const struct mtd_ooblayout_ops ato25d1ga_ooblayout = { + .ecc = ato25d1ga_ooblayout_ecc, + .free = ato25d1ga_ooblayout_free, +}; + + +static const struct spinand_info ato_spinand_table[] = { + SPINAND_INFO("ATO25D1GA", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0x12), + NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1), + NAND_ECCREQ(1, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&ato25d1ga_ooblayout, NULL)), +}; + +static const struct spinand_manufacturer_ops ato_spinand_manuf_ops = { +}; + +const struct spinand_manufacturer ato_spinand_manufacturer = { + .id = SPINAND_MFR_ATO, + .name = "ATO", + .chips = ato_spinand_table, + .nchips = ARRAY_SIZE(ato_spinand_table), + .ops = &ato_spinand_manuf_ops, +}; diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index d5b685d1605e..9d73910a7ae8 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -927,6 +927,7 @@ static const struct nand_ops spinand_ops = { }; static const struct spinand_manufacturer *spinand_manufacturers[] = { + &ato_spinand_manufacturer, &gigadevice_spinand_manufacturer, ¯onix_spinand_manufacturer, µn_spinand_manufacturer, diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 5584d3bb6556..6d3392a7edc6 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -260,6 +260,7 @@ struct spinand_manufacturer { }; /* SPI NAND manufacturers */ +extern const struct spinand_manufacturer ato_spinand_manufacturer; extern const struct spinand_manufacturer gigadevice_spinand_manufacturer; extern const struct spinand_manufacturer macronix_spinand_manufacturer; extern const struct spinand_manufacturer micron_spinand_manufacturer; -- cgit v1.2.3 From 39d649602be2ceb3f8f8e98483d09e4d71133c6a Mon Sep 17 00:00:00 2001 From: Clément Léger Date: Wed, 1 Jun 2022 10:17:58 +0200 Subject: of: constify of_property_check_flags() prop argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This argument is not modified and thus can be set as const. Signed-off-by: Clément Léger Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20220601081801.348571-2-clement.leger@bootlin.com --- include/linux/of.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/of.h b/include/linux/of.h index f0a5d6b10c5a..d74fd82a6963 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -207,7 +207,7 @@ static inline void of_node_clear_flag(struct device_node *n, unsigned long flag) } #if defined(CONFIG_OF_DYNAMIC) || defined(CONFIG_SPARC) -static inline int of_property_check_flag(struct property *p, unsigned long flag) +static inline int of_property_check_flag(const struct property *p, unsigned long flag) { return test_bit(flag, &p->_flags); } @@ -814,7 +814,8 @@ static inline void of_node_clear_flag(struct device_node *n, unsigned long flag) { } -static inline int of_property_check_flag(struct property *p, unsigned long flag) +static inline int of_property_check_flag(const struct property *p, + unsigned long flag) { return 0; } -- cgit v1.2.3 From ff8372a467fa28752610f57a6512c5365413faa2 Mon Sep 17 00:00:00 2001 From: Menglong Dong Date: Mon, 6 Jun 2022 10:24:34 +0800 Subject: net: skb: move enum skb_drop_reason to standalone header file As the skb drop reasons are getting more and more, move the enum 'skb_drop_reason' and related function to the standalone header 'dropreason.h', as Jakub Kicinski suggested. Signed-off-by: Menglong Dong Signed-off-by: Paolo Abeni --- include/linux/skbuff.h | 179 +-------------------------------------------- include/net/dropreason.h | 184 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 178 deletions(-) create mode 100644 include/net/dropreason.h (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d3d10556f0fa..82edf0359ab3 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -43,6 +43,7 @@ #include #endif #include +#include /** * DOC: skb checksums @@ -337,184 +338,6 @@ struct sk_buff_head { struct sk_buff; -/* The reason of skb drop, which is used in kfree_skb_reason(). - * en...maybe they should be splited by group? - * - * Each item here should also be in 'TRACE_SKB_DROP_REASON', which is - * used to translate the reason to string. - */ -enum skb_drop_reason { - SKB_NOT_DROPPED_YET = 0, - SKB_DROP_REASON_NOT_SPECIFIED, /* drop reason is not specified */ - SKB_DROP_REASON_NO_SOCKET, /* socket not found */ - SKB_DROP_REASON_PKT_TOO_SMALL, /* packet size is too small */ - SKB_DROP_REASON_TCP_CSUM, /* TCP checksum error */ - SKB_DROP_REASON_SOCKET_FILTER, /* dropped by socket filter */ - SKB_DROP_REASON_UDP_CSUM, /* UDP checksum error */ - SKB_DROP_REASON_NETFILTER_DROP, /* dropped by netfilter */ - SKB_DROP_REASON_OTHERHOST, /* packet don't belong to current - * host (interface is in promisc - * mode) - */ - SKB_DROP_REASON_IP_CSUM, /* IP checksum error */ - SKB_DROP_REASON_IP_INHDR, /* there is something wrong with - * IP header (see - * IPSTATS_MIB_INHDRERRORS) - */ - SKB_DROP_REASON_IP_RPFILTER, /* IP rpfilter validate failed. - * see the document for rp_filter - * in ip-sysctl.rst for more - * information - */ - SKB_DROP_REASON_UNICAST_IN_L2_MULTICAST, /* destination address of L2 - * is multicast, but L3 is - * unicast. - */ - SKB_DROP_REASON_XFRM_POLICY, /* xfrm policy check failed */ - SKB_DROP_REASON_IP_NOPROTO, /* no support for IP protocol */ - SKB_DROP_REASON_SOCKET_RCVBUFF, /* socket receive buff is full */ - SKB_DROP_REASON_PROTO_MEM, /* proto memory limition, such as - * udp packet drop out of - * udp_memory_allocated. - */ - SKB_DROP_REASON_TCP_MD5NOTFOUND, /* no MD5 hash and one - * expected, corresponding - * to LINUX_MIB_TCPMD5NOTFOUND - */ - SKB_DROP_REASON_TCP_MD5UNEXPECTED, /* MD5 hash and we're not - * expecting one, corresponding - * to LINUX_MIB_TCPMD5UNEXPECTED - */ - SKB_DROP_REASON_TCP_MD5FAILURE, /* MD5 hash and its wrong, - * corresponding to - * LINUX_MIB_TCPMD5FAILURE - */ - SKB_DROP_REASON_SOCKET_BACKLOG, /* failed to add skb to socket - * backlog (see - * LINUX_MIB_TCPBACKLOGDROP) - */ - SKB_DROP_REASON_TCP_FLAGS, /* TCP flags invalid */ - SKB_DROP_REASON_TCP_ZEROWINDOW, /* TCP receive window size is zero, - * see LINUX_MIB_TCPZEROWINDOWDROP - */ - SKB_DROP_REASON_TCP_OLD_DATA, /* the TCP data reveived is already - * received before (spurious retrans - * may happened), see - * LINUX_MIB_DELAYEDACKLOST - */ - SKB_DROP_REASON_TCP_OVERWINDOW, /* the TCP data is out of window, - * the seq of the first byte exceed - * the right edges of receive - * window - */ - SKB_DROP_REASON_TCP_OFOMERGE, /* the data of skb is already in - * the ofo queue, corresponding to - * LINUX_MIB_TCPOFOMERGE - */ - SKB_DROP_REASON_TCP_RFC7323_PAWS, /* PAWS check, corresponding to - * LINUX_MIB_PAWSESTABREJECTED - */ - SKB_DROP_REASON_TCP_INVALID_SEQUENCE, /* Not acceptable SEQ field */ - SKB_DROP_REASON_TCP_RESET, /* Invalid RST packet */ - SKB_DROP_REASON_TCP_INVALID_SYN, /* Incoming packet has unexpected SYN flag */ - SKB_DROP_REASON_TCP_CLOSE, /* TCP socket in CLOSE state */ - SKB_DROP_REASON_TCP_FASTOPEN, /* dropped by FASTOPEN request socket */ - SKB_DROP_REASON_TCP_OLD_ACK, /* TCP ACK is old, but in window */ - SKB_DROP_REASON_TCP_TOO_OLD_ACK, /* TCP ACK is too old */ - SKB_DROP_REASON_TCP_ACK_UNSENT_DATA, /* TCP ACK for data we haven't sent yet */ - SKB_DROP_REASON_TCP_OFO_QUEUE_PRUNE, /* pruned from TCP OFO queue */ - SKB_DROP_REASON_TCP_OFO_DROP, /* data already in receive queue */ - SKB_DROP_REASON_IP_OUTNOROUTES, /* route lookup failed */ - SKB_DROP_REASON_BPF_CGROUP_EGRESS, /* dropped by - * BPF_PROG_TYPE_CGROUP_SKB - * eBPF program - */ - SKB_DROP_REASON_IPV6DISABLED, /* IPv6 is disabled on the device */ - SKB_DROP_REASON_NEIGH_CREATEFAIL, /* failed to create neigh - * entry - */ - SKB_DROP_REASON_NEIGH_FAILED, /* neigh entry in failed state */ - SKB_DROP_REASON_NEIGH_QUEUEFULL, /* arp_queue for neigh - * entry is full - */ - SKB_DROP_REASON_NEIGH_DEAD, /* neigh entry is dead */ - SKB_DROP_REASON_TC_EGRESS, /* dropped in TC egress HOOK */ - SKB_DROP_REASON_QDISC_DROP, /* dropped by qdisc when packet - * outputting (failed to enqueue to - * current qdisc) - */ - SKB_DROP_REASON_CPU_BACKLOG, /* failed to enqueue the skb to - * the per CPU backlog queue. This - * can be caused by backlog queue - * full (see netdev_max_backlog in - * net.rst) or RPS flow limit - */ - SKB_DROP_REASON_XDP, /* dropped by XDP in input path */ - SKB_DROP_REASON_TC_INGRESS, /* dropped in TC ingress HOOK */ - SKB_DROP_REASON_UNHANDLED_PROTO, /* protocol not implemented - * or not supported - */ - SKB_DROP_REASON_SKB_CSUM, /* sk_buff checksum computation - * error - */ - SKB_DROP_REASON_SKB_GSO_SEG, /* gso segmentation error */ - SKB_DROP_REASON_SKB_UCOPY_FAULT, /* failed to copy data from - * user space, e.g., via - * zerocopy_sg_from_iter() - * or skb_orphan_frags_rx() - */ - SKB_DROP_REASON_DEV_HDR, /* device driver specific - * header/metadata is invalid - */ - /* the device is not ready to xmit/recv due to any of its data - * structure that is not up/ready/initialized, e.g., the IFF_UP is - * not set, or driver specific tun->tfiles[txq] is not initialized - */ - SKB_DROP_REASON_DEV_READY, - SKB_DROP_REASON_FULL_RING, /* ring buffer is full */ - SKB_DROP_REASON_NOMEM, /* error due to OOM */ - SKB_DROP_REASON_HDR_TRUNC, /* failed to trunc/extract the header - * from networking data, e.g., failed - * to pull the protocol header from - * frags via pskb_may_pull() - */ - SKB_DROP_REASON_TAP_FILTER, /* dropped by (ebpf) filter directly - * attached to tun/tap, e.g., via - * TUNSETFILTEREBPF - */ - SKB_DROP_REASON_TAP_TXFILTER, /* dropped by tx filter implemented - * at tun/tap, e.g., check_filter() - */ - SKB_DROP_REASON_ICMP_CSUM, /* ICMP checksum error */ - SKB_DROP_REASON_INVALID_PROTO, /* the packet doesn't follow RFC - * 2211, such as a broadcasts - * ICMP_TIMESTAMP - */ - SKB_DROP_REASON_IP_INADDRERRORS, /* host unreachable, corresponding - * to IPSTATS_MIB_INADDRERRORS - */ - SKB_DROP_REASON_IP_INNOROUTES, /* network unreachable, corresponding - * to IPSTATS_MIB_INADDRERRORS - */ - SKB_DROP_REASON_PKT_TOO_BIG, /* packet size is too big (maybe exceed - * the MTU) - */ - SKB_DROP_REASON_MAX, -}; - -#define SKB_DR_INIT(name, reason) \ - enum skb_drop_reason name = SKB_DROP_REASON_##reason -#define SKB_DR(name) \ - SKB_DR_INIT(name, NOT_SPECIFIED) -#define SKB_DR_SET(name, reason) \ - (name = SKB_DROP_REASON_##reason) -#define SKB_DR_OR(name, reason) \ - do { \ - if (name == SKB_DROP_REASON_NOT_SPECIFIED || \ - name == SKB_NOT_DROPPED_YET) \ - SKB_DR_SET(name, reason); \ - } while (0) - /* To allow 64K frame to be packed as single skb without frag_list we * require 64K/PAGE_SIZE pages plus 1 additional page to allow for * buffers which do not start on a page boundary. diff --git a/include/net/dropreason.h b/include/net/dropreason.h new file mode 100644 index 000000000000..ecd18b7b1364 --- /dev/null +++ b/include/net/dropreason.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef _LINUX_DROPREASON_H +#define _LINUX_DROPREASON_H + +/* The reason of skb drop, which is used in kfree_skb_reason(). + * en...maybe they should be splited by group? + * + * Each item here should also be in 'TRACE_SKB_DROP_REASON', which is + * used to translate the reason to string. + */ +enum skb_drop_reason { + SKB_NOT_DROPPED_YET = 0, + SKB_DROP_REASON_NOT_SPECIFIED, /* drop reason is not specified */ + SKB_DROP_REASON_NO_SOCKET, /* socket not found */ + SKB_DROP_REASON_PKT_TOO_SMALL, /* packet size is too small */ + SKB_DROP_REASON_TCP_CSUM, /* TCP checksum error */ + SKB_DROP_REASON_SOCKET_FILTER, /* dropped by socket filter */ + SKB_DROP_REASON_UDP_CSUM, /* UDP checksum error */ + SKB_DROP_REASON_NETFILTER_DROP, /* dropped by netfilter */ + SKB_DROP_REASON_OTHERHOST, /* packet don't belong to current + * host (interface is in promisc + * mode) + */ + SKB_DROP_REASON_IP_CSUM, /* IP checksum error */ + SKB_DROP_REASON_IP_INHDR, /* there is something wrong with + * IP header (see + * IPSTATS_MIB_INHDRERRORS) + */ + SKB_DROP_REASON_IP_RPFILTER, /* IP rpfilter validate failed. + * see the document for rp_filter + * in ip-sysctl.rst for more + * information + */ + SKB_DROP_REASON_UNICAST_IN_L2_MULTICAST, /* destination address of L2 + * is multicast, but L3 is + * unicast. + */ + SKB_DROP_REASON_XFRM_POLICY, /* xfrm policy check failed */ + SKB_DROP_REASON_IP_NOPROTO, /* no support for IP protocol */ + SKB_DROP_REASON_SOCKET_RCVBUFF, /* socket receive buff is full */ + SKB_DROP_REASON_PROTO_MEM, /* proto memory limition, such as + * udp packet drop out of + * udp_memory_allocated. + */ + SKB_DROP_REASON_TCP_MD5NOTFOUND, /* no MD5 hash and one + * expected, corresponding + * to LINUX_MIB_TCPMD5NOTFOUND + */ + SKB_DROP_REASON_TCP_MD5UNEXPECTED, /* MD5 hash and we're not + * expecting one, corresponding + * to LINUX_MIB_TCPMD5UNEXPECTED + */ + SKB_DROP_REASON_TCP_MD5FAILURE, /* MD5 hash and its wrong, + * corresponding to + * LINUX_MIB_TCPMD5FAILURE + */ + SKB_DROP_REASON_SOCKET_BACKLOG, /* failed to add skb to socket + * backlog (see + * LINUX_MIB_TCPBACKLOGDROP) + */ + SKB_DROP_REASON_TCP_FLAGS, /* TCP flags invalid */ + SKB_DROP_REASON_TCP_ZEROWINDOW, /* TCP receive window size is zero, + * see LINUX_MIB_TCPZEROWINDOWDROP + */ + SKB_DROP_REASON_TCP_OLD_DATA, /* the TCP data reveived is already + * received before (spurious retrans + * may happened), see + * LINUX_MIB_DELAYEDACKLOST + */ + SKB_DROP_REASON_TCP_OVERWINDOW, /* the TCP data is out of window, + * the seq of the first byte exceed + * the right edges of receive + * window + */ + SKB_DROP_REASON_TCP_OFOMERGE, /* the data of skb is already in + * the ofo queue, corresponding to + * LINUX_MIB_TCPOFOMERGE + */ + SKB_DROP_REASON_TCP_RFC7323_PAWS, /* PAWS check, corresponding to + * LINUX_MIB_PAWSESTABREJECTED + */ + SKB_DROP_REASON_TCP_INVALID_SEQUENCE, /* Not acceptable SEQ field */ + SKB_DROP_REASON_TCP_RESET, /* Invalid RST packet */ + SKB_DROP_REASON_TCP_INVALID_SYN, /* Incoming packet has unexpected SYN flag */ + SKB_DROP_REASON_TCP_CLOSE, /* TCP socket in CLOSE state */ + SKB_DROP_REASON_TCP_FASTOPEN, /* dropped by FASTOPEN request socket */ + SKB_DROP_REASON_TCP_OLD_ACK, /* TCP ACK is old, but in window */ + SKB_DROP_REASON_TCP_TOO_OLD_ACK, /* TCP ACK is too old */ + SKB_DROP_REASON_TCP_ACK_UNSENT_DATA, /* TCP ACK for data we haven't sent yet */ + SKB_DROP_REASON_TCP_OFO_QUEUE_PRUNE, /* pruned from TCP OFO queue */ + SKB_DROP_REASON_TCP_OFO_DROP, /* data already in receive queue */ + SKB_DROP_REASON_IP_OUTNOROUTES, /* route lookup failed */ + SKB_DROP_REASON_BPF_CGROUP_EGRESS, /* dropped by + * BPF_PROG_TYPE_CGROUP_SKB + * eBPF program + */ + SKB_DROP_REASON_IPV6DISABLED, /* IPv6 is disabled on the device */ + SKB_DROP_REASON_NEIGH_CREATEFAIL, /* failed to create neigh + * entry + */ + SKB_DROP_REASON_NEIGH_FAILED, /* neigh entry in failed state */ + SKB_DROP_REASON_NEIGH_QUEUEFULL, /* arp_queue for neigh + * entry is full + */ + SKB_DROP_REASON_NEIGH_DEAD, /* neigh entry is dead */ + SKB_DROP_REASON_TC_EGRESS, /* dropped in TC egress HOOK */ + SKB_DROP_REASON_QDISC_DROP, /* dropped by qdisc when packet + * outputting (failed to enqueue to + * current qdisc) + */ + SKB_DROP_REASON_CPU_BACKLOG, /* failed to enqueue the skb to + * the per CPU backlog queue. This + * can be caused by backlog queue + * full (see netdev_max_backlog in + * net.rst) or RPS flow limit + */ + SKB_DROP_REASON_XDP, /* dropped by XDP in input path */ + SKB_DROP_REASON_TC_INGRESS, /* dropped in TC ingress HOOK */ + SKB_DROP_REASON_UNHANDLED_PROTO, /* protocol not implemented + * or not supported + */ + SKB_DROP_REASON_SKB_CSUM, /* sk_buff checksum computation + * error + */ + SKB_DROP_REASON_SKB_GSO_SEG, /* gso segmentation error */ + SKB_DROP_REASON_SKB_UCOPY_FAULT, /* failed to copy data from + * user space, e.g., via + * zerocopy_sg_from_iter() + * or skb_orphan_frags_rx() + */ + SKB_DROP_REASON_DEV_HDR, /* device driver specific + * header/metadata is invalid + */ + /* the device is not ready to xmit/recv due to any of its data + * structure that is not up/ready/initialized, e.g., the IFF_UP is + * not set, or driver specific tun->tfiles[txq] is not initialized + */ + SKB_DROP_REASON_DEV_READY, + SKB_DROP_REASON_FULL_RING, /* ring buffer is full */ + SKB_DROP_REASON_NOMEM, /* error due to OOM */ + SKB_DROP_REASON_HDR_TRUNC, /* failed to trunc/extract the header + * from networking data, e.g., failed + * to pull the protocol header from + * frags via pskb_may_pull() + */ + SKB_DROP_REASON_TAP_FILTER, /* dropped by (ebpf) filter directly + * attached to tun/tap, e.g., via + * TUNSETFILTEREBPF + */ + SKB_DROP_REASON_TAP_TXFILTER, /* dropped by tx filter implemented + * at tun/tap, e.g., check_filter() + */ + SKB_DROP_REASON_ICMP_CSUM, /* ICMP checksum error */ + SKB_DROP_REASON_INVALID_PROTO, /* the packet doesn't follow RFC + * 2211, such as a broadcasts + * ICMP_TIMESTAMP + */ + SKB_DROP_REASON_IP_INADDRERRORS, /* host unreachable, corresponding + * to IPSTATS_MIB_INADDRERRORS + */ + SKB_DROP_REASON_IP_INNOROUTES, /* network unreachable, corresponding + * to IPSTATS_MIB_INADDRERRORS + */ + SKB_DROP_REASON_PKT_TOO_BIG, /* packet size is too big (maybe exceed + * the MTU) + */ + SKB_DROP_REASON_MAX, +}; + +#define SKB_DR_INIT(name, reason) \ + enum skb_drop_reason name = SKB_DROP_REASON_##reason +#define SKB_DR(name) \ + SKB_DR_INIT(name, NOT_SPECIFIED) +#define SKB_DR_SET(name, reason) \ + (name = SKB_DROP_REASON_##reason) +#define SKB_DR_OR(name, reason) \ + do { \ + if (name == SKB_DROP_REASON_NOT_SPECIFIED || \ + name == SKB_NOT_DROPPED_YET) \ + SKB_DR_SET(name, reason); \ + } while (0) + +#endif -- cgit v1.2.3 From 5f69a6577bc33d8f6d6bbe02bccdeb357b287f56 Mon Sep 17 00:00:00 2001 From: Chen Wandun Date: Thu, 26 May 2022 20:26:56 +0800 Subject: psi: dont alloc memory for psi by default Memory about struct psi_group is allocated by default for each cgroup even if psi_disabled is true, in this case, these allocated memory is waste, so alloc memory for struct psi_group only when psi_disabled is false. Signed-off-by: Chen Wandun Acked-by: Johannes Weiner Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 2 +- include/linux/cgroup.h | 2 +- kernel/cgroup/cgroup.c | 8 ++++---- kernel/sched/psi.c | 19 +++++++++++++------ 4 files changed, 19 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 1bfcfb1af352..672de25e3ec8 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -475,7 +475,7 @@ struct cgroup { struct work_struct release_agent_work; /* used to track pressure stalls */ - struct psi_group psi; + struct psi_group *psi; /* used to store eBPF programs */ struct cgroup_bpf bpf; diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 0d1ada8968d7..ed53bfe7c46c 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -674,7 +674,7 @@ static inline void pr_cont_cgroup_path(struct cgroup *cgrp) static inline struct psi_group *cgroup_psi(struct cgroup *cgrp) { - return &cgrp->psi; + return cgrp->psi; } bool cgroup_psi_enabled(void); diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 1779ccddb734..90a654cb8a1e 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3609,21 +3609,21 @@ static int cpu_stat_show(struct seq_file *seq, void *v) static int cgroup_io_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgrp = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; + struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; return psi_show(seq, psi, PSI_IO); } static int cgroup_memory_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgrp = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; + struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; return psi_show(seq, psi, PSI_MEM); } static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgrp = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; + struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; return psi_show(seq, psi, PSI_CPU); } @@ -3649,7 +3649,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, return -EBUSY; } - psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; + psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; new = psi_trigger_create(psi, buf, nbytes, res); if (IS_ERR(new)) { cgroup_put(cgrp); diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index a337f3e35997..ec66b40bdd40 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -957,10 +957,16 @@ int psi_cgroup_alloc(struct cgroup *cgroup) if (static_branch_likely(&psi_disabled)) return 0; - cgroup->psi.pcpu = alloc_percpu(struct psi_group_cpu); - if (!cgroup->psi.pcpu) + cgroup->psi = kmalloc(sizeof(struct psi_group), GFP_KERNEL); + if (!cgroup->psi) return -ENOMEM; - group_init(&cgroup->psi); + + cgroup->psi->pcpu = alloc_percpu(struct psi_group_cpu); + if (!cgroup->psi->pcpu) { + kfree(cgroup->psi); + return -ENOMEM; + } + group_init(cgroup->psi); return 0; } @@ -969,10 +975,11 @@ void psi_cgroup_free(struct cgroup *cgroup) if (static_branch_likely(&psi_disabled)) return; - cancel_delayed_work_sync(&cgroup->psi.avgs_work); - free_percpu(cgroup->psi.pcpu); + cancel_delayed_work_sync(&cgroup->psi->avgs_work); + free_percpu(cgroup->psi->pcpu); /* All triggers must be removed by now */ - WARN_ONCE(cgroup->psi.poll_states, "psi: trigger leak\n"); + WARN_ONCE(cgroup->psi->poll_states, "psi: trigger leak\n"); + kfree(cgroup->psi); } /** -- cgit v1.2.3 From 6089fb325cf737eeb2c4d236c94697112ca860da Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Mon, 6 Jun 2022 23:26:00 -0700 Subject: bpf: Add btf enum64 support Currently, BTF only supports upto 32bit enum value with BTF_KIND_ENUM. But in kernel, some enum indeed has 64bit values, e.g., in uapi bpf.h, we have enum { BPF_F_INDEX_MASK = 0xffffffffULL, BPF_F_CURRENT_CPU = BPF_F_INDEX_MASK, BPF_F_CTXLEN_MASK = (0xfffffULL << 32), }; In this case, BTF_KIND_ENUM will encode the value of BPF_F_CTXLEN_MASK as 0, which certainly is incorrect. This patch added a new btf kind, BTF_KIND_ENUM64, which permits 64bit value to cover the above use case. The BTF_KIND_ENUM64 has the following three fields followed by the common type: struct bpf_enum64 { __u32 nume_off; __u32 val_lo32; __u32 val_hi32; }; Currently, btf type section has an alignment of 4 as all element types are u32. Representing the value with __u64 will introduce a pad for bpf_enum64 and may also introduce misalignment for the 64bit value. Hence, two members of val_hi32 and val_lo32 are chosen to avoid these issues. The kflag is also introduced for BTF_KIND_ENUM and BTF_KIND_ENUM64 to indicate whether the value is signed or unsigned. The kflag intends to provide consistent output of BTF C fortmat with the original source code. For example, the original BTF_KIND_ENUM bit value is 0xffffffff. The format C has two choices, printing out 0xffffffff or -1 and current libbpf prints out as unsigned value. But if the signedness is preserved in btf, the value can be printed the same as the original source code. The kflag value 0 means unsigned values, which is consistent to the default by libbpf and should also cover most cases as well. The new BTF_KIND_ENUM64 is intended to support the enum value represented as 64bit value. But it can represent all BTF_KIND_ENUM values as well. The compiler ([1]) and pahole will generate BTF_KIND_ENUM64 only if the value has to be represented with 64 bits. In addition, a static inline function btf_kind_core_compat() is introduced which will be used later when libbpf relo_core.c changed. Here the kernel shares the same relo_core.c with libbpf. [1] https://reviews.llvm.org/D124641 Acked-by: Andrii Nakryiko Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20220607062600.3716578-1-yhs@fb.com Signed-off-by: Alexei Starovoitov --- include/linux/btf.h | 28 ++++++++ include/uapi/linux/btf.h | 17 ++++- kernel/bpf/btf.c | 142 +++++++++++++++++++++++++++++++++++++---- kernel/bpf/verifier.c | 2 +- tools/include/uapi/linux/btf.h | 17 ++++- 5 files changed, 185 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index 2611cea2c2b6..1bfed7fa0428 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -177,6 +177,19 @@ static inline bool btf_type_is_enum(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM; } +static inline bool btf_is_any_enum(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM || + BTF_INFO_KIND(t->info) == BTF_KIND_ENUM64; +} + +static inline bool btf_kind_core_compat(const struct btf_type *t1, + const struct btf_type *t2) +{ + return BTF_INFO_KIND(t1->info) == BTF_INFO_KIND(t2->info) || + (btf_is_any_enum(t1) && btf_is_any_enum(t2)); +} + static inline bool str_is_empty(const char *s) { return !s || !s[0]; @@ -192,6 +205,16 @@ static inline bool btf_is_enum(const struct btf_type *t) return btf_kind(t) == BTF_KIND_ENUM; } +static inline bool btf_is_enum64(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_ENUM64; +} + +static inline u64 btf_enum64_value(const struct btf_enum64 *e) +{ + return ((u64)e->val_hi32 << 32) | e->val_lo32; +} + static inline bool btf_is_composite(const struct btf_type *t) { u16 kind = btf_kind(t); @@ -332,6 +355,11 @@ static inline struct btf_enum *btf_enum(const struct btf_type *t) return (struct btf_enum *)(t + 1); } +static inline struct btf_enum64 *btf_enum64(const struct btf_type *t) +{ + return (struct btf_enum64 *)(t + 1); +} + static inline const struct btf_var_secinfo *btf_type_var_secinfo( const struct btf_type *t) { diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h index a9162a6c0284..ec1798b6d3ff 100644 --- a/include/uapi/linux/btf.h +++ b/include/uapi/linux/btf.h @@ -36,10 +36,10 @@ struct btf_type { * bits 24-28: kind (e.g. int, ptr, array...etc) * bits 29-30: unused * bit 31: kind_flag, currently used by - * struct, union and fwd + * struct, union, enum, fwd and enum64 */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION and DATASEC. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, @@ -63,7 +63,7 @@ enum { BTF_KIND_ARRAY = 3, /* Array */ BTF_KIND_STRUCT = 4, /* Struct */ BTF_KIND_UNION = 5, /* Union */ - BTF_KIND_ENUM = 6, /* Enumeration */ + BTF_KIND_ENUM = 6, /* Enumeration up to 32-bit values */ BTF_KIND_FWD = 7, /* Forward */ BTF_KIND_TYPEDEF = 8, /* Typedef */ BTF_KIND_VOLATILE = 9, /* Volatile */ @@ -76,6 +76,7 @@ enum { BTF_KIND_FLOAT = 16, /* Floating point */ BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ + BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, @@ -186,4 +187,14 @@ struct btf_decl_tag { __s32 component_idx; }; +/* BTF_KIND_ENUM64 is followed by multiple "struct btf_enum64". + * The exact number of btf_enum64 is stored in the vlen (of the + * info in "struct btf_type"). + */ +struct btf_enum64 { + __u32 name_off; + __u32 val_lo32; + __u32 val_hi32; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7bccaa4646e5..6c0d8480e15c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -309,6 +309,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_FLOAT] = "FLOAT", [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", + [BTF_KIND_ENUM64] = "ENUM64", }; const char *btf_type_str(const struct btf_type *t) @@ -666,6 +667,7 @@ static bool btf_type_has_size(const struct btf_type *t) case BTF_KIND_ENUM: case BTF_KIND_DATASEC: case BTF_KIND_FLOAT: + case BTF_KIND_ENUM64: return true; } @@ -711,6 +713,11 @@ static const struct btf_decl_tag *btf_type_decl_tag(const struct btf_type *t) return (const struct btf_decl_tag *)(t + 1); } +static const struct btf_enum64 *btf_type_enum64(const struct btf_type *t) +{ + return (const struct btf_enum64 *)(t + 1); +} + static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t) { return kind_ops[BTF_INFO_KIND(t->info)]; @@ -1019,6 +1026,7 @@ static const char *btf_show_name(struct btf_show *show) parens = "{"; break; case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: prefix = "enum"; break; default: @@ -1834,6 +1842,7 @@ __btf_resolve_size(const struct btf *btf, const struct btf_type *type, case BTF_KIND_UNION: case BTF_KIND_ENUM: case BTF_KIND_FLOAT: + case BTF_KIND_ENUM64: size = type->size; goto resolved; @@ -3670,6 +3679,7 @@ static s32 btf_enum_check_meta(struct btf_verifier_env *env, { const struct btf_enum *enums = btf_type_enum(t); struct btf *btf = env->btf; + const char *fmt_str; u16 i, nr_enums; u32 meta_needed; @@ -3683,11 +3693,6 @@ static s32 btf_enum_check_meta(struct btf_verifier_env *env, return -EINVAL; } - if (btf_type_kflag(t)) { - btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); - return -EINVAL; - } - if (t->size > 8 || !is_power_of_2(t->size)) { btf_verifier_log_type(env, t, "Unexpected size"); return -EINVAL; @@ -3718,7 +3723,8 @@ static s32 btf_enum_check_meta(struct btf_verifier_env *env, if (env->log.level == BPF_LOG_KERNEL) continue; - btf_verifier_log(env, "\t%s val=%d\n", + fmt_str = btf_type_kflag(t) ? "\t%s val=%d\n" : "\t%s val=%u\n"; + btf_verifier_log(env, fmt_str, __btf_name_by_offset(btf, enums[i].name_off), enums[i].val); } @@ -3759,7 +3765,10 @@ static void btf_enum_show(const struct btf *btf, const struct btf_type *t, return; } - btf_show_type_value(show, "%d", v); + if (btf_type_kflag(t)) + btf_show_type_value(show, "%d", v); + else + btf_show_type_value(show, "%u", v); btf_show_end_type(show); } @@ -3772,6 +3781,109 @@ static struct btf_kind_operations enum_ops = { .show = btf_enum_show, }; +static s32 btf_enum64_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + const struct btf_enum64 *enums = btf_type_enum64(t); + struct btf *btf = env->btf; + const char *fmt_str; + u16 i, nr_enums; + u32 meta_needed; + + nr_enums = btf_type_vlen(t); + meta_needed = nr_enums * sizeof(*enums); + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->size > 8 || !is_power_of_2(t->size)) { + btf_verifier_log_type(env, t, "Unexpected size"); + return -EINVAL; + } + + /* enum type either no name or a valid one */ + if (t->name_off && + !btf_name_valid_identifier(env->btf, t->name_off)) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + for (i = 0; i < nr_enums; i++) { + if (!btf_name_offset_valid(btf, enums[i].name_off)) { + btf_verifier_log(env, "\tInvalid name_offset:%u", + enums[i].name_off); + return -EINVAL; + } + + /* enum member must have a valid name */ + if (!enums[i].name_off || + !btf_name_valid_identifier(btf, enums[i].name_off)) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + if (env->log.level == BPF_LOG_KERNEL) + continue; + + fmt_str = btf_type_kflag(t) ? "\t%s val=%lld\n" : "\t%s val=%llu\n"; + btf_verifier_log(env, fmt_str, + __btf_name_by_offset(btf, enums[i].name_off), + btf_enum64_value(enums + i)); + } + + return meta_needed; +} + +static void btf_enum64_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) +{ + const struct btf_enum64 *enums = btf_type_enum64(t); + u32 i, nr_enums = btf_type_vlen(t); + void *safe_data; + s64 v; + + safe_data = btf_show_start_type(show, t, type_id, data); + if (!safe_data) + return; + + v = *(u64 *)safe_data; + + for (i = 0; i < nr_enums; i++) { + if (v != btf_enum64_value(enums + i)) + continue; + + btf_show_type_value(show, "%s", + __btf_name_by_offset(btf, + enums[i].name_off)); + + btf_show_end_type(show); + return; + } + + if (btf_type_kflag(t)) + btf_show_type_value(show, "%lld", v); + else + btf_show_type_value(show, "%llu", v); + btf_show_end_type(show); +} + +static struct btf_kind_operations enum64_ops = { + .check_meta = btf_enum64_check_meta, + .resolve = btf_df_resolve, + .check_member = btf_enum_check_member, + .check_kflag_member = btf_enum_check_kflag_member, + .log_details = btf_enum_log, + .show = btf_enum64_show, +}; + static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, const struct btf_type *t, u32 meta_left) @@ -4438,6 +4550,7 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { [BTF_KIND_FLOAT] = &float_ops, [BTF_KIND_DECL_TAG] = &decl_tag_ops, [BTF_KIND_TYPE_TAG] = &modifier_ops, + [BTF_KIND_ENUM64] = &enum64_ops, }; static s32 btf_check_meta(struct btf_verifier_env *env, @@ -5299,7 +5412,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, /* skip modifiers */ while (btf_type_is_modifier(t)) t = btf_type_by_id(btf, t->type); - if (btf_type_is_small_int(t) || btf_type_is_enum(t)) + if (btf_type_is_small_int(t) || btf_is_any_enum(t)) /* accessing a scalar */ return true; if (!btf_type_is_ptr(t)) { @@ -5763,7 +5876,7 @@ static int __get_type_size(struct btf *btf, u32 btf_id, if (btf_type_is_ptr(t)) /* kernel size of pointer. Not BPF's size of pointer*/ return sizeof(void *); - if (btf_type_is_int(t) || btf_type_is_enum(t)) + if (btf_type_is_int(t) || btf_is_any_enum(t)) return t->size; *bad_type = t; return -EINVAL; @@ -5911,7 +6024,7 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, * to context only. And only global functions can be replaced. * Hence type check only those types. */ - if (btf_type_is_int(t1) || btf_type_is_enum(t1)) + if (btf_type_is_int(t1) || btf_is_any_enum(t1)) continue; if (!btf_type_is_ptr(t1)) { bpf_log(log, @@ -6408,7 +6521,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, t = btf_type_by_id(btf, t->type); while (btf_type_is_modifier(t)) t = btf_type_by_id(btf, t->type); - if (!btf_type_is_int(t) && !btf_type_is_enum(t)) { + if (!btf_type_is_int(t) && !btf_is_any_enum(t)) { bpf_log(log, "Global function %s() doesn't return scalar. Only those are supported.\n", tname); @@ -6423,7 +6536,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) t = btf_type_by_id(btf, t->type); - if (btf_type_is_int(t) || btf_type_is_enum(t)) { + if (btf_type_is_int(t) || btf_is_any_enum(t)) { reg->type = SCALAR_VALUE; continue; } @@ -7335,6 +7448,7 @@ recur: case BTF_KIND_UNION: case BTF_KIND_ENUM: case BTF_KIND_FWD: + case BTF_KIND_ENUM64: return 1; case BTF_KIND_INT: /* just reject deprecated bitfield-like integers; all other @@ -7387,10 +7501,10 @@ recur: * field-based relocations. This function assumes that root types were already * checked for name match. Beyond that initial root-level name check, names * are completely ignored. Compatibility rules are as follows: - * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but + * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs/ENUM64s are considered compatible, but * kind should match for local and target types (i.e., STRUCT is not * compatible with UNION); - * - for ENUMs, the size is ignored; + * - for ENUMs/ENUM64s, the size is ignored; * - for INT, size and signedness are ignored; * - for ARRAY, dimensionality is ignored, element types are checked for * compatibility recursively; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index aedac2ac02b9..2d2872682278 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10901,7 +10901,7 @@ static int check_btf_func(struct bpf_verifier_env *env, goto err_free; ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); scalar_return = - btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); + btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type); if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); goto err_free; diff --git a/tools/include/uapi/linux/btf.h b/tools/include/uapi/linux/btf.h index a9162a6c0284..ec1798b6d3ff 100644 --- a/tools/include/uapi/linux/btf.h +++ b/tools/include/uapi/linux/btf.h @@ -36,10 +36,10 @@ struct btf_type { * bits 24-28: kind (e.g. int, ptr, array...etc) * bits 29-30: unused * bit 31: kind_flag, currently used by - * struct, union and fwd + * struct, union, enum, fwd and enum64 */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION and DATASEC. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, @@ -63,7 +63,7 @@ enum { BTF_KIND_ARRAY = 3, /* Array */ BTF_KIND_STRUCT = 4, /* Struct */ BTF_KIND_UNION = 5, /* Union */ - BTF_KIND_ENUM = 6, /* Enumeration */ + BTF_KIND_ENUM = 6, /* Enumeration up to 32-bit values */ BTF_KIND_FWD = 7, /* Forward */ BTF_KIND_TYPEDEF = 8, /* Typedef */ BTF_KIND_VOLATILE = 9, /* Volatile */ @@ -76,6 +76,7 @@ enum { BTF_KIND_FLOAT = 16, /* Floating point */ BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ + BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, @@ -186,4 +187,14 @@ struct btf_decl_tag { __s32 component_idx; }; +/* BTF_KIND_ENUM64 is followed by multiple "struct btf_enum64". + * The exact number of btf_enum64 is stored in the vlen (of the + * info in "struct btf_type"). + */ +struct btf_enum64 { + __u32 name_off; + __u32 val_lo32; + __u32 val_hi32; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ -- cgit v1.2.3 From 0e3c3b901c00364198d31482fa2552ccf2d5c899 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 6 Jun 2022 18:42:59 -0400 Subject: No need of likely/unlikely on calls of check_copy_size() it's inline and unlikely() inside of it (including the implicit one in WARN_ON_ONCE()) suffice to convince the compiler that getting false from check_copy_size() is unlikely. Spotted-by: Jens Axboe Reviewed-by: Christoph Hellwig Reviewed-by: Christian Brauner (Microsoft) Signed-off-by: Al Viro --- arch/powerpc/include/asm/uaccess.h | 2 +- arch/s390/include/asm/uaccess.h | 4 ++-- include/linux/uaccess.h | 4 ++-- include/linux/uio.h | 15 ++++++--------- 4 files changed, 11 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 9b82b38ff867..105f200b1e31 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -348,7 +348,7 @@ copy_mc_to_kernel(void *to, const void *from, unsigned long size) static inline unsigned long __must_check copy_mc_to_user(void __user *to, const void *from, unsigned long n) { - if (likely(check_copy_size(from, n, true))) { + if (check_copy_size(from, n, true)) { if (access_ok(to, n)) { allow_write_to_user(to, n); n = copy_mc_generic((void *)to, from, n); diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index f4511e21d646..c2c9995466e0 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -39,7 +39,7 @@ _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned static __always_inline unsigned long __must_check copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key) { - if (likely(check_copy_size(to, n, false))) + if (check_copy_size(to, n, false)) n = _copy_from_user_key(to, from, n, key); return n; } @@ -50,7 +50,7 @@ _copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned l static __always_inline unsigned long __must_check copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key) { - if (likely(check_copy_size(from, n, true))) + if (check_copy_size(from, n, true)) n = _copy_to_user_key(to, from, n, key); return n; } diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 5a328cf02b75..47e5d374c7eb 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -148,7 +148,7 @@ _copy_to_user(void __user *, const void *, unsigned long); static __always_inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) { - if (likely(check_copy_size(to, n, false))) + if (check_copy_size(to, n, false)) n = _copy_from_user(to, from, n); return n; } @@ -156,7 +156,7 @@ copy_from_user(void *to, const void __user *from, unsigned long n) static __always_inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n) { - if (likely(check_copy_size(from, n, true))) + if (check_copy_size(from, n, true)) n = _copy_to_user(to, from, n); return n; } diff --git a/include/linux/uio.h b/include/linux/uio.h index 739285fe5a2f..76d305f3d4c2 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -156,19 +156,17 @@ static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset, static __always_inline __must_check size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - if (unlikely(!check_copy_size(addr, bytes, true))) - return 0; - else + if (check_copy_size(addr, bytes, true)) return _copy_to_iter(addr, bytes, i); + return 0; } static __always_inline __must_check size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) { - if (unlikely(!check_copy_size(addr, bytes, false))) - return 0; - else + if (check_copy_size(addr, bytes, false)) return _copy_from_iter(addr, bytes, i); + return 0; } static __always_inline __must_check @@ -184,10 +182,9 @@ bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) static __always_inline __must_check size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { - if (unlikely(!check_copy_size(addr, bytes, false))) - return 0; - else + if (check_copy_size(addr, bytes, false)) return _copy_from_iter_nocache(addr, bytes, i); + return 0; } static __always_inline __must_check -- cgit v1.2.3 From b1d288d9c3c5ca28df062214656a59cf7ee370e0 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 6 Jun 2022 20:18:03 +0000 Subject: platform/chrome: cros_ec_proto: Rename cros_ec_command function cros_ec_command() is the name of a function as well as a struct, as such it can confuse indexing tools (like ctags). Avoid this by renaming it to cros_ec_cmd(). Update all the callsites to use the new name. This patch is a find-and-replace, so should not introduce any functional changes. Suggested-by: Stephen Boyd Signed-off-by: Prashant Malani Acked-by: Lee Jones Reviewed-by: Stephen Boyd Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220606201825.763788-3-pmalani@chromium.org --- drivers/mfd/cros_ec_dev.c | 4 +-- drivers/platform/chrome/cros_ec_proto.c | 22 ++++++++-------- drivers/platform/chrome/cros_ec_typec.c | 39 ++++++++++++++--------------- drivers/platform/chrome/cros_usbpd_notify.c | 4 +-- drivers/regulator/cros-ec-regulator.c | 24 +++++++++--------- include/linux/platform_data/cros_ec_proto.h | 2 +- 6 files changed, 47 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c index 596731caf407..02d4271dfe06 100644 --- a/drivers/mfd/cros_ec_dev.c +++ b/drivers/mfd/cros_ec_dev.c @@ -250,8 +250,8 @@ static int ec_device_probe(struct platform_device *pdev) * The PCHG device cannot be detected by sending EC_FEATURE_GET_CMD, but * it can be detected by querying the number of peripheral chargers. */ - retval = cros_ec_command(ec->ec_dev, 0, EC_CMD_PCHG_COUNT, NULL, 0, - &pchg_count, sizeof(pchg_count)); + retval = cros_ec_cmd(ec->ec_dev, 0, EC_CMD_PCHG_COUNT, NULL, 0, + &pchg_count, sizeof(pchg_count)); if (retval >= 0 && pchg_count.port_count) { retval = mfd_add_hotplug_devices(ec->dev, cros_ec_pchg_cells, diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index 13ced9d2dd71..b6bea183ee28 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -860,8 +860,8 @@ bool cros_ec_check_features(struct cros_ec_dev *ec, int feature) if (features->flags[0] == -1U && features->flags[1] == -1U) { /* features bitmap not read yet */ - ret = cros_ec_command(ec->ec_dev, 0, EC_CMD_GET_FEATURES + ec->cmd_offset, - NULL, 0, features, sizeof(*features)); + ret = cros_ec_cmd(ec->ec_dev, 0, EC_CMD_GET_FEATURES + ec->cmd_offset, + NULL, 0, features, sizeof(*features)); if (ret < 0) { dev_warn(ec->dev, "cannot get EC features: %d\n", ret); memset(features, 0, sizeof(*features)); @@ -942,7 +942,7 @@ int cros_ec_get_sensor_count(struct cros_ec_dev *ec) EXPORT_SYMBOL_GPL(cros_ec_get_sensor_count); /** - * cros_ec_command - Send a command to the EC. + * cros_ec_cmd - Send a command to the EC. * * @ec_dev: EC device * @version: EC command version @@ -954,13 +954,13 @@ EXPORT_SYMBOL_GPL(cros_ec_get_sensor_count); * * Return: >= 0 on success, negative error number on failure. */ -int cros_ec_command(struct cros_ec_device *ec_dev, - unsigned int version, - int command, - void *outdata, - int outsize, - void *indata, - int insize) +int cros_ec_cmd(struct cros_ec_device *ec_dev, + unsigned int version, + int command, + void *outdata, + int outsize, + void *indata, + int insize) { struct cros_ec_command *msg; int ret; @@ -987,4 +987,4 @@ error: kfree(msg); return ret; } -EXPORT_SYMBOL_GPL(cros_ec_command); +EXPORT_SYMBOL_GPL(cros_ec_cmd); diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 7cb2e35c4ded..d6088ba447af 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -525,8 +525,8 @@ static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, enum typec_orientation orientation; int ret; - ret = cros_ec_command(typec->ec, 0, EC_CMD_USB_PD_MUX_INFO, - &req, sizeof(req), &resp, sizeof(resp)); + ret = cros_ec_cmd(typec->ec, 0, EC_CMD_USB_PD_MUX_INFO, + &req, sizeof(req), &resp, sizeof(resp)); if (ret < 0) { dev_warn(typec->dev, "Failed to get mux info for port: %d, err = %d\n", port_num, ret); @@ -585,8 +585,8 @@ mux_ack: /* Sending Acknowledgment to EC */ mux_ack.port = port_num; - if (cros_ec_command(typec->ec, 0, EC_CMD_USB_PD_MUX_ACK, &mux_ack, - sizeof(mux_ack), NULL, 0) < 0) + if (cros_ec_cmd(typec->ec, 0, EC_CMD_USB_PD_MUX_ACK, &mux_ack, + sizeof(mux_ack), NULL, 0) < 0) dev_warn(typec->dev, "Failed to send Mux ACK to EC for port: %d\n", port_num); @@ -754,8 +754,8 @@ static int cros_typec_handle_sop_prime_disc(struct cros_typec_data *typec, int p int ret = 0; memset(disc, 0, EC_PROTO2_MAX_RESPONSE_SIZE); - ret = cros_ec_command(typec->ec, 0, EC_CMD_TYPEC_DISCOVERY, &req, sizeof(req), - disc, EC_PROTO2_MAX_RESPONSE_SIZE); + ret = cros_ec_cmd(typec->ec, 0, EC_CMD_TYPEC_DISCOVERY, &req, sizeof(req), + disc, EC_PROTO2_MAX_RESPONSE_SIZE); if (ret < 0) { dev_err(typec->dev, "Failed to get SOP' discovery data for port: %d\n", port_num); goto sop_prime_disc_exit; @@ -837,8 +837,8 @@ static int cros_typec_handle_sop_disc(struct cros_typec_data *typec, int port_nu typec_partner_set_pd_revision(port->partner, pd_revision); memset(sop_disc, 0, EC_PROTO2_MAX_RESPONSE_SIZE); - ret = cros_ec_command(typec->ec, 0, EC_CMD_TYPEC_DISCOVERY, &req, sizeof(req), - sop_disc, EC_PROTO2_MAX_RESPONSE_SIZE); + ret = cros_ec_cmd(typec->ec, 0, EC_CMD_TYPEC_DISCOVERY, &req, sizeof(req), + sop_disc, EC_PROTO2_MAX_RESPONSE_SIZE); if (ret < 0) { dev_err(typec->dev, "Failed to get SOP discovery data for port: %d\n", port_num); goto disc_exit; @@ -870,8 +870,8 @@ static int cros_typec_send_clear_event(struct cros_typec_data *typec, int port_n .clear_events_mask = events_mask, }; - return cros_ec_command(typec->ec, 0, EC_CMD_TYPEC_CONTROL, &req, - sizeof(req), NULL, 0); + return cros_ec_cmd(typec->ec, 0, EC_CMD_TYPEC_CONTROL, &req, + sizeof(req), NULL, 0); } static void cros_typec_handle_status(struct cros_typec_data *typec, int port_num) @@ -882,8 +882,8 @@ static void cros_typec_handle_status(struct cros_typec_data *typec, int port_num }; int ret; - ret = cros_ec_command(typec->ec, 0, EC_CMD_TYPEC_STATUS, &req, sizeof(req), - &resp, sizeof(resp)); + ret = cros_ec_cmd(typec->ec, 0, EC_CMD_TYPEC_STATUS, &req, sizeof(req), + &resp, sizeof(resp)); if (ret < 0) { dev_warn(typec->dev, "EC_CMD_TYPEC_STATUS failed for port: %d\n", port_num); return; @@ -960,9 +960,9 @@ static int cros_typec_port_update(struct cros_typec_data *typec, int port_num) req.mux = USB_PD_CTRL_MUX_NO_CHANGE; req.swap = USB_PD_CTRL_SWAP_NONE; - ret = cros_ec_command(typec->ec, typec->pd_ctrl_ver, - EC_CMD_USB_PD_CONTROL, &req, sizeof(req), - &resp, sizeof(resp)); + ret = cros_ec_cmd(typec->ec, typec->pd_ctrl_ver, + EC_CMD_USB_PD_CONTROL, &req, sizeof(req), + &resp, sizeof(resp)); if (ret < 0) return ret; @@ -997,9 +997,8 @@ static int cros_typec_get_cmd_version(struct cros_typec_data *typec) /* We're interested in the PD control command version. */ req_v1.cmd = EC_CMD_USB_PD_CONTROL; - ret = cros_ec_command(typec->ec, 1, EC_CMD_GET_CMD_VERSIONS, - &req_v1, sizeof(req_v1), &resp, - sizeof(resp)); + ret = cros_ec_cmd(typec->ec, 1, EC_CMD_GET_CMD_VERSIONS, + &req_v1, sizeof(req_v1), &resp, sizeof(resp)); if (ret < 0) return ret; @@ -1090,8 +1089,8 @@ static int cros_typec_probe(struct platform_device *pdev) typec->typec_cmd_supported = cros_ec_check_features(ec_dev, EC_FEATURE_TYPEC_CMD); typec->needs_mux_ack = cros_ec_check_features(ec_dev, EC_FEATURE_TYPEC_MUX_REQUIRE_AP_ACK); - ret = cros_ec_command(typec->ec, 0, EC_CMD_USB_PD_PORTS, NULL, 0, - &resp, sizeof(resp)); + ret = cros_ec_cmd(typec->ec, 0, EC_CMD_USB_PD_PORTS, NULL, 0, + &resp, sizeof(resp)); if (ret < 0) return ret; diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c index 91ce6be91aac..4b5a81c9dc6d 100644 --- a/drivers/platform/chrome/cros_usbpd_notify.c +++ b/drivers/platform/chrome/cros_usbpd_notify.c @@ -71,8 +71,8 @@ static void cros_usbpd_get_event_and_notify(struct device *dev, } /* Check for PD host events on EC. */ - ret = cros_ec_command(ec_dev, 0, EC_CMD_PD_HOST_EVENT_STATUS, - NULL, 0, &host_event_status, sizeof(host_event_status)); + ret = cros_ec_cmd(ec_dev, 0, EC_CMD_PD_HOST_EVENT_STATUS, + NULL, 0, &host_event_status, sizeof(host_event_status)); if (ret < 0) { dev_warn(dev, "Can't get host event status (err: %d)\n", ret); goto send_notify; diff --git a/drivers/regulator/cros-ec-regulator.c b/drivers/regulator/cros-ec-regulator.c index 1c5fc74a4776..1591636f86c3 100644 --- a/drivers/regulator/cros-ec-regulator.c +++ b/drivers/regulator/cros-ec-regulator.c @@ -30,8 +30,8 @@ static int cros_ec_regulator_enable(struct regulator_dev *dev) .enable = 1, }; - return cros_ec_command(data->ec_dev, 0, EC_CMD_REGULATOR_ENABLE, &cmd, - sizeof(cmd), NULL, 0); + return cros_ec_cmd(data->ec_dev, 0, EC_CMD_REGULATOR_ENABLE, &cmd, + sizeof(cmd), NULL, 0); } static int cros_ec_regulator_disable(struct regulator_dev *dev) @@ -42,8 +42,8 @@ static int cros_ec_regulator_disable(struct regulator_dev *dev) .enable = 0, }; - return cros_ec_command(data->ec_dev, 0, EC_CMD_REGULATOR_ENABLE, &cmd, - sizeof(cmd), NULL, 0); + return cros_ec_cmd(data->ec_dev, 0, EC_CMD_REGULATOR_ENABLE, &cmd, + sizeof(cmd), NULL, 0); } static int cros_ec_regulator_is_enabled(struct regulator_dev *dev) @@ -55,8 +55,8 @@ static int cros_ec_regulator_is_enabled(struct regulator_dev *dev) struct ec_response_regulator_is_enabled resp; int ret; - ret = cros_ec_command(data->ec_dev, 0, EC_CMD_REGULATOR_IS_ENABLED, &cmd, - sizeof(cmd), &resp, sizeof(resp)); + ret = cros_ec_cmd(data->ec_dev, 0, EC_CMD_REGULATOR_IS_ENABLED, &cmd, + sizeof(cmd), &resp, sizeof(resp)); if (ret < 0) return ret; return resp.enabled; @@ -82,8 +82,8 @@ static int cros_ec_regulator_get_voltage(struct regulator_dev *dev) struct ec_response_regulator_get_voltage resp; int ret; - ret = cros_ec_command(data->ec_dev, 0, EC_CMD_REGULATOR_GET_VOLTAGE, &cmd, - sizeof(cmd), &resp, sizeof(resp)); + ret = cros_ec_cmd(data->ec_dev, 0, EC_CMD_REGULATOR_GET_VOLTAGE, &cmd, + sizeof(cmd), &resp, sizeof(resp)); if (ret < 0) return ret; return resp.voltage_mv * 1000; @@ -108,8 +108,8 @@ static int cros_ec_regulator_set_voltage(struct regulator_dev *dev, int min_uV, if (min_mV > max_mV) return -EINVAL; - return cros_ec_command(data->ec_dev, 0, EC_CMD_REGULATOR_SET_VOLTAGE, &cmd, - sizeof(cmd), NULL, 0); + return cros_ec_cmd(data->ec_dev, 0, EC_CMD_REGULATOR_SET_VOLTAGE, &cmd, + sizeof(cmd), NULL, 0); } static const struct regulator_ops cros_ec_regulator_voltage_ops = { @@ -130,8 +130,8 @@ static int cros_ec_regulator_init_info(struct device *dev, struct ec_response_regulator_get_info resp; int ret; - ret = cros_ec_command(data->ec_dev, 0, EC_CMD_REGULATOR_GET_INFO, &cmd, - sizeof(cmd), &resp, sizeof(resp)); + ret = cros_ec_cmd(data->ec_dev, 0, EC_CMD_REGULATOR_GET_INFO, &cmd, + sizeof(cmd), &resp, sizeof(resp)); if (ret < 0) return ret; diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index 138fd912c808..816da4eef3e5 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -231,7 +231,7 @@ bool cros_ec_check_features(struct cros_ec_dev *ec, int feature); int cros_ec_get_sensor_count(struct cros_ec_dev *ec); -int cros_ec_command(struct cros_ec_device *ec_dev, unsigned int version, int command, void *outdata, +int cros_ec_cmd(struct cros_ec_device *ec_dev, unsigned int version, int command, void *outdata, int outsize, void *indata, int insize); /** -- cgit v1.2.3 From f87e15fbf6d8cdb51f953338d41a4a52ad1aca14 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 6 Jun 2022 20:18:05 +0000 Subject: platform/chrome: cros_ec_proto: Update size arg types cros_ec_cmd() takes 2 size arguments. Update them to be of the more appropriate type size_t. Suggested-by: Stephen Boyd Signed-off-by: Prashant Malani Reviewed-by: Stephen Boyd Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220606201825.763788-4-pmalani@chromium.org --- drivers/platform/chrome/cros_ec_proto.c | 4 ++-- include/linux/platform_data/cros_ec_proto.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index b6bea183ee28..cefabfe45551 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -958,9 +958,9 @@ int cros_ec_cmd(struct cros_ec_device *ec_dev, unsigned int version, int command, void *outdata, - int outsize, + size_t outsize, void *indata, - int insize) + size_t insize) { struct cros_ec_command *msg; int ret; diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index 816da4eef3e5..85e29300f63d 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -232,7 +232,7 @@ bool cros_ec_check_features(struct cros_ec_dev *ec, int feature); int cros_ec_get_sensor_count(struct cros_ec_dev *ec); int cros_ec_cmd(struct cros_ec_device *ec_dev, unsigned int version, int command, void *outdata, - int outsize, void *indata, int insize); + size_t outsize, void *indata, size_t insize); /** * cros_ec_get_time_ns() - Return time in ns. -- cgit v1.2.3 From 5dfac65b621733e69b789150a0a3f1bf2f9095a3 Mon Sep 17 00:00:00 2001 From: David Jander Date: Wed, 8 Jun 2022 17:33:09 +0200 Subject: spi: : Add missing documentation for struct members Fixes these "make htmldocs" warnings: include/linux/spi/spi.h:82: warning: Function parameter or member 'syncp' not described in 'spi_statistics' include/linux/spi/spi.h:213: warning: Function parameter or member 'pcpu_statistics' not described in 'spi_device' include/linux/spi/spi.h:676: warning: Function parameter or member 'pcpu_statistics' not described in 'spi_controller' Fixes: 6598b91b5ac3 ("spi: spi.c: Convert statistics to per-cpu u64_stats_t") Reported-by: Stephen Rothwell Signed-off-by: David Jander Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20220608153309.2899565-1-david@protonic.nl Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index eac8d3caf954..2e63b4935deb 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -35,7 +35,8 @@ extern struct bus_type spi_bus_type; /** * struct spi_statistics - statistics for spi transfers - * @lock: lock protecting this structure + * @syncp: seqcount to protect members in this struct for per-cpu udate + * on 32-bit systems * * @messages: number of spi-messages handled * @transfers: number of spi_transfers handled @@ -155,7 +156,7 @@ extern int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer); * @cs_inactive: delay to be introduced by the controller after CS is * deasserted. If @cs_change_delay is used from @spi_transfer, then the * two delays will be added up. - * @statistics: statistics for the spi_device + * @pcpu_statistics: statistics for the spi_device * * A @spi_device is used to interchange data between an SPI slave * (usually a discrete chip) and CPU memory. @@ -439,7 +440,7 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @max_native_cs: When cs_gpiods is used, and this field is filled in, * spi_register_controller() will validate all native CS (including the * unused native CS) against this value. - * @statistics: statistics for the spi_controller + * @pcpu_statistics: statistics for the spi_controller * @dma_tx: DMA transmit channel * @dma_rx: DMA receive channel * @dummy_rx: dummy receive buffer for full-duplex devices -- cgit v1.2.3 From 0c90466a7985d39355f743e9cd2139da3e86c4d8 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Fri, 3 Jun 2022 23:07:45 +0200 Subject: mtd: hyperbus: Make hyperbus_unregister_device() return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only thing that could theoretically fail in that function is mtd_device_unregister(). However it's not supposed to fail and when used correctly it doesn't. So wail loudly if it does anyhow. This matches how other drivers (e.g. nand/raw/nandsim.c) use mtd_device_unregister(). This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20220603210758.148493-2-u.kleine-koenig@pengutronix.de --- drivers/mtd/hyperbus/hbmc-am654.c | 6 +++--- drivers/mtd/hyperbus/hyperbus-core.c | 8 ++------ drivers/mtd/hyperbus/rpc-if.c | 5 +++-- include/linux/mtd/hyperbus.h | 4 +--- 4 files changed, 9 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/mtd/hyperbus/hbmc-am654.c b/drivers/mtd/hyperbus/hbmc-am654.c index a3439b791eeb..a6161ce340d4 100644 --- a/drivers/mtd/hyperbus/hbmc-am654.c +++ b/drivers/mtd/hyperbus/hbmc-am654.c @@ -233,16 +233,16 @@ static int am654_hbmc_remove(struct platform_device *pdev) { struct am654_hbmc_priv *priv = platform_get_drvdata(pdev); struct am654_hbmc_device_priv *dev_priv = priv->hbdev.priv; - int ret; - ret = hyperbus_unregister_device(&priv->hbdev); + hyperbus_unregister_device(&priv->hbdev); + if (priv->mux_ctrl) mux_control_deselect(priv->mux_ctrl); if (dev_priv->rx_chan) dma_release_channel(dev_priv->rx_chan); - return ret; + return 0; } static const struct of_device_id am654_hbmc_dt_ids[] = { diff --git a/drivers/mtd/hyperbus/hyperbus-core.c b/drivers/mtd/hyperbus/hyperbus-core.c index 2f9fc4e17d53..4d8047d43e48 100644 --- a/drivers/mtd/hyperbus/hyperbus-core.c +++ b/drivers/mtd/hyperbus/hyperbus-core.c @@ -126,16 +126,12 @@ int hyperbus_register_device(struct hyperbus_device *hbdev) } EXPORT_SYMBOL_GPL(hyperbus_register_device); -int hyperbus_unregister_device(struct hyperbus_device *hbdev) +void hyperbus_unregister_device(struct hyperbus_device *hbdev) { - int ret = 0; - if (hbdev && hbdev->mtd) { - ret = mtd_device_unregister(hbdev->mtd); + WARN_ON(mtd_device_unregister(hbdev->mtd)); map_destroy(hbdev->mtd); } - - return ret; } EXPORT_SYMBOL_GPL(hyperbus_unregister_device); diff --git a/drivers/mtd/hyperbus/rpc-if.c b/drivers/mtd/hyperbus/rpc-if.c index 6e08ec1d4f09..15a0be63ede1 100644 --- a/drivers/mtd/hyperbus/rpc-if.c +++ b/drivers/mtd/hyperbus/rpc-if.c @@ -153,11 +153,12 @@ static int rpcif_hb_probe(struct platform_device *pdev) static int rpcif_hb_remove(struct platform_device *pdev) { struct rpcif_hyperbus *hyperbus = platform_get_drvdata(pdev); - int error = hyperbus_unregister_device(&hyperbus->hbdev); + + hyperbus_unregister_device(&hyperbus->hbdev); rpcif_disable_rpm(&hyperbus->rpc); - return error; + return 0; } static struct platform_driver rpcif_platform_driver = { diff --git a/include/linux/mtd/hyperbus.h b/include/linux/mtd/hyperbus.h index 0ce612428aea..bb6b7121a542 100644 --- a/include/linux/mtd/hyperbus.h +++ b/include/linux/mtd/hyperbus.h @@ -89,9 +89,7 @@ int hyperbus_register_device(struct hyperbus_device *hbdev); /** * hyperbus_unregister_device - deregister HyperBus slave memory device * @hbdev: hyperbus_device to be unregistered - * - * Return: 0 for success, others for failure. */ -int hyperbus_unregister_device(struct hyperbus_device *hbdev); +void hyperbus_unregister_device(struct hyperbus_device *hbdev); #endif /* __LINUX_MTD_HYPERBUS_H__ */ -- cgit v1.2.3 From 0949ee75da6c918fcbd567e1bfa4943a56ab4e5d Mon Sep 17 00:00:00 2001 From: Javier Martinez Canillas Date: Tue, 7 Jun 2022 20:23:34 +0200 Subject: firmware: sysfb: Make sysfb_create_simplefb() return a pdev pointer This function just returned 0 on success or an errno code on error, but it could be useful for sysfb_init() callers to have a pointer to the device. Signed-off-by: Javier Martinez Canillas Reviewed-by: Daniel Vetter Reviewed-by: Thomas Zimmermann Link: https://patchwork.freedesktop.org/patch/msgid/20220607182338.344270-2-javierm@redhat.com --- drivers/firmware/sysfb.c | 4 ++-- drivers/firmware/sysfb_simplefb.c | 16 ++++++++-------- include/linux/sysfb.h | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/sysfb.c b/drivers/firmware/sysfb.c index 2bfbb05f7d89..b032f40a92de 100644 --- a/drivers/firmware/sysfb.c +++ b/drivers/firmware/sysfb.c @@ -46,8 +46,8 @@ static __init int sysfb_init(void) /* try to create a simple-framebuffer device */ compatible = sysfb_parse_mode(si, &mode); if (compatible) { - ret = sysfb_create_simplefb(si, &mode); - if (!ret) + pd = sysfb_create_simplefb(si, &mode); + if (!IS_ERR(pd)) return 0; } diff --git a/drivers/firmware/sysfb_simplefb.c b/drivers/firmware/sysfb_simplefb.c index bda8712bfd8c..a353e27f83f5 100644 --- a/drivers/firmware/sysfb_simplefb.c +++ b/drivers/firmware/sysfb_simplefb.c @@ -57,8 +57,8 @@ __init bool sysfb_parse_mode(const struct screen_info *si, return false; } -__init int sysfb_create_simplefb(const struct screen_info *si, - const struct simplefb_platform_data *mode) +__init struct platform_device *sysfb_create_simplefb(const struct screen_info *si, + const struct simplefb_platform_data *mode) { struct platform_device *pd; struct resource res; @@ -76,7 +76,7 @@ __init int sysfb_create_simplefb(const struct screen_info *si, base |= (u64)si->ext_lfb_base << 32; if (!base || (u64)(resource_size_t)base != base) { printk(KERN_DEBUG "sysfb: inaccessible VRAM base\n"); - return -EINVAL; + return ERR_PTR(-EINVAL); } /* @@ -93,7 +93,7 @@ __init int sysfb_create_simplefb(const struct screen_info *si, length = mode->height * mode->stride; if (length > size) { printk(KERN_WARNING "sysfb: VRAM smaller than advertised\n"); - return -EINVAL; + return ERR_PTR(-EINVAL); } length = PAGE_ALIGN(length); @@ -104,11 +104,11 @@ __init int sysfb_create_simplefb(const struct screen_info *si, res.start = base; res.end = res.start + length - 1; if (res.end <= res.start) - return -EINVAL; + return ERR_PTR(-EINVAL); pd = platform_device_alloc("simple-framebuffer", 0); if (!pd) - return -ENOMEM; + return ERR_PTR(-ENOMEM); sysfb_apply_efi_quirks(pd); @@ -124,10 +124,10 @@ __init int sysfb_create_simplefb(const struct screen_info *si, if (ret) goto err_put_device; - return 0; + return pd; err_put_device: platform_device_put(pd); - return ret; + return ERR_PTR(ret); } diff --git a/include/linux/sysfb.h b/include/linux/sysfb.h index b0dcfa26d07b..708152e9037b 100644 --- a/include/linux/sysfb.h +++ b/include/linux/sysfb.h @@ -72,8 +72,8 @@ static inline void sysfb_apply_efi_quirks(struct platform_device *pd) bool sysfb_parse_mode(const struct screen_info *si, struct simplefb_platform_data *mode); -int sysfb_create_simplefb(const struct screen_info *si, - const struct simplefb_platform_data *mode); +struct platform_device *sysfb_create_simplefb(const struct screen_info *si, + const struct simplefb_platform_data *mode); #else /* CONFIG_SYSFB_SIMPLE */ @@ -83,10 +83,10 @@ static inline bool sysfb_parse_mode(const struct screen_info *si, return false; } -static inline int sysfb_create_simplefb(const struct screen_info *si, - const struct simplefb_platform_data *mode) +static inline struct platform_device *sysfb_create_simplefb(const struct screen_info *si, + const struct simplefb_platform_data *mode) { - return -EINVAL; + return ERR_PTR(-EINVAL); } #endif /* CONFIG_SYSFB_SIMPLE */ -- cgit v1.2.3 From bc824922b264aff40eba8c160972ee07a95e7dd4 Mon Sep 17 00:00:00 2001 From: Javier Martinez Canillas Date: Tue, 7 Jun 2022 20:23:35 +0200 Subject: firmware: sysfb: Add sysfb_disable() helper function This can be used by subsystems to unregister a platform device registered by sysfb and also to disable future platform device registration in sysfb. Suggested-by: Daniel Vetter Signed-off-by: Javier Martinez Canillas Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20220607182338.344270-3-javierm@redhat.com --- .../driver-api/firmware/other_interfaces.rst | 6 +++ drivers/firmware/sysfb.c | 54 +++++++++++++++++++--- include/linux/sysfb.h | 12 +++++ 3 files changed, 66 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/firmware/other_interfaces.rst b/Documentation/driver-api/firmware/other_interfaces.rst index b81794e0cfbb..06ac89adaafb 100644 --- a/Documentation/driver-api/firmware/other_interfaces.rst +++ b/Documentation/driver-api/firmware/other_interfaces.rst @@ -13,6 +13,12 @@ EDD Interfaces .. kernel-doc:: drivers/firmware/edd.c :internal: +Generic System Framebuffers Interface +------------------------------------- + +.. kernel-doc:: drivers/firmware/sysfb.c + :export: + Intel Stratix10 SoC Service Layer --------------------------------- Some features of the Intel Stratix10 SoC require a level of privilege diff --git a/drivers/firmware/sysfb.c b/drivers/firmware/sysfb.c index b032f40a92de..1f276f108cc9 100644 --- a/drivers/firmware/sysfb.c +++ b/drivers/firmware/sysfb.c @@ -34,21 +34,59 @@ #include #include +static struct platform_device *pd; +static DEFINE_MUTEX(disable_lock); +static bool disabled; + +static bool sysfb_unregister(void) +{ + if (IS_ERR_OR_NULL(pd)) + return false; + + platform_device_unregister(pd); + pd = NULL; + + return true; +} + +/** + * sysfb_disable() - disable the Generic System Framebuffers support + * + * This disables the registration of system framebuffer devices that match the + * generic drivers that make use of the system framebuffer set up by firmware. + * + * It also unregisters a device if this was already registered by sysfb_init(). + * + * Context: The function can sleep. A @disable_lock mutex is acquired to serialize + * against sysfb_init(), that registers a system framebuffer device. + */ +void sysfb_disable(void) +{ + mutex_lock(&disable_lock); + sysfb_unregister(); + disabled = true; + mutex_unlock(&disable_lock); +} +EXPORT_SYMBOL_GPL(sysfb_disable); + static __init int sysfb_init(void) { struct screen_info *si = &screen_info; struct simplefb_platform_data mode; - struct platform_device *pd; const char *name; bool compatible; - int ret; + int ret = 0; + + mutex_lock(&disable_lock); + if (disabled) + goto unlock_mutex; /* try to create a simple-framebuffer device */ compatible = sysfb_parse_mode(si, &mode); if (compatible) { pd = sysfb_create_simplefb(si, &mode); if (!IS_ERR(pd)) - return 0; + goto unlock_mutex; } /* if the FB is incompatible, create a legacy framebuffer device */ @@ -60,8 +98,10 @@ static __init int sysfb_init(void) name = "platform-framebuffer"; pd = platform_device_alloc(name, 0); - if (!pd) - return -ENOMEM; + if (!pd) { + ret = -ENOMEM; + goto unlock_mutex; + } sysfb_apply_efi_quirks(pd); @@ -73,9 +113,11 @@ static __init int sysfb_init(void) if (ret) goto err; - return 0; + goto unlock_mutex; err: platform_device_put(pd); +unlock_mutex: + mutex_unlock(&disable_lock); return ret; } diff --git a/include/linux/sysfb.h b/include/linux/sysfb.h index 708152e9037b..8ba8b5be5567 100644 --- a/include/linux/sysfb.h +++ b/include/linux/sysfb.h @@ -55,6 +55,18 @@ struct efifb_dmi_info { int flags; }; +#ifdef CONFIG_SYSFB + +void sysfb_disable(void); + +#else /* CONFIG_SYSFB */ + +static inline void sysfb_disable(void) +{ +} + +#endif /* CONFIG_SYSFB */ + #ifdef CONFIG_EFI extern struct efifb_dmi_info efifb_dmi_list[]; -- cgit v1.2.3 From ea7f0f777d28db6e500a05836f2a9d467c7012de Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Thu, 9 Jun 2022 08:49:37 +0000 Subject: platform/chrome: cros_ec_commands: fix compile errors Fix compile errors when including cros_ec_commands.h solely. 1. cros_ec_commands.h:587:9: error: unknown type name 'uint8_t' 587 | uint8_t flags; | ^~~~~~~ 2. cros_ec_commands.h:1105:43: error: implicit declaration of function 'BIT' 1105 | EC_COMMS_STATUS_PROCESSING = BIT(0), | ^~~ Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220609084957.3684698-2-tzungbi@kernel.org --- include/linux/platform_data/cros_ec_commands.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index e59f51c41a1c..f13568b3e247 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -13,8 +13,8 @@ #ifndef __CROS_EC_COMMANDS_H #define __CROS_EC_COMMANDS_H - - +#include +#include #define BUILD_ASSERT(_cond) -- cgit v1.2.3 From 3db0c9e5de7bd9dbe52580eb9752b2b3049e38da Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Thu, 9 Jun 2022 08:49:39 +0000 Subject: platform/chrome: use macros for passthru indexes Move passthru indexes for EC and PD devices to common header. Also use them instead of literal constants. Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220609084957.3684698-4-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec.c | 3 --- drivers/platform/chrome/cros_ec_proto.c | 6 +++--- drivers/platform/chrome/cros_ec_proto_test.c | 15 ++++++++++----- drivers/platform/chrome/cros_ec_trace.h | 8 ++++---- include/linux/platform_data/cros_ec_proto.h | 3 +++ 5 files changed, 20 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index b3e94cdf7d1a..e51a3f2176c7 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -19,9 +19,6 @@ #include "cros_ec.h" -#define CROS_EC_DEV_EC_INDEX 0 -#define CROS_EC_DEV_PD_INDEX 1 - static struct cros_ec_platform ec_p = { .ec_name = CROS_EC_DEV_NAME, .cmd_offset = EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_EC_INDEX), diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index cefabfe45551..cfa3dacce4e5 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -433,7 +433,7 @@ int cros_ec_query_all(struct cros_ec_device *ec_dev) /* First try sending with proto v3. */ ec_dev->proto_version = 3; - ret = cros_ec_host_command_proto_query(ec_dev, 0, proto_msg); + ret = cros_ec_host_command_proto_query(ec_dev, CROS_EC_DEV_EC_INDEX, proto_msg); if (ret == 0) { proto_info = (struct ec_response_get_protocol_info *) @@ -459,7 +459,7 @@ int cros_ec_query_all(struct cros_ec_device *ec_dev) /* * Check for PD */ - ret = cros_ec_host_command_proto_query(ec_dev, 1, proto_msg); + ret = cros_ec_host_command_proto_query(ec_dev, CROS_EC_DEV_PD_INDEX, proto_msg); if (ret) { dev_dbg(ec_dev->dev, "no PD chip found: %d\n", ret); @@ -609,7 +609,7 @@ int cros_ec_cmd_xfer(struct cros_ec_device *ec_dev, struct cros_ec_command *msg) msg->insize = ec_dev->max_response; } - if (msg->command < EC_CMD_PASSTHRU_OFFSET(1)) { + if (msg->command < EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX)) { if (msg->outsize > ec_dev->max_request) { dev_err(ec_dev->dev, "request of size %u is too big (max: %u)\n", diff --git a/drivers/platform/chrome/cros_ec_proto_test.c b/drivers/platform/chrome/cros_ec_proto_test.c index 675306c16d47..f8dbfb0d8dc8 100644 --- a/drivers/platform/chrome/cros_ec_proto_test.c +++ b/drivers/platform/chrome/cros_ec_proto_test.c @@ -281,7 +281,8 @@ static void cros_ec_proto_test_query_all_normal(struct kunit *test) KUNIT_EXPECT_EQ(test, mock->msg.version, 0); KUNIT_EXPECT_EQ(test, mock->msg.command, - EC_CMD_PASSTHRU_OFFSET(1) | EC_CMD_GET_PROTOCOL_INFO); + EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX) | + EC_CMD_GET_PROTOCOL_INFO); KUNIT_EXPECT_EQ(test, mock->msg.insize, sizeof(struct ec_response_get_protocol_info)); KUNIT_EXPECT_EQ(test, mock->msg.outsize, 0); @@ -396,7 +397,8 @@ static void cros_ec_proto_test_query_all_no_pd_return_error(struct kunit *test) KUNIT_EXPECT_EQ(test, mock->msg.version, 0); KUNIT_EXPECT_EQ(test, mock->msg.command, - EC_CMD_PASSTHRU_OFFSET(1) | EC_CMD_GET_PROTOCOL_INFO); + EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX) | + EC_CMD_GET_PROTOCOL_INFO); KUNIT_EXPECT_EQ(test, mock->msg.insize, sizeof(struct ec_response_get_protocol_info)); KUNIT_EXPECT_EQ(test, mock->msg.outsize, 0); @@ -685,7 +687,8 @@ static void cros_ec_proto_test_query_all_no_mkbp(struct kunit *test) KUNIT_EXPECT_EQ(test, mock->msg.version, 0); KUNIT_EXPECT_EQ(test, mock->msg.command, - EC_CMD_PASSTHRU_OFFSET(1) | EC_CMD_GET_PROTOCOL_INFO); + EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX) | + EC_CMD_GET_PROTOCOL_INFO); KUNIT_EXPECT_EQ(test, mock->msg.insize, sizeof(struct ec_response_get_protocol_info)); KUNIT_EXPECT_EQ(test, mock->msg.outsize, 0); @@ -783,7 +786,8 @@ static void cros_ec_proto_test_query_all_no_host_sleep(struct kunit *test) KUNIT_EXPECT_EQ(test, mock->msg.version, 0); KUNIT_EXPECT_EQ(test, mock->msg.command, - EC_CMD_PASSTHRU_OFFSET(1) | EC_CMD_GET_PROTOCOL_INFO); + EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX) | + EC_CMD_GET_PROTOCOL_INFO); KUNIT_EXPECT_EQ(test, mock->msg.insize, sizeof(struct ec_response_get_protocol_info)); KUNIT_EXPECT_EQ(test, mock->msg.outsize, 0); @@ -889,7 +893,8 @@ static void cros_ec_proto_test_query_all_default_wake_mask_return_error(struct k KUNIT_EXPECT_EQ(test, mock->msg.version, 0); KUNIT_EXPECT_EQ(test, mock->msg.command, - EC_CMD_PASSTHRU_OFFSET(1) | EC_CMD_GET_PROTOCOL_INFO); + EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX) | + EC_CMD_GET_PROTOCOL_INFO); KUNIT_EXPECT_EQ(test, mock->msg.insize, sizeof(struct ec_response_get_protocol_info)); KUNIT_EXPECT_EQ(test, mock->msg.outsize, 0); diff --git a/drivers/platform/chrome/cros_ec_trace.h b/drivers/platform/chrome/cros_ec_trace.h index 9bb5cd2c98b8..d7e407de88df 100644 --- a/drivers/platform/chrome/cros_ec_trace.h +++ b/drivers/platform/chrome/cros_ec_trace.h @@ -30,8 +30,8 @@ TRACE_EVENT(cros_ec_request_start, ), TP_fast_assign( __entry->version = cmd->version; - __entry->offset = cmd->command / EC_CMD_PASSTHRU_OFFSET(1); - __entry->command = cmd->command % EC_CMD_PASSTHRU_OFFSET(1); + __entry->offset = cmd->command / EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX); + __entry->command = cmd->command % EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX); __entry->outsize = cmd->outsize; __entry->insize = cmd->insize; ), @@ -55,8 +55,8 @@ TRACE_EVENT(cros_ec_request_done, ), TP_fast_assign( __entry->version = cmd->version; - __entry->offset = cmd->command / EC_CMD_PASSTHRU_OFFSET(1); - __entry->command = cmd->command % EC_CMD_PASSTHRU_OFFSET(1); + __entry->offset = cmd->command / EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX); + __entry->command = cmd->command % EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX); __entry->outsize = cmd->outsize; __entry->insize = cmd->insize; __entry->result = cmd->result; diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index 85e29300f63d..c82a8285d936 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -21,6 +21,9 @@ #define CROS_EC_DEV_SCP_NAME "cros_scp" #define CROS_EC_DEV_TP_NAME "cros_tp" +#define CROS_EC_DEV_EC_INDEX 0 +#define CROS_EC_DEV_PD_INDEX 1 + /* * The EC is unresponsive for a time after a reboot command. Add a * simple delay to make sure that the bus stays locked. -- cgit v1.2.3 From d62607c3fe45911b2331fac073355a8c914bbde2 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 7 Jun 2022 21:39:55 -0700 Subject: net: rename reference+tracking helpers Netdev reference helpers have a dev_ prefix for historic reasons. Renaming the old helpers would be too much churn but we can rename the tracking ones which are relatively recent and should be the default for new code. Rename: dev_hold_track() -> netdev_hold() dev_put_track() -> netdev_put() dev_replace_track() -> netdev_ref_replace() Link: https://lore.kernel.org/r/20220608043955.919359-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/eql.c | 4 ++-- drivers/net/macsec.c | 4 ++-- drivers/net/macvlan.c | 4 ++-- drivers/net/netconsole.c | 2 +- drivers/net/vrf.c | 8 ++++---- include/linux/netdevice.h | 24 ++++++++++++------------ include/net/xfrm.h | 2 +- net/8021q/vlan_dev.c | 4 ++-- net/ax25/af_ax25.c | 7 ++++--- net/ax25/ax25_dev.c | 6 +++--- net/bridge/br_if.c | 10 +++++----- net/core/dev.c | 10 +++++----- net/core/dev_ioctl.c | 4 ++-- net/core/drop_monitor.c | 5 +++-- net/core/dst.c | 8 ++++---- net/core/failover.c | 4 ++-- net/core/link_watch.c | 2 +- net/core/neighbour.c | 18 +++++++++--------- net/core/net-sysfs.c | 8 ++++---- net/core/netpoll.c | 2 +- net/core/pktgen.c | 6 +++--- net/ethtool/ioctl.c | 4 ++-- net/ethtool/netlink.c | 6 +++--- net/ethtool/netlink.h | 2 +- net/ipv4/devinet.c | 4 ++-- net/ipv4/fib_semantics.c | 11 ++++++----- net/ipv4/ipmr.c | 2 +- net/ipv4/route.c | 7 +++---- net/ipv4/xfrm4_policy.c | 2 +- net/ipv6/addrconf.c | 4 ++-- net/ipv6/addrconf_core.c | 2 +- net/ipv6/ip6_gre.c | 8 ++++---- net/ipv6/ip6_tunnel.c | 4 ++-- net/ipv6/ip6_vti.c | 4 ++-- net/ipv6/ip6mr.c | 2 +- net/ipv6/route.c | 10 +++++----- net/ipv6/sit.c | 4 ++-- net/ipv6/xfrm6_policy.c | 4 ++-- net/llc/af_llc.c | 2 +- net/openvswitch/vport-netdev.c | 6 +++--- net/packet/af_packet.c | 12 ++++++------ net/sched/act_mirred.c | 6 +++--- net/sched/sch_api.c | 2 +- net/sched/sch_generic.c | 11 ++++++----- net/smc/smc_pnet.c | 7 ++++--- net/switchdev/switchdev.c | 4 ++-- net/tipc/bearer.c | 4 ++-- net/xfrm/xfrm_device.c | 2 +- 48 files changed, 141 insertions(+), 137 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/eql.c b/drivers/net/eql.c index 557ca8ff9dec..ca3e4700a813 100644 --- a/drivers/net/eql.c +++ b/drivers/net/eql.c @@ -225,7 +225,7 @@ static void eql_kill_one_slave(slave_queue_t *queue, slave_t *slave) list_del(&slave->list); queue->num_slaves--; slave->dev->flags &= ~IFF_SLAVE; - dev_put_track(slave->dev, &slave->dev_tracker); + netdev_put(slave->dev, &slave->dev_tracker); kfree(slave); } @@ -399,7 +399,7 @@ static int __eql_insert_slave(slave_queue_t *queue, slave_t *slave) if (duplicate_slave) eql_kill_one_slave(queue, duplicate_slave); - dev_hold_track(slave->dev, &slave->dev_tracker, GFP_ATOMIC); + netdev_hold(slave->dev, &slave->dev_tracker, GFP_ATOMIC); list_add(&slave->list, &queue->all_slaves); queue->num_slaves++; slave->dev->flags |= IFF_SLAVE; diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 817577e713d7..815738c0e067 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -3462,7 +3462,7 @@ static int macsec_dev_init(struct net_device *dev) memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len); /* Get macsec's reference to real_dev */ - dev_hold_track(real_dev, &macsec->dev_tracker, GFP_KERNEL); + netdev_hold(real_dev, &macsec->dev_tracker, GFP_KERNEL); return 0; } @@ -3710,7 +3710,7 @@ static void macsec_free_netdev(struct net_device *dev) free_percpu(macsec->secy.tx_sc.stats); /* Get rid of the macsec's reference to real_dev */ - dev_put_track(macsec->real_dev, &macsec->dev_tracker); + netdev_put(macsec->real_dev, &macsec->dev_tracker); } static void macsec_setup(struct net_device *dev) diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index eff75beb1395..5b46a6526fc6 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -915,7 +915,7 @@ static int macvlan_init(struct net_device *dev) port->count += 1; /* Get macvlan's reference to lowerdev */ - dev_hold_track(lowerdev, &vlan->dev_tracker, GFP_KERNEL); + netdev_hold(lowerdev, &vlan->dev_tracker, GFP_KERNEL); return 0; } @@ -1185,7 +1185,7 @@ static void macvlan_dev_free(struct net_device *dev) struct macvlan_dev *vlan = netdev_priv(dev); /* Get rid of the macvlan's reference to lowerdev */ - dev_put_track(vlan->lowerdev, &vlan->dev_tracker); + netdev_put(vlan->lowerdev, &vlan->dev_tracker); } void macvlan_common_setup(struct net_device *dev) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index ab8cd5551020..ddac61d79145 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -721,7 +721,7 @@ restart: __netpoll_cleanup(&nt->np); spin_lock_irqsave(&target_list_lock, flags); - dev_put_track(nt->np.dev, &nt->np.dev_tracker); + netdev_put(nt->np.dev, &nt->np.dev_tracker); nt->np.dev = NULL; nt->enabled = false; stopped = true; diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index cfc30ce4c6e1..40445a12c682 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -814,8 +814,8 @@ static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf) */ if (rt6) { dst = &rt6->dst; - dev_replace_track(dst->dev, net->loopback_dev, - &dst->dev_tracker, GFP_KERNEL); + netdev_ref_replace(dst->dev, net->loopback_dev, + &dst->dev_tracker, GFP_KERNEL); dst->dev = net->loopback_dev; dst_release(dst); } @@ -1061,8 +1061,8 @@ static void vrf_rtable_release(struct net_device *dev, struct net_vrf *vrf) */ if (rth) { dst = &rth->dst; - dev_replace_track(dst->dev, net->loopback_dev, - &dst->dev_tracker, GFP_KERNEL); + netdev_ref_replace(dst->dev, net->loopback_dev, + &dst->dev_tracker, GFP_KERNEL); dst->dev = net->loopback_dev; dst_release(dst); } diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index f615a66c89e9..e2e5088888b1 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3981,8 +3981,8 @@ static inline void netdev_tracker_free(struct net_device *dev, #endif } -static inline void dev_hold_track(struct net_device *dev, - netdevice_tracker *tracker, gfp_t gfp) +static inline void netdev_hold(struct net_device *dev, + netdevice_tracker *tracker, gfp_t gfp) { if (dev) { __dev_hold(dev); @@ -3990,8 +3990,8 @@ static inline void dev_hold_track(struct net_device *dev, } } -static inline void dev_put_track(struct net_device *dev, - netdevice_tracker *tracker) +static inline void netdev_put(struct net_device *dev, + netdevice_tracker *tracker) { if (dev) { netdev_tracker_free(dev, tracker); @@ -4004,11 +4004,11 @@ static inline void dev_put_track(struct net_device *dev, * @dev: network device * * Hold reference to device to keep it from being freed. - * Try using dev_hold_track() instead. + * Try using netdev_hold() instead. */ static inline void dev_hold(struct net_device *dev) { - dev_hold_track(dev, NULL, GFP_ATOMIC); + netdev_hold(dev, NULL, GFP_ATOMIC); } /** @@ -4016,17 +4016,17 @@ static inline void dev_hold(struct net_device *dev) * @dev: network device * * Release reference to device to allow it to be freed. - * Try using dev_put_track() instead. + * Try using netdev_put() instead. */ static inline void dev_put(struct net_device *dev) { - dev_put_track(dev, NULL); + netdev_put(dev, NULL); } -static inline void dev_replace_track(struct net_device *odev, - struct net_device *ndev, - netdevice_tracker *tracker, - gfp_t gfp) +static inline void netdev_ref_replace(struct net_device *odev, + struct net_device *ndev, + netdevice_tracker *tracker, + gfp_t gfp) { if (odev) netdev_tracker_free(odev, tracker); diff --git a/include/net/xfrm.h b/include/net/xfrm.h index c39d910d4b45..9287712ad977 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -1923,7 +1923,7 @@ static inline void xfrm_dev_state_free(struct xfrm_state *x) if (dev->xfrmdev_ops->xdo_dev_state_free) dev->xfrmdev_ops->xdo_dev_state_free(x); xso->dev = NULL; - dev_put_track(dev, &xso->dev_tracker); + netdev_put(dev, &xso->dev_tracker); } } #else diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 839f2020b015..e3dff2df6f54 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -615,7 +615,7 @@ static int vlan_dev_init(struct net_device *dev) return -ENOMEM; /* Get vlan's reference to real_dev */ - dev_hold_track(real_dev, &vlan->dev_tracker, GFP_KERNEL); + netdev_hold(real_dev, &vlan->dev_tracker, GFP_KERNEL); return 0; } @@ -852,7 +852,7 @@ static void vlan_dev_free(struct net_device *dev) vlan->vlan_pcpu_stats = NULL; /* Get rid of the vlan's reference to real_dev */ - dev_put_track(vlan->real_dev, &vlan->dev_tracker); + netdev_put(vlan->real_dev, &vlan->dev_tracker); } void vlan_setup(struct net_device *dev) diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c index 95393bb2760b..1a5c0b071aa3 100644 --- a/net/ax25/af_ax25.c +++ b/net/ax25/af_ax25.c @@ -102,7 +102,8 @@ again: ax25_disconnect(s, ENETUNREACH); s->ax25_dev = NULL; if (sk->sk_socket) { - dev_put_track(ax25_dev->dev, &ax25_dev->dev_tracker); + netdev_put(ax25_dev->dev, + &ax25_dev->dev_tracker); ax25_dev_put(ax25_dev); } ax25_cb_del(s); @@ -1065,7 +1066,7 @@ static int ax25_release(struct socket *sock) del_timer_sync(&ax25->t3timer); del_timer_sync(&ax25->idletimer); } - dev_put_track(ax25_dev->dev, &ax25_dev->dev_tracker); + netdev_put(ax25_dev->dev, &ax25_dev->dev_tracker); ax25_dev_put(ax25_dev); } @@ -1146,7 +1147,7 @@ static int ax25_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) if (ax25_dev) { ax25_fillin_cb(ax25, ax25_dev); - dev_hold_track(ax25_dev->dev, &ax25_dev->dev_tracker, GFP_ATOMIC); + netdev_hold(ax25_dev->dev, &ax25_dev->dev_tracker, GFP_ATOMIC); } done: diff --git a/net/ax25/ax25_dev.c b/net/ax25/ax25_dev.c index 95a76d571c44..ab88b6ac5401 100644 --- a/net/ax25/ax25_dev.c +++ b/net/ax25/ax25_dev.c @@ -60,7 +60,7 @@ void ax25_dev_device_up(struct net_device *dev) refcount_set(&ax25_dev->refcount, 1); dev->ax25_ptr = ax25_dev; ax25_dev->dev = dev; - dev_hold_track(dev, &ax25_dev->dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &ax25_dev->dev_tracker, GFP_ATOMIC); ax25_dev->forward = NULL; ax25_dev->device_up = true; @@ -136,7 +136,7 @@ unlock_put: spin_unlock_bh(&ax25_dev_lock); ax25_dev_put(ax25_dev); dev->ax25_ptr = NULL; - dev_put_track(dev, &ax25_dev->dev_tracker); + netdev_put(dev, &ax25_dev->dev_tracker); ax25_dev_put(ax25_dev); } @@ -205,7 +205,7 @@ void __exit ax25_dev_free(void) ax25_dev = ax25_dev_list; while (ax25_dev != NULL) { s = ax25_dev; - dev_put_track(ax25_dev->dev, &ax25_dev->dev_tracker); + netdev_put(ax25_dev->dev, &ax25_dev->dev_tracker); ax25_dev = ax25_dev->next; kfree(s); } diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index 47fcbade7389..a84a7cfb9d6d 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -274,7 +274,7 @@ static void destroy_nbp(struct net_bridge_port *p) p->br = NULL; p->dev = NULL; - dev_put_track(dev, &p->dev_tracker); + netdev_put(dev, &p->dev_tracker); kobject_put(&p->kobj); } @@ -423,7 +423,7 @@ static struct net_bridge_port *new_nbp(struct net_bridge *br, return ERR_PTR(-ENOMEM); p->br = br; - dev_hold_track(dev, &p->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &p->dev_tracker, GFP_KERNEL); p->dev = dev; p->path_cost = port_cost(dev); p->priority = 0x8000 >> BR_PORT_BITS; @@ -434,7 +434,7 @@ static struct net_bridge_port *new_nbp(struct net_bridge *br, br_stp_port_timer_init(p); err = br_multicast_add_port(p); if (err) { - dev_put_track(dev, &p->dev_tracker); + netdev_put(dev, &p->dev_tracker); kfree(p); p = ERR_PTR(err); } @@ -615,7 +615,7 @@ int br_add_if(struct net_bridge *br, struct net_device *dev, err = dev_set_allmulti(dev, 1); if (err) { br_multicast_del_port(p); - dev_put_track(dev, &p->dev_tracker); + netdev_put(dev, &p->dev_tracker); kfree(p); /* kobject not yet init'd, manually free */ goto err1; } @@ -725,7 +725,7 @@ err3: sysfs_remove_link(br->ifobj, p->dev->name); err2: br_multicast_del_port(p); - dev_put_track(dev, &p->dev_tracker); + netdev_put(dev, &p->dev_tracker); kobject_put(&p->kobj); dev_set_allmulti(dev, -1); err1: diff --git a/net/core/dev.c b/net/core/dev.c index 08ce317fcec8..c9d96b85a825 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -7463,7 +7463,7 @@ static int __netdev_adjacent_dev_insert(struct net_device *dev, adj->ref_nr = 1; adj->private = private; adj->ignore = false; - dev_hold_track(adj_dev, &adj->dev_tracker, GFP_KERNEL); + netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL); pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); @@ -7492,7 +7492,7 @@ remove_symlinks: if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); free_adj: - dev_put_track(adj_dev, &adj->dev_tracker); + netdev_put(adj_dev, &adj->dev_tracker); kfree(adj); return ret; @@ -7534,7 +7534,7 @@ static void __netdev_adjacent_dev_remove(struct net_device *dev, list_del_rcu(&adj->list); pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", adj_dev->name, dev->name, adj_dev->name); - dev_put_track(adj_dev, &adj->dev_tracker); + netdev_put(adj_dev, &adj->dev_tracker); kfree_rcu(adj, rcu); } @@ -10062,7 +10062,7 @@ int register_netdevice(struct net_device *dev) dev_init_scheduler(dev); - dev_hold_track(dev, &dev->dev_registered_tracker, GFP_KERNEL); + netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL); list_netdevice(dev); add_device_randomness(dev->dev_addr, dev->addr_len); @@ -10868,7 +10868,7 @@ void unregister_netdevice_many(struct list_head *head) synchronize_net(); list_for_each_entry(dev, head, unreg_list) { - dev_put_track(dev, &dev->dev_registered_tracker); + netdev_put(dev, &dev->dev_registered_tracker); net_set_todo(dev); } diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index 4f6be442ae7e..7674bb9f3076 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -384,10 +384,10 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, void __user *data, return -ENODEV; if (!netif_is_bridge_master(dev)) return -EOPNOTSUPP; - dev_hold_track(dev, &dev_tracker, GFP_KERNEL); + netdev_hold(dev, &dev_tracker, GFP_KERNEL); rtnl_unlock(); err = br_ioctl_call(net, netdev_priv(dev), cmd, ifr, NULL); - dev_put_track(dev, &dev_tracker); + netdev_put(dev, &dev_tracker); rtnl_lock(); return err; diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 4ad1decce724..804d02fc245f 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -864,7 +864,8 @@ net_dm_hw_metadata_copy(const struct devlink_trap_metadata *metadata) } hw_metadata->input_dev = metadata->input_dev; - dev_hold_track(hw_metadata->input_dev, &hw_metadata->dev_tracker, GFP_ATOMIC); + netdev_hold(hw_metadata->input_dev, &hw_metadata->dev_tracker, + GFP_ATOMIC); return hw_metadata; @@ -880,7 +881,7 @@ free_hw_metadata: static void net_dm_hw_metadata_free(struct devlink_trap_metadata *hw_metadata) { - dev_put_track(hw_metadata->input_dev, &hw_metadata->dev_tracker); + netdev_put(hw_metadata->input_dev, &hw_metadata->dev_tracker); kfree(hw_metadata->fa_cookie); kfree(hw_metadata->trap_name); kfree(hw_metadata->trap_group_name); diff --git a/net/core/dst.c b/net/core/dst.c index d16c2c9bfebd..bc9c9be4e080 100644 --- a/net/core/dst.c +++ b/net/core/dst.c @@ -49,7 +49,7 @@ void dst_init(struct dst_entry *dst, struct dst_ops *ops, unsigned short flags) { dst->dev = dev; - dev_hold_track(dev, &dst->dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &dst->dev_tracker, GFP_ATOMIC); dst->ops = ops; dst_init_metrics(dst, dst_default_metrics.metrics, true); dst->expires = 0UL; @@ -117,7 +117,7 @@ struct dst_entry *dst_destroy(struct dst_entry * dst) if (dst->ops->destroy) dst->ops->destroy(dst); - dev_put_track(dst->dev, &dst->dev_tracker); + netdev_put(dst->dev, &dst->dev_tracker); lwtstate_put(dst->lwtstate); @@ -159,8 +159,8 @@ void dst_dev_put(struct dst_entry *dst) dst->input = dst_discard; dst->output = dst_discard_out; dst->dev = blackhole_netdev; - dev_replace_track(dev, blackhole_netdev, &dst->dev_tracker, - GFP_ATOMIC); + netdev_ref_replace(dev, blackhole_netdev, &dst->dev_tracker, + GFP_ATOMIC); } EXPORT_SYMBOL(dst_dev_put); diff --git a/net/core/failover.c b/net/core/failover.c index dcaa92a85ea2..864d2d83eff4 100644 --- a/net/core/failover.c +++ b/net/core/failover.c @@ -252,7 +252,7 @@ struct failover *failover_register(struct net_device *dev, return ERR_PTR(-ENOMEM); rcu_assign_pointer(failover->ops, ops); - dev_hold_track(dev, &failover->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &failover->dev_tracker, GFP_KERNEL); dev->priv_flags |= IFF_FAILOVER; rcu_assign_pointer(failover->failover_dev, dev); @@ -285,7 +285,7 @@ void failover_unregister(struct failover *failover) failover_dev->name); failover_dev->priv_flags &= ~IFF_FAILOVER; - dev_put_track(failover_dev, &failover->dev_tracker); + netdev_put(failover_dev, &failover->dev_tracker); spin_lock(&failover_lock); list_del(&failover->list); diff --git a/net/core/link_watch.c b/net/core/link_watch.c index a244d3bade7d..aa6cb1f90966 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -110,7 +110,7 @@ static void linkwatch_add_event(struct net_device *dev) spin_lock_irqsave(&lweventlist_lock, flags); if (list_empty(&dev->link_watch_list)) { list_add_tail(&dev->link_watch_list, &lweventlist); - dev_hold_track(dev, &dev->linkwatch_dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &dev->linkwatch_dev_tracker, GFP_ATOMIC); } spin_unlock_irqrestore(&lweventlist_lock, flags); } diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 54625287ee5b..d8ec70622ecb 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -624,7 +624,7 @@ ___neigh_create(struct neigh_table *tbl, const void *pkey, memcpy(n->primary_key, pkey, key_len); n->dev = dev; - dev_hold_track(dev, &n->dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &n->dev_tracker, GFP_ATOMIC); /* Protocol specific setup. */ if (tbl->constructor && (error = tbl->constructor(n)) < 0) { @@ -770,10 +770,10 @@ struct pneigh_entry * pneigh_lookup(struct neigh_table *tbl, write_pnet(&n->net, net); memcpy(n->key, pkey, key_len); n->dev = dev; - dev_hold_track(dev, &n->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &n->dev_tracker, GFP_KERNEL); if (tbl->pconstructor && tbl->pconstructor(n)) { - dev_put_track(dev, &n->dev_tracker); + netdev_put(dev, &n->dev_tracker); kfree(n); n = NULL; goto out; @@ -805,7 +805,7 @@ int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey, write_unlock_bh(&tbl->lock); if (tbl->pdestructor) tbl->pdestructor(n); - dev_put_track(n->dev, &n->dev_tracker); + netdev_put(n->dev, &n->dev_tracker); kfree(n); return 0; } @@ -838,7 +838,7 @@ static int pneigh_ifdown_and_unlock(struct neigh_table *tbl, n->next = NULL; if (tbl->pdestructor) tbl->pdestructor(n); - dev_put_track(n->dev, &n->dev_tracker); + netdev_put(n->dev, &n->dev_tracker); kfree(n); } return -ENOENT; @@ -879,7 +879,7 @@ void neigh_destroy(struct neighbour *neigh) if (dev->netdev_ops->ndo_neigh_destroy) dev->netdev_ops->ndo_neigh_destroy(dev, neigh); - dev_put_track(dev, &neigh->dev_tracker); + netdev_put(dev, &neigh->dev_tracker); neigh_parms_put(neigh->parms); neigh_dbg(2, "neigh %p is destroyed\n", neigh); @@ -1671,13 +1671,13 @@ struct neigh_parms *neigh_parms_alloc(struct net_device *dev, refcount_set(&p->refcnt, 1); p->reachable_time = neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME)); - dev_hold_track(dev, &p->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &p->dev_tracker, GFP_KERNEL); p->dev = dev; write_pnet(&p->net, net); p->sysctl_table = NULL; if (ops->ndo_neigh_setup && ops->ndo_neigh_setup(dev, p)) { - dev_put_track(dev, &p->dev_tracker); + netdev_put(dev, &p->dev_tracker); kfree(p); return NULL; } @@ -1708,7 +1708,7 @@ void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms) list_del(&parms->list); parms->dead = 1; write_unlock_bh(&tbl->lock); - dev_put_track(parms->dev, &parms->dev_tracker); + netdev_put(parms->dev, &parms->dev_tracker); call_rcu(&parms->rcu_head, neigh_rcu_free_parms); } EXPORT_SYMBOL(neigh_parms_release); diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index e319e242dddf..d49fc974e630 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -1016,7 +1016,7 @@ static void rx_queue_release(struct kobject *kobj) #endif memset(kobj, 0, sizeof(*kobj)); - dev_put_track(queue->dev, &queue->dev_tracker); + netdev_put(queue->dev, &queue->dev_tracker); } static const void *rx_queue_namespace(struct kobject *kobj) @@ -1056,7 +1056,7 @@ static int rx_queue_add_kobject(struct net_device *dev, int index) /* Kobject_put later will trigger rx_queue_release call which * decreases dev refcount: Take that reference here */ - dev_hold_track(queue->dev, &queue->dev_tracker, GFP_KERNEL); + netdev_hold(queue->dev, &queue->dev_tracker, GFP_KERNEL); kobj->kset = dev->queues_kset; error = kobject_init_and_add(kobj, &rx_queue_ktype, NULL, @@ -1619,7 +1619,7 @@ static void netdev_queue_release(struct kobject *kobj) struct netdev_queue *queue = to_netdev_queue(kobj); memset(kobj, 0, sizeof(*kobj)); - dev_put_track(queue->dev, &queue->dev_tracker); + netdev_put(queue->dev, &queue->dev_tracker); } static const void *netdev_queue_namespace(struct kobject *kobj) @@ -1659,7 +1659,7 @@ static int netdev_queue_add_kobject(struct net_device *dev, int index) /* Kobject_put later will trigger netdev_queue_release call * which decreases dev refcount: Take that reference here */ - dev_hold_track(queue->dev, &queue->dev_tracker, GFP_KERNEL); + netdev_hold(queue->dev, &queue->dev_tracker, GFP_KERNEL); kobj->kset = dev->queues_kset; error = kobject_init_and_add(kobj, &netdev_queue_ktype, NULL, diff --git a/net/core/netpoll.c b/net/core/netpoll.c index db724463e7cd..5d27067b72d5 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -853,7 +853,7 @@ void netpoll_cleanup(struct netpoll *np) if (!np->dev) goto out; __netpoll_cleanup(np); - dev_put_track(np->dev, &np->dev_tracker); + netdev_put(np->dev, &np->dev_tracker); np->dev = NULL; out: rtnl_unlock(); diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 84b62cd7bc57..88906ba6d9a7 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -2100,7 +2100,7 @@ static int pktgen_setup_dev(const struct pktgen_net *pn, /* Clean old setups */ if (pkt_dev->odev) { - dev_put_track(pkt_dev->odev, &pkt_dev->dev_tracker); + netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); pkt_dev->odev = NULL; } @@ -3807,7 +3807,7 @@ static int pktgen_add_device(struct pktgen_thread *t, const char *ifname) return add_dev_to_thread(t, pkt_dev); out2: - dev_put_track(pkt_dev->odev, &pkt_dev->dev_tracker); + netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); out1: #ifdef CONFIG_XFRM free_SAs(pkt_dev); @@ -3901,7 +3901,7 @@ static int pktgen_remove_device(struct pktgen_thread *t, /* Dis-associate from the interface */ if (pkt_dev->odev) { - dev_put_track(pkt_dev->odev, &pkt_dev->dev_tracker); + netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); pkt_dev->odev = NULL; } diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 326e14ee05db..d05ff6a17a1f 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -2010,7 +2010,7 @@ static int ethtool_phys_id(struct net_device *dev, void __user *useraddr) * removal of the device. */ busy = true; - dev_hold_track(dev, &dev_tracker, GFP_KERNEL); + netdev_hold(dev, &dev_tracker, GFP_KERNEL); rtnl_unlock(); if (rc == 0) { @@ -2034,7 +2034,7 @@ static int ethtool_phys_id(struct net_device *dev, void __user *useraddr) } rtnl_lock(); - dev_put_track(dev, &dev_tracker); + netdev_put(dev, &dev_tracker); busy = false; (void) ops->set_phys_id(dev, ETHTOOL_ID_INACTIVE); diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 5fe8f4ae2ceb..e26079e11835 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -402,7 +402,7 @@ static int ethnl_default_doit(struct sk_buff *skb, struct genl_info *info) ops->cleanup_data(reply_data); genlmsg_end(rskb, reply_payload); - dev_put_track(req_info->dev, &req_info->dev_tracker); + netdev_put(req_info->dev, &req_info->dev_tracker); kfree(reply_data); kfree(req_info); return genlmsg_reply(rskb, info); @@ -414,7 +414,7 @@ err_cleanup: if (ops->cleanup_data) ops->cleanup_data(reply_data); err_dev: - dev_put_track(req_info->dev, &req_info->dev_tracker); + netdev_put(req_info->dev, &req_info->dev_tracker); kfree(reply_data); kfree(req_info); return ret; @@ -550,7 +550,7 @@ static int ethnl_default_start(struct netlink_callback *cb) * same parser as for non-dump (doit) requests is used, it * would take reference to the device if it finds one */ - dev_put_track(req_info->dev, &req_info->dev_tracker); + netdev_put(req_info->dev, &req_info->dev_tracker); req_info->dev = NULL; } if (ret < 0) diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index 7919ddb2371c..c0d587611854 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -237,7 +237,7 @@ struct ethnl_req_info { static inline void ethnl_parse_header_dev_put(struct ethnl_req_info *req_info) { - dev_put_track(req_info->dev, &req_info->dev_tracker); + netdev_put(req_info->dev, &req_info->dev_tracker); } /** diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index b2366ad540e6..92b778e423df 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -244,7 +244,7 @@ void in_dev_finish_destroy(struct in_device *idev) #ifdef NET_REFCNT_DEBUG pr_debug("%s: %p=%s\n", __func__, idev, dev ? dev->name : "NIL"); #endif - dev_put_track(dev, &idev->dev_tracker); + netdev_put(dev, &idev->dev_tracker); if (!idev->dead) pr_err("Freeing alive in_device %p\n", idev); else @@ -272,7 +272,7 @@ static struct in_device *inetdev_init(struct net_device *dev) if (IPV4_DEVCONF(in_dev->cnf, FORWARDING)) dev_disable_lro(dev); /* Reference in_dev->dev */ - dev_hold_track(dev, &in_dev->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &in_dev->dev_tracker, GFP_KERNEL); /* Account for reference dev->ip_ptr (below) */ refcount_set(&in_dev->refcnt, 1); diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index a57ba23571c9..a5439a8414d4 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -211,7 +211,7 @@ static void rt_fibinfo_free_cpus(struct rtable __rcu * __percpu *rtp) void fib_nh_common_release(struct fib_nh_common *nhc) { - dev_put_track(nhc->nhc_dev, &nhc->nhc_dev_tracker); + netdev_put(nhc->nhc_dev, &nhc->nhc_dev_tracker); lwtstate_put(nhc->nhc_lwtstate); rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output); rt_fibinfo_free(&nhc->nhc_rth_input); @@ -1057,7 +1057,8 @@ static int fib_check_nh_v6_gw(struct net *net, struct fib_nh *nh, err = ipv6_stub->fib6_nh_init(net, &fib6_nh, &cfg, GFP_KERNEL, extack); if (!err) { nh->fib_nh_dev = fib6_nh.fib_nh_dev; - dev_hold_track(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, GFP_KERNEL); + netdev_hold(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, + GFP_KERNEL); nh->fib_nh_oif = nh->fib_nh_dev->ifindex; nh->fib_nh_scope = RT_SCOPE_LINK; @@ -1141,7 +1142,7 @@ static int fib_check_nh_v4_gw(struct net *net, struct fib_nh *nh, u32 table, if (!netif_carrier_ok(dev)) nh->fib_nh_flags |= RTNH_F_LINKDOWN; nh->fib_nh_dev = dev; - dev_hold_track(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); nh->fib_nh_scope = RT_SCOPE_LINK; return 0; } @@ -1195,7 +1196,7 @@ static int fib_check_nh_v4_gw(struct net *net, struct fib_nh *nh, u32 table, "No egress device for nexthop gateway"); goto out; } - dev_hold_track(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); if (!netif_carrier_ok(dev)) nh->fib_nh_flags |= RTNH_F_LINKDOWN; err = (dev->flags & IFF_UP) ? 0 : -ENETDOWN; @@ -1229,7 +1230,7 @@ static int fib_check_nh_nongw(struct net *net, struct fib_nh *nh, } nh->fib_nh_dev = in_dev->dev; - dev_hold_track(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); + netdev_hold(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); nh->fib_nh_scope = RT_SCOPE_HOST; if (!netif_carrier_ok(nh->fib_nh_dev)) nh->fib_nh_flags |= RTNH_F_LINKDOWN; diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 13e6329784fb..8324e541d193 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -691,7 +691,7 @@ static int vif_delete(struct mr_table *mrt, int vifi, int notify, if (v->flags & (VIFF_TUNNEL | VIFF_REGISTER) && !notify) unregister_netdevice_queue(dev, head); - dev_put_track(dev, &v->dev_tracker); + netdev_put(dev, &v->dev_tracker); return 0; } diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 356f535f3443..2d16bcc7d346 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -1550,9 +1550,8 @@ void rt_flush_dev(struct net_device *dev) if (rt->dst.dev != dev) continue; rt->dst.dev = blackhole_netdev; - dev_replace_track(dev, blackhole_netdev, - &rt->dst.dev_tracker, - GFP_ATOMIC); + netdev_ref_replace(dev, blackhole_netdev, + &rt->dst.dev_tracker, GFP_ATOMIC); list_move(&rt->rt_uncached, &ul->quarantine); } spin_unlock_bh(&ul->lock); @@ -2851,7 +2850,7 @@ struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_or new->output = dst_discard_out; new->dev = net->loopback_dev; - dev_hold_track(new->dev, &new->dev_tracker, GFP_ATOMIC); + netdev_hold(new->dev, &new->dev_tracker, GFP_ATOMIC); rt->rt_is_input = ort->rt_is_input; rt->rt_iif = ort->rt_iif; diff --git a/net/ipv4/xfrm4_policy.c b/net/ipv4/xfrm4_policy.c index 6fde0b184791..3d0dfa6cf9f9 100644 --- a/net/ipv4/xfrm4_policy.c +++ b/net/ipv4/xfrm4_policy.c @@ -75,7 +75,7 @@ static int xfrm4_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, xdst->u.rt.rt_iif = fl4->flowi4_iif; xdst->u.dst.dev = dev; - dev_hold_track(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC); /* Sheit... I remember I did this right. Apparently, * it was magically lost, so this code needs audit */ diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 1b1932502e9e..3497ad1362c0 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -398,13 +398,13 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev) if (ndev->cnf.forwarding) dev_disable_lro(dev); /* We refer to the device */ - dev_hold_track(dev, &ndev->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &ndev->dev_tracker, GFP_KERNEL); if (snmp6_alloc_dev(ndev) < 0) { netdev_dbg(dev, "%s: cannot allocate memory for statistics\n", __func__); neigh_parms_release(&nd_tbl, ndev->nd_parms); - dev_put_track(dev, &ndev->dev_tracker); + netdev_put(dev, &ndev->dev_tracker); kfree(ndev); return ERR_PTR(err); } diff --git a/net/ipv6/addrconf_core.c b/net/ipv6/addrconf_core.c index 881d1477d24a..507a8353a6bd 100644 --- a/net/ipv6/addrconf_core.c +++ b/net/ipv6/addrconf_core.c @@ -263,7 +263,7 @@ void in6_dev_finish_destroy(struct inet6_dev *idev) #ifdef NET_REFCNT_DEBUG pr_debug("%s: %s\n", __func__, dev ? dev->name : "NIL"); #endif - dev_put_track(dev, &idev->dev_tracker); + netdev_put(dev, &idev->dev_tracker); if (!idev->dead) { pr_warn("Freeing alive inet6 device %p\n", idev); return; diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 4e37f7c29900..3e22cbe5966a 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -398,7 +398,7 @@ static void ip6erspan_tunnel_uninit(struct net_device *dev) ip6erspan_tunnel_unlink_md(ign, t); ip6gre_tunnel_unlink(ign, t); dst_cache_reset(&t->dst_cache); - dev_put_track(dev, &t->dev_tracker); + netdev_put(dev, &t->dev_tracker); } static void ip6gre_tunnel_uninit(struct net_device *dev) @@ -411,7 +411,7 @@ static void ip6gre_tunnel_uninit(struct net_device *dev) if (ign->fb_tunnel_dev == dev) WRITE_ONCE(ign->fb_tunnel_dev, NULL); dst_cache_reset(&t->dst_cache); - dev_put_track(dev, &t->dev_tracker); + netdev_put(dev, &t->dev_tracker); } @@ -1495,7 +1495,7 @@ static int ip6gre_tunnel_init_common(struct net_device *dev) } ip6gre_tnl_init_features(dev); - dev_hold_track(dev, &tunnel->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &tunnel->dev_tracker, GFP_KERNEL); return 0; cleanup_dst_cache_init: @@ -1887,7 +1887,7 @@ static int ip6erspan_tap_init(struct net_device *dev) dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; ip6erspan_tnl_link_config(tunnel, 1); - dev_hold_track(dev, &tunnel->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &tunnel->dev_tracker, GFP_KERNEL); return 0; cleanup_dst_cache_init: diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 19325b7600bb..689de5eb604e 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -381,7 +381,7 @@ ip6_tnl_dev_uninit(struct net_device *dev) else ip6_tnl_unlink(ip6n, t); dst_cache_reset(&t->dst_cache); - dev_put_track(dev, &t->dev_tracker); + netdev_put(dev, &t->dev_tracker); } /** @@ -1889,7 +1889,7 @@ ip6_tnl_dev_init_gen(struct net_device *dev) dev->min_mtu = ETH_MIN_MTU; dev->max_mtu = IP6_MAX_MTU - dev->hard_header_len; - dev_hold_track(dev, &t->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &t->dev_tracker, GFP_KERNEL); return 0; destroy_dst: diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c index 3a434d75925c..8fe59a79e800 100644 --- a/net/ipv6/ip6_vti.c +++ b/net/ipv6/ip6_vti.c @@ -293,7 +293,7 @@ static void vti6_dev_uninit(struct net_device *dev) RCU_INIT_POINTER(ip6n->tnls_wc[0], NULL); else vti6_tnl_unlink(ip6n, t); - dev_put_track(dev, &t->dev_tracker); + netdev_put(dev, &t->dev_tracker); } static int vti6_input_proto(struct sk_buff *skb, int nexthdr, __be32 spi, @@ -936,7 +936,7 @@ static inline int vti6_dev_init_gen(struct net_device *dev) dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); if (!dev->tstats) return -ENOMEM; - dev_hold_track(dev, &t->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &t->dev_tracker, GFP_KERNEL); return 0; } diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 4e74bc61a3db..d4aad41c9849 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -741,7 +741,7 @@ static int mif6_delete(struct mr_table *mrt, int vifi, int notify, if ((v->flags & MIFF_REGISTER) && !notify) unregister_netdevice_queue(dev, head); - dev_put_track(dev, &v->dev_tracker); + netdev_put(dev, &v->dev_tracker); return 0; } diff --git a/net/ipv6/route.c b/net/ipv6/route.c index d25dc83bac62..0be01a4d48c1 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -182,9 +182,9 @@ static void rt6_uncached_list_flush_dev(struct net_device *dev) if (rt_dev == dev) { rt->dst.dev = blackhole_netdev; - dev_replace_track(rt_dev, blackhole_netdev, - &rt->dst.dev_tracker, - GFP_ATOMIC); + netdev_ref_replace(rt_dev, blackhole_netdev, + &rt->dst.dev_tracker, + GFP_ATOMIC); handled = true; } if (handled) @@ -607,7 +607,7 @@ static void rt6_probe_deferred(struct work_struct *w) addrconf_addr_solict_mult(&work->target, &mcaddr); ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0); - dev_put_track(work->dev, &work->dev_tracker); + netdev_put(work->dev, &work->dev_tracker); kfree(work); } @@ -661,7 +661,7 @@ static void rt6_probe(struct fib6_nh *fib6_nh) } else { INIT_WORK(&work->work, rt6_probe_deferred); work->target = *nh_gw; - dev_hold_track(dev, &work->dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC); work->dev = dev; schedule_work(&work->work); } diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index c0b138c20992..4f1721865fda 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -521,7 +521,7 @@ static void ipip6_tunnel_uninit(struct net_device *dev) ipip6_tunnel_del_prl(tunnel, NULL); } dst_cache_reset(&tunnel->dst_cache); - dev_put_track(dev, &tunnel->dev_tracker); + netdev_put(dev, &tunnel->dev_tracker); } static int ipip6_err(struct sk_buff *skb, u32 info) @@ -1463,7 +1463,7 @@ static int ipip6_tunnel_init(struct net_device *dev) dev->tstats = NULL; return err; } - dev_hold_track(dev, &tunnel->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &tunnel->dev_tracker, GFP_KERNEL); return 0; } diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index e64e427a51cf..4a4b0e49ec92 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -73,11 +73,11 @@ static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, struct rt6_info *rt = (struct rt6_info *)xdst->route; xdst->u.dst.dev = dev; - dev_hold_track(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC); xdst->u.rt6.rt6i_idev = in6_dev_get(dev); if (!xdst->u.rt6.rt6i_idev) { - dev_put_track(dev, &xdst->u.dst.dev_tracker); + netdev_put(dev, &xdst->u.dst.dev_tracker); return -ENODEV; } diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index 7f555d2e5357..da7fe94bea2e 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -224,7 +224,7 @@ static int llc_ui_release(struct socket *sock) } else { release_sock(sk); } - dev_put_track(llc->dev, &llc->dev_tracker); + netdev_put(llc->dev, &llc->dev_tracker); sock_put(sk); llc_sk_free(sk); out: diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c index b498dac4e1e0..2f61d5bdce1a 100644 --- a/net/openvswitch/vport-netdev.c +++ b/net/openvswitch/vport-netdev.c @@ -115,7 +115,7 @@ error_master_upper_dev_unlink: error_unlock: rtnl_unlock(); error_put: - dev_put_track(vport->dev, &vport->dev_tracker); + netdev_put(vport->dev, &vport->dev_tracker); error_free_vport: ovs_vport_free(vport); return ERR_PTR(err); @@ -137,7 +137,7 @@ static void vport_netdev_free(struct rcu_head *rcu) { struct vport *vport = container_of(rcu, struct vport, rcu); - dev_put_track(vport->dev, &vport->dev_tracker); + netdev_put(vport->dev, &vport->dev_tracker); ovs_vport_free(vport); } @@ -173,7 +173,7 @@ void ovs_netdev_tunnel_destroy(struct vport *vport) */ if (vport->dev->reg_state == NETREG_REGISTERED) rtnl_delete_link(vport->dev); - dev_put_track(vport->dev, &vport->dev_tracker); + netdev_put(vport->dev, &vport->dev_tracker); vport->dev = NULL; rtnl_unlock(); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index ca6e92a22923..d08c4728523b 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -3134,7 +3134,7 @@ static int packet_release(struct socket *sock) packet_cached_dev_reset(po); if (po->prot_hook.dev) { - dev_put_track(po->prot_hook.dev, &po->prot_hook.dev_tracker); + netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker); po->prot_hook.dev = NULL; } spin_unlock(&po->bind_lock); @@ -3235,15 +3235,15 @@ static int packet_do_bind(struct sock *sk, const char *name, int ifindex, WRITE_ONCE(po->num, proto); po->prot_hook.type = proto; - dev_put_track(po->prot_hook.dev, &po->prot_hook.dev_tracker); + netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker); if (unlikely(unlisted)) { po->prot_hook.dev = NULL; WRITE_ONCE(po->ifindex, -1); packet_cached_dev_reset(po); } else { - dev_hold_track(dev, &po->prot_hook.dev_tracker, - GFP_ATOMIC); + netdev_hold(dev, &po->prot_hook.dev_tracker, + GFP_ATOMIC); po->prot_hook.dev = dev; WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0); packet_cached_dev_assign(po, dev); @@ -4167,8 +4167,8 @@ static int packet_notifier(struct notifier_block *this, if (msg == NETDEV_UNREGISTER) { packet_cached_dev_reset(po); WRITE_ONCE(po->ifindex, -1); - dev_put_track(po->prot_hook.dev, - &po->prot_hook.dev_tracker); + netdev_put(po->prot_hook.dev, + &po->prot_hook.dev_tracker); po->prot_hook.dev = NULL; } spin_unlock(&po->bind_lock); diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index ebb92fb072ab..a1d70cf86843 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -79,7 +79,7 @@ static void tcf_mirred_release(struct tc_action *a) /* last reference to action, no need to lock */ dev = rcu_dereference_protected(m->tcfm_dev, 1); - dev_put_track(dev, &m->tcfm_dev_tracker); + netdev_put(dev, &m->tcfm_dev_tracker); } static const struct nla_policy mirred_policy[TCA_MIRRED_MAX + 1] = { @@ -181,7 +181,7 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla, mac_header_xmit = dev_is_mac_header_xmit(ndev); odev = rcu_replace_pointer(m->tcfm_dev, ndev, lockdep_is_held(&m->tcf_lock)); - dev_put_track(odev, &m->tcfm_dev_tracker); + netdev_put(odev, &m->tcfm_dev_tracker); netdev_tracker_alloc(ndev, &m->tcfm_dev_tracker, GFP_ATOMIC); m->tcfm_mac_header_xmit = mac_header_xmit; } @@ -402,7 +402,7 @@ static int mirred_device_event(struct notifier_block *unused, list_for_each_entry(m, &mirred_list, tcfm_list) { spin_lock_bh(&m->tcf_lock); if (tcf_mirred_dev_dereference(m) == dev) { - dev_put_track(dev, &m->tcfm_dev_tracker); + netdev_put(dev, &m->tcfm_dev_tracker); /* Note : no rcu grace period necessary, as * net_device are already rcu protected. */ diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index e3c0e8ea2dbb..bf87b50837a8 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -1292,7 +1292,7 @@ err_out5: if (ops->destroy) ops->destroy(sch); err_out3: - dev_put_track(dev, &sch->dev_tracker); + netdev_put(dev, &sch->dev_tracker); qdisc_free(sch); err_out2: module_put(ops->owner); diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index dba0b3e24af5..cc6eabee2830 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -541,7 +541,7 @@ static void dev_watchdog(struct timer_list *t) spin_unlock(&dev->tx_global_lock); if (release) - dev_put_track(dev, &dev->watchdog_dev_tracker); + netdev_put(dev, &dev->watchdog_dev_tracker); } void __netdev_watchdog_up(struct net_device *dev) @@ -551,7 +551,8 @@ void __netdev_watchdog_up(struct net_device *dev) dev->watchdog_timeo = 5*HZ; if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo))) - dev_hold_track(dev, &dev->watchdog_dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &dev->watchdog_dev_tracker, + GFP_ATOMIC); } } EXPORT_SYMBOL_GPL(__netdev_watchdog_up); @@ -565,7 +566,7 @@ static void dev_watchdog_down(struct net_device *dev) { netif_tx_lock_bh(dev); if (del_timer(&dev->watchdog_timer)) - dev_put_track(dev, &dev->watchdog_dev_tracker); + netdev_put(dev, &dev->watchdog_dev_tracker); netif_tx_unlock_bh(dev); } @@ -975,7 +976,7 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, sch->enqueue = ops->enqueue; sch->dequeue = ops->dequeue; sch->dev_queue = dev_queue; - dev_hold_track(dev, &sch->dev_tracker, GFP_KERNEL); + netdev_hold(dev, &sch->dev_tracker, GFP_KERNEL); refcount_set(&sch->refcnt, 1); return sch; @@ -1067,7 +1068,7 @@ static void qdisc_destroy(struct Qdisc *qdisc) ops->destroy(qdisc); module_put(ops->owner); - dev_put_track(qdisc_dev(qdisc), &qdisc->dev_tracker); + netdev_put(qdisc_dev(qdisc), &qdisc->dev_tracker); trace_qdisc_destroy(qdisc); diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c index 7055ed10e316..4c3bf6db7038 100644 --- a/net/smc/smc_pnet.c +++ b/net/smc/smc_pnet.c @@ -120,7 +120,8 @@ static int smc_pnet_remove_by_pnetid(struct net *net, char *pnet_name) smc_pnet_match(pnetelem->pnet_name, pnet_name)) { list_del(&pnetelem->list); if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev) { - dev_put_track(pnetelem->ndev, &pnetelem->dev_tracker); + netdev_put(pnetelem->ndev, + &pnetelem->dev_tracker); pr_warn_ratelimited("smc: net device %s " "erased user defined " "pnetid %.16s\n", @@ -196,7 +197,7 @@ static int smc_pnet_add_by_ndev(struct net_device *ndev) list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) { if (pnetelem->type == SMC_PNET_ETH && !pnetelem->ndev && !strncmp(pnetelem->eth_name, ndev->name, IFNAMSIZ)) { - dev_hold_track(ndev, &pnetelem->dev_tracker, GFP_ATOMIC); + netdev_hold(ndev, &pnetelem->dev_tracker, GFP_ATOMIC); pnetelem->ndev = ndev; rc = 0; pr_warn_ratelimited("smc: adding net device %s with " @@ -227,7 +228,7 @@ static int smc_pnet_remove_by_ndev(struct net_device *ndev) mutex_lock(&pnettable->lock); list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) { if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev == ndev) { - dev_put_track(pnetelem->ndev, &pnetelem->dev_tracker); + netdev_put(pnetelem->ndev, &pnetelem->dev_tracker); pnetelem->ndev = NULL; rc = 0; pr_warn_ratelimited("smc: removing net device %s with " diff --git a/net/switchdev/switchdev.c b/net/switchdev/switchdev.c index 474f76383033..8cc42aea19c7 100644 --- a/net/switchdev/switchdev.c +++ b/net/switchdev/switchdev.c @@ -64,7 +64,7 @@ void switchdev_deferred_process(void) while ((dfitem = switchdev_deferred_dequeue())) { dfitem->func(dfitem->dev, dfitem->data); - dev_put_track(dfitem->dev, &dfitem->dev_tracker); + netdev_put(dfitem->dev, &dfitem->dev_tracker); kfree(dfitem); } } @@ -91,7 +91,7 @@ static int switchdev_deferred_enqueue(struct net_device *dev, dfitem->dev = dev; dfitem->func = func; memcpy(dfitem->data, data, data_len); - dev_hold_track(dev, &dfitem->dev_tracker, GFP_ATOMIC); + netdev_hold(dev, &dfitem->dev_tracker, GFP_ATOMIC); spin_lock_bh(&deferred_lock); list_add_tail(&dfitem->list, &deferred); spin_unlock_bh(&deferred_lock); diff --git a/net/tipc/bearer.c b/net/tipc/bearer.c index 932c87b98eca..35cac7733fd3 100644 --- a/net/tipc/bearer.c +++ b/net/tipc/bearer.c @@ -788,7 +788,7 @@ int tipc_attach_loopback(struct net *net) if (!dev) return -ENODEV; - dev_hold_track(dev, &tn->loopback_pt.dev_tracker, GFP_KERNEL); + netdev_hold(dev, &tn->loopback_pt.dev_tracker, GFP_KERNEL); tn->loopback_pt.dev = dev; tn->loopback_pt.type = htons(ETH_P_TIPC); tn->loopback_pt.func = tipc_loopback_rcv_pkt; @@ -801,7 +801,7 @@ void tipc_detach_loopback(struct net *net) struct tipc_net *tn = tipc_net(net); dev_remove_pack(&tn->loopback_pt); - dev_put_track(net->loopback_dev, &tn->loopback_pt.dev_tracker); + netdev_put(net->loopback_dev, &tn->loopback_pt.dev_tracker); } /* Caller should hold rtnl_lock to protect the bearer */ diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 35c7e89b2e7d..637ca8838436 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -275,7 +275,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, xso->dev = NULL; xso->dir = 0; xso->real_dev = NULL; - dev_put_track(dev, &xso->dev_tracker); + netdev_put(dev, &xso->dev_tracker); if (err != -EOPNOTSUPP) return err; -- cgit v1.2.3 From 09cca53c1656960c38c04bb12da30f61952ca660 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 8 Jun 2022 08:46:32 -0700 Subject: vlan: adopt u64_stats_t As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing. Add READ_ONCE() when reading rx_errors & tx_dropped. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- drivers/net/macvlan.c | 18 +++++++++--------- include/linux/if_macvlan.h | 6 +++--- include/linux/if_vlan.h | 10 +++++----- net/8021q/vlan_core.c | 6 +++--- net/8021q/vlan_dev.c | 18 +++++++++--------- 5 files changed, 29 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 5b46a6526fc6..1080d6ebff63 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -575,8 +575,8 @@ static netdev_tx_t macvlan_start_xmit(struct sk_buff *skb, pcpu_stats = this_cpu_ptr(vlan->pcpu_stats); u64_stats_update_begin(&pcpu_stats->syncp); - pcpu_stats->tx_packets++; - pcpu_stats->tx_bytes += len; + u64_stats_inc(&pcpu_stats->tx_packets); + u64_stats_add(&pcpu_stats->tx_bytes, len); u64_stats_update_end(&pcpu_stats->syncp); } else { this_cpu_inc(vlan->pcpu_stats->tx_dropped); @@ -949,11 +949,11 @@ static void macvlan_dev_get_stats64(struct net_device *dev, p = per_cpu_ptr(vlan->pcpu_stats, i); do { start = u64_stats_fetch_begin_irq(&p->syncp); - rx_packets = p->rx_packets; - rx_bytes = p->rx_bytes; - rx_multicast = p->rx_multicast; - tx_packets = p->tx_packets; - tx_bytes = p->tx_bytes; + rx_packets = u64_stats_read(&p->rx_packets); + rx_bytes = u64_stats_read(&p->rx_bytes); + rx_multicast = u64_stats_read(&p->rx_multicast); + tx_packets = u64_stats_read(&p->tx_packets); + tx_bytes = u64_stats_read(&p->tx_bytes); } while (u64_stats_fetch_retry_irq(&p->syncp, start)); stats->rx_packets += rx_packets; @@ -964,8 +964,8 @@ static void macvlan_dev_get_stats64(struct net_device *dev, /* rx_errors & tx_dropped are u32, updated * without syncp protection. */ - rx_errors += p->rx_errors; - tx_dropped += p->tx_dropped; + rx_errors += READ_ONCE(p->rx_errors); + tx_dropped += READ_ONCE(p->tx_dropped); } stats->rx_errors = rx_errors; stats->rx_dropped = rx_errors; diff --git a/include/linux/if_macvlan.h b/include/linux/if_macvlan.h index b42294739063..523025106a64 100644 --- a/include/linux/if_macvlan.h +++ b/include/linux/if_macvlan.h @@ -46,10 +46,10 @@ static inline void macvlan_count_rx(const struct macvlan_dev *vlan, pcpu_stats = get_cpu_ptr(vlan->pcpu_stats); u64_stats_update_begin(&pcpu_stats->syncp); - pcpu_stats->rx_packets++; - pcpu_stats->rx_bytes += len; + u64_stats_inc(&pcpu_stats->rx_packets); + u64_stats_add(&pcpu_stats->rx_bytes, len); if (multicast) - pcpu_stats->rx_multicast++; + u64_stats_inc(&pcpu_stats->rx_multicast); u64_stats_update_end(&pcpu_stats->syncp); put_cpu_ptr(vlan->pcpu_stats); } else { diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 2be4dd7e90a9..e00c4ee81ff7 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -118,11 +118,11 @@ static inline void vlan_drop_rx_stag_filter_info(struct net_device *dev) * @tx_dropped: number of tx drops */ struct vlan_pcpu_stats { - u64 rx_packets; - u64 rx_bytes; - u64 rx_multicast; - u64 tx_packets; - u64 tx_bytes; + u64_stats_t rx_packets; + u64_stats_t rx_bytes; + u64_stats_t rx_multicast; + u64_stats_t tx_packets; + u64_stats_t tx_bytes; struct u64_stats_sync syncp; u32 rx_errors; u32 tx_dropped; diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c index acf8c791f320..5aa8144101dc 100644 --- a/net/8021q/vlan_core.c +++ b/net/8021q/vlan_core.c @@ -63,10 +63,10 @@ bool vlan_do_receive(struct sk_buff **skbp) rx_stats = this_cpu_ptr(vlan_dev_priv(vlan_dev)->vlan_pcpu_stats); u64_stats_update_begin(&rx_stats->syncp); - rx_stats->rx_packets++; - rx_stats->rx_bytes += skb->len; + u64_stats_inc(&rx_stats->rx_packets); + u64_stats_add(&rx_stats->rx_bytes, skb->len); if (skb->pkt_type == PACKET_MULTICAST) - rx_stats->rx_multicast++; + u64_stats_inc(&rx_stats->rx_multicast); u64_stats_update_end(&rx_stats->syncp); return true; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index e3dff2df6f54..035812b0461c 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -128,8 +128,8 @@ static netdev_tx_t vlan_dev_hard_start_xmit(struct sk_buff *skb, stats = this_cpu_ptr(vlan->vlan_pcpu_stats); u64_stats_update_begin(&stats->syncp); - stats->tx_packets++; - stats->tx_bytes += len; + u64_stats_inc(&stats->tx_packets); + u64_stats_add(&stats->tx_bytes, len); u64_stats_update_end(&stats->syncp); } else { this_cpu_inc(vlan->vlan_pcpu_stats->tx_dropped); @@ -713,11 +713,11 @@ static void vlan_dev_get_stats64(struct net_device *dev, p = per_cpu_ptr(vlan_dev_priv(dev)->vlan_pcpu_stats, i); do { start = u64_stats_fetch_begin_irq(&p->syncp); - rxpackets = p->rx_packets; - rxbytes = p->rx_bytes; - rxmulticast = p->rx_multicast; - txpackets = p->tx_packets; - txbytes = p->tx_bytes; + rxpackets = u64_stats_read(&p->rx_packets); + rxbytes = u64_stats_read(&p->rx_bytes); + rxmulticast = u64_stats_read(&p->rx_multicast); + txpackets = u64_stats_read(&p->tx_packets); + txbytes = u64_stats_read(&p->tx_bytes); } while (u64_stats_fetch_retry_irq(&p->syncp, start)); stats->rx_packets += rxpackets; @@ -726,8 +726,8 @@ static void vlan_dev_get_stats64(struct net_device *dev, stats->tx_packets += txpackets; stats->tx_bytes += txbytes; /* rx_errors & tx_dropped are u32 */ - rx_errors += p->rx_errors; - tx_dropped += p->tx_dropped; + rx_errors += READ_ONCE(p->rx_errors); + tx_dropped += READ_ONCE(p->tx_dropped); } stats->rx_errors = rx_errors; stats->tx_dropped = tx_dropped; -- cgit v1.2.3 From 9962acefbcb92736c268aafe5f52200948f60f3e Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 8 Jun 2022 08:46:37 -0700 Subject: net: adopt u64_stats_t in struct pcpu_sw_netstats As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- drivers/net/macsec.c | 8 ++++---- drivers/net/usb/usbnet.c | 8 ++++---- drivers/net/vxlan/vxlan_core.c | 8 ++++---- include/linux/netdevice.h | 16 ++++++++-------- include/net/ip_tunnels.h | 4 ++-- net/bridge/br_netlink.c | 8 ++++---- net/bridge/br_vlan.c | 36 ++++++++++++++++++++---------------- net/core/dev.c | 18 +++++++++--------- net/dsa/slave.c | 8 ++++---- 9 files changed, 59 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 815738c0e067..c881e1bf6f6e 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -523,8 +523,8 @@ static void count_tx(struct net_device *dev, int ret, int len) struct pcpu_sw_netstats *stats = this_cpu_ptr(dev->tstats); u64_stats_update_begin(&stats->syncp); - stats->tx_packets++; - stats->tx_bytes += len; + u64_stats_inc(&stats->tx_packets); + u64_stats_add(&stats->tx_bytes, len); u64_stats_update_end(&stats->syncp); } } @@ -825,8 +825,8 @@ static void count_rx(struct net_device *dev, int len) struct pcpu_sw_netstats *stats = this_cpu_ptr(dev->tstats); u64_stats_update_begin(&stats->syncp); - stats->rx_packets++; - stats->rx_bytes += len; + u64_stats_inc(&stats->rx_packets); + u64_stats_add(&stats->rx_bytes, len); u64_stats_update_end(&stats->syncp); } diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index 1cb6dab3e2d0..dc79811535c2 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -337,8 +337,8 @@ void usbnet_skb_return (struct usbnet *dev, struct sk_buff *skb) skb->protocol = eth_type_trans (skb, dev->net); flags = u64_stats_update_begin_irqsave(&stats64->syncp); - stats64->rx_packets++; - stats64->rx_bytes += skb->len; + u64_stats_inc(&stats64->rx_packets); + u64_stats_add(&stats64->rx_bytes, skb->len); u64_stats_update_end_irqrestore(&stats64->syncp, flags); netif_dbg(dev, rx_status, dev->net, "< rx, len %zu, type 0x%x\n", @@ -1258,8 +1258,8 @@ static void tx_complete (struct urb *urb) unsigned long flags; flags = u64_stats_update_begin_irqsave(&stats64->syncp); - stats64->tx_packets += entry->packets; - stats64->tx_bytes += entry->length; + u64_stats_add(&stats64->tx_packets, entry->packets); + u64_stats_add(&stats64->tx_bytes, entry->length); u64_stats_update_end_irqrestore(&stats64->syncp, flags); } else { dev->net->stats.tx_errors++; diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index 265d4a0245e7..8b0710b576c2 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -2385,15 +2385,15 @@ static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan, vxlan_snoop(dev, &loopback, eth_hdr(skb)->h_source, 0, vni); u64_stats_update_begin(&tx_stats->syncp); - tx_stats->tx_packets++; - tx_stats->tx_bytes += len; + u64_stats_inc(&tx_stats->tx_packets); + u64_stats_add(&tx_stats->tx_bytes, len); u64_stats_update_end(&tx_stats->syncp); vxlan_vnifilter_count(src_vxlan, vni, NULL, VXLAN_VNI_STATS_TX, len); if (__netif_rx(skb) == NET_RX_SUCCESS) { u64_stats_update_begin(&rx_stats->syncp); - rx_stats->rx_packets++; - rx_stats->rx_bytes += len; + u64_stats_inc(&rx_stats->rx_packets); + u64_stats_add(&rx_stats->rx_bytes, len); u64_stats_update_end(&rx_stats->syncp); vxlan_vnifilter_count(dst_vxlan, vni, NULL, VXLAN_VNI_STATS_RX, len); diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e2e5088888b1..89afa4f7747d 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2636,10 +2636,10 @@ struct packet_offload { /* often modified stats are per-CPU, other are shared (netdev->stats) */ struct pcpu_sw_netstats { - u64 rx_packets; - u64 rx_bytes; - u64 tx_packets; - u64 tx_bytes; + u64_stats_t rx_packets; + u64_stats_t rx_bytes; + u64_stats_t tx_packets; + u64_stats_t tx_bytes; struct u64_stats_sync syncp; } __aligned(4 * sizeof(u64)); @@ -2656,8 +2656,8 @@ static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int l struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats); u64_stats_update_begin(&tstats->syncp); - tstats->rx_bytes += len; - tstats->rx_packets++; + u64_stats_add(&tstats->rx_bytes, len); + u64_stats_inc(&tstats->rx_packets); u64_stats_update_end(&tstats->syncp); } @@ -2668,8 +2668,8 @@ static inline void dev_sw_netstats_tx_add(struct net_device *dev, struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats); u64_stats_update_begin(&tstats->syncp); - tstats->tx_bytes += len; - tstats->tx_packets += packets; + u64_stats_add(&tstats->tx_bytes, len); + u64_stats_add(&tstats->tx_packets, packets); u64_stats_update_end(&tstats->syncp); } diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index c24fa934221d..70cbc4a72669 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -456,8 +456,8 @@ static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len) struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); u64_stats_update_begin(&tstats->syncp); - tstats->tx_bytes += pkt_len; - tstats->tx_packets++; + u64_stats_add(&tstats->tx_bytes, pkt_len); + u64_stats_inc(&tstats->tx_packets); u64_stats_update_end(&tstats->syncp); put_cpu_ptr(tstats); } else { diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c index bb01776d2d88..1ef14a099c6b 100644 --- a/net/bridge/br_netlink.c +++ b/net/bridge/br_netlink.c @@ -1770,10 +1770,10 @@ static int br_fill_linkxstats(struct sk_buff *skb, if (v->vid == pvid) vxi.flags |= BRIDGE_VLAN_INFO_PVID; br_vlan_get_stats(v, &stats); - vxi.rx_bytes = stats.rx_bytes; - vxi.rx_packets = stats.rx_packets; - vxi.tx_bytes = stats.tx_bytes; - vxi.tx_packets = stats.tx_packets; + vxi.rx_bytes = u64_stats_read(&stats.rx_bytes); + vxi.rx_packets = u64_stats_read(&stats.rx_packets); + vxi.tx_bytes = u64_stats_read(&stats.tx_bytes); + vxi.tx_packets = u64_stats_read(&stats.tx_packets); if (nla_put(skb, BRIDGE_XSTATS_VLAN, sizeof(vxi), &vxi)) goto nla_put_failure; diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index 0f5e75ccac79..6e53dc991409 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -505,8 +505,8 @@ struct sk_buff *br_handle_vlan(struct net_bridge *br, if (br_opt_get(br, BROPT_VLAN_STATS_ENABLED)) { stats = this_cpu_ptr(v->stats); u64_stats_update_begin(&stats->syncp); - stats->tx_bytes += skb->len; - stats->tx_packets++; + u64_stats_add(&stats->tx_bytes, skb->len); + u64_stats_inc(&stats->tx_packets); u64_stats_update_end(&stats->syncp); } @@ -624,8 +624,8 @@ static bool __allowed_ingress(const struct net_bridge *br, if (br_opt_get(br, BROPT_VLAN_STATS_ENABLED)) { stats = this_cpu_ptr(v->stats); u64_stats_update_begin(&stats->syncp); - stats->rx_bytes += skb->len; - stats->rx_packets++; + u64_stats_add(&stats->rx_bytes, skb->len); + u64_stats_inc(&stats->rx_packets); u64_stats_update_end(&stats->syncp); } @@ -1379,16 +1379,16 @@ void br_vlan_get_stats(const struct net_bridge_vlan *v, cpu_stats = per_cpu_ptr(v->stats, i); do { start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); - rxpackets = cpu_stats->rx_packets; - rxbytes = cpu_stats->rx_bytes; - txbytes = cpu_stats->tx_bytes; - txpackets = cpu_stats->tx_packets; + rxpackets = u64_stats_read(&cpu_stats->rx_packets); + rxbytes = u64_stats_read(&cpu_stats->rx_bytes); + txbytes = u64_stats_read(&cpu_stats->tx_bytes); + txpackets = u64_stats_read(&cpu_stats->tx_packets); } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); - stats->rx_packets += rxpackets; - stats->rx_bytes += rxbytes; - stats->tx_bytes += txbytes; - stats->tx_packets += txpackets; + u64_stats_add(&stats->rx_packets, rxpackets); + u64_stats_add(&stats->rx_bytes, rxbytes); + u64_stats_add(&stats->tx_bytes, txbytes); + u64_stats_add(&stats->tx_packets, txpackets); } } @@ -1779,14 +1779,18 @@ static bool br_vlan_stats_fill(struct sk_buff *skb, return false; br_vlan_get_stats(v, &stats); - if (nla_put_u64_64bit(skb, BRIDGE_VLANDB_STATS_RX_BYTES, stats.rx_bytes, + if (nla_put_u64_64bit(skb, BRIDGE_VLANDB_STATS_RX_BYTES, + u64_stats_read(&stats.rx_bytes), BRIDGE_VLANDB_STATS_PAD) || nla_put_u64_64bit(skb, BRIDGE_VLANDB_STATS_RX_PACKETS, - stats.rx_packets, BRIDGE_VLANDB_STATS_PAD) || - nla_put_u64_64bit(skb, BRIDGE_VLANDB_STATS_TX_BYTES, stats.tx_bytes, + u64_stats_read(&stats.rx_packets), + BRIDGE_VLANDB_STATS_PAD) || + nla_put_u64_64bit(skb, BRIDGE_VLANDB_STATS_TX_BYTES, + u64_stats_read(&stats.tx_bytes), BRIDGE_VLANDB_STATS_PAD) || nla_put_u64_64bit(skb, BRIDGE_VLANDB_STATS_TX_PACKETS, - stats.tx_packets, BRIDGE_VLANDB_STATS_PAD)) + u64_stats_read(&stats.tx_packets), + BRIDGE_VLANDB_STATS_PAD)) goto out_err; nla_nest_end(skb, nest); diff --git a/net/core/dev.c b/net/core/dev.c index c9d96b85a825..511cf5d3aade 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10459,23 +10459,23 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, int cpu; for_each_possible_cpu(cpu) { + u64 rx_packets, rx_bytes, tx_packets, tx_bytes; const struct pcpu_sw_netstats *stats; - struct pcpu_sw_netstats tmp; unsigned int start; stats = per_cpu_ptr(netstats, cpu); do { start = u64_stats_fetch_begin_irq(&stats->syncp); - tmp.rx_packets = stats->rx_packets; - tmp.rx_bytes = stats->rx_bytes; - tmp.tx_packets = stats->tx_packets; - tmp.tx_bytes = stats->tx_bytes; + rx_packets = u64_stats_read(&stats->rx_packets); + rx_bytes = u64_stats_read(&stats->rx_bytes); + tx_packets = u64_stats_read(&stats->tx_packets); + tx_bytes = u64_stats_read(&stats->tx_bytes); } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); - s->rx_packets += tmp.rx_packets; - s->rx_bytes += tmp.rx_bytes; - s->tx_packets += tmp.tx_packets; - s->tx_bytes += tmp.tx_bytes; + s->rx_packets += rx_packets; + s->rx_bytes += rx_bytes; + s->tx_packets += tx_packets; + s->tx_bytes += tx_bytes; } } EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats); diff --git a/net/dsa/slave.c b/net/dsa/slave.c index 801a5d445833..2e1ac638d135 100644 --- a/net/dsa/slave.c +++ b/net/dsa/slave.c @@ -935,10 +935,10 @@ static void dsa_slave_get_ethtool_stats(struct net_device *dev, s = per_cpu_ptr(dev->tstats, i); do { start = u64_stats_fetch_begin_irq(&s->syncp); - tx_packets = s->tx_packets; - tx_bytes = s->tx_bytes; - rx_packets = s->rx_packets; - rx_bytes = s->rx_bytes; + tx_packets = u64_stats_read(&s->tx_packets); + tx_bytes = u64_stats_read(&s->tx_bytes); + rx_packets = u64_stats_read(&s->rx_packets); + rx_bytes = u64_stats_read(&s->rx_bytes); } while (u64_stats_fetch_retry_irq(&s->syncp, start)); data[0] += tx_packets; data[1] += tx_bytes; -- cgit v1.2.3 From 9ec321aba2eaca109139a2a67c65ede7f07bd8ec Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 8 Jun 2022 08:46:40 -0700 Subject: team: adopt u64_stats_t As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- drivers/net/team/team.c | 26 +++++++++++++------------- include/linux/if_team.h | 10 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c index b07dde6f0abf..aac133a1e27a 100644 --- a/drivers/net/team/team.c +++ b/drivers/net/team/team.c @@ -749,10 +749,10 @@ static rx_handler_result_t team_handle_frame(struct sk_buff **pskb) pcpu_stats = this_cpu_ptr(team->pcpu_stats); u64_stats_update_begin(&pcpu_stats->syncp); - pcpu_stats->rx_packets++; - pcpu_stats->rx_bytes += skb->len; + u64_stats_inc(&pcpu_stats->rx_packets); + u64_stats_add(&pcpu_stats->rx_bytes, skb->len); if (skb->pkt_type == PACKET_MULTICAST) - pcpu_stats->rx_multicast++; + u64_stats_inc(&pcpu_stats->rx_multicast); u64_stats_update_end(&pcpu_stats->syncp); skb->dev = team->dev; @@ -1720,8 +1720,8 @@ static netdev_tx_t team_xmit(struct sk_buff *skb, struct net_device *dev) pcpu_stats = this_cpu_ptr(team->pcpu_stats); u64_stats_update_begin(&pcpu_stats->syncp); - pcpu_stats->tx_packets++; - pcpu_stats->tx_bytes += len; + u64_stats_inc(&pcpu_stats->tx_packets); + u64_stats_add(&pcpu_stats->tx_bytes, len); u64_stats_update_end(&pcpu_stats->syncp); } else { this_cpu_inc(team->pcpu_stats->tx_dropped); @@ -1854,11 +1854,11 @@ team_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) p = per_cpu_ptr(team->pcpu_stats, i); do { start = u64_stats_fetch_begin_irq(&p->syncp); - rx_packets = p->rx_packets; - rx_bytes = p->rx_bytes; - rx_multicast = p->rx_multicast; - tx_packets = p->tx_packets; - tx_bytes = p->tx_bytes; + rx_packets = u64_stats_read(&p->rx_packets); + rx_bytes = u64_stats_read(&p->rx_bytes); + rx_multicast = u64_stats_read(&p->rx_multicast); + tx_packets = u64_stats_read(&p->tx_packets); + tx_bytes = u64_stats_read(&p->tx_bytes); } while (u64_stats_fetch_retry_irq(&p->syncp, start)); stats->rx_packets += rx_packets; @@ -1870,9 +1870,9 @@ team_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) * rx_dropped, tx_dropped & rx_nohandler are u32, * updated without syncp protection. */ - rx_dropped += p->rx_dropped; - tx_dropped += p->tx_dropped; - rx_nohandler += p->rx_nohandler; + rx_dropped += READ_ONCE(p->rx_dropped); + tx_dropped += READ_ONCE(p->tx_dropped); + rx_nohandler += READ_ONCE(p->rx_nohandler); } stats->rx_dropped = rx_dropped; stats->tx_dropped = tx_dropped; diff --git a/include/linux/if_team.h b/include/linux/if_team.h index add607943c95..fc985e5c739d 100644 --- a/include/linux/if_team.h +++ b/include/linux/if_team.h @@ -12,11 +12,11 @@ #include struct team_pcpu_stats { - u64 rx_packets; - u64 rx_bytes; - u64 rx_multicast; - u64 tx_packets; - u64 tx_bytes; + u64_stats_t rx_packets; + u64_stats_t rx_bytes; + u64_stats_t rx_multicast; + u64_stats_t tx_packets; + u64_stats_t tx_bytes; struct u64_stats_sync syncp; u32 rx_dropped; u32 tx_dropped; -- cgit v1.2.3 From 21ab562c1f65e583b5d89081acaf096805522482 Mon Sep 17 00:00:00 2001 From: Po Hao Huang Date: Wed, 8 Jun 2022 19:32:22 +0800 Subject: ieee80211: add trigger frame definition Define trigger stype of control frame, and its checking function, struct and trigger type within common_info of trigger. Signed-off-by: Po Hao Huang Signed-off-by: Ping-Ke Shih Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/20220608113224.11193-2-pkshih@realtek.com --- include/linux/ieee80211.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 75d40acb60c1..5c65ae6b8154 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -76,6 +76,7 @@ #define IEEE80211_STYPE_ACTION 0x00D0 /* control */ +#define IEEE80211_STYPE_TRIGGER 0x0020 #define IEEE80211_STYPE_CTL_EXT 0x0060 #define IEEE80211_STYPE_BACK_REQ 0x0080 #define IEEE80211_STYPE_BACK 0x0090 @@ -295,6 +296,17 @@ static inline u16 ieee80211_sn_sub(u16 sn1, u16 sn2) #define IEEE80211_HT_CTL_LEN 4 +/* trigger type within common_info of trigger frame */ +#define IEEE80211_TRIGGER_TYPE_MASK 0xf +#define IEEE80211_TRIGGER_TYPE_BASIC 0x0 +#define IEEE80211_TRIGGER_TYPE_BFRP 0x1 +#define IEEE80211_TRIGGER_TYPE_MU_BAR 0x2 +#define IEEE80211_TRIGGER_TYPE_MU_RTS 0x3 +#define IEEE80211_TRIGGER_TYPE_BSRP 0x4 +#define IEEE80211_TRIGGER_TYPE_GCR_MU_BAR 0x5 +#define IEEE80211_TRIGGER_TYPE_BQRP 0x6 +#define IEEE80211_TRIGGER_TYPE_NFRP 0x7 + struct ieee80211_hdr { __le16 frame_control; __le16 duration_id; @@ -324,6 +336,15 @@ struct ieee80211_qos_hdr { __le16 qos_ctrl; } __packed __aligned(2); +struct ieee80211_trigger { + __le16 frame_control; + __le16 duration; + u8 ra[ETH_ALEN]; + u8 ta[ETH_ALEN]; + __le64 common_info; + u8 variable[]; +} __packed __aligned(2); + /** * ieee80211_has_tods - check if IEEE80211_FCTL_TODS is set * @fc: frame control bytes in little-endian byteorder @@ -729,6 +750,16 @@ static inline bool ieee80211_is_qos_nullfunc(__le16 fc) cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC); } +/** + * ieee80211_is_trigger - check if frame is trigger frame + * @fc: frame control field in little-endian byteorder + */ +static inline bool ieee80211_is_trigger(__le16 fc) +{ + return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) == + cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_TRIGGER); +} + /** * ieee80211_is_any_nullfunc - check if frame is regular or QoS nullfunc frame * @fc: frame control bytes in little-endian byteorder -- cgit v1.2.3 From e3fa404a261bb8b7795b62be1b9af7b7dc1030f6 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 28 May 2022 16:59:17 +0200 Subject: USB: Follow-up to SPDX identifiers addition - remove now useless comments All these files have been updated in the commit given in the Fixes: tag below. When the SPDX-License-Identifier: has been added, the corresponding text at the beginning of the files has not been deleted. All these texts are about GPL-2.0, with different variation in the wording. Remove these now useless lines to save some LoC. Fixes: 5fd54ace4721 ("USB: add SPDX identifiers to all remaining files in drivers/usb/") Signed-off-by: Christophe JAILLET Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/audio-v2.h | 3 --- include/linux/usb/audio.h | 3 --- include/linux/usb/cdc-wdm.h | 4 ---- include/linux/usb/cdc.h | 4 ---- include/linux/usb/gadget.h | 2 -- include/linux/usb/input.h | 4 ---- include/linux/usb/isp1301.h | 10 ---------- include/linux/usb/m66592.h | 14 -------------- include/linux/usb/of.h | 2 -- include/linux/usb/r8a66597.h | 14 -------------- include/linux/usb/serial.h | 5 ----- include/linux/usb/storage.h | 2 -- include/linux/usb/tegra_usb_phy.h | 10 ---------- include/linux/usb/ulpi.h | 4 ---- include/linux/usb/xhci-dbgp.h | 4 ---- 15 files changed, 85 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/audio-v2.h b/include/linux/usb/audio-v2.h index 8fc2abd7aecb..ca796dc1a984 100644 --- a/include/linux/usb/audio-v2.h +++ b/include/linux/usb/audio-v2.h @@ -2,9 +2,6 @@ /* * Copyright (c) 2010 Daniel Mack * - * This software is distributed under the terms of the GNU General Public - * License ("GPL") version 2, as published by the Free Software Foundation. - * * This file holds USB constants and structures defined * by the USB Device Class Definition for Audio Devices in version 2.0. * Comments below reference relevant sections of the documents contained diff --git a/include/linux/usb/audio.h b/include/linux/usb/audio.h index 170acd500ea1..0747b24a1a7c 100644 --- a/include/linux/usb/audio.h +++ b/include/linux/usb/audio.h @@ -6,9 +6,6 @@ * Developed for Thumtronics by Grey Innovation * Ben Williamson * - * This software is distributed under the terms of the GNU General Public - * License ("GPL") version 2, as published by the Free Software Foundation. - * * This file holds USB constants and structures defined * by the USB Device Class Definition for Audio Devices. * Comments below reference relevant sections of that document: diff --git a/include/linux/usb/cdc-wdm.h b/include/linux/usb/cdc-wdm.h index 9f5a51f79ba5..85417f00a89a 100644 --- a/include/linux/usb/cdc-wdm.h +++ b/include/linux/usb/cdc-wdm.h @@ -3,10 +3,6 @@ * USB CDC Device Management subdriver * * Copyright (c) 2012 Bjørn Mork - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. */ #ifndef __LINUX_USB_CDC_WDM_H diff --git a/include/linux/usb/cdc.h b/include/linux/usb/cdc.h index 35d784cf32a4..0af0db51bc89 100644 --- a/include/linux/usb/cdc.h +++ b/include/linux/usb/cdc.h @@ -3,10 +3,6 @@ * USB CDC common helpers * * Copyright (c) 2015 Oliver Neukum - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. */ #ifndef __LINUX_USB_CDC_H #define __LINUX_USB_CDC_H diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index 3ad58b7a0824..dc3092cea99e 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -10,8 +10,6 @@ * * (C) Copyright 2002-2004 by David Brownell * All Rights Reserved. - * - * This software is licensed under the GNU GPL version 2. */ #ifndef __LINUX_USB_GADGET_H diff --git a/include/linux/usb/input.h b/include/linux/usb/input.h index 974befa72ac0..5e759b2cf551 100644 --- a/include/linux/usb/input.h +++ b/include/linux/usb/input.h @@ -1,10 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2005 Dmitry Torokhov - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published by - * the Free Software Foundation. */ #ifndef __LINUX_USB_INPUT_H diff --git a/include/linux/usb/isp1301.h b/include/linux/usb/isp1301.h index dedb3b2473e8..fa986b926a12 100644 --- a/include/linux/usb/isp1301.h +++ b/include/linux/usb/isp1301.h @@ -3,16 +3,6 @@ * NXP ISP1301 USB transceiver driver * * Copyright (C) 2012 Roland Stigge - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * */ #ifndef __LINUX_USB_ISP1301_H diff --git a/include/linux/usb/m66592.h b/include/linux/usb/m66592.h index 2dfe68183495..5f04de2b47fd 100644 --- a/include/linux/usb/m66592.h +++ b/include/linux/usb/m66592.h @@ -3,20 +3,6 @@ * M66592 driver platform data * * Copyright (C) 2009 Renesas Solutions Corp. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * */ #ifndef __LINUX_USB_M66592_H diff --git a/include/linux/usb/of.h b/include/linux/usb/of.h index dba55ccb9b53..98487fd7ab11 100644 --- a/include/linux/usb/of.h +++ b/include/linux/usb/of.h @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * OF helpers for usb devices. - * - * This file is released under the GPLv2 */ #ifndef __LINUX_USB_OF_H diff --git a/include/linux/usb/r8a66597.h b/include/linux/usb/r8a66597.h index c0753d026bbf..f0fa7ddadbaa 100644 --- a/include/linux/usb/r8a66597.h +++ b/include/linux/usb/r8a66597.h @@ -5,20 +5,6 @@ * Copyright (C) 2009 Renesas Solutions Corp. * * Author : Yoshihiro Shimoda - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * */ #ifndef __LINUX_USB_R8A66597_H diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index 16ea5a4cc586..8ea319f89e1f 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -4,11 +4,6 @@ * * Copyright (C) 1999 - 2012 * Greg Kroah-Hartman (greg@kroah.com) - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * */ #ifndef __LINUX_USB_SERIAL_H diff --git a/include/linux/usb/storage.h b/include/linux/usb/storage.h index e0240f864548..2827ce72e502 100644 --- a/include/linux/usb/storage.h +++ b/include/linux/usb/storage.h @@ -9,8 +9,6 @@ * * This file contains definitions taken from the * USB Mass Storage Class Specification Overview - * - * Distributed under the terms of the GNU GPL, version two. */ /* Storage subclass codes */ diff --git a/include/linux/usb/tegra_usb_phy.h b/include/linux/usb/tegra_usb_phy.h index d3e65eb9e16f..46e73584b6e6 100644 --- a/include/linux/usb/tegra_usb_phy.h +++ b/include/linux/usb/tegra_usb_phy.h @@ -1,16 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2010 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * */ #ifndef __TEGRA_USB_PHY_H diff --git a/include/linux/usb/ulpi.h b/include/linux/usb/ulpi.h index 36c2982780ad..5050f502c1ed 100644 --- a/include/linux/usb/ulpi.h +++ b/include/linux/usb/ulpi.h @@ -3,10 +3,6 @@ * ulpi.h -- ULPI defines and function prorotypes * * Copyright (C) 2010 Nokia Corporation - * - * This software is distributed under the terms of the GNU General - * Public License ("GPL") as published by the Free Software Foundation, - * version 2 of that License. */ #ifndef __LINUX_USB_ULPI_H diff --git a/include/linux/usb/xhci-dbgp.h b/include/linux/usb/xhci-dbgp.h index 01fe768873f9..171fd74b1cfc 100644 --- a/include/linux/usb/xhci-dbgp.h +++ b/include/linux/usb/xhci-dbgp.h @@ -5,10 +5,6 @@ * Copyright (C) 2016 Intel Corporation * * Author: Lu Baolu - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __LINUX_XHCI_DBGP_H -- cgit v1.2.3 From 3e00a22fdc9a9457f1c64d885532710e174babd8 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 28 May 2022 17:13:56 +0200 Subject: USB: Follow-up to SPDX GPL-2.0+ identifiers addition - remove now useless comments All these files have been updated in the commit given in the Fixes: tag below. When the SPDX-License-Identifier: has been added, the corresponding text at the beginning of the files has not been deleted. All these texts are about GPL-2.0+, with different variation in the wording. Remove these now useless lines to save some LoC. Fixes: 5fd54ace4721 ("USB: add SPDX identifiers to all remaining files in drivers/usb/") Signed-off-by: Christophe JAILLET Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/c67x00.h | 15 --------------- include/linux/usb/composite.h | 14 -------------- include/linux/usb/ehci_def.h | 14 -------------- include/linux/usb/ehci_pdriver.h | 14 -------------- include/linux/usb/g_hid.h | 14 -------------- include/linux/usb/hcd.h | 14 -------------- include/linux/usb/musb-ux500.h | 10 ---------- include/linux/usb/net2280.h | 14 -------------- include/linux/usb/ohci_pdriver.h | 14 -------------- include/linux/usb/otg-fsm.h | 17 ++--------------- include/linux/usb/phy_companion.h | 10 ---------- include/linux/usb/rndis_host.h | 14 -------------- include/linux/usb/usb338x.h | 11 ----------- include/linux/usb/usbnet.h | 14 -------------- 14 files changed, 2 insertions(+), 187 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/c67x00.h b/include/linux/usb/c67x00.h index 2fc39e3b7281..45e0757e58f3 100644 --- a/include/linux/usb/c67x00.h +++ b/include/linux/usb/c67x00.h @@ -3,21 +3,6 @@ * usb_c67x00.h: platform definitions for the Cypress C67X00 USB chip * * Copyright (C) 2006-2008 Barco N.V. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA. */ #ifndef _LINUX_USB_C67X00_H diff --git a/include/linux/usb/composite.h b/include/linux/usb/composite.h index 9d2762279286..43ac3fa760db 100644 --- a/include/linux/usb/composite.h +++ b/include/linux/usb/composite.h @@ -3,20 +3,6 @@ * composite.h -- framework for usb gadgets which are composite devices * * Copyright (C) 2006-2008 David Brownell - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef __LINUX_USB_COMPOSITE_H diff --git a/include/linux/usb/ehci_def.h b/include/linux/usb/ehci_def.h index c892c5bc6638..fbabadd3b372 100644 --- a/include/linux/usb/ehci_def.h +++ b/include/linux/usb/ehci_def.h @@ -1,20 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ /* * Copyright (c) 2001-2002 by David Brownell - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef __LINUX_USB_EHCI_DEF_H diff --git a/include/linux/usb/ehci_pdriver.h b/include/linux/usb/ehci_pdriver.h index 89fc901e778f..0f1b166f5aa0 100644 --- a/include/linux/usb/ehci_pdriver.h +++ b/include/linux/usb/ehci_pdriver.h @@ -1,20 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ /* * Copyright (C) 2012 Hauke Mehrtens - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef __USB_CORE_EHCI_PDRIVER_H diff --git a/include/linux/usb/g_hid.h b/include/linux/usb/g_hid.h index 7581e488c237..d56bfedeb079 100644 --- a/include/linux/usb/g_hid.h +++ b/include/linux/usb/g_hid.h @@ -3,20 +3,6 @@ * g_hid.h -- Header file for USB HID gadget driver * * Copyright (C) 2010 Fabien Chouteau - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef __LINUX_USB_G_HID_H diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 2c1fc9212cf2..37ccb31b1d40 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -1,20 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ /* * Copyright (c) 2001-2002 by David Brownell - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef __USB_CORE_HCD_H diff --git a/include/linux/usb/musb-ux500.h b/include/linux/usb/musb-ux500.h index c4b7ad9850ca..d60dcfc56b5a 100644 --- a/include/linux/usb/musb-ux500.h +++ b/include/linux/usb/musb-ux500.h @@ -1,16 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ /* * Copyright (C) 2013 ST-Ericsson AB - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __MUSB_UX500_H__ diff --git a/include/linux/usb/net2280.h b/include/linux/usb/net2280.h index 08b85caecfaf..f29fe6a1f415 100644 --- a/include/linux/usb/net2280.h +++ b/include/linux/usb/net2280.h @@ -5,20 +5,6 @@ * * Copyright (C) 2002 NetChip Technology, Inc. (http://www.netchip.com) * Copyright (C) 2003 David Brownell - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef __LINUX_USB_NET2280_H diff --git a/include/linux/usb/ohci_pdriver.h b/include/linux/usb/ohci_pdriver.h index 7eb16cf587ee..2447c78b1766 100644 --- a/include/linux/usb/ohci_pdriver.h +++ b/include/linux/usb/ohci_pdriver.h @@ -1,20 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ /* * Copyright (C) 2012 Hauke Mehrtens - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef __USB_CORE_OHCI_PDRIVER_H diff --git a/include/linux/usb/otg-fsm.h b/include/linux/usb/otg-fsm.h index 784659d4dc99..6135d076c53d 100644 --- a/include/linux/usb/otg-fsm.h +++ b/include/linux/usb/otg-fsm.h @@ -1,19 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ -/* Copyright (C) 2007,2008 Freescale Semiconductor, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. +/* + * Copyright (C) 2007,2008 Freescale Semiconductor, Inc. */ #ifndef __LINUX_USB_OTG_FSM_H diff --git a/include/linux/usb/phy_companion.h b/include/linux/usb/phy_companion.h index 263196f05015..862aaeca2319 100644 --- a/include/linux/usb/phy_companion.h +++ b/include/linux/usb/phy_companion.h @@ -3,18 +3,8 @@ * phy-companion.h -- phy companion to indicate the comparator part of PHY * * Copyright (C) 2012 Texas Instruments Incorporated - https://www.ti.com - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. * * Author: Kishon Vijay Abraham I - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * */ #ifndef __DRIVERS_PHY_COMPANION_H diff --git a/include/linux/usb/rndis_host.h b/include/linux/usb/rndis_host.h index cc42db51bbba..489cfb1d00f6 100644 --- a/include/linux/usb/rndis_host.h +++ b/include/linux/usb/rndis_host.h @@ -2,20 +2,6 @@ /* * Host Side support for RNDIS Networking Links * Copyright (C) 2005 by David Brownell - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef __LINUX_USB_RNDIS_HOST_H diff --git a/include/linux/usb/usb338x.h b/include/linux/usb/usb338x.h index 20020c1336d5..70a7e3cdb3c9 100644 --- a/include/linux/usb/usb338x.h +++ b/include/linux/usb/usb338x.h @@ -6,17 +6,6 @@ * Copyright (C) 2002 NetChip Technology, Inc. (http://www.netchip.com) * Copyright (C) 2003 David Brownell * Copyright (C) 2014 Ricardo Ribalda - Qtechnology/AS - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * */ #ifndef __LINUX_USB_USB338X_H diff --git a/include/linux/usb/usbnet.h b/include/linux/usb/usbnet.h index 1b4d72d5e891..4f5608cbd9ab 100644 --- a/include/linux/usb/usbnet.h +++ b/include/linux/usb/usbnet.h @@ -4,20 +4,6 @@ * * Copyright (C) 2000-2005 by David Brownell * Copyright (C) 2003-2005 David Hollis - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef __LINUX_USB_USBNET_H -- cgit v1.2.3 From 4173f018aae16b6496d292c234b858241f85254f Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 7 Jun 2022 12:49:12 +0200 Subject: tty/vt: consolemap: rename and document struct uni_pagedir struct uni_pagedir contains 32 unicode page directories, so the name of the structure is a bit misleading. Rename the structure to uni_pagedict, so it looks like this: struct uni_pagedict -> 32 page dirs -> 32 rows -> 64 glyphs Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20220607104946.18710-2-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/consolemap.c | 47 +++++++++++++++++++++++++----------------- drivers/video/console/vgacon.c | 4 ++-- include/linux/console_struct.h | 6 +++--- 3 files changed, 33 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c index 839d75d1a6c0..5acafeea9afc 100644 --- a/drivers/tty/vt/consolemap.c +++ b/drivers/tty/vt/consolemap.c @@ -186,17 +186,26 @@ static unsigned short translations[][256] = { static int inv_translate[MAX_NR_CONSOLES]; -struct uni_pagedir { - u16 **uni_pgdir[32]; +/** + * struct uni_pagedict -- unicode directory + * + * @uni_pgdir: 32*32*64 table with glyphs + * @refcount: reference count of this structure + * @sum: checksum + * @inverse_translations: best-effort inverse mapping + * @inverse_trans_unicode: best-effort inverse mapping to unicode + */ +struct uni_pagedict { + u16 **uni_pgdir[32]; unsigned long refcount; unsigned long sum; unsigned char *inverse_translations[4]; u16 *inverse_trans_unicode; }; -static struct uni_pagedir *dflt; +static struct uni_pagedict *dflt; -static void set_inverse_transl(struct vc_data *conp, struct uni_pagedir *p, int i) +static void set_inverse_transl(struct vc_data *conp, struct uni_pagedict *p, int i) { int j, glyph; unsigned short *t = translations[i]; @@ -221,7 +230,7 @@ static void set_inverse_transl(struct vc_data *conp, struct uni_pagedir *p, int } static void set_inverse_trans_unicode(struct vc_data *conp, - struct uni_pagedir *p) + struct uni_pagedict *p) { int i, j, k, glyph; u16 **p1, *p2; @@ -270,7 +279,7 @@ unsigned short *set_translate(int m, struct vc_data *vc) */ u16 inverse_translate(const struct vc_data *conp, int glyph, int use_unicode) { - struct uni_pagedir *p; + struct uni_pagedict *p; int m; if (glyph < 0 || glyph >= MAX_GLYPH) return 0; @@ -297,7 +306,7 @@ EXPORT_SYMBOL_GPL(inverse_translate); static void update_user_maps(void) { int i; - struct uni_pagedir *p, *q = NULL; + struct uni_pagedict *p, *q = NULL; for (i = 0; i < MAX_NR_CONSOLES; i++) { if (!vc_cons_allocated(i)) @@ -393,7 +402,7 @@ int con_get_trans_new(ushort __user * arg) extern u8 dfont_unicount[]; /* Defined in console_defmap.c */ extern u16 dfont_unitable[]; -static void con_release_unimap(struct uni_pagedir *p) +static void con_release_unimap(struct uni_pagedict *p) { u16 **p1; int i, j; @@ -419,7 +428,7 @@ static void con_release_unimap(struct uni_pagedir *p) /* Caller must hold the console lock */ void con_free_unimap(struct vc_data *vc) { - struct uni_pagedir *p; + struct uni_pagedict *p; p = *vc->vc_uni_pagedir_loc; if (!p) @@ -431,10 +440,10 @@ void con_free_unimap(struct vc_data *vc) kfree(p); } -static int con_unify_unimap(struct vc_data *conp, struct uni_pagedir *p) +static int con_unify_unimap(struct vc_data *conp, struct uni_pagedict *p) { int i, j, k; - struct uni_pagedir *q; + struct uni_pagedict *q; for (i = 0; i < MAX_NR_CONSOLES; i++) { if (!vc_cons_allocated(i)) @@ -472,7 +481,7 @@ static int con_unify_unimap(struct vc_data *conp, struct uni_pagedir *p) } static int -con_insert_unipair(struct uni_pagedir *p, u_short unicode, u_short fontpos) +con_insert_unipair(struct uni_pagedict *p, u_short unicode, u_short fontpos) { int i, n; u16 **p1, *p2; @@ -503,7 +512,7 @@ con_insert_unipair(struct uni_pagedir *p, u_short unicode, u_short fontpos) /* Caller must hold the lock */ static int con_do_clear_unimap(struct vc_data *vc) { - struct uni_pagedir *p, *q; + struct uni_pagedict *p, *q; p = *vc->vc_uni_pagedir_loc; if (!p || --p->refcount) { @@ -536,7 +545,7 @@ int con_clear_unimap(struct vc_data *vc) int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list) { int err = 0, err1, i; - struct uni_pagedir *p, *q; + struct uni_pagedict *p, *q; struct unipair *unilist, *plist; if (!ct) @@ -569,7 +578,7 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list) /* * Since refcount was > 1, con_clear_unimap() allocated a - * a new uni_pagedir for this vc. Re: p != q + * a new uni_pagedict for this vc. Re: p != q */ q = *vc->vc_uni_pagedir_loc; @@ -660,7 +669,7 @@ int con_set_default_unimap(struct vc_data *vc) { int i, j, err = 0, err1; u16 *q; - struct uni_pagedir *p; + struct uni_pagedict *p; if (dflt) { p = *vc->vc_uni_pagedir_loc; @@ -714,7 +723,7 @@ EXPORT_SYMBOL(con_set_default_unimap); */ int con_copy_unimap(struct vc_data *dst_vc, struct vc_data *src_vc) { - struct uni_pagedir *q; + struct uni_pagedict *q; if (!*src_vc->vc_uni_pagedir_loc) return -EINVAL; @@ -739,7 +748,7 @@ int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct, struct uni int i, j, k, ret = 0; ushort ect; u16 **p1, *p2; - struct uni_pagedir *p; + struct uni_pagedict *p; struct unipair *unilist; unilist = kvmalloc_array(ct, sizeof(struct unipair), GFP_KERNEL); @@ -810,7 +819,7 @@ conv_uni_to_pc(struct vc_data *conp, long ucs) { int h; u16 **p1, *p2; - struct uni_pagedir *p; + struct uni_pagedict *p; /* Only 16-bit codes supported at this time */ if (ucs > 0xffff) diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 576612f18d59..058a78b8dbcf 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -75,7 +75,7 @@ static void vgacon_scrolldelta(struct vc_data *c, int lines); static int vgacon_set_origin(struct vc_data *c); static void vgacon_save_screen(struct vc_data *c); static void vgacon_invert_region(struct vc_data *c, u16 * p, int count); -static struct uni_pagedir *vgacon_uni_pagedir; +static struct uni_pagedict *vgacon_uni_pagedir; static int vgacon_refcount; /* Description of the hardware situation */ @@ -342,7 +342,7 @@ static const char *vgacon_startup(void) static void vgacon_init(struct vc_data *c, int init) { - struct uni_pagedir *p; + struct uni_pagedict *p; /* * We cannot be loaded as a module, therefore init will be 1 diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h index d5b9c8d40c18..f75033f0277f 100644 --- a/include/linux/console_struct.h +++ b/include/linux/console_struct.h @@ -17,7 +17,7 @@ #include #include -struct uni_pagedir; +struct uni_pagedict; struct uni_screen; #define NPAR 16 @@ -157,8 +157,8 @@ struct vc_data { unsigned int vc_bell_duration; /* Console bell duration */ unsigned short vc_cur_blink_ms; /* Cursor blink duration */ struct vc_data **vc_display_fg; /* [!] Ptr to var holding fg console for this display */ - struct uni_pagedir *vc_uni_pagedir; - struct uni_pagedir **vc_uni_pagedir_loc; /* [!] Location of uni_pagedir variable for this console */ + struct uni_pagedict *vc_uni_pagedir; + struct uni_pagedict **vc_uni_pagedir_loc; /* [!] Location of uni_pagedict variable for this console */ struct uni_screen *vc_uni_screen; /* unicode screen content */ /* additional information is in vt_kern.h */ }; -- cgit v1.2.3 From 0b75f7968d61566d150747512344cabaa59e54c6 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 7 Jun 2022 12:49:15 +0200 Subject: tty/vt: consolemap: remove extern from function decls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extern keyword is not needed for function declarations. Remove it, so that the consolemap header conforms to other tty headers. Reviewed-by: Ilpo Järvinen Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20220607104946.18710-5-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/consolemap.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/consolemap.h b/include/linux/consolemap.h index bcfce748c9d8..abc28e4450bf 100644 --- a/include/linux/consolemap.h +++ b/include/linux/consolemap.h @@ -17,12 +17,11 @@ #ifdef CONFIG_CONSOLE_TRANSLATIONS struct vc_data; -extern u16 inverse_translate(const struct vc_data *conp, int glyph, - int use_unicode); -extern unsigned short *set_translate(int m, struct vc_data *vc); -extern int conv_uni_to_pc(struct vc_data *conp, long ucs); -extern u32 conv_8bit_to_uni(unsigned char c); -extern int conv_uni_to_8bit(u32 uni); +u16 inverse_translate(const struct vc_data *conp, int glyph, int use_unicode); +unsigned short *set_translate(int m, struct vc_data *vc); +int conv_uni_to_pc(struct vc_data *conp, long ucs); +u32 conv_8bit_to_uni(unsigned char c); +int conv_uni_to_8bit(u32 uni); void console_map_init(void); #else #define inverse_translate(conp, glyph, uni) ((uint16_t)glyph) -- cgit v1.2.3 From f827c754f9b6247f4d4f9cbb508b22bff2cf8e6d Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 7 Jun 2022 12:49:16 +0200 Subject: tty/vt: consolemap: convert macros to static inlines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit changes !CONFIG_CONSOLE_TRANSLATIONS definitions to real (inline) functions. So the commit: * makes type checking much stronger, * removes the need of many parentheses and casts, and * makes the code more readable. Reviewed-by: Ilpo Järvinen Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20220607104946.18710-6-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/consolemap.h | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/consolemap.h b/include/linux/consolemap.h index abc28e4450bf..98171dbed51f 100644 --- a/include/linux/consolemap.h +++ b/include/linux/consolemap.h @@ -14,9 +14,9 @@ #include -#ifdef CONFIG_CONSOLE_TRANSLATIONS struct vc_data; +#ifdef CONFIG_CONSOLE_TRANSLATIONS u16 inverse_translate(const struct vc_data *conp, int glyph, int use_unicode); unsigned short *set_translate(int m, struct vc_data *vc); int conv_uni_to_pc(struct vc_data *conp, long ucs); @@ -24,12 +24,33 @@ u32 conv_8bit_to_uni(unsigned char c); int conv_uni_to_8bit(u32 uni); void console_map_init(void); #else -#define inverse_translate(conp, glyph, uni) ((uint16_t)glyph) -#define set_translate(m, vc) ((unsigned short *)NULL) -#define conv_uni_to_pc(conp, ucs) ((int) (ucs > 0xff ? -1: ucs)) -#define conv_8bit_to_uni(c) ((uint32_t)(c)) -#define conv_uni_to_8bit(c) ((int) ((c) & 0xff)) -#define console_map_init(c) do { ; } while (0) +static inline u16 inverse_translate(const struct vc_data *conp, int glyph, + int use_unicode) +{ + return glyph; +} + +static inline unsigned short *set_translate(int m, struct vc_data *vc) +{ + return NULL; +} + +static inline int conv_uni_to_pc(struct vc_data *conp, long ucs) +{ + return ucs > 0xff ? -1 : ucs; +} + +static inline u32 conv_8bit_to_uni(unsigned char c) +{ + return c; +} + +static inline int conv_uni_to_8bit(u32 uni) +{ + return uni & 0xff; +} + +static inline void console_map_init(void) { } #endif /* CONFIG_CONSOLE_TRANSLATIONS */ #endif /* __LINUX_CONSOLEMAP_H__ */ -- cgit v1.2.3 From d9ebb906a45adc71a62aceb1e3608c847cafa660 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 7 Jun 2022 12:49:17 +0200 Subject: tty/vt: consolemap: make parameters of inverse_translate() saner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - int use_unicode -> bool: it's used as bool at some places already, so make it explicit. - int glyph -> u16: every caller passes a u16 in. So make it explicit too. And remove a negative check from inverse_translate() as it never could be negative. Reviewed-by: Ilpo Järvinen Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20220607104946.18710-7-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/accessibility/braille/braille_console.c | 2 +- drivers/accessibility/speakup/main.c | 2 +- drivers/tty/vt/consolemap.c | 4 ++-- drivers/tty/vt/selection.c | 3 ++- drivers/tty/vt/vt.c | 2 +- include/linux/consolemap.h | 6 +++--- 6 files changed, 10 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/accessibility/braille/braille_console.c b/drivers/accessibility/braille/braille_console.c index fdc6b593f500..c4d54a5326b1 100644 --- a/drivers/accessibility/braille/braille_console.c +++ b/drivers/accessibility/braille/braille_console.c @@ -131,7 +131,7 @@ static void vc_refresh(struct vc_data *vc) for (i = 0; i < WIDTH; i++) { u16 glyph = screen_glyph(vc, 2 * (vc_x + i) + vc_y * vc->vc_size_row); - buf[i] = inverse_translate(vc, glyph, 1); + buf[i] = inverse_translate(vc, glyph, true); } braille_write(buf); } diff --git a/drivers/accessibility/speakup/main.c b/drivers/accessibility/speakup/main.c index d726537fa16c..f52265293482 100644 --- a/drivers/accessibility/speakup/main.c +++ b/drivers/accessibility/speakup/main.c @@ -470,7 +470,7 @@ static u16 get_char(struct vc_data *vc, u16 *pos, u_char *attribs) c |= 0x100; } - ch = inverse_translate(vc, c, 1); + ch = inverse_translate(vc, c, true); *attribs = (w & 0xff00) >> 8; } return ch; diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c index fb61158f4dc6..157c7f936294 100644 --- a/drivers/tty/vt/consolemap.c +++ b/drivers/tty/vt/consolemap.c @@ -281,12 +281,12 @@ unsigned short *set_translate(int m, struct vc_data *vc) * was active. * Still, it is now possible to a certain extent to cut and paste non-ASCII. */ -u16 inverse_translate(const struct vc_data *conp, int glyph, int use_unicode) +u16 inverse_translate(const struct vc_data *conp, u16 glyph, bool use_unicode) { struct uni_pagedict *p; int m; - if (glyph < 0 || glyph >= MAX_GLYPH) + if (glyph >= MAX_GLYPH) return 0; p = *conp->vc_uni_pagedir_loc; diff --git a/drivers/tty/vt/selection.c b/drivers/tty/vt/selection.c index f7755e73696e..6ef22f01cc51 100644 --- a/drivers/tty/vt/selection.c +++ b/drivers/tty/vt/selection.c @@ -68,7 +68,8 @@ sel_pos(int n, bool unicode) { if (unicode) return screen_glyph_unicode(vc_sel.cons, n / 2); - return inverse_translate(vc_sel.cons, screen_glyph(vc_sel.cons, n), 0); + return inverse_translate(vc_sel.cons, screen_glyph(vc_sel.cons, n), + false); } /** diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index f8c87c4d7399..1ea1c11c42fd 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -4741,7 +4741,7 @@ u32 screen_glyph_unicode(const struct vc_data *vc, int n) if (uniscr) return uniscr->lines[n / vc->vc_cols][n % vc->vc_cols]; - return inverse_translate(vc, screen_glyph(vc, n * 2), 1); + return inverse_translate(vc, screen_glyph(vc, n * 2), true); } EXPORT_SYMBOL_GPL(screen_glyph_unicode); diff --git a/include/linux/consolemap.h b/include/linux/consolemap.h index 98171dbed51f..1ff2bf55eb85 100644 --- a/include/linux/consolemap.h +++ b/include/linux/consolemap.h @@ -17,15 +17,15 @@ struct vc_data; #ifdef CONFIG_CONSOLE_TRANSLATIONS -u16 inverse_translate(const struct vc_data *conp, int glyph, int use_unicode); +u16 inverse_translate(const struct vc_data *conp, u16 glyph, bool use_unicode); unsigned short *set_translate(int m, struct vc_data *vc); int conv_uni_to_pc(struct vc_data *conp, long ucs); u32 conv_8bit_to_uni(unsigned char c); int conv_uni_to_8bit(u32 uni); void console_map_init(void); #else -static inline u16 inverse_translate(const struct vc_data *conp, int glyph, - int use_unicode) +static inline u16 inverse_translate(const struct vc_data *conp, u16 glyph, + bool use_unicode) { return glyph; } -- cgit v1.2.3 From 5a904a936b407624cd1ff5ee3f1675ca3d2366a5 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 7 Jun 2022 12:49:27 +0200 Subject: tty/vt: consolemap: introduce enum translation_map and use it Again, instead of magic constants in the code, declare an enum and be a little bit more explicit. Both in the translations definition and in the loops etc. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20220607104946.18710-17-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/consolemap.c | 39 ++++++++++++++++++++------------------- include/linux/consolemap.h | 18 ++++++++++++------ 2 files changed, 32 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c index 92b5dddb00d9..80536687acef 100644 --- a/drivers/tty/vt/consolemap.c +++ b/drivers/tty/vt/consolemap.c @@ -38,7 +38,7 @@ static unsigned short translations[][256] = { /* 8-bit Latin-1 mapped to Unicode -- trivial mapping */ - { + [LAT1_MAP] = { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, @@ -71,9 +71,9 @@ static unsigned short translations[][256] = { 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff - }, + }, /* VT100 graphics mapped to Unicode */ - { + [GRAF_MAP] = { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, @@ -108,8 +108,8 @@ static unsigned short translations[][256] = { 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff }, /* IBM Codepage 437 mapped to Unicode */ - { - 0x0000, 0x263a, 0x263b, 0x2665, 0x2666, 0x2663, 0x2660, 0x2022, + [IBMPC_MAP] = { + 0x0000, 0x263a, 0x263b, 0x2665, 0x2666, 0x2663, 0x2660, 0x2022, 0x25d8, 0x25cb, 0x25d9, 0x2642, 0x2640, 0x266a, 0x266b, 0x263c, 0x25b6, 0x25c0, 0x2195, 0x203c, 0x00b6, 0x00a7, 0x25ac, 0x21a8, 0x2191, 0x2193, 0x2192, 0x2190, 0x221f, 0x2194, 0x25b2, 0x25bc, @@ -141,9 +141,9 @@ static unsigned short translations[][256] = { 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229, 0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248, 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0 - }, + }, /* User mapping -- default to codes for direct font mapping */ - { + [USER_MAP] = { 0xf000, 0xf001, 0xf002, 0xf003, 0xf004, 0xf005, 0xf006, 0xf007, 0xf008, 0xf009, 0xf00a, 0xf00b, 0xf00c, 0xf00d, 0xf00e, 0xf00f, 0xf010, 0xf011, 0xf012, 0xf013, 0xf014, 0xf015, 0xf016, 0xf017, @@ -184,7 +184,7 @@ static unsigned short translations[][256] = { #define MAX_GLYPH 512 /* Max possible glyph value */ -static int inv_translate[MAX_NR_CONSOLES]; +static enum translation_map inv_translate[MAX_NR_CONSOLES]; #define UNI_DIRS 32U #define UNI_DIR_ROWS 32U @@ -208,24 +208,25 @@ struct uni_pagedict { u16 **uni_pgdir[UNI_DIRS]; unsigned long refcount; unsigned long sum; - unsigned char *inverse_translations[4]; + unsigned char *inverse_translations[LAST_MAP + 1]; u16 *inverse_trans_unicode; }; static struct uni_pagedict *dflt; -static void set_inverse_transl(struct vc_data *conp, struct uni_pagedict *p, int i) +static void set_inverse_transl(struct vc_data *conp, struct uni_pagedict *p, + enum translation_map m) { int j, glyph; - unsigned short *t = translations[i]; + unsigned short *t = translations[m]; unsigned char *q; if (!p) return; - q = p->inverse_translations[i]; + q = p->inverse_translations[m]; if (!q) { - q = p->inverse_translations[i] = kmalloc(MAX_GLYPH, GFP_KERNEL); + q = p->inverse_translations[m] = kmalloc(MAX_GLYPH, GFP_KERNEL); if (!q) return; } @@ -276,7 +277,7 @@ static void set_inverse_trans_unicode(struct vc_data *conp, } } -unsigned short *set_translate(int m, struct vc_data *vc) +unsigned short *set_translate(enum translation_map m, struct vc_data *vc) { inv_translate[vc->vc_num] = m; return translations[m]; @@ -292,7 +293,7 @@ unsigned short *set_translate(int m, struct vc_data *vc) u16 inverse_translate(const struct vc_data *conp, u16 glyph, bool use_unicode) { struct uni_pagedict *p; - int m; + enum translation_map m; if (glyph >= MAX_GLYPH) return 0; @@ -669,8 +670,8 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list) if (con_unify_unimap(vc, p)) goto out_unlock; - for (i = 0; i <= 3; i++) - set_inverse_transl(vc, p, i); /* Update inverse translations */ + for (enum translation_map m = FIRST_MAP; m <= LAST_MAP; m++) + set_inverse_transl(vc, p, m); /* Update inverse translations */ set_inverse_trans_unicode(vc, p); out_unlock: @@ -731,8 +732,8 @@ int con_set_default_unimap(struct vc_data *vc) return err; } - for (i = 0; i <= 3; i++) - set_inverse_transl(vc, p, i); /* Update all inverse translations */ + for (enum translation_map m = FIRST_MAP; m <= LAST_MAP; m++) + set_inverse_transl(vc, p, m); /* Update all inverse translations */ set_inverse_trans_unicode(vc, p); dflt = p; return err; diff --git a/include/linux/consolemap.h b/include/linux/consolemap.h index 1ff2bf55eb85..c35db4896c37 100644 --- a/include/linux/consolemap.h +++ b/include/linux/consolemap.h @@ -7,10 +7,15 @@ #ifndef __LINUX_CONSOLEMAP_H__ #define __LINUX_CONSOLEMAP_H__ -#define LAT1_MAP 0 -#define GRAF_MAP 1 -#define IBMPC_MAP 2 -#define USER_MAP 3 +enum translation_map { + LAT1_MAP, + GRAF_MAP, + IBMPC_MAP, + USER_MAP, + + FIRST_MAP = LAT1_MAP, + LAST_MAP = USER_MAP, +}; #include @@ -18,7 +23,7 @@ struct vc_data; #ifdef CONFIG_CONSOLE_TRANSLATIONS u16 inverse_translate(const struct vc_data *conp, u16 glyph, bool use_unicode); -unsigned short *set_translate(int m, struct vc_data *vc); +unsigned short *set_translate(enum translation_map m, struct vc_data *vc); int conv_uni_to_pc(struct vc_data *conp, long ucs); u32 conv_8bit_to_uni(unsigned char c); int conv_uni_to_8bit(u32 uni); @@ -30,7 +35,8 @@ static inline u16 inverse_translate(const struct vc_data *conp, u16 glyph, return glyph; } -static inline unsigned short *set_translate(int m, struct vc_data *vc) +static inline unsigned short *set_translate(enum translation_map m, + struct vc_data *vc) { return NULL; } -- cgit v1.2.3 From 8322b1f527159de578aab277629296575a11eb3c Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Mon, 6 Jun 2022 13:03:58 +0300 Subject: serial: Add uart_rs485_config() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A few serial drivers make a call to rs485_config() themselves (all these seem to relate to init). Convert them all to use a common helper which makes it easy to make adjustments on tasks related to it as serial_rs485 struct sanitization is going to be added. In pci_fintek_setup() (in 8250_pci.c), the rs485_config() call was made with NULL, however, it can be changed to pass uart_port's rs485 struct. No other callers should pass NULL into rs485_config() so the NULL check can now be eliminated. Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220606100433.13793-2-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_pci.c | 6 ++---- drivers/tty/serial/8250/8250_port.c | 2 +- drivers/tty/serial/fsl_lpuart.c | 2 +- drivers/tty/serial/imx.c | 2 +- drivers/tty/serial/serial_core.c | 6 ++++++ include/linux/serial_core.h | 1 + 6 files changed, 12 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index a17619db7939..fb0a49e39072 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -1562,9 +1562,7 @@ static int pci_fintek_rs485_config(struct uart_port *port, pci_read_config_byte(pci_dev, 0x40 + 8 * *index + 7, &setting); - if (!rs485) - rs485 = &port->rs485; - else if (rs485->flags & SER_RS485_ENABLED) + if (rs485->flags & SER_RS485_ENABLED) memset(rs485->padding, 0, sizeof(rs485->padding)); else memset(rs485, 0, sizeof(*rs485)); @@ -1689,7 +1687,7 @@ static int pci_fintek_init(struct pci_dev *dev) * pciserial_resume_ports() */ port = serial8250_get_port(priv->line[i]); - pci_fintek_rs485_config(&port->port, NULL); + uart_rs485_config(&port->port); } else { /* First init without port data * force init to RS232 Mode diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 30e0aaf52adc..af550a4a27f8 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -3191,7 +3191,7 @@ static void serial8250_config_port(struct uart_port *port, int flags) autoconfig(up); if (port->rs485.flags & SER_RS485_ENABLED) - port->rs485_config(port, &port->rs485); + uart_rs485_config(port); /* if access method is AU, it is a 16550 with a quirk */ if (port->type == PORT_16550A && port->iotype == UPIO_AU) diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c index 0d6e62f6bb07..509a7912fa9d 100644 --- a/drivers/tty/serial/fsl_lpuart.c +++ b/drivers/tty/serial/fsl_lpuart.c @@ -2724,7 +2724,7 @@ static int lpuart_probe(struct platform_device *pdev) sport->port.rs485.delay_rts_after_send) dev_err(&pdev->dev, "driver doesn't support RTS delays\n"); - sport->port.rs485_config(&sport->port, &sport->port.rs485); + uart_rs485_config(&sport->port); ret = devm_request_irq(&pdev->dev, sport->port.irq, handler, 0, DRIVER_NAME, sport); diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 30edb35a6a15..17fb9a57078b 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -2338,7 +2338,7 @@ static int imx_uart_probe(struct platform_device *pdev) dev_err(&pdev->dev, "low-active RTS not possible when receiver is off, enabling receiver\n"); - imx_uart_rs485_config(&sport->port, &sport->port.rs485); + uart_rs485_config(&sport->port); /* Disable interrupts before requesting them */ ucr1 = imx_uart_readl(sport, UCR1); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 9a85b41caa0a..8466181db4e9 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -1276,6 +1276,12 @@ static int uart_get_icount(struct tty_struct *tty, return 0; } +int uart_rs485_config(struct uart_port *port) +{ + return port->rs485_config(port, &port->rs485); +} +EXPORT_SYMBOL_GPL(uart_rs485_config); + static int uart_get_rs485_config(struct uart_port *port, struct serial_rs485 __user *rs485) { diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index cbd5070bc87f..d3ebb4db2d80 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -592,4 +592,5 @@ static inline int uart_handle_break(struct uart_port *port) !((cflag) & CLOCAL)) int uart_get_rs485_mode(struct uart_port *port); +int uart_rs485_config(struct uart_port *port); #endif /* LINUX_SERIAL_CORE_H */ -- cgit v1.2.3 From 8925c31c1ac2f1e05da988581f2a70a2a8c4d638 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Mon, 6 Jun 2022 13:04:00 +0300 Subject: serial: Add rs485_supported to uart_port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preparing to move serial_rs485 struct sanitization into serial core, each driver has to provide what fields/flags it supports. This information is pointed into by rs485_supported. Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220606100433.13793-4-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_core.c | 1 + include/linux/serial_core.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include/linux') diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index b0320de3379c..90ddc8924811 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -1003,6 +1003,7 @@ int serial8250_register_8250_port(const struct uart_8250_port *up) uart->port.throttle = up->port.throttle; uart->port.unthrottle = up->port.unthrottle; uart->port.rs485_config = up->port.rs485_config; + uart->port.rs485_supported = up->port.rs485_supported; uart->port.rs485 = up->port.rs485; uart->rs485_start_tx = up->rs485_start_tx; uart->rs485_stop_tx = up->rs485_stop_tx; diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index d3ebb4db2d80..5518b70177b3 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -254,6 +254,7 @@ struct uart_port { struct attribute_group *attr_group; /* port specific attributes */ const struct attribute_group **tty_groups; /* all attributes (serial core use only) */ struct serial_rs485 rs485; + const struct serial_rs485 *rs485_supported; /* Supported mask for serial_rs485 */ struct gpio_desc *rs485_term_gpio; /* enable RS485 bus termination */ struct serial_iso7816 iso7816; void *private_data; /* generic platform data pointer */ -- cgit v1.2.3 From 6bb6fa6908ebd3cb4e14cd4f0ce272ec885d2eb0 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Mon, 6 Jun 2022 18:36:51 +0300 Subject: tty: Implement lookahead to process XON/XOFF timely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When tty is not read from, XON/XOFF may get stuck into an intermediate buffer. As those characters are there to do software flow-control, it is not very useful. In the case where neither end reads from ttys, the receiving ends might not be able receive the XOFF characters and just keep sending more data to the opposite direction. This problem is almost guaranteed to occur with DMA which sends data in large chunks. If TTY is slow to process characters, that is, eats less than given amount in receive_buf, invoke lookahead for the rest of the chars to process potential XON/XOFF characters. We need to keep track of how many characters have been processed by the lookahead to avoid processing the flow control char again on the normal path. Bookkeeping occurs parallel on two layers (tty_buffer and n_tty) to avoid passing the lookahead_count through the whole call chain. When a flow-control char is processed, two things must occur: a) it must not be treated as normal char b) if not yet processed, flow-control actions need to be taken The return value of n_tty_receive_char_flow_ctrl() tells caller a), and b) is kept internal to n_tty_receive_char_flow_ctrl(). If characters were previous looked ahead, __receive_buf() makes two calls to the appropriate n_tty_receive_buf_* function. First call is made with lookahead_done=true for the characters that were subject to lookahead earlier and then with lookahead=false for the new characters. Either of the calls might be skipped when it has no characters to handle. Reported-by: Gilles Buloz Reviewed-by: Andy Shevchenko Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220606153652.63554-2-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/n_tty.c | 91 ++++++++++++++++++++++++++++++++++++++-------- drivers/tty/tty_buffer.c | 59 +++++++++++++++++++++++++----- drivers/tty/tty_port.c | 21 +++++++++++ include/linux/tty_buffer.h | 1 + include/linux/tty_ldisc.h | 14 +++++++ include/linux/tty_port.h | 2 + 6 files changed, 163 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c index 640c9e871044..8d80384df874 100644 --- a/drivers/tty/n_tty.c +++ b/drivers/tty/n_tty.c @@ -118,6 +118,9 @@ struct n_tty_data { size_t read_tail; size_t line_start; + /* # of chars looked ahead (to find software flow control chars) */ + size_t lookahead_count; + /* protected by output lock */ unsigned int column; unsigned int canon_column; @@ -333,6 +336,8 @@ static void reset_buffer_flags(struct n_tty_data *ldata) ldata->erasing = 0; bitmap_zero(ldata->read_flags, N_TTY_BUF_SIZE); ldata->push = 0; + + ldata->lookahead_count = 0; } static void n_tty_packet_mode_flush(struct tty_struct *tty) @@ -1225,12 +1230,30 @@ static bool n_tty_is_char_flow_ctrl(struct tty_struct *tty, unsigned char c) return c == START_CHAR(tty) || c == STOP_CHAR(tty); } -/* Returns true if c is consumed as flow-control character */ -static bool n_tty_receive_char_flow_ctrl(struct tty_struct *tty, unsigned char c) +/** + * n_tty_receive_char_flow_ctrl - receive flow control chars + * @tty: terminal device + * @c: character + * @lookahead_done: lookahead has processed this character already + * + * Receive and process flow control character actions. + * + * In case lookahead for flow control chars already handled the character in + * advance to the normal receive, the actions are skipped during normal + * receive. + * + * Returns true if @c is consumed as flow-control character, the character + * must not be treated as normal character. + */ +static bool n_tty_receive_char_flow_ctrl(struct tty_struct *tty, unsigned char c, + bool lookahead_done) { if (!n_tty_is_char_flow_ctrl(tty, c)) return false; + if (lookahead_done) + return true; + if (c == START_CHAR(tty)) { start_tty(tty); process_echoes(tty); @@ -1242,11 +1265,12 @@ static bool n_tty_receive_char_flow_ctrl(struct tty_struct *tty, unsigned char c return true; } -static void n_tty_receive_char_special(struct tty_struct *tty, unsigned char c) +static void n_tty_receive_char_special(struct tty_struct *tty, unsigned char c, + bool lookahead_done) { struct n_tty_data *ldata = tty->disc_data; - if (I_IXON(tty) && n_tty_receive_char_flow_ctrl(tty, c)) + if (I_IXON(tty) && n_tty_receive_char_flow_ctrl(tty, c, lookahead_done)) return; if (L_ISIG(tty)) { @@ -1401,7 +1425,8 @@ static void n_tty_receive_char(struct tty_struct *tty, unsigned char c) put_tty_queue(c, ldata); } -static void n_tty_receive_char_closing(struct tty_struct *tty, unsigned char c) +static void n_tty_receive_char_closing(struct tty_struct *tty, unsigned char c, + bool lookahead_done) { if (I_ISTRIP(tty)) c &= 0x7f; @@ -1409,9 +1434,12 @@ static void n_tty_receive_char_closing(struct tty_struct *tty, unsigned char c) c = tolower(c); if (I_IXON(tty)) { - if (c == STOP_CHAR(tty)) - stop_tty(tty); - else if (c == START_CHAR(tty) || + if (c == STOP_CHAR(tty)) { + if (!lookahead_done) + stop_tty(tty); + } else if (c == START_CHAR(tty) && lookahead_done) { + return; + } else if (c == START_CHAR(tty) || (tty->flow.stopped && !tty->flow.tco_stopped && I_IXANY(tty) && c != INTR_CHAR(tty) && c != QUIT_CHAR(tty) && c != SUSP_CHAR(tty))) { @@ -1457,6 +1485,27 @@ n_tty_receive_char_lnext(struct tty_struct *tty, unsigned char c, char flag) n_tty_receive_char_flagged(tty, c, flag); } +/* Caller must ensure count > 0 */ +static void n_tty_lookahead_flow_ctrl(struct tty_struct *tty, const unsigned char *cp, + const unsigned char *fp, unsigned int count) +{ + struct n_tty_data *ldata = tty->disc_data; + unsigned char flag = TTY_NORMAL; + + ldata->lookahead_count += count; + + if (!I_IXON(tty)) + return; + + while (count--) { + if (fp) + flag = *fp++; + if (likely(flag == TTY_NORMAL)) + n_tty_receive_char_flow_ctrl(tty, *cp, false); + cp++; + } +} + static void n_tty_receive_buf_real_raw(struct tty_struct *tty, const unsigned char *cp, const char *fp, int count) @@ -1496,7 +1545,7 @@ n_tty_receive_buf_raw(struct tty_struct *tty, const unsigned char *cp, static void n_tty_receive_buf_closing(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count) + const char *fp, int count, bool lookahead_done) { char flag = TTY_NORMAL; @@ -1504,12 +1553,12 @@ n_tty_receive_buf_closing(struct tty_struct *tty, const unsigned char *cp, if (fp) flag = *fp++; if (likely(flag == TTY_NORMAL)) - n_tty_receive_char_closing(tty, *cp++); + n_tty_receive_char_closing(tty, *cp++, lookahead_done); } } static void n_tty_receive_buf_standard(struct tty_struct *tty, - const unsigned char *cp, const char *fp, int count) + const unsigned char *cp, const char *fp, int count, bool lookahead_done) { struct n_tty_data *ldata = tty->disc_data; char flag = TTY_NORMAL; @@ -1540,7 +1589,7 @@ static void n_tty_receive_buf_standard(struct tty_struct *tty, } if (test_bit(c, ldata->char_map)) - n_tty_receive_char_special(tty, c); + n_tty_receive_char_special(tty, c, lookahead_done); else n_tty_receive_char(tty, c); } @@ -1551,21 +1600,30 @@ static void __receive_buf(struct tty_struct *tty, const unsigned char *cp, { struct n_tty_data *ldata = tty->disc_data; bool preops = I_ISTRIP(tty) || (I_IUCLC(tty) && L_IEXTEN(tty)); + size_t la_count = min_t(size_t, ldata->lookahead_count, count); if (ldata->real_raw) n_tty_receive_buf_real_raw(tty, cp, fp, count); else if (ldata->raw || (L_EXTPROC(tty) && !preops)) n_tty_receive_buf_raw(tty, cp, fp, count); - else if (tty->closing && !L_EXTPROC(tty)) - n_tty_receive_buf_closing(tty, cp, fp, count); - else { - n_tty_receive_buf_standard(tty, cp, fp, count); + else if (tty->closing && !L_EXTPROC(tty)) { + if (la_count > 0) + n_tty_receive_buf_closing(tty, cp, fp, la_count, true); + if (count > la_count) + n_tty_receive_buf_closing(tty, cp, fp, count - la_count, false); + } else { + if (la_count > 0) + n_tty_receive_buf_standard(tty, cp, fp, la_count, true); + if (count > la_count) + n_tty_receive_buf_standard(tty, cp, fp, count - la_count, false); flush_echoes(tty); if (tty->ops->flush_chars) tty->ops->flush_chars(tty); } + ldata->lookahead_count -= la_count; + if (ldata->icanon && !L_EXTPROC(tty)) return; @@ -2446,6 +2504,7 @@ static struct tty_ldisc_ops n_tty_ops = { .receive_buf = n_tty_receive_buf, .write_wakeup = n_tty_write_wakeup, .receive_buf2 = n_tty_receive_buf2, + .lookahead_buf = n_tty_lookahead_flow_ctrl, }; /** diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c index bfa431a8e690..754fa43670cc 100644 --- a/drivers/tty/tty_buffer.c +++ b/drivers/tty/tty_buffer.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -104,6 +105,7 @@ static void tty_buffer_reset(struct tty_buffer *p, size_t size) p->size = size; p->next = NULL; p->commit = 0; + p->lookahead = 0; p->read = 0; p->flags = 0; } @@ -234,6 +236,7 @@ void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) buf->head = next; } buf->head->read = buf->head->commit; + buf->head->lookahead = buf->head->read; if (ld && ld->ops->flush_buffer) ld->ops->flush_buffer(tty); @@ -276,13 +279,15 @@ static int __tty_buffer_request_room(struct tty_port *port, size_t size, if (n != NULL) { n->flags = flags; buf->tail = n; - /* paired w/ acquire in flush_to_ldisc(); ensures - * flush_to_ldisc() sees buffer data. + /* + * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() + * ensures they see all buffer data. */ smp_store_release(&b->commit, b->used); - /* paired w/ acquire in flush_to_ldisc(); ensures the - * latest commit value can be read before the head is - * advanced to the next buffer + /* + * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() + * ensures the latest commit value can be read before the head + * is advanced to the next buffer. */ smp_store_release(&b->next, n); } else if (change) @@ -459,6 +464,40 @@ int tty_ldisc_receive_buf(struct tty_ldisc *ld, const unsigned char *p, } EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf); +static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head) +{ + head->lookahead = max(head->lookahead, head->read); + + while (head) { + struct tty_buffer *next; + unsigned char *p, *f = NULL; + unsigned int count; + + /* + * Paired w/ release in __tty_buffer_request_room(); + * ensures commit value read is not stale if the head + * is advancing to the next buffer. + */ + next = smp_load_acquire(&head->next); + /* + * Paired w/ release in __tty_buffer_request_room() or in + * tty_buffer_flush(); ensures we see the committed buffer data. + */ + count = smp_load_acquire(&head->commit) - head->lookahead; + if (!count) { + head = next; + continue; + } + + p = char_buf_ptr(head, head->lookahead); + if (~head->flags & TTYB_NORMAL) + f = flag_buf_ptr(head, head->lookahead); + + port->client_ops->lookahead_buf(port, p, f, count); + head->lookahead += count; + } +} + static int receive_buf(struct tty_port *port, struct tty_buffer *head, int count) { @@ -496,7 +535,7 @@ static void flush_to_ldisc(struct work_struct *work) while (1) { struct tty_buffer *head = buf->head; struct tty_buffer *next; - int count; + int count, rcvd; /* Ldisc or user is trying to gain exclusive access */ if (atomic_read(&buf->priority)) @@ -519,10 +558,12 @@ static void flush_to_ldisc(struct work_struct *work) continue; } - count = receive_buf(port, head, count); - if (!count) + rcvd = receive_buf(port, head, count); + head->read += rcvd; + if (rcvd < count) + lookahead_bufs(port, head); + if (!rcvd) break; - head->read += count; if (need_resched()) cond_resched(); diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c index 880608a65773..dce08a6d7b5e 100644 --- a/drivers/tty/tty_port.c +++ b/drivers/tty/tty_port.c @@ -43,6 +43,26 @@ static int tty_port_default_receive_buf(struct tty_port *port, return ret; } +static void tty_port_default_lookahead_buf(struct tty_port *port, const unsigned char *p, + const unsigned char *f, unsigned int count) +{ + struct tty_struct *tty; + struct tty_ldisc *disc; + + tty = READ_ONCE(port->itty); + if (!tty) + return; + + disc = tty_ldisc_ref(tty); + if (!disc) + return; + + if (disc->ops->lookahead_buf) + disc->ops->lookahead_buf(disc->tty, p, f, count); + + tty_ldisc_deref(disc); +} + static void tty_port_default_wakeup(struct tty_port *port) { struct tty_struct *tty = tty_port_tty_get(port); @@ -55,6 +75,7 @@ static void tty_port_default_wakeup(struct tty_port *port) const struct tty_port_client_operations tty_port_default_client_ops = { .receive_buf = tty_port_default_receive_buf, + .lookahead_buf = tty_port_default_lookahead_buf, .write_wakeup = tty_port_default_wakeup, }; EXPORT_SYMBOL_GPL(tty_port_default_client_ops); diff --git a/include/linux/tty_buffer.h b/include/linux/tty_buffer.h index 3b9d77604291..1796648c2907 100644 --- a/include/linux/tty_buffer.h +++ b/include/linux/tty_buffer.h @@ -15,6 +15,7 @@ struct tty_buffer { int used; int size; int commit; + int lookahead; /* Lazy update on recv, can become less than "read" */ int read; int flags; /* Data points here */ diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index e85002b56752..33678e1936f6 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -186,6 +186,18 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * indicate all data received is %TTY_NORMAL. If assigned, prefer this * function for automatic flow control. * + * @lookahead_buf: [DRV] ``void ()(struct tty_struct *tty, + * const unsigned char *cp, const char *fp, int count) + * + * This function is called by the low-level tty driver for characters + * not eaten by ->receive_buf() or ->receive_buf2(). It is useful for + * processing high-priority characters such as software flow-control + * characters that could otherwise get stuck into the intermediate + * buffer until tty has room to receive them. Ldisc must be able to + * handle later a ->receive_buf() or ->receive_buf2() call for the + * same characters (e.g. by skipping the actions for high-priority + * characters already handled by ->lookahead_buf()). + * * @owner: module containting this ldisc (for reference counting) * * This structure defines the interface between the tty line discipline @@ -229,6 +241,8 @@ struct tty_ldisc_ops { void (*dcd_change)(struct tty_struct *tty, unsigned int status); int (*receive_buf2)(struct tty_struct *tty, const unsigned char *cp, const char *fp, int count); + void (*lookahead_buf)(struct tty_struct *tty, const unsigned char *cp, + const unsigned char *fp, unsigned int count); struct module *owner; }; diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h index 58e9619116b7..fa3c3bdaa234 100644 --- a/include/linux/tty_port.h +++ b/include/linux/tty_port.h @@ -40,6 +40,8 @@ struct tty_port_operations { struct tty_port_client_operations { int (*receive_buf)(struct tty_port *port, const unsigned char *, const unsigned char *, size_t); + void (*lookahead_buf)(struct tty_port *port, const unsigned char *cp, + const unsigned char *fp, unsigned int count); void (*write_wakeup)(struct tty_port *port); }; -- cgit v1.2.3 From 67b9d64139e13621d3ab8bb0daad7602e5fe0778 Mon Sep 17 00:00:00 2001 From: David Jander Date: Thu, 9 Jun 2022 14:13:34 +0200 Subject: spi: Fix per-cpu stats access on 32 bit systems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 32 bit systems, the following kernel BUG is hit: BUG: using smp_processor_id() in preemptible [00000000] code: swapper/0/1 caller is debug_smp_processor_id+0x18/0x24 CPU: 1 PID: 1 Comm: swapper/0 Not tainted 5.19.0-rc1-00001-g6ae0aec8a366 #181 Hardware name: Freescale i.MX6 Quad/DualLite (Device Tree) Backtrace: dump_backtrace from show_stack+0x20/0x24 r7:81024ffd r6:00000000 r5:81024ffd r4:60000013 show_stack from dump_stack_lvl+0x60/0x78 dump_stack_lvl from dump_stack+0x14/0x1c r7:81024ffd r6:80f652de r5:80bec180 r4:819a2500 dump_stack from check_preemption_disabled+0xc8/0xf0 check_preemption_disabled from debug_smp_processor_id+0x18/0x24 r8:8119b7e0 r7:81205534 r6:819f5c00 r5:819f4c00 r4:c083d724 debug_smp_processor_id from __spi_sync+0x78/0x220 __spi_sync from spi_sync+0x34/0x4c r10:bb7bf4e0 r9:c083d724 r8:00000007 r7:81a068c0 r6:822a83c0 r5:c083d724 r4:819f4c00 spi_sync from spi_mem_exec_op+0x338/0x370 r5:000000b4 r4:c083d910 spi_mem_exec_op from spi_nor_read_id+0x98/0xdc r10:bb7bf4e0 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:82358040 r4:819f7c40 spi_nor_read_id from spi_nor_detect+0x38/0x114 r7:82358040 r6:00000000 r5:819f7c40 r4:819f7c40 spi_nor_detect from spi_nor_scan+0x11c/0xbec r10:bb7bf4e0 r9:00000000 r8:00000000 r7:c083da4c r6:00000000 r5:00010101 r4:819f7c40 spi_nor_scan from spi_nor_probe+0x10c/0x2d0 r10:bb7bf4e0 r9:bb7bf4d0 r8:00000000 r7:819f4c00 r6:00000000 r5:00000000 r4:819f7c40 per-cpu access needs to be guarded against preemption. Fixes: 6598b91b5ac3 ("spi: spi.c: Convert statistics to per-cpu u64_stats_t") Reported-by: Marc Kleine-Budde Signed-off-by: David Jander Tested-by: Nícolas F. R. A. Prado Link: https://lore.kernel.org/r/20220609121334.2984808-1-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 5 ++++- include/linux/spi/spi.h | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index d94822bf3cec..ac61824b87b5 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -314,11 +314,13 @@ static void spi_statistics_add_transfer_stats(struct spi_statistics *pcpu_stats, struct spi_controller *ctlr) { int l2len = min(fls(xfer->len), SPI_STATISTICS_HISTO_SIZE) - 1; - struct spi_statistics *stats = this_cpu_ptr(pcpu_stats); + struct spi_statistics *stats; if (l2len < 0) l2len = 0; + get_cpu(); + stats = this_cpu_ptr(pcpu_stats); u64_stats_update_begin(&stats->syncp); u64_stats_inc(&stats->transfers); @@ -333,6 +335,7 @@ static void spi_statistics_add_transfer_stats(struct spi_statistics *pcpu_stats, u64_stats_add(&stats->bytes_rx, xfer->len); u64_stats_update_end(&stats->syncp); + put_cpu(); } /* diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 2e63b4935deb..c96f526d9a20 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -84,18 +84,24 @@ struct spi_statistics { #define SPI_STATISTICS_ADD_TO_FIELD(pcpu_stats, field, count) \ do { \ - struct spi_statistics *__lstats = this_cpu_ptr(pcpu_stats); \ + struct spi_statistics *__lstats; \ + get_cpu(); \ + __lstats = this_cpu_ptr(pcpu_stats); \ u64_stats_update_begin(&__lstats->syncp); \ u64_stats_add(&__lstats->field, count); \ u64_stats_update_end(&__lstats->syncp); \ + put_cpu(); \ } while (0) #define SPI_STATISTICS_INCREMENT_FIELD(pcpu_stats, field) \ do { \ - struct spi_statistics *__lstats = this_cpu_ptr(pcpu_stats); \ + struct spi_statistics *__lstats; \ + get_cpu(); \ + __lstats = this_cpu_ptr(pcpu_stats); \ u64_stats_update_begin(&__lstats->syncp); \ u64_stats_inc(&__lstats->field); \ u64_stats_update_end(&__lstats->syncp); \ + put_cpu(); \ } while (0) /** -- cgit v1.2.3 From a6546f89eac9bfad4b50a7f8ccfbefe0ae01e7df Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 7 Jun 2022 16:11:10 +0200 Subject: treewide: Replace GPLv2 boilerplate/reference with SPDX - gpl-2.0_8.RULE Based on the normalized pattern: this program is free software you can redistribute it and/or modify it under the terms of the gnu general public license version 2 as published by the free software foundation extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference. Reviewed-by: Allison Randal Signed-off-by: Thomas Gleixner Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/vfio-mediated-device.rst | 4 +--- arch/arm/boot/bootp/bootp.lds | 5 +---- include/linux/input/elan-i2c-ids.h | 5 +---- 3 files changed, 3 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst index eded8719180f..66bd00d7aecf 100644 --- a/Documentation/driver-api/vfio-mediated-device.rst +++ b/Documentation/driver-api/vfio-mediated-device.rst @@ -1,3 +1,4 @@ +.. SPDX-License-Identifier: GPL-2.0-only .. include:: ===================== @@ -8,9 +9,6 @@ VFIO Mediated devices :Author: Neo Jia :Author: Kirti Wankhede -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License version 2 as -published by the Free Software Foundation. Virtual Function I/O (VFIO) Mediated devices[1] diff --git a/arch/arm/boot/bootp/bootp.lds b/arch/arm/boot/bootp/bootp.lds index fc54394f4340..160128186bf8 100644 --- a/arch/arm/boot/bootp/bootp.lds +++ b/arch/arm/boot/bootp/bootp.lds @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * linux/arch/arm/boot/bootp/bootp.lds * * Copyright (C) 2000-2002 Russell King - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ OUTPUT_ARCH(arm) ENTRY(_start) diff --git a/include/linux/input/elan-i2c-ids.h b/include/linux/input/elan-i2c-ids.h index 520858d12680..51cca17ee94c 100644 --- a/include/linux/input/elan-i2c-ids.h +++ b/include/linux/input/elan-i2c-ids.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Elan I2C/SMBus Touchpad device whitelist * @@ -11,10 +12,6 @@ * copyright (c) 2011-2012 Cypress Semiconductor, Inc. * copyright (c) 2011-2012 Google, Inc. * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * * Trademarks are the property of their respective owners. */ -- cgit v1.2.3 From 2aec85b26f39fa9e036c5872950c0ef9b479a1ec Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 7 Jun 2022 16:11:13 +0200 Subject: treewide: Replace GPLv2 boilerplate/reference with SPDX - gpl-2.0_30.RULE (part 2) Based on the normalized pattern: this program is free software you can redistribute it and/or modify it under the terms of the gnu general public license as published by the free software foundation version 2 this program is distributed as is without any warranty of any kind whether express or implied without even the implied warranty of merchantability or fitness for a particular purpose see the gnu general public license for more details extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference. Reviewed-by: Allison Randal Signed-off-by: Thomas Gleixner Signed-off-by: Greg Kroah-Hartman --- drivers/dma/bcm-sba-raid.c | 14 ++------------ drivers/dma/ti/edma.c | 10 +--------- drivers/gpio/gpio-bcm-kona.c | 10 +--------- drivers/gpio/gpio-brcmstb.c | 14 ++------------ drivers/i2c/busses/i2c-bcm-iproc.c | 14 ++------------ drivers/i2c/busses/i2c-bcm-kona.c | 14 ++------------ drivers/i2c/busses/i2c-brcmstb.c | 14 ++------------ drivers/input/keyboard/bcm-keypad.c | 14 ++------------ drivers/input/misc/gpio_decoder.c | 10 +--------- drivers/input/touchscreen/bcm_iproc_tsc.c | 9 +-------- drivers/irqchip/irq-keystone.c | 10 +--------- drivers/mailbox/bcm-flexrm-mailbox.c | 14 ++------------ drivers/media/i2c/adv7343_regs.h | 10 +--------- drivers/media/i2c/adv7393_regs.h | 10 +--------- drivers/media/platform/ti/davinci/vpif.h | 11 +---------- drivers/media/platform/ti/davinci/vpif_display.h | 10 +--------- drivers/memory/ti-emif-sram-pm.S | 10 +--------- drivers/mfd/lp873x.c | 10 +--------- drivers/mfd/tps65217.c | 10 +--------- drivers/mmc/host/sdhci-bcm-kona.c | 14 ++------------ drivers/mmc/host/sdhci-iproc.c | 14 ++------------ drivers/net/can/ti_hecc.c | 11 +---------- drivers/nvmem/bcm-ocotp.c | 14 ++------------ drivers/phy/broadcom/phy-bcm-cygnus-pcie.c | 14 ++------------ drivers/phy/broadcom/phy-bcm-ns2-pcie.c | 14 ++------------ drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c | 14 ++------------ drivers/phy/motorola/phy-cpcap-usb.c | 10 +--------- drivers/phy/ti/phy-dm816x-usb.c | 11 +---------- drivers/pinctrl/bcm/pinctrl-bcm281xx.c | 14 ++------------ drivers/pinctrl/bcm/pinctrl-cygnus-mux.c | 14 ++------------ drivers/pinctrl/bcm/pinctrl-ns2-mux.c | 10 +--------- drivers/pinctrl/bcm/pinctrl-nsp-gpio.c | 14 ++------------ drivers/pinctrl/bcm/pinctrl-nsp-mux.c | 10 +--------- drivers/power/reset/brcm-kona-reset.c | 14 ++------------ drivers/power/reset/brcmstb-reboot.c | 14 ++------------ drivers/ptp/ptp_dte.c | 14 ++------------ drivers/pwm/pwm-bcm-iproc.c | 14 ++------------ drivers/pwm/pwm-bcm-kona.c | 14 ++------------ drivers/regulator/cpcap-regulator.c | 10 +--------- drivers/regulator/isl6271a-regulator.c | 10 +--------- drivers/regulator/tps65023-regulator.c | 10 +--------- drivers/regulator/tps6507x-regulator.c | 10 +--------- drivers/regulator/tps65217-regulator.c | 10 +--------- drivers/soc/ti/knav_dma.c | 10 +--------- drivers/uio/uio_pruss.c | 10 +--------- drivers/video/backlight/tps65217_bl.c | 10 +--------- include/dt-bindings/clock/bcm21664.h | 10 +--------- include/dt-bindings/clock/bcm281xx.h | 10 +--------- include/linux/mfd/lp873x.h | 10 +--------- include/linux/mfd/tps65217.h | 10 +--------- include/linux/platform_data/davinci_asp.h | 10 +--------- include/linux/platform_data/gpio-davinci.h | 10 +--------- include/linux/platform_data/uio_dmem_genirq.h | 10 +--------- include/linux/platform_data/uio_pruss.h | 10 +--------- include/linux/reset/bcm63xx_pmb.h | 10 +--------- include/linux/soc/ti/knav_dma.h | 10 +--------- include/linux/soc/ti/knav_qmss.h | 10 +--------- include/linux/sram.h | 14 ++------------ include/linux/ti-emif-sram.h | 10 +--------- include/linux/wkup_m3_ipc.h | 10 +--------- include/media/i2c/adv7343.h | 10 +--------- include/media/i2c/adv7393.h | 10 +--------- net/hsr/hsr_debugfs.c | 10 +--------- sound/soc/bcm/cygnus-pcm.c | 14 ++------------ sound/soc/bcm/cygnus-ssp.c | 14 ++------------ sound/soc/bcm/cygnus-ssp.h | 14 ++------------ 66 files changed, 91 insertions(+), 671 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index 64239da02e74..064761289a73 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2017 Broadcom /* * Broadcom SBA RAID Driver diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 3ea8ef7f57df..4cbca80ee16e 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI EDMA DMA engine driver * * Copyright 2012 Texas Instruments - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/gpio/gpio-bcm-kona.c b/drivers/gpio/gpio-bcm-kona.c index e84474494429..70770429ba48 100644 --- a/drivers/gpio/gpio-bcm-kona.c +++ b/drivers/gpio/gpio-bcm-kona.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Broadcom Kona GPIO Driver * * Author: Broadcom Corporation * Copyright (C) 2012-2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c index 6b7439b44690..85429dd6b3b6 100644 --- a/drivers/gpio/gpio-brcmstb.c +++ b/drivers/gpio/gpio-brcmstb.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2015-2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2015-2017 Broadcom #include #include diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c index 6304d1dd2dd6..85d8a6b04885 100644 --- a/drivers/i2c/busses/i2c-bcm-iproc.c +++ b/drivers/i2c/busses/i2c-bcm-iproc.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014 Broadcom Corporation #include #include diff --git a/drivers/i2c/busses/i2c-bcm-kona.c b/drivers/i2c/busses/i2c-bcm-kona.c index 8e350f20cde0..16bf41f1f086 100644 --- a/drivers/i2c/busses/i2c-bcm-kona.c +++ b/drivers/i2c/busses/i2c-bcm-kona.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2013 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2013 Broadcom Corporation #include #include diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c index b00f35c0b066..ff63ed5ec961 100644 --- a/drivers/i2c/busses/i2c-brcmstb.c +++ b/drivers/i2c/busses/i2c-brcmstb.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014 Broadcom Corporation #include #include diff --git a/drivers/input/keyboard/bcm-keypad.c b/drivers/input/keyboard/bcm-keypad.c index 166d6023a538..56a919ec23b5 100644 --- a/drivers/input/keyboard/bcm-keypad.c +++ b/drivers/input/keyboard/bcm-keypad.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014 Broadcom Corporation #include #include diff --git a/drivers/input/misc/gpio_decoder.c b/drivers/input/misc/gpio_decoder.c index 145826a1a9a1..ee668eba302f 100644 --- a/drivers/input/misc/gpio_decoder.c +++ b/drivers/input/misc/gpio_decoder.c @@ -1,15 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/ * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * * A generic driver to read multiple gpio lines and translate the * encoded numeric value into an input event. */ diff --git a/drivers/input/touchscreen/bcm_iproc_tsc.c b/drivers/input/touchscreen/bcm_iproc_tsc.c index 7de1fd24ce36..35e2fe9911a4 100644 --- a/drivers/input/touchscreen/bcm_iproc_tsc.c +++ b/drivers/input/touchscreen/bcm_iproc_tsc.c @@ -1,14 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2015 Broadcom Corporation * -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License as -* published by the Free Software Foundation version 2. -* -* This program is distributed "as is" WITHOUT ANY WARRANTY of any -* kind, whether express or implied; without even the implied warranty -* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. */ #include #include diff --git a/drivers/irqchip/irq-keystone.c b/drivers/irqchip/irq-keystone.c index d47c8041e5bc..ba9792e60329 100644 --- a/drivers/irqchip/irq-keystone.c +++ b/drivers/irqchip/irq-keystone.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Texas Instruments Keystone IRQ controller IP driver * * Copyright (C) 2014 Texas Instruments, Inc. * Author: Sajesh Kumar Saran * Grygorii Strashko - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c index 22acb51531cb..fda16f76401e 100644 --- a/drivers/mailbox/bcm-flexrm-mailbox.c +++ b/drivers/mailbox/bcm-flexrm-mailbox.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2017 Broadcom /* * Broadcom FlexRM Mailbox Driver diff --git a/drivers/media/i2c/adv7343_regs.h b/drivers/media/i2c/adv7343_regs.h index 2f04ce4b9118..e0357e6272e3 100644 --- a/drivers/media/i2c/adv7343_regs.h +++ b/drivers/media/i2c/adv7343_regs.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * ADV7343 encoder related structure and register definitions * * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed .as is. WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef ADV7343_REGS_H diff --git a/drivers/media/i2c/adv7393_regs.h b/drivers/media/i2c/adv7393_regs.h index 78968330f0be..6eb8732b5324 100644 --- a/drivers/media/i2c/adv7393_regs.h +++ b/drivers/media/i2c/adv7393_regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * ADV7393 encoder related structure and register definitions * @@ -7,15 +8,6 @@ * Based on ADV7343 driver, * * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed .as is. WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef ADV7393_REGS_H diff --git a/drivers/media/platform/ti/davinci/vpif.h b/drivers/media/platform/ti/davinci/vpif.h index c6d1d890478a..651943e3e375 100644 --- a/drivers/media/platform/ti/davinci/vpif.h +++ b/drivers/media/platform/ti/davinci/vpif.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * VPIF header file * * Copyright (C) 2009 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed .as is. WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef VPIF_H @@ -685,4 +677,3 @@ struct vpif_params { }; #endif /* End of #ifndef VPIF_H */ - diff --git a/drivers/media/platform/ti/davinci/vpif_display.h b/drivers/media/platform/ti/davinci/vpif_display.h index f98062e79167..f27474e0fc36 100644 --- a/drivers/media/platform/ti/davinci/vpif_display.h +++ b/drivers/media/platform/ti/davinci/vpif_display.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * VPIF display header file * * Copyright (C) 2009 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed .as is. WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef VPIF_DISPLAY_H diff --git a/drivers/memory/ti-emif-sram-pm.S b/drivers/memory/ti-emif-sram-pm.S index d1c83bd5b98e..9bcac35c3304 100644 --- a/drivers/memory/ti-emif-sram-pm.S +++ b/drivers/memory/ti-emif-sram-pm.S @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Low level PM code for TI EMIF * * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/ * Dave Gerlach - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/mfd/lp873x.c b/drivers/mfd/lp873x.c index 858c9e0a49a4..b6166dec492d 100644 --- a/drivers/mfd/lp873x.c +++ b/drivers/mfd/lp873x.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2016 Texas Instruments Incorporated - https://www.ti.com/ * * Author: Keerthy - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/mfd/tps65217.c b/drivers/mfd/tps65217.c index 8027b0a9e14f..8e8da204a02e 100644 --- a/drivers/mfd/tps65217.c +++ b/drivers/mfd/tps65217.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * tps65217.c * * TPS65217 chip family multi-function driver * * Copyright (C) 2011 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/mmc/host/sdhci-bcm-kona.c b/drivers/mmc/host/sdhci-bcm-kona.c index 4d4aac85cc7a..61a12f2f7f03 100644 --- a/drivers/mmc/host/sdhci-bcm-kona.c +++ b/drivers/mmc/host/sdhci-bcm-kona.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2013 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2013 Broadcom Corporation #include #include diff --git a/drivers/mmc/host/sdhci-iproc.c b/drivers/mmc/host/sdhci-iproc.c index 032bf852397f..6db35b1b8557 100644 --- a/drivers/mmc/host/sdhci-iproc.c +++ b/drivers/mmc/host/sdhci-iproc.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014 Broadcom Corporation /* * iProc SDHCI platform driver diff --git a/drivers/net/can/ti_hecc.c b/drivers/net/can/ti_hecc.c index debe17bfd0f0..7ce26cdbeb77 100644 --- a/drivers/net/can/ti_hecc.c +++ b/drivers/net/can/ti_hecc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI HECC (CAN) device driver * @@ -6,16 +7,6 @@ * * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/ * Copyright (C) 2019 Jeroen Hofstee - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed as is WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * */ #include diff --git a/drivers/nvmem/bcm-ocotp.c b/drivers/nvmem/bcm-ocotp.c index dfea96c52463..a128c7f5e351 100644 --- a/drivers/nvmem/bcm-ocotp.c +++ b/drivers/nvmem/bcm-ocotp.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2016 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2016 Broadcom #include #include diff --git a/drivers/phy/broadcom/phy-bcm-cygnus-pcie.c b/drivers/phy/broadcom/phy-bcm-cygnus-pcie.c index 548e46776100..cc29b08e49eb 100644 --- a/drivers/phy/broadcom/phy-bcm-cygnus-pcie.c +++ b/drivers/phy/broadcom/phy-bcm-cygnus-pcie.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2015 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2015 Broadcom Corporation #include #include diff --git a/drivers/phy/broadcom/phy-bcm-ns2-pcie.c b/drivers/phy/broadcom/phy-bcm-ns2-pcie.c index 9e7434a0d3e0..2eaa41f8fc70 100644 --- a/drivers/phy/broadcom/phy-bcm-ns2-pcie.c +++ b/drivers/phy/broadcom/phy-bcm-ns2-pcie.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2016 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2016 Broadcom #include #include diff --git a/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c b/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c index 65a399acc845..36ad02c33ac5 100644 --- a/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c +++ b/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2017 Broadcom #include #include diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c index 6ee478bc5211..2f8210167b77 100644 --- a/drivers/phy/motorola/phy-cpcap-usb.c +++ b/drivers/phy/motorola/phy-cpcap-usb.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Motorola CPCAP PMIC USB PHY driver * Copyright (C) 2017 Tony Lindgren @@ -5,15 +6,6 @@ * Some parts based on earlier Motorola Linux kernel tree code in * board-mapphone-usb.c and cpcap-usb-det.c: * Copyright (C) 2007 - 2011 Motorola, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/phy/ti/phy-dm816x-usb.c b/drivers/phy/ti/phy-dm816x-usb.c index 9fe6ea6fdae5..fb619908f912 100644 --- a/drivers/phy/ti/phy-dm816x-usb.c +++ b/drivers/phy/ti/phy-dm816x-usb.c @@ -1,13 +1,4 @@ -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only #include #include diff --git a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c index 9ab1f427286a..fd52a83387ef 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2013-2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2013-2017 Broadcom #include #include diff --git a/drivers/pinctrl/bcm/pinctrl-cygnus-mux.c b/drivers/pinctrl/bcm/pinctrl-cygnus-mux.c index 4344c5732400..5251460f6327 100644 --- a/drivers/pinctrl/bcm/pinctrl-cygnus-mux.c +++ b/drivers/pinctrl/bcm/pinctrl-cygnus-mux.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014-2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014-2017 Broadcom /* * Broadcom Cygnus IOMUX driver diff --git a/drivers/pinctrl/bcm/pinctrl-ns2-mux.c b/drivers/pinctrl/bcm/pinctrl-ns2-mux.c index 0fe4a1fcdf00..960e253f0be4 100644 --- a/drivers/pinctrl/bcm/pinctrl-ns2-mux.c +++ b/drivers/pinctrl/bcm/pinctrl-ns2-mux.c @@ -1,13 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2016 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. * * This file contains the Northstar2 IOMUX driver that supports group * based PINMUX configuration. The PWM is functional only when the diff --git a/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c b/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c index 643dbd315033..3c792bf03bda 100644 --- a/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c +++ b/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014-2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014-2017 Broadcom /* * This file contains the Broadcom Northstar Plus (NSP) GPIO driver that diff --git a/drivers/pinctrl/bcm/pinctrl-nsp-mux.c b/drivers/pinctrl/bcm/pinctrl-nsp-mux.c index f1d60a708815..db8f79920ff0 100644 --- a/drivers/pinctrl/bcm/pinctrl-nsp-mux.c +++ b/drivers/pinctrl/bcm/pinctrl-nsp-mux.c @@ -1,13 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2015 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. * * This file contains the Northstar plus (NSP) IOMUX driver that supports * group based PINMUX configuration. The Northstar plus IOMUX controller diff --git a/drivers/power/reset/brcm-kona-reset.c b/drivers/power/reset/brcm-kona-reset.c index 8eaa959d8be6..3de024e3ceb7 100644 --- a/drivers/power/reset/brcm-kona-reset.c +++ b/drivers/power/reset/brcm-kona-reset.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2016 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2016 Broadcom #include #include diff --git a/drivers/power/reset/brcmstb-reboot.c b/drivers/power/reset/brcmstb-reboot.c index 884b53c483c0..0f2944dc9355 100644 --- a/drivers/power/reset/brcmstb-reboot.c +++ b/drivers/power/reset/brcmstb-reboot.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2013 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2013 Broadcom Corporation #include #include diff --git a/drivers/ptp/ptp_dte.c b/drivers/ptp/ptp_dte.c index 82d31ba32690..8641fd060491 100644 --- a/drivers/ptp/ptp_dte.c +++ b/drivers/ptp/ptp_dte.c @@ -1,15 +1,5 @@ -/* - * Copyright 2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2017 Broadcom #include #include diff --git a/drivers/pwm/pwm-bcm-iproc.c b/drivers/pwm/pwm-bcm-iproc.c index 0226bf697f09..7251037d4dd5 100644 --- a/drivers/pwm/pwm-bcm-iproc.c +++ b/drivers/pwm/pwm-bcm-iproc.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2016 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2016 Broadcom #include #include diff --git a/drivers/pwm/pwm-bcm-kona.c b/drivers/pwm/pwm-bcm-kona.c index f171169c1c1f..4fa6e249e4cf 100644 --- a/drivers/pwm/pwm-bcm-kona.c +++ b/drivers/pwm/pwm-bcm-kona.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014 Broadcom Corporation #include #include diff --git a/drivers/regulator/cpcap-regulator.c b/drivers/regulator/cpcap-regulator.c index 79b3eb3222c6..b0c225d98631 100644 --- a/drivers/regulator/cpcap-regulator.c +++ b/drivers/regulator/cpcap-regulator.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Motorola CPCAP PMIC regulator driver * @@ -6,15 +7,6 @@ * * Rewritten for mainline kernel to use device tree and regmap * Copyright (C) 2017 Tony Lindgren - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/regulator/isl6271a-regulator.c b/drivers/regulator/isl6271a-regulator.c index 6f28bba81d13..591a64e1ca61 100644 --- a/drivers/regulator/isl6271a-regulator.c +++ b/drivers/regulator/isl6271a-regulator.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * isl6271a-regulator.c * * Support for Intersil ISL6271A voltage regulator * * Copyright (C) 2010 Marek Vasut - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any kind, - * whether express or implied; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. */ #include diff --git a/drivers/regulator/tps65023-regulator.c b/drivers/regulator/tps65023-regulator.c index f25806531c7e..d24333344f93 100644 --- a/drivers/regulator/tps65023-regulator.c +++ b/drivers/regulator/tps65023-regulator.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * tps65023-regulator.c * * Supports TPS65023 Regulator * * Copyright (C) 2009 Texas Instrument Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any kind, - * whether express or implied; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. */ #include diff --git a/drivers/regulator/tps6507x-regulator.c b/drivers/regulator/tps6507x-regulator.c index eafbc2bb4b57..b83816ee6867 100644 --- a/drivers/regulator/tps6507x-regulator.c +++ b/drivers/regulator/tps6507x-regulator.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * tps6507x-regulator.c * * Regulator driver for TPS65073 PMIC * * Copyright (C) 2009 Texas Instrument Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any kind, - * whether express or implied; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. */ #include diff --git a/drivers/regulator/tps65217-regulator.c b/drivers/regulator/tps65217-regulator.c index e88ed96f4744..6bb5b02e19e2 100644 --- a/drivers/regulator/tps65217-regulator.c +++ b/drivers/regulator/tps65217-regulator.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * tps65217-regulator.c * * Regulator driver for TPS65217 PMIC * * Copyright (C) 2011 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/soc/ti/knav_dma.c b/drivers/soc/ti/knav_dma.c index d756591de973..84afebd355be 100644 --- a/drivers/soc/ti/knav_dma.c +++ b/drivers/soc/ti/knav_dma.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014 Texas Instruments Incorporated * Authors: Santosh Shilimkar * Sandeep Nair * Cyril Chemparathy - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/uio/uio_pruss.c b/drivers/uio/uio_pruss.c index e9096f53b4cc..83966dbd3bbf 100644 --- a/drivers/uio/uio_pruss.c +++ b/drivers/uio/uio_pruss.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Programmable Real-Time Unit Sub System (PRUSS) UIO driver (uio_pruss) * @@ -5,15 +6,6 @@ * and DDR RAM to user space for applications interacting with PRUSS firmware * * Copyright (C) 2010-11 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include #include diff --git a/drivers/video/backlight/tps65217_bl.c b/drivers/video/backlight/tps65217_bl.c index 8457166f357f..d96d713fe7db 100644 --- a/drivers/video/backlight/tps65217_bl.c +++ b/drivers/video/backlight/tps65217_bl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * tps65217_bl.c * @@ -5,15 +6,6 @@ * * Copyright (C) 2012 Matthias Kaehlcke * Author: Matthias Kaehlcke - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/include/dt-bindings/clock/bcm21664.h b/include/dt-bindings/clock/bcm21664.h index 5a7f0e4750a8..7c7492742f3d 100644 --- a/include/dt-bindings/clock/bcm21664.h +++ b/include/dt-bindings/clock/bcm21664.h @@ -1,15 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2013 Broadcom Corporation * Copyright 2013 Linaro Limited - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _CLOCK_BCM21664_H diff --git a/include/dt-bindings/clock/bcm281xx.h b/include/dt-bindings/clock/bcm281xx.h index a763460cf1af..d74ca42112e7 100644 --- a/include/dt-bindings/clock/bcm281xx.h +++ b/include/dt-bindings/clock/bcm281xx.h @@ -1,15 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2013 Broadcom Corporation * Copyright 2013 Linaro Limited - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _CLOCK_BCM281XX_H diff --git a/include/linux/mfd/lp873x.h b/include/linux/mfd/lp873x.h index 5546688c7da7..fe8174cc8637 100644 --- a/include/linux/mfd/lp873x.h +++ b/include/linux/mfd/lp873x.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Functions to access LP873X power management chip. * * Copyright (C) 2016 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __LINUX_MFD_LP873X_H diff --git a/include/linux/mfd/tps65217.h b/include/linux/mfd/tps65217.h index db7091824ed0..877d9c41c53d 100644 --- a/include/linux/mfd/tps65217.h +++ b/include/linux/mfd/tps65217.h @@ -1,18 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * linux/mfd/tps65217.h * * Functions to access TPS65217 power management chip. * * Copyright (C) 2011 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __LINUX_MFD_TPS65217_H diff --git a/include/linux/platform_data/davinci_asp.h b/include/linux/platform_data/davinci_asp.h index 76b13ef67562..c8645b2ed3c0 100644 --- a/include/linux/platform_data/davinci_asp.h +++ b/include/linux/platform_data/davinci_asp.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * TI DaVinci Audio Serial Port support * * Copyright (C) 2012 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __DAVINCI_ASP_H diff --git a/include/linux/platform_data/gpio-davinci.h b/include/linux/platform_data/gpio-davinci.h index e182a46e609f..b82e44662efe 100644 --- a/include/linux/platform_data/gpio-davinci.h +++ b/include/linux/platform_data/gpio-davinci.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * DaVinci GPIO Platform Related Defines * * Copyright (C) 2013 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __DAVINCI_GPIO_PLATFORM_H diff --git a/include/linux/platform_data/uio_dmem_genirq.h b/include/linux/platform_data/uio_dmem_genirq.h index 973c1bb32168..c8f6de685306 100644 --- a/include/linux/platform_data/uio_dmem_genirq.h +++ b/include/linux/platform_data/uio_dmem_genirq.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * include/linux/platform_data/uio_dmem_genirq.h * * Copyright (C) 2012 Damian Hobson-Garcia - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _UIO_DMEM_GENIRQ_H diff --git a/include/linux/platform_data/uio_pruss.h b/include/linux/platform_data/uio_pruss.h index 31f2e22661bc..f76fa393b802 100644 --- a/include/linux/platform_data/uio_pruss.h +++ b/include/linux/platform_data/uio_pruss.h @@ -1,18 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * include/linux/platform_data/uio_pruss.h * * Platform data for uio_pruss driver * * Copyright (C) 2010-11 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _UIO_PRUSS_H_ diff --git a/include/linux/reset/bcm63xx_pmb.h b/include/linux/reset/bcm63xx_pmb.h index bb4af7b5eb36..c77b6999518a 100644 --- a/include/linux/reset/bcm63xx_pmb.h +++ b/include/linux/reset/bcm63xx_pmb.h @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Broadcom BCM63xx Processor Monitor Bus shared routines (SMP and reset) * * Copyright (C) 2015, Broadcom Corporation * Author: Florian Fainelli - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __BCM63XX_PMB_H #define __BCM63XX_PMB_H diff --git a/include/linux/soc/ti/knav_dma.h b/include/linux/soc/ti/knav_dma.h index 7127ec301537..18d806a8e52c 100644 --- a/include/linux/soc/ti/knav_dma.h +++ b/include/linux/soc/ti/knav_dma.h @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2014 Texas Instruments Incorporated * Authors: Sandeep Nair - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __SOC_TI_KEYSTONE_NAVIGATOR_DMA_H__ diff --git a/include/linux/soc/ti/knav_qmss.h b/include/linux/soc/ti/knav_qmss.h index c75ef99c99ca..175f466ebcc3 100644 --- a/include/linux/soc/ti/knav_qmss.h +++ b/include/linux/soc/ti/knav_qmss.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Keystone Navigator Queue Management Sub-System header * @@ -5,15 +6,6 @@ * Author: Sandeep Nair * Cyril Chemparathy * Santosh Shilimkar - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __SOC_TI_KNAV_QMSS_H__ diff --git a/include/linux/sram.h b/include/linux/sram.h index 4fb405fb0480..d7dee19505c6 100644 --- a/include/linux/sram.h +++ b/include/linux/sram.h @@ -1,15 +1,5 @@ -/* - * Generic SRAM Driver Interface - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Generic SRAM Driver Interface */ #ifndef __LINUX_SRAM_H__ #define __LINUX_SRAM_H__ diff --git a/include/linux/ti-emif-sram.h b/include/linux/ti-emif-sram.h index 2fc854155c27..441b2988e66a 100644 --- a/include/linux/ti-emif-sram.h +++ b/include/linux/ti-emif-sram.h @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * TI AM33XX EMIF Routines * * Copyright (C) 2016-2017 Texas Instruments Inc. * Dave Gerlach - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __LINUX_TI_EMIF_H #define __LINUX_TI_EMIF_H diff --git a/include/linux/wkup_m3_ipc.h b/include/linux/wkup_m3_ipc.h index 26d1eb058fa3..5e1b26f988e2 100644 --- a/include/linux/wkup_m3_ipc.h +++ b/include/linux/wkup_m3_ipc.h @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * TI Wakeup M3 for AMx3 SoCs Power Management Routines * * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Dave Gerlach - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _LINUX_WKUP_M3_IPC_H diff --git a/include/media/i2c/adv7343.h b/include/media/i2c/adv7343.h index b8937035c5d3..d35d3e925795 100644 --- a/include/media/i2c/adv7343.h +++ b/include/media/i2c/adv7343.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * ADV7343 header file * * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed .as is. WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef ADV7343_H diff --git a/include/media/i2c/adv7393.h b/include/media/i2c/adv7393.h index b28edf351842..c73b36321d06 100644 --- a/include/media/i2c/adv7393.h +++ b/include/media/i2c/adv7393.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * ADV7393 header file * @@ -7,15 +8,6 @@ * Based on ADV7343 driver, * * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed .as is. WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef ADV7393_H diff --git a/net/hsr/hsr_debugfs.c b/net/hsr/hsr_debugfs.c index fe6094e9a2db..de476a417631 100644 --- a/net/hsr/hsr_debugfs.c +++ b/net/hsr/hsr_debugfs.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * debugfs code for HSR & PRP * Copyright (C) 2019 Texas Instruments Incorporated * * Author(s): * Murali Karicheri - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include #include diff --git a/sound/soc/bcm/cygnus-pcm.c b/sound/soc/bcm/cygnus-pcm.c index 3abeaf0f1b1c..8f488f92936b 100644 --- a/sound/soc/bcm/cygnus-pcm.c +++ b/sound/soc/bcm/cygnus-pcm.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014-2015 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014-2015 Broadcom Corporation #include #include #include diff --git a/sound/soc/bcm/cygnus-ssp.c b/sound/soc/bcm/cygnus-ssp.c index 9698f4531c90..b0254e724ec9 100644 --- a/sound/soc/bcm/cygnus-ssp.c +++ b/sound/soc/bcm/cygnus-ssp.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014-2015 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2014-2015 Broadcom Corporation #include #include #include diff --git a/sound/soc/bcm/cygnus-ssp.h b/sound/soc/bcm/cygnus-ssp.h index 33dd34305928..74152b2d770d 100644 --- a/sound/soc/bcm/cygnus-ssp.h +++ b/sound/soc/bcm/cygnus-ssp.h @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2014-2015 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2014-2015 Broadcom Corporation */ #ifndef __CYGNUS_SSP_H__ #define __CYGNUS_SSP_H__ -- cgit v1.2.3 From 1accad5e74635d7f9d994e692649fbe736afe150 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 7 Jun 2022 16:11:20 +0200 Subject: treewide: Replace GPLv2 boilerplate/reference with SPDX - gpl-2.0_149.RULE Based on the normalized pattern: netapp provides this source code under the gpl v2 license the gpl v2 license is available at https://opensource org/licenses/gpl-license php this software is provided by the copyright holders and contributors as is and any express or implied warranties including but not limited to the implied warranties of merchantability and fitness for a particular purpose are disclaimed in no event shall the copyright owner or contributors be liable for any direct indirect incidental special exemplary or consequential damages (including but not limited to procurement of substitute goods or services loss of use data or profits or business interruption) however caused and on any theory of liability whether in contract strict liability or tort (including negligence or otherwise) arising in any way out of the use of this software even if advised of the possibility of such damage extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference. Reviewed-by: Allison Randal Signed-off-by: Thomas Gleixner Signed-off-by: Greg Kroah-Hartman --- include/linux/sunrpc/bc_xprt.h | 17 +---------------- net/sunrpc/backchannel_rqst.c | 16 +--------------- net/sunrpc/sunrpc.h | 16 +--------------- 3 files changed, 3 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h index f07c334c599f..db30a159f9d5 100644 --- a/include/linux/sunrpc/bc_xprt.h +++ b/include/linux/sunrpc/bc_xprt.h @@ -1,22 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /****************************************************************************** (c) 2008 NetApp. All Rights Reserved. -NetApp provides this source code under the GPL v2 License. -The GPL v2 license is available at -https://opensource.org/licenses/gpl-license.php. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ @@ -83,4 +69,3 @@ static inline void xprt_free_bc_request(struct rpc_rqst *req) } #endif /* CONFIG_SUNRPC_BACKCHANNEL */ #endif /* _LINUX_SUNRPC_BC_XPRT_H */ - diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c index 5a6b61dcdf2d..253a54c2fcfe 100644 --- a/net/sunrpc/backchannel_rqst.c +++ b/net/sunrpc/backchannel_rqst.c @@ -1,23 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /****************************************************************************** (c) 2007 Network Appliance, Inc. All Rights Reserved. (c) 2009 NetApp. All Rights Reserved. -NetApp provides this source code under the GPL v2 License. -The GPL v2 license is available at -https://opensource.org/licenses/gpl-license.php. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h index 2f59464e6524..d4a362c9e4b3 100644 --- a/net/sunrpc/sunrpc.h +++ b/net/sunrpc/sunrpc.h @@ -1,22 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /****************************************************************************** (c) 2008 NetApp. All Rights Reserved. -NetApp provides this source code under the GPL v2 License. -The GPL v2 license is available at -https://opensource.org/licenses/gpl-license.php. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -- cgit v1.2.3 From 298b95f111be85f2d5a18dc0177eb9a64130f9b4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 7 Jun 2022 16:11:21 +0200 Subject: treewide: Replace GPLv2 boilerplate/reference with SPDX - gpl-2.0_152.RULE Based on the normalized pattern: this software is distributed under the terms of the gnu general public license ( gpl ) version 2 as published by the free software foundation this software is provided by the copyright holders and contributors as is and any express or implied warranties including but not limited to the implied warranties of merchantability and fitness for a particular purpose are disclaimed in no event shall the copyright owner or contributors be liable for any direct indirect incidental special exemplary or consequential damages (including but not limited to procurement of substitute goods or services loss of use data or profits or business interruption) however caused and on any theory of liability whether in contract strict liability or tort (including negligence or otherwise) arising in any way out of the use of this software even if advised of the possibility of such damage extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference. Reviewed-by: Allison Randal Signed-off-by: Thomas Gleixner Signed-off-by: Greg Kroah-Hartman --- include/linux/platform_data/usb-omap.h | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/usb-omap.h b/include/linux/platform_data/usb-omap.h index 5e70d667031c..580978e468f8 100644 --- a/include/linux/platform_data/usb-omap.h +++ b/include/linux/platform_data/usb-omap.h @@ -1,22 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * usb-omap.h - Platform data for the various OMAP USB IPs * * Copyright (C) 2012 Texas Instruments Incorporated - https://www.ti.com - * - * This software is distributed under the terms of the GNU General Public - * License ("GPL") version 2, as published by the Free Software Foundation. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. */ #define OMAP3_HS_USB_PORTS 3 -- cgit v1.2.3 From abd462747539bbc630e50e8be59e3aebca63441d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 7 Jun 2022 16:11:31 +0200 Subject: treewide: Replace GPLv2 boilerplate/reference with SPDX - gpl-2.0_319.RULE Based on the normalized pattern: this program is free software you can redistribute it and/or modify it under the terms of the gnu general public license version 2 as published by the free software foundation this program is distributed as is without any warranty of any kind whether expressed or implied without even the implied warranty of merchantability or fitness for a particular purpose see the gnu general public license version 2 for more details extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference. Reviewed-by: Allison Randal Signed-off-by: Thomas Gleixner Signed-off-by: Greg Kroah-Hartman --- drivers/gpio/gpio-lp873x.c | 10 +--------- drivers/gpio/gpio-lp87565.c | 10 +--------- drivers/gpio/gpio-pisosr.c | 10 +--------- drivers/gpio/gpio-tpic2810.c | 10 +--------- drivers/mfd/tps65086.c | 10 +--------- drivers/mfd/tps65218.c | 10 +--------- drivers/mfd/tps65912-core.c | 10 +--------- drivers/mfd/tps65912-i2c.c | 10 +--------- drivers/mfd/tps65912-spi.c | 10 +--------- drivers/regulator/lp873x-regulator.c | 10 +--------- drivers/regulator/tps65086-regulator.c | 10 +--------- drivers/regulator/tps65218-regulator.c | 10 +--------- drivers/regulator/tps65912-regulator.c | 10 +--------- include/linux/mfd/tps65086.h | 10 +--------- include/linux/mfd/tps65218.h | 10 +--------- include/linux/mfd/tps65912.h | 10 +--------- 16 files changed, 16 insertions(+), 144 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpio-lp873x.c b/drivers/gpio/gpio-lp873x.c index 70fad87ff2db..5c79ba1f229c 100644 --- a/drivers/gpio/gpio-lp873x.c +++ b/drivers/gpio/gpio-lp873x.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/ * Keerthy * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65218 driver */ diff --git a/drivers/gpio/gpio-lp87565.c b/drivers/gpio/gpio-lp87565.c index fcde6708b5df..d3ce027de081 100644 --- a/drivers/gpio/gpio-lp87565.c +++ b/drivers/gpio/gpio-lp87565.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/ * Keerthy * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the LP873X driver */ diff --git a/drivers/gpio/gpio-pisosr.c b/drivers/gpio/gpio-pisosr.c index 81a47ae09ff8..67071bea08c2 100644 --- a/drivers/gpio/gpio-pisosr.c +++ b/drivers/gpio/gpio-pisosr.c @@ -1,15 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com/ * Andrew F. Davis - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. */ #include diff --git a/drivers/gpio/gpio-tpic2810.c b/drivers/gpio/gpio-tpic2810.c index 99d5a84a9129..a09b1e69b072 100644 --- a/drivers/gpio/gpio-tpic2810.c +++ b/drivers/gpio/gpio-tpic2810.c @@ -1,15 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com/ * Andrew F. Davis - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. */ #include diff --git a/drivers/mfd/tps65086.c b/drivers/mfd/tps65086.c index 3bd5728844a0..cbae9777a24e 100644 --- a/drivers/mfd/tps65086.c +++ b/drivers/mfd/tps65086.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65912 driver */ diff --git a/drivers/mfd/tps65218.c b/drivers/mfd/tps65218.c index 167e9fc308ef..49bb8fd168f8 100644 --- a/drivers/mfd/tps65218.c +++ b/drivers/mfd/tps65218.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Driver for TPS65218 Integrated power management chipsets * * Copyright (C) 2014 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. */ #include diff --git a/drivers/mfd/tps65912-core.c b/drivers/mfd/tps65912-core.c index c282a05e7146..7d994b8a5965 100644 --- a/drivers/mfd/tps65912-core.c +++ b/drivers/mfd/tps65912-core.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Core functions for TI TPS65912x PMICs * * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65218 driver and the previous TPS65912 driver by * Margarita Olaya Cabrera */ diff --git a/drivers/mfd/tps65912-i2c.c b/drivers/mfd/tps65912-i2c.c index 06eb2784d322..afb7f7d97dc0 100644 --- a/drivers/mfd/tps65912-i2c.c +++ b/drivers/mfd/tps65912-i2c.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * I2C access driver for TI TPS65912x PMICs * * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65218 driver and the previous TPS65912 driver by * Margarita Olaya Cabrera */ diff --git a/drivers/mfd/tps65912-spi.c b/drivers/mfd/tps65912-spi.c index bba38fbc781d..9e976f9c6bbe 100644 --- a/drivers/mfd/tps65912-spi.c +++ b/drivers/mfd/tps65912-spi.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * SPI access driver for TI TPS65912x PMICs * * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65218 driver and the previous TPS65912 driver by * Margarita Olaya Cabrera */ diff --git a/drivers/regulator/lp873x-regulator.c b/drivers/regulator/lp873x-regulator.c index c38387e0fbb2..d6e597922cb5 100644 --- a/drivers/regulator/lp873x-regulator.c +++ b/drivers/regulator/lp873x-regulator.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Regulator driver for LP873X PMIC * * Copyright (C) 2016 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. */ #include diff --git a/drivers/regulator/tps65086-regulator.c b/drivers/regulator/tps65086-regulator.c index 070c956216b0..f1bc54c825dd 100644 --- a/drivers/regulator/tps65086-regulator.c +++ b/drivers/regulator/tps65086-regulator.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * * Author: Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65912 driver */ diff --git a/drivers/regulator/tps65218-regulator.c b/drivers/regulator/tps65218-regulator.c index fa263545a70e..48809c3b3abc 100644 --- a/drivers/regulator/tps65218-regulator.c +++ b/drivers/regulator/tps65218-regulator.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * tps65218-regulator.c * * Regulator driver for TPS65218 PMIC * * Copyright (C) 2014 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. */ #include diff --git a/drivers/regulator/tps65912-regulator.c b/drivers/regulator/tps65912-regulator.c index b52d4f2874b7..76f90202ae09 100644 --- a/drivers/regulator/tps65912-regulator.c +++ b/drivers/regulator/tps65912-regulator.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Regulator driver for TI TPS65912x PMICs * * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65218 driver and the previous TPS65912 driver by * Margarita Olaya Cabrera */ diff --git a/include/linux/mfd/tps65086.h b/include/linux/mfd/tps65086.h index e0a417e53766..16f87cccc003 100644 --- a/include/linux/mfd/tps65086.h +++ b/include/linux/mfd/tps65086.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65912 driver */ diff --git a/include/linux/mfd/tps65218.h b/include/linux/mfd/tps65218.h index 122e24ddbd4b..2946be2f15f3 100644 --- a/include/linux/mfd/tps65218.h +++ b/include/linux/mfd/tps65218.h @@ -1,18 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * linux/mfd/tps65218.h * * Functions to access TPS65218 power management chip. * * Copyright (C) 2014 Texas Instruments Incorporated - https://www.ti.com/ - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. */ #ifndef __LINUX_MFD_TPS65218_H diff --git a/include/linux/mfd/tps65912.h b/include/linux/mfd/tps65912.h index 8a61386cb8c1..860ec0a16c96 100644 --- a/include/linux/mfd/tps65912.h +++ b/include/linux/mfd/tps65912.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65218 driver and the previous TPS65912 driver by * Margarita Olaya Cabrera */ -- cgit v1.2.3 From 5a729246e57eac410e4a13f5aba66ae2dc552632 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 7 Jun 2022 16:11:32 +0200 Subject: treewide: Replace GPLv2 boilerplate/reference with SPDX - gpl-2.0_320.RULE Based on the normalized pattern: this program is free software you can redistribute it and/or modify it under the terms of the gnu general public license version 2 as published by the free software foundation this program is distributed as is without any warranty of any kind whether express or implied without even the implied warranty of merchantability or fitness for a particular purpose see the gnu general public license for more details extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference. Reviewed-by: Allison Randal Signed-off-by: Thomas Gleixner Signed-off-by: Greg Kroah-Hartman --- arch/arm/mach-omap1/timer.c | 10 +--------- arch/arm/mach-omap2/display.c | 10 +--------- arch/arm/mach-omap2/omap_opp_data.h | 10 +--------- arch/arm/mach-omap2/opp3xxx_data.c | 10 +--------- arch/arm/mach-omap2/opp4xxx_data.c | 10 +--------- arch/arm/mach-versatile/spc.c | 10 +--------- drivers/bus/omap_l3_noc.c | 10 +--------- drivers/bus/omap_l3_noc.h | 10 +--------- drivers/clk/keystone/sci-clk.c | 10 +--------- drivers/clk/ti/apll.c | 10 +--------- drivers/clk/ti/autoidle.c | 10 +--------- drivers/clk/ti/clk-dra7-atl.c | 10 +--------- drivers/clk/ti/clk.c | 10 +--------- drivers/clk/ti/clkctrl.c | 10 +--------- drivers/clk/ti/clkt_dflt.c | 10 +--------- drivers/clk/ti/clockdomain.c | 10 +--------- drivers/clk/ti/composite.c | 10 +--------- drivers/clk/ti/divider.c | 10 +--------- drivers/clk/ti/dpll.c | 10 +--------- drivers/clk/ti/fixed-factor.c | 10 +--------- drivers/clk/ti/gate.c | 10 +--------- drivers/clk/ti/interface.c | 10 +--------- drivers/clk/ti/mux.c | 10 +--------- drivers/cpufreq/scpi-cpufreq.c | 10 +--------- drivers/input/misc/palmas-pwrbutton.c | 10 +--------- drivers/input/misc/tps65218-pwrbutton.c | 10 +--------- drivers/misc/sram-exec.c | 10 +--------- drivers/power/supply/cpcap-battery.c | 10 +--------- drivers/regulator/ti-abb-regulator.c | 10 +--------- drivers/reset/reset-ti-sci.c | 10 +--------- drivers/reset/reset-ti-syscon.c | 10 +--------- drivers/thermal/hisi_thermal.c | 10 +--------- include/dt-bindings/clock/ti-dra7-atl.h | 10 +--------- include/dt-bindings/pinctrl/hisi.h | 10 +--------- include/dt-bindings/pinctrl/keystone.h | 10 +--------- include/linux/clk/ti.h | 10 +--------- include/linux/pm_wakeirq.h | 14 ++------------ include/linux/soc/ti/ti-msgmgr.h | 10 +--------- 38 files changed, 39 insertions(+), 345 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/mach-omap1/timer.c b/arch/arm/mach-omap1/timer.c index 9ed64345f06e..f5cd4bbf7566 100644 --- a/arch/arm/mach-omap1/timer.c +++ b/arch/arm/mach-omap1/timer.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP1 Dual-Mode Timers - platform device registration * @@ -9,15 +10,6 @@ * Copyright (C) 2011 Texas Instruments Incorporated - https://www.ti.com/ * Tarun Kanti DebBarma * Thara Gopinath - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c index 21413a9b7b6c..47149b278ad5 100644 --- a/arch/arm/mach-omap2/display.c +++ b/arch/arm/mach-omap2/display.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP2plus display device setup / initialization. * * Copyright (C) 2010 Texas Instruments Incorporated - https://www.ti.com/ * Senthilvadivu Guruswamy * Sumit Semwal - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/arch/arm/mach-omap2/omap_opp_data.h b/arch/arm/mach-omap2/omap_opp_data.h index 533dd643069a..88375ab38e31 100644 --- a/arch/arm/mach-omap2/omap_opp_data.h +++ b/arch/arm/mach-omap2/omap_opp_data.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * OMAP SoC specific OPP Data helpers * @@ -6,15 +7,6 @@ * Kevin Hilman * Copyright (C) 2010 Nokia Corporation. * Eduardo Valentin - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __ARCH_ARM_MACH_OMAP2_OMAP_OPP_DATA_H #define __ARCH_ARM_MACH_OMAP2_OMAP_OPP_DATA_H diff --git a/arch/arm/mach-omap2/opp3xxx_data.c b/arch/arm/mach-omap2/opp3xxx_data.c index b610c5fb423b..90257e2fb3d6 100644 --- a/arch/arm/mach-omap2/opp3xxx_data.c +++ b/arch/arm/mach-omap2/opp3xxx_data.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP3 OPP table definitions. * @@ -7,15 +8,6 @@ * Copyright (C) 2010-2011 Nokia Corporation. * Eduardo Valentin * Paul Walmsley - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/arch/arm/mach-omap2/opp4xxx_data.c b/arch/arm/mach-omap2/opp4xxx_data.c index d937c5ef41c6..a9851886017d 100644 --- a/arch/arm/mach-omap2/opp4xxx_data.c +++ b/arch/arm/mach-omap2/opp4xxx_data.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP4 OPP table definitions. * @@ -8,15 +9,6 @@ * Copyright (C) 2010-2011 Nokia Corporation. * Eduardo Valentin * Paul Walmsley - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/arch/arm/mach-versatile/spc.c b/arch/arm/mach-versatile/spc.c index 6e6985e756af..5e44170e1a9a 100644 --- a/arch/arm/mach-versatile/spc.c +++ b/arch/arm/mach-versatile/spc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Versatile Express Serial Power Controller (SPC) support * @@ -6,15 +7,6 @@ * Authors: Sudeep KarkadaNagesha * Achin Gupta * Lorenzo Pieralisi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/bus/omap_l3_noc.c b/drivers/bus/omap_l3_noc.c index dcfb32ee5cb6..eb1ba6319fda 100644 --- a/drivers/bus/omap_l3_noc.c +++ b/drivers/bus/omap_l3_noc.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP L3 Interconnect error handling driver * * Copyright (C) 2011-2015 Texas Instruments Incorporated - http://www.ti.com/ * Santosh Shilimkar * Sricharan - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include #include diff --git a/drivers/bus/omap_l3_noc.h b/drivers/bus/omap_l3_noc.h index 73431f81da28..bb3eebd3465d 100644 --- a/drivers/bus/omap_l3_noc.h +++ b/drivers/bus/omap_l3_noc.h @@ -1,18 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * OMAP L3 Interconnect error handling driver header * * Copyright (C) 2011-2015 Texas Instruments Incorporated - http://www.ti.com/ * Santosh Shilimkar * sricharan - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __OMAP_L3_NOC_H #define __OMAP_L3_NOC_H diff --git a/drivers/clk/keystone/sci-clk.c b/drivers/clk/keystone/sci-clk.c index 7e1b136e71ae..d4b4e74e22da 100644 --- a/drivers/clk/keystone/sci-clk.c +++ b/drivers/clk/keystone/sci-clk.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * SCI Clock driver for keystone based devices * * Copyright (C) 2015-2016 Texas Instruments Incorporated - https://www.ti.com/ * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include #include diff --git a/drivers/clk/ti/apll.c b/drivers/clk/ti/apll.c index e4db6b9a55c6..dd0709c9c249 100644 --- a/drivers/clk/ti/apll.c +++ b/drivers/clk/ti/apll.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP APLL clock support * * Copyright (C) 2013 Texas Instruments, Inc. * * J Keerthy - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/autoidle.c b/drivers/clk/ti/autoidle.c index d6e5f1511ace..27e6b9cb1881 100644 --- a/drivers/clk/ti/autoidle.c +++ b/drivers/clk/ti/autoidle.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI clock autoidle support * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/clk-dra7-atl.c b/drivers/clk/ti/clk-dra7-atl.c index aa0950c4f498..f0f5bf68b6d2 100644 --- a/drivers/clk/ti/clk-dra7-atl.c +++ b/drivers/clk/ti/clk-dra7-atl.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * DRA7 ATL (Audio Tracking Logic) clock driver * * Copyright (C) 2013 Texas Instruments, Inc. * * Peter Ujfalusi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c index 3463579220b5..ef2a445c63a3 100644 --- a/drivers/clk/ti/clk.c +++ b/drivers/clk/ti/clk.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI clock support * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/clkctrl.c b/drivers/clk/ti/clkctrl.c index 617360e20d86..3b5ebae3740f 100644 --- a/drivers/clk/ti/clkctrl.c +++ b/drivers/clk/ti/clkctrl.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP clkctrl clock support * * Copyright (C) 2017 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/clkt_dflt.c b/drivers/clk/ti/clkt_dflt.c index 91751dd26b16..a756ab1a5856 100644 --- a/drivers/clk/ti/clkt_dflt.c +++ b/drivers/clk/ti/clkt_dflt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Default clock type * @@ -8,15 +9,6 @@ * Richard Woodruff * Paul Walmsley * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c index 24179c907774..c897ad7e681e 100644 --- a/drivers/clk/ti/clockdomain.c +++ b/drivers/clk/ti/clockdomain.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP clockdomain support * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/composite.c b/drivers/clk/ti/composite.c index 779b9900f636..77b771dd050a 100644 --- a/drivers/clk/ti/composite.c +++ b/drivers/clk/ti/composite.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI composite clock support * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c index 9fbea0997b43..488d3da60c31 100644 --- a/drivers/clk/ti/divider.c +++ b/drivers/clk/ti/divider.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI Divider Clock * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c index 7c6dc8449b22..8ed43bc6b7cc 100644 --- a/drivers/clk/ti/dpll.c +++ b/drivers/clk/ti/dpll.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP DPLL clock support * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/fixed-factor.c b/drivers/clk/ti/fixed-factor.c index 8cb00d0af966..c80cee0f5d3d 100644 --- a/drivers/clk/ti/fixed-factor.c +++ b/drivers/clk/ti/fixed-factor.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI Fixed Factor Clock * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c index 0033de9beb4c..307702921431 100644 --- a/drivers/clk/ti/gate.c +++ b/drivers/clk/ti/gate.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP gate clock support * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/interface.c b/drivers/clk/ti/interface.c index dd2b455183a9..f47beeea211e 100644 --- a/drivers/clk/ti/interface.c +++ b/drivers/clk/ti/interface.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * OMAP interface clock support * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c index 15de513d2d81..46b45b3e8319 100644 --- a/drivers/clk/ti/mux.c +++ b/drivers/clk/ti/mux.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI Multiplexer Clock * * Copyright (C) 2013 Texas Instruments, Inc. * * Tero Kristo - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c index bda3e7d42964..fd2c16821d54 100644 --- a/drivers/cpufreq/scpi-cpufreq.c +++ b/drivers/cpufreq/scpi-cpufreq.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * System Control and Power Interface (SCPI) based CPUFreq Interface driver * * Copyright (C) 2015 ARM Ltd. * Sudeep Holla - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/input/misc/palmas-pwrbutton.c b/drivers/input/misc/palmas-pwrbutton.c index 2213e06b611d..465e6693077a 100644 --- a/drivers/input/misc/palmas-pwrbutton.c +++ b/drivers/input/misc/palmas-pwrbutton.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Texas Instruments' Palmas Power Button Input Driver * * Copyright (C) 2012-2014 Texas Instruments Incorporated - http://www.ti.com/ * Girish S Ghongdemath * Nishanth Menon - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/input/misc/tps65218-pwrbutton.c b/drivers/input/misc/tps65218-pwrbutton.c index f011447c44fb..fc450fce0932 100644 --- a/drivers/input/misc/tps65218-pwrbutton.c +++ b/drivers/input/misc/tps65218-pwrbutton.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Texas Instruments' TPS65217 and TPS65218 Power Button Input Driver * * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com/ * Author: Felipe Balbi * Author: Marcin Niestroj - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/misc/sram-exec.c b/drivers/misc/sram-exec.c index 6cc31789b38d..a948e95d4375 100644 --- a/drivers/misc/sram-exec.c +++ b/drivers/misc/sram-exec.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * SRAM protect-exec region helper functions * * Copyright (C) 2017 Texas Instruments Incorporated - https://www.ti.com/ * Dave Gerlach - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index ae284bdd6cc3..d98d9244e394 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Battery driver for CPCAP PMIC * @@ -7,15 +8,6 @@ * drivers: * * Copyright (C) 2009-2010 Motorola, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/regulator/ti-abb-regulator.c b/drivers/regulator/ti-abb-regulator.c index bd7b2f287250..cf18452386ca 100644 --- a/drivers/regulator/ti-abb-regulator.c +++ b/drivers/regulator/ti-abb-regulator.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Texas Instruments SoC Adaptive Body Bias(ABB) Regulator * @@ -7,15 +8,6 @@ * Copyright (C) 2012-2013 Texas Instruments, Inc. * Andrii Tseglytskyi * Nishanth Menon - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include #include diff --git a/drivers/reset/reset-ti-sci.c b/drivers/reset/reset-ti-sci.c index b799aefad547..cc01fa5b0bea 100644 --- a/drivers/reset/reset-ti-sci.c +++ b/drivers/reset/reset-ti-sci.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Texas Instrument's System Control Interface (TI-SCI) reset driver * * Copyright (C) 2015-2017 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/reset/reset-ti-syscon.c b/drivers/reset/reset-ti-syscon.c index 2b92775d58f0..f0dd7ffc3b72 100644 --- a/drivers/reset/reset-ti-syscon.c +++ b/drivers/reset/reset-ti-syscon.c @@ -1,18 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI SYSCON regmap reset driver * * Copyright (C) 2015-2016 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis * Suman Anna - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/drivers/thermal/hisi_thermal.c b/drivers/thermal/hisi_thermal.c index b29ab09040d5..19a242c69ce6 100644 --- a/drivers/thermal/hisi_thermal.c +++ b/drivers/thermal/hisi_thermal.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * HiSilicon thermal sensor driver * @@ -6,15 +7,6 @@ * * Xinwei Kong * Leo Yan - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/include/dt-bindings/clock/ti-dra7-atl.h b/include/dt-bindings/clock/ti-dra7-atl.h index 42dd4164f6f4..b0e71e3cce95 100644 --- a/include/dt-bindings/clock/ti-dra7-atl.h +++ b/include/dt-bindings/clock/ti-dra7-atl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * This header provides constants for DRA7 ATL (Audio Tracking Logic) * @@ -6,15 +7,6 @@ * Copyright (C) 2013 Texas Instruments, Inc. * * Peter Ujfalusi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _DT_BINDINGS_CLK_DRA7_ATL_H diff --git a/include/dt-bindings/pinctrl/hisi.h b/include/dt-bindings/pinctrl/hisi.h index 93064c750c8c..2175ec89c82f 100644 --- a/include/dt-bindings/pinctrl/hisi.h +++ b/include/dt-bindings/pinctrl/hisi.h @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * This header provides constants for hisilicon pinctrl bindings. * * Copyright (c) 2015 HiSilicon Limited. * Copyright (c) 2015 Linaro Limited. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _DT_BINDINGS_PINCTRL_HISI_H diff --git a/include/dt-bindings/pinctrl/keystone.h b/include/dt-bindings/pinctrl/keystone.h index 7f97d776a8ff..66f8aecada53 100644 --- a/include/dt-bindings/pinctrl/keystone.h +++ b/include/dt-bindings/pinctrl/keystone.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * This header provides constants for Keystone pinctrl bindings. * * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _DT_BINDINGS_PINCTRL_KEYSTONE_H diff --git a/include/linux/clk/ti.h b/include/linux/clk/ti.h index 3486f20a3753..cbfcbf186ce3 100644 --- a/include/linux/clk/ti.h +++ b/include/linux/clk/ti.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * TI clock drivers support * * Copyright (C) 2013 Texas Instruments, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __LINUX_CLK_TI_H__ #define __LINUX_CLK_TI_H__ diff --git a/include/linux/pm_wakeirq.h b/include/linux/pm_wakeirq.h index e63a63aa47a3..dd42d16945d0 100644 --- a/include/linux/pm_wakeirq.h +++ b/include/linux/pm_wakeirq.h @@ -1,15 +1,5 @@ -/* - * pm_wakeirq.h - Device wakeirq helper functions - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* pm_wakeirq.h - Device wakeirq helper functions */ #ifndef _LINUX_PM_WAKEIRQ_H #define _LINUX_PM_WAKEIRQ_H diff --git a/include/linux/soc/ti/ti-msgmgr.h b/include/linux/soc/ti/ti-msgmgr.h index 69a8d7682c4b..543da257a5f2 100644 --- a/include/linux/soc/ti/ti-msgmgr.h +++ b/include/linux/soc/ti/ti-msgmgr.h @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Texas Instruments' Message Manager * * Copyright (C) 2015-2022 Texas Instruments Incorporated - https://www.ti.com/ * Nishanth Menon - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef TI_MSGMGR_H -- cgit v1.2.3 From 35ba63b8f6d07d353159505423cfca5a4378a11c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 6 Jun 2022 10:41:05 +0200 Subject: vme: move back to staging The VME subsystem graduated from staging into a top-level subsystem in 2012, with commit db3b9e990e75 ("Staging: VME: move VME drivers out of staging") stating: The VME device drivers have not moved out yet due to some API questions they are still working through, that should happen soon, hopefully. However, this never happened: maintenance of drivers/vme effectively stopped in 2017, with all subsequent changes being treewide cleanups. No hardware driver remains in staging, only the limited user-level access, and I just removed one of the two bridge drivers and the only remaining board. drivers/staging/vme/devices/ was recently moved to drivers/staging/vme_user/, but as the vme_user driver is the only one remaining for this subsystem, it is easier to just move the remaining three source files into this directory rather than keeping the original hierarchy. Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20220606084109.4108188-3-arnd@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/vme.rst | 4 +- MAINTAINERS | 4 +- drivers/Kconfig | 2 - drivers/Makefile | 1 - drivers/staging/vme_user/Kconfig | 27 + drivers/staging/vme_user/Makefile | 3 + drivers/staging/vme_user/vme.c | 2015 +++++++++++++++++++++++++ drivers/staging/vme_user/vme.h | 190 +++ drivers/staging/vme_user/vme_bridge.h | 190 +++ drivers/staging/vme_user/vme_fake.c | 1305 ++++++++++++++++ drivers/staging/vme_user/vme_tsi148.c | 2661 +++++++++++++++++++++++++++++++++ drivers/staging/vme_user/vme_tsi148.h | 1407 +++++++++++++++++ drivers/staging/vme_user/vme_user.c | 2 +- drivers/vme/Kconfig | 16 - drivers/vme/Makefile | 7 - drivers/vme/bridges/Kconfig | 17 - drivers/vme/bridges/Makefile | 3 - drivers/vme/bridges/vme_fake.c | 1305 ---------------- drivers/vme/bridges/vme_tsi148.c | 2661 --------------------------------- drivers/vme/bridges/vme_tsi148.h | 1407 ----------------- drivers/vme/vme.c | 2015 ------------------------- drivers/vme/vme_bridge.h | 190 --- include/linux/vme.h | 190 --- 23 files changed, 7802 insertions(+), 7820 deletions(-) create mode 100644 drivers/staging/vme_user/vme.c create mode 100644 drivers/staging/vme_user/vme.h create mode 100644 drivers/staging/vme_user/vme_bridge.h create mode 100644 drivers/staging/vme_user/vme_fake.c create mode 100644 drivers/staging/vme_user/vme_tsi148.c create mode 100644 drivers/staging/vme_user/vme_tsi148.h delete mode 100644 drivers/vme/Kconfig delete mode 100644 drivers/vme/Makefile delete mode 100644 drivers/vme/bridges/Kconfig delete mode 100644 drivers/vme/bridges/Makefile delete mode 100644 drivers/vme/bridges/vme_fake.c delete mode 100644 drivers/vme/bridges/vme_tsi148.c delete mode 100644 drivers/vme/bridges/vme_tsi148.h delete mode 100644 drivers/vme/vme.c delete mode 100644 drivers/vme/vme_bridge.h delete mode 100644 include/linux/vme.h (limited to 'include/linux') diff --git a/Documentation/driver-api/vme.rst b/Documentation/driver-api/vme.rst index def139c13410..c0b475369de0 100644 --- a/Documentation/driver-api/vme.rst +++ b/Documentation/driver-api/vme.rst @@ -290,8 +290,8 @@ The function :c:func:`vme_bus_num` returns the bus ID of the provided bridge. VME API ------- -.. kernel-doc:: include/linux/vme.h +.. kernel-doc:: drivers/staging/vme_user/vme.h :internal: -.. kernel-doc:: drivers/vme/vme.c +.. kernel-doc:: drivers/staging/vme_user/vme.c :export: diff --git a/MAINTAINERS b/MAINTAINERS index 60e5e4007844..7097b824ba18 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -21230,12 +21230,10 @@ M: Martyn Welch M: Manohar Vanga M: Greg Kroah-Hartman L: linux-kernel@vger.kernel.org -S: Maintained +S: Odd fixes T: git git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git F: Documentation/driver-api/vme.rst F: drivers/staging/vme_user/ -F: drivers/vme/ -F: include/linux/vme* VM SOCKETS (AF_VSOCK) M: Stefano Garzarella diff --git a/drivers/Kconfig b/drivers/Kconfig index b6a172d32a7d..19ee995bd0ae 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -183,8 +183,6 @@ source "drivers/iio/Kconfig" source "drivers/ntb/Kconfig" -source "drivers/vme/Kconfig" - source "drivers/pwm/Kconfig" source "drivers/irqchip/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index 9a30842b22c5..dadf2678277f 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -165,7 +165,6 @@ obj-$(CONFIG_PM_DEVFREQ) += devfreq/ obj-$(CONFIG_EXTCON) += extcon/ obj-$(CONFIG_MEMORY) += memory/ obj-$(CONFIG_IIO) += iio/ -obj-$(CONFIG_VME_BUS) += vme/ obj-$(CONFIG_IPACK_BUS) += ipack/ obj-$(CONFIG_NTB) += ntb/ obj-$(CONFIG_POWERCAP) += powercap/ diff --git a/drivers/staging/vme_user/Kconfig b/drivers/staging/vme_user/Kconfig index e8b4461bf27f..c8eabf8f40f1 100644 --- a/drivers/staging/vme_user/Kconfig +++ b/drivers/staging/vme_user/Kconfig @@ -1,4 +1,29 @@ # SPDX-License-Identifier: GPL-2.0 +menuconfig VME_BUS + bool "VME bridge support" + depends on STAGING && PCI + help + If you say Y here you get support for the VME bridge Framework. + +if VME_BUS + +comment "VME Bridge Drivers" + +config VME_TSI148 + tristate "Tempe" + depends on HAS_DMA + help + If you say Y here you get support for the Tundra TSI148 VME bridge + chip. + +config VME_FAKE + tristate "Fake" + help + If you say Y here you get support for the fake VME bridge. This + provides a virtualised VME Bus for devices with no VME bridge. This + is mainly useful for VME development (in the absence of VME + hardware). + comment "VME Device Drivers" config VME_USER @@ -11,3 +36,5 @@ config VME_USER To compile this driver as a module, choose M here. The module will be called vme_user. If unsure, say N. + +endif diff --git a/drivers/staging/vme_user/Makefile b/drivers/staging/vme_user/Makefile index 5380115139b0..8dcc6938ce5c 100644 --- a/drivers/staging/vme_user/Makefile +++ b/drivers/staging/vme_user/Makefile @@ -3,4 +3,7 @@ # Makefile for the VME device drivers. # +obj-$(CONFIG_VME_BUS) += vme.o obj-$(CONFIG_VME_USER) += vme_user.o +obj-$(CONFIG_VME_TSI148) += vme_tsi148.o +obj-$(CONFIG_VME_FAKE) += vme_fake.o diff --git a/drivers/staging/vme_user/vme.c b/drivers/staging/vme_user/vme.c new file mode 100644 index 000000000000..b5555683a069 --- /dev/null +++ b/drivers/staging/vme_user/vme.c @@ -0,0 +1,2015 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * VME Bridge Framework + * + * Author: Martyn Welch + * Copyright 2008 GE Intelligent Platforms Embedded Systems, Inc. + * + * Based on work by Tom Armistead and Ajit Prem + * Copyright 2004 Motorola Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vme.h" +#include "vme_bridge.h" + +/* Bitmask and list of registered buses both protected by common mutex */ +static unsigned int vme_bus_numbers; +static LIST_HEAD(vme_bus_list); +static DEFINE_MUTEX(vme_buses_lock); + +static int __init vme_init(void); + +static struct vme_dev *dev_to_vme_dev(struct device *dev) +{ + return container_of(dev, struct vme_dev, dev); +} + +/* + * Find the bridge that the resource is associated with. + */ +static struct vme_bridge *find_bridge(struct vme_resource *resource) +{ + /* Get list to search */ + switch (resource->type) { + case VME_MASTER: + return list_entry(resource->entry, struct vme_master_resource, + list)->parent; + case VME_SLAVE: + return list_entry(resource->entry, struct vme_slave_resource, + list)->parent; + case VME_DMA: + return list_entry(resource->entry, struct vme_dma_resource, + list)->parent; + case VME_LM: + return list_entry(resource->entry, struct vme_lm_resource, + list)->parent; + default: + printk(KERN_ERR "Unknown resource type\n"); + return NULL; + } +} + +/** + * vme_alloc_consistent - Allocate contiguous memory. + * @resource: Pointer to VME resource. + * @size: Size of allocation required. + * @dma: Pointer to variable to store physical address of allocation. + * + * Allocate a contiguous block of memory for use by the driver. This is used to + * create the buffers for the slave windows. + * + * Return: Virtual address of allocation on success, NULL on failure. + */ +void *vme_alloc_consistent(struct vme_resource *resource, size_t size, + dma_addr_t *dma) +{ + struct vme_bridge *bridge; + + if (!resource) { + printk(KERN_ERR "No resource\n"); + return NULL; + } + + bridge = find_bridge(resource); + if (!bridge) { + printk(KERN_ERR "Can't find bridge\n"); + return NULL; + } + + if (!bridge->parent) { + printk(KERN_ERR "Dev entry NULL for bridge %s\n", bridge->name); + return NULL; + } + + if (!bridge->alloc_consistent) { + printk(KERN_ERR "alloc_consistent not supported by bridge %s\n", + bridge->name); + return NULL; + } + + return bridge->alloc_consistent(bridge->parent, size, dma); +} +EXPORT_SYMBOL(vme_alloc_consistent); + +/** + * vme_free_consistent - Free previously allocated memory. + * @resource: Pointer to VME resource. + * @size: Size of allocation to free. + * @vaddr: Virtual address of allocation. + * @dma: Physical address of allocation. + * + * Free previously allocated block of contiguous memory. + */ +void vme_free_consistent(struct vme_resource *resource, size_t size, + void *vaddr, dma_addr_t dma) +{ + struct vme_bridge *bridge; + + if (!resource) { + printk(KERN_ERR "No resource\n"); + return; + } + + bridge = find_bridge(resource); + if (!bridge) { + printk(KERN_ERR "Can't find bridge\n"); + return; + } + + if (!bridge->parent) { + printk(KERN_ERR "Dev entry NULL for bridge %s\n", bridge->name); + return; + } + + if (!bridge->free_consistent) { + printk(KERN_ERR "free_consistent not supported by bridge %s\n", + bridge->name); + return; + } + + bridge->free_consistent(bridge->parent, size, vaddr, dma); +} +EXPORT_SYMBOL(vme_free_consistent); + +/** + * vme_get_size - Helper function returning size of a VME window + * @resource: Pointer to VME slave or master resource. + * + * Determine the size of the VME window provided. This is a helper + * function, wrappering the call to vme_master_get or vme_slave_get + * depending on the type of window resource handed to it. + * + * Return: Size of the window on success, zero on failure. + */ +size_t vme_get_size(struct vme_resource *resource) +{ + int enabled, retval; + unsigned long long base, size; + dma_addr_t buf_base; + u32 aspace, cycle, dwidth; + + switch (resource->type) { + case VME_MASTER: + retval = vme_master_get(resource, &enabled, &base, &size, + &aspace, &cycle, &dwidth); + if (retval) + return 0; + + return size; + case VME_SLAVE: + retval = vme_slave_get(resource, &enabled, &base, &size, + &buf_base, &aspace, &cycle); + if (retval) + return 0; + + return size; + case VME_DMA: + return 0; + default: + printk(KERN_ERR "Unknown resource type\n"); + return 0; + } +} +EXPORT_SYMBOL(vme_get_size); + +int vme_check_window(u32 aspace, unsigned long long vme_base, + unsigned long long size) +{ + int retval = 0; + + if (vme_base + size < size) + return -EINVAL; + + switch (aspace) { + case VME_A16: + if (vme_base + size > VME_A16_MAX) + retval = -EFAULT; + break; + case VME_A24: + if (vme_base + size > VME_A24_MAX) + retval = -EFAULT; + break; + case VME_A32: + if (vme_base + size > VME_A32_MAX) + retval = -EFAULT; + break; + case VME_A64: + /* The VME_A64_MAX limit is actually U64_MAX + 1 */ + break; + case VME_CRCSR: + if (vme_base + size > VME_CRCSR_MAX) + retval = -EFAULT; + break; + case VME_USER1: + case VME_USER2: + case VME_USER3: + case VME_USER4: + /* User Defined */ + break; + default: + printk(KERN_ERR "Invalid address space\n"); + retval = -EINVAL; + break; + } + + return retval; +} +EXPORT_SYMBOL(vme_check_window); + +static u32 vme_get_aspace(int am) +{ + switch (am) { + case 0x29: + case 0x2D: + return VME_A16; + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + return VME_A24; + case 0x8: + case 0x9: + case 0xA: + case 0xB: + case 0xC: + case 0xD: + case 0xE: + case 0xF: + return VME_A32; + case 0x0: + case 0x1: + case 0x3: + return VME_A64; + } + + return 0; +} + +/** + * vme_slave_request - Request a VME slave window resource. + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * @address: Required VME address space. + * @cycle: Required VME data transfer cycle type. + * + * Request use of a VME window resource capable of being set for the requested + * address space and data transfer cycle. + * + * Return: Pointer to VME resource on success, NULL on failure. + */ +struct vme_resource *vme_slave_request(struct vme_dev *vdev, u32 address, + u32 cycle) +{ + struct vme_bridge *bridge; + struct list_head *slave_pos = NULL; + struct vme_slave_resource *allocated_image = NULL; + struct vme_slave_resource *slave_image = NULL; + struct vme_resource *resource = NULL; + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + goto err_bus; + } + + /* Loop through slave resources */ + list_for_each(slave_pos, &bridge->slave_resources) { + slave_image = list_entry(slave_pos, + struct vme_slave_resource, list); + + if (!slave_image) { + printk(KERN_ERR "Registered NULL Slave resource\n"); + continue; + } + + /* Find an unlocked and compatible image */ + mutex_lock(&slave_image->mtx); + if (((slave_image->address_attr & address) == address) && + ((slave_image->cycle_attr & cycle) == cycle) && + (slave_image->locked == 0)) { + + slave_image->locked = 1; + mutex_unlock(&slave_image->mtx); + allocated_image = slave_image; + break; + } + mutex_unlock(&slave_image->mtx); + } + + /* No free image */ + if (!allocated_image) + goto err_image; + + resource = kmalloc(sizeof(*resource), GFP_KERNEL); + if (!resource) + goto err_alloc; + + resource->type = VME_SLAVE; + resource->entry = &allocated_image->list; + + return resource; + +err_alloc: + /* Unlock image */ + mutex_lock(&slave_image->mtx); + slave_image->locked = 0; + mutex_unlock(&slave_image->mtx); +err_image: +err_bus: + return NULL; +} +EXPORT_SYMBOL(vme_slave_request); + +/** + * vme_slave_set - Set VME slave window configuration. + * @resource: Pointer to VME slave resource. + * @enabled: State to which the window should be configured. + * @vme_base: Base address for the window. + * @size: Size of the VME window. + * @buf_base: Based address of buffer used to provide VME slave window storage. + * @aspace: VME address space for the VME window. + * @cycle: VME data transfer cycle type for the VME window. + * + * Set configuration for provided VME slave window. + * + * Return: Zero on success, -EINVAL if operation is not supported on this + * device, if an invalid resource has been provided or invalid + * attributes are provided. Hardware specific errors may also be + * returned. + */ +int vme_slave_set(struct vme_resource *resource, int enabled, + unsigned long long vme_base, unsigned long long size, + dma_addr_t buf_base, u32 aspace, u32 cycle) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_slave_resource *image; + int retval; + + if (resource->type != VME_SLAVE) { + printk(KERN_ERR "Not a slave resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_slave_resource, list); + + if (!bridge->slave_set) { + printk(KERN_ERR "Function not supported\n"); + return -ENOSYS; + } + + if (!(((image->address_attr & aspace) == aspace) && + ((image->cycle_attr & cycle) == cycle))) { + printk(KERN_ERR "Invalid attributes\n"); + return -EINVAL; + } + + retval = vme_check_window(aspace, vme_base, size); + if (retval) + return retval; + + return bridge->slave_set(image, enabled, vme_base, size, buf_base, + aspace, cycle); +} +EXPORT_SYMBOL(vme_slave_set); + +/** + * vme_slave_get - Retrieve VME slave window configuration. + * @resource: Pointer to VME slave resource. + * @enabled: Pointer to variable for storing state. + * @vme_base: Pointer to variable for storing window base address. + * @size: Pointer to variable for storing window size. + * @buf_base: Pointer to variable for storing slave buffer base address. + * @aspace: Pointer to variable for storing VME address space. + * @cycle: Pointer to variable for storing VME data transfer cycle type. + * + * Return configuration for provided VME slave window. + * + * Return: Zero on success, -EINVAL if operation is not supported on this + * device or if an invalid resource has been provided. + */ +int vme_slave_get(struct vme_resource *resource, int *enabled, + unsigned long long *vme_base, unsigned long long *size, + dma_addr_t *buf_base, u32 *aspace, u32 *cycle) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_slave_resource *image; + + if (resource->type != VME_SLAVE) { + printk(KERN_ERR "Not a slave resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_slave_resource, list); + + if (!bridge->slave_get) { + printk(KERN_ERR "vme_slave_get not supported\n"); + return -EINVAL; + } + + return bridge->slave_get(image, enabled, vme_base, size, buf_base, + aspace, cycle); +} +EXPORT_SYMBOL(vme_slave_get); + +/** + * vme_slave_free - Free VME slave window + * @resource: Pointer to VME slave resource. + * + * Free the provided slave resource so that it may be reallocated. + */ +void vme_slave_free(struct vme_resource *resource) +{ + struct vme_slave_resource *slave_image; + + if (resource->type != VME_SLAVE) { + printk(KERN_ERR "Not a slave resource\n"); + return; + } + + slave_image = list_entry(resource->entry, struct vme_slave_resource, + list); + if (!slave_image) { + printk(KERN_ERR "Can't find slave resource\n"); + return; + } + + /* Unlock image */ + mutex_lock(&slave_image->mtx); + if (slave_image->locked == 0) + printk(KERN_ERR "Image is already free\n"); + + slave_image->locked = 0; + mutex_unlock(&slave_image->mtx); + + /* Free up resource memory */ + kfree(resource); +} +EXPORT_SYMBOL(vme_slave_free); + +/** + * vme_master_request - Request a VME master window resource. + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * @address: Required VME address space. + * @cycle: Required VME data transfer cycle type. + * @dwidth: Required VME data transfer width. + * + * Request use of a VME window resource capable of being set for the requested + * address space, data transfer cycle and width. + * + * Return: Pointer to VME resource on success, NULL on failure. + */ +struct vme_resource *vme_master_request(struct vme_dev *vdev, u32 address, + u32 cycle, u32 dwidth) +{ + struct vme_bridge *bridge; + struct list_head *master_pos = NULL; + struct vme_master_resource *allocated_image = NULL; + struct vme_master_resource *master_image = NULL; + struct vme_resource *resource = NULL; + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + goto err_bus; + } + + /* Loop through master resources */ + list_for_each(master_pos, &bridge->master_resources) { + master_image = list_entry(master_pos, + struct vme_master_resource, list); + + if (!master_image) { + printk(KERN_WARNING "Registered NULL master resource\n"); + continue; + } + + /* Find an unlocked and compatible image */ + spin_lock(&master_image->lock); + if (((master_image->address_attr & address) == address) && + ((master_image->cycle_attr & cycle) == cycle) && + ((master_image->width_attr & dwidth) == dwidth) && + (master_image->locked == 0)) { + + master_image->locked = 1; + spin_unlock(&master_image->lock); + allocated_image = master_image; + break; + } + spin_unlock(&master_image->lock); + } + + /* Check to see if we found a resource */ + if (!allocated_image) { + printk(KERN_ERR "Can't find a suitable resource\n"); + goto err_image; + } + + resource = kmalloc(sizeof(*resource), GFP_KERNEL); + if (!resource) + goto err_alloc; + + resource->type = VME_MASTER; + resource->entry = &allocated_image->list; + + return resource; + +err_alloc: + /* Unlock image */ + spin_lock(&master_image->lock); + master_image->locked = 0; + spin_unlock(&master_image->lock); +err_image: +err_bus: + return NULL; +} +EXPORT_SYMBOL(vme_master_request); + +/** + * vme_master_set - Set VME master window configuration. + * @resource: Pointer to VME master resource. + * @enabled: State to which the window should be configured. + * @vme_base: Base address for the window. + * @size: Size of the VME window. + * @aspace: VME address space for the VME window. + * @cycle: VME data transfer cycle type for the VME window. + * @dwidth: VME data transfer width for the VME window. + * + * Set configuration for provided VME master window. + * + * Return: Zero on success, -EINVAL if operation is not supported on this + * device, if an invalid resource has been provided or invalid + * attributes are provided. Hardware specific errors may also be + * returned. + */ +int vme_master_set(struct vme_resource *resource, int enabled, + unsigned long long vme_base, unsigned long long size, u32 aspace, + u32 cycle, u32 dwidth) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_master_resource *image; + int retval; + + if (resource->type != VME_MASTER) { + printk(KERN_ERR "Not a master resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_master_resource, list); + + if (!bridge->master_set) { + printk(KERN_WARNING "vme_master_set not supported\n"); + return -EINVAL; + } + + if (!(((image->address_attr & aspace) == aspace) && + ((image->cycle_attr & cycle) == cycle) && + ((image->width_attr & dwidth) == dwidth))) { + printk(KERN_WARNING "Invalid attributes\n"); + return -EINVAL; + } + + retval = vme_check_window(aspace, vme_base, size); + if (retval) + return retval; + + return bridge->master_set(image, enabled, vme_base, size, aspace, + cycle, dwidth); +} +EXPORT_SYMBOL(vme_master_set); + +/** + * vme_master_get - Retrieve VME master window configuration. + * @resource: Pointer to VME master resource. + * @enabled: Pointer to variable for storing state. + * @vme_base: Pointer to variable for storing window base address. + * @size: Pointer to variable for storing window size. + * @aspace: Pointer to variable for storing VME address space. + * @cycle: Pointer to variable for storing VME data transfer cycle type. + * @dwidth: Pointer to variable for storing VME data transfer width. + * + * Return configuration for provided VME master window. + * + * Return: Zero on success, -EINVAL if operation is not supported on this + * device or if an invalid resource has been provided. + */ +int vme_master_get(struct vme_resource *resource, int *enabled, + unsigned long long *vme_base, unsigned long long *size, u32 *aspace, + u32 *cycle, u32 *dwidth) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_master_resource *image; + + if (resource->type != VME_MASTER) { + printk(KERN_ERR "Not a master resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_master_resource, list); + + if (!bridge->master_get) { + printk(KERN_WARNING "%s not supported\n", __func__); + return -EINVAL; + } + + return bridge->master_get(image, enabled, vme_base, size, aspace, + cycle, dwidth); +} +EXPORT_SYMBOL(vme_master_get); + +/** + * vme_master_read - Read data from VME space into a buffer. + * @resource: Pointer to VME master resource. + * @buf: Pointer to buffer where data should be transferred. + * @count: Number of bytes to transfer. + * @offset: Offset into VME master window at which to start transfer. + * + * Perform read of count bytes of data from location on VME bus which maps into + * the VME master window at offset to buf. + * + * Return: Number of bytes read, -EINVAL if resource is not a VME master + * resource or read operation is not supported. -EFAULT returned if + * invalid offset is provided. Hardware specific errors may also be + * returned. + */ +ssize_t vme_master_read(struct vme_resource *resource, void *buf, size_t count, + loff_t offset) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_master_resource *image; + size_t length; + + if (!bridge->master_read) { + printk(KERN_WARNING "Reading from resource not supported\n"); + return -EINVAL; + } + + if (resource->type != VME_MASTER) { + printk(KERN_ERR "Not a master resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_master_resource, list); + + length = vme_get_size(resource); + + if (offset > length) { + printk(KERN_WARNING "Invalid Offset\n"); + return -EFAULT; + } + + if ((offset + count) > length) + count = length - offset; + + return bridge->master_read(image, buf, count, offset); + +} +EXPORT_SYMBOL(vme_master_read); + +/** + * vme_master_write - Write data out to VME space from a buffer. + * @resource: Pointer to VME master resource. + * @buf: Pointer to buffer holding data to transfer. + * @count: Number of bytes to transfer. + * @offset: Offset into VME master window at which to start transfer. + * + * Perform write of count bytes of data from buf to location on VME bus which + * maps into the VME master window at offset. + * + * Return: Number of bytes written, -EINVAL if resource is not a VME master + * resource or write operation is not supported. -EFAULT returned if + * invalid offset is provided. Hardware specific errors may also be + * returned. + */ +ssize_t vme_master_write(struct vme_resource *resource, void *buf, + size_t count, loff_t offset) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_master_resource *image; + size_t length; + + if (!bridge->master_write) { + printk(KERN_WARNING "Writing to resource not supported\n"); + return -EINVAL; + } + + if (resource->type != VME_MASTER) { + printk(KERN_ERR "Not a master resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_master_resource, list); + + length = vme_get_size(resource); + + if (offset > length) { + printk(KERN_WARNING "Invalid Offset\n"); + return -EFAULT; + } + + if ((offset + count) > length) + count = length - offset; + + return bridge->master_write(image, buf, count, offset); +} +EXPORT_SYMBOL(vme_master_write); + +/** + * vme_master_rmw - Perform read-modify-write cycle. + * @resource: Pointer to VME master resource. + * @mask: Bits to be compared and swapped in operation. + * @compare: Bits to be compared with data read from offset. + * @swap: Bits to be swapped in data read from offset. + * @offset: Offset into VME master window at which to perform operation. + * + * Perform read-modify-write cycle on provided location: + * - Location on VME bus is read. + * - Bits selected by mask are compared with compare. + * - Where a selected bit matches that in compare and are selected in swap, + * the bit is swapped. + * - Result written back to location on VME bus. + * + * Return: Bytes written on success, -EINVAL if resource is not a VME master + * resource or RMW operation is not supported. Hardware specific + * errors may also be returned. + */ +unsigned int vme_master_rmw(struct vme_resource *resource, unsigned int mask, + unsigned int compare, unsigned int swap, loff_t offset) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_master_resource *image; + + if (!bridge->master_rmw) { + printk(KERN_WARNING "Writing to resource not supported\n"); + return -EINVAL; + } + + if (resource->type != VME_MASTER) { + printk(KERN_ERR "Not a master resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_master_resource, list); + + return bridge->master_rmw(image, mask, compare, swap, offset); +} +EXPORT_SYMBOL(vme_master_rmw); + +/** + * vme_master_mmap - Mmap region of VME master window. + * @resource: Pointer to VME master resource. + * @vma: Pointer to definition of user mapping. + * + * Memory map a region of the VME master window into user space. + * + * Return: Zero on success, -EINVAL if resource is not a VME master + * resource or -EFAULT if map exceeds window size. Other generic mmap + * errors may also be returned. + */ +int vme_master_mmap(struct vme_resource *resource, struct vm_area_struct *vma) +{ + struct vme_master_resource *image; + phys_addr_t phys_addr; + unsigned long vma_size; + + if (resource->type != VME_MASTER) { + pr_err("Not a master resource\n"); + return -EINVAL; + } + + image = list_entry(resource->entry, struct vme_master_resource, list); + phys_addr = image->bus_resource.start + (vma->vm_pgoff << PAGE_SHIFT); + vma_size = vma->vm_end - vma->vm_start; + + if (phys_addr + vma_size > image->bus_resource.end + 1) { + pr_err("Map size cannot exceed the window size\n"); + return -EFAULT; + } + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + return vm_iomap_memory(vma, phys_addr, vma->vm_end - vma->vm_start); +} +EXPORT_SYMBOL(vme_master_mmap); + +/** + * vme_master_free - Free VME master window + * @resource: Pointer to VME master resource. + * + * Free the provided master resource so that it may be reallocated. + */ +void vme_master_free(struct vme_resource *resource) +{ + struct vme_master_resource *master_image; + + if (resource->type != VME_MASTER) { + printk(KERN_ERR "Not a master resource\n"); + return; + } + + master_image = list_entry(resource->entry, struct vme_master_resource, + list); + if (!master_image) { + printk(KERN_ERR "Can't find master resource\n"); + return; + } + + /* Unlock image */ + spin_lock(&master_image->lock); + if (master_image->locked == 0) + printk(KERN_ERR "Image is already free\n"); + + master_image->locked = 0; + spin_unlock(&master_image->lock); + + /* Free up resource memory */ + kfree(resource); +} +EXPORT_SYMBOL(vme_master_free); + +/** + * vme_dma_request - Request a DMA controller. + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * @route: Required src/destination combination. + * + * Request a VME DMA controller with capability to perform transfers bewteen + * requested source/destination combination. + * + * Return: Pointer to VME DMA resource on success, NULL on failure. + */ +struct vme_resource *vme_dma_request(struct vme_dev *vdev, u32 route) +{ + struct vme_bridge *bridge; + struct list_head *dma_pos = NULL; + struct vme_dma_resource *allocated_ctrlr = NULL; + struct vme_dma_resource *dma_ctrlr = NULL; + struct vme_resource *resource = NULL; + + /* XXX Not checking resource attributes */ + printk(KERN_ERR "No VME resource Attribute tests done\n"); + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + goto err_bus; + } + + /* Loop through DMA resources */ + list_for_each(dma_pos, &bridge->dma_resources) { + dma_ctrlr = list_entry(dma_pos, + struct vme_dma_resource, list); + if (!dma_ctrlr) { + printk(KERN_ERR "Registered NULL DMA resource\n"); + continue; + } + + /* Find an unlocked and compatible controller */ + mutex_lock(&dma_ctrlr->mtx); + if (((dma_ctrlr->route_attr & route) == route) && + (dma_ctrlr->locked == 0)) { + + dma_ctrlr->locked = 1; + mutex_unlock(&dma_ctrlr->mtx); + allocated_ctrlr = dma_ctrlr; + break; + } + mutex_unlock(&dma_ctrlr->mtx); + } + + /* Check to see if we found a resource */ + if (!allocated_ctrlr) + goto err_ctrlr; + + resource = kmalloc(sizeof(*resource), GFP_KERNEL); + if (!resource) + goto err_alloc; + + resource->type = VME_DMA; + resource->entry = &allocated_ctrlr->list; + + return resource; + +err_alloc: + /* Unlock image */ + mutex_lock(&dma_ctrlr->mtx); + dma_ctrlr->locked = 0; + mutex_unlock(&dma_ctrlr->mtx); +err_ctrlr: +err_bus: + return NULL; +} +EXPORT_SYMBOL(vme_dma_request); + +/** + * vme_new_dma_list - Create new VME DMA list. + * @resource: Pointer to VME DMA resource. + * + * Create a new VME DMA list. It is the responsibility of the user to free + * the list once it is no longer required with vme_dma_list_free(). + * + * Return: Pointer to new VME DMA list, NULL on allocation failure or invalid + * VME DMA resource. + */ +struct vme_dma_list *vme_new_dma_list(struct vme_resource *resource) +{ + struct vme_dma_list *dma_list; + + if (resource->type != VME_DMA) { + printk(KERN_ERR "Not a DMA resource\n"); + return NULL; + } + + dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL); + if (!dma_list) + return NULL; + + INIT_LIST_HEAD(&dma_list->entries); + dma_list->parent = list_entry(resource->entry, + struct vme_dma_resource, + list); + mutex_init(&dma_list->mtx); + + return dma_list; +} +EXPORT_SYMBOL(vme_new_dma_list); + +/** + * vme_dma_pattern_attribute - Create "Pattern" type VME DMA list attribute. + * @pattern: Value to use used as pattern + * @type: Type of pattern to be written. + * + * Create VME DMA list attribute for pattern generation. It is the + * responsibility of the user to free used attributes using + * vme_dma_free_attribute(). + * + * Return: Pointer to VME DMA attribute, NULL on failure. + */ +struct vme_dma_attr *vme_dma_pattern_attribute(u32 pattern, u32 type) +{ + struct vme_dma_attr *attributes; + struct vme_dma_pattern *pattern_attr; + + attributes = kmalloc(sizeof(*attributes), GFP_KERNEL); + if (!attributes) + goto err_attr; + + pattern_attr = kmalloc(sizeof(*pattern_attr), GFP_KERNEL); + if (!pattern_attr) + goto err_pat; + + attributes->type = VME_DMA_PATTERN; + attributes->private = (void *)pattern_attr; + + pattern_attr->pattern = pattern; + pattern_attr->type = type; + + return attributes; + +err_pat: + kfree(attributes); +err_attr: + return NULL; +} +EXPORT_SYMBOL(vme_dma_pattern_attribute); + +/** + * vme_dma_pci_attribute - Create "PCI" type VME DMA list attribute. + * @address: PCI base address for DMA transfer. + * + * Create VME DMA list attribute pointing to a location on PCI for DMA + * transfers. It is the responsibility of the user to free used attributes + * using vme_dma_free_attribute(). + * + * Return: Pointer to VME DMA attribute, NULL on failure. + */ +struct vme_dma_attr *vme_dma_pci_attribute(dma_addr_t address) +{ + struct vme_dma_attr *attributes; + struct vme_dma_pci *pci_attr; + + /* XXX Run some sanity checks here */ + + attributes = kmalloc(sizeof(*attributes), GFP_KERNEL); + if (!attributes) + goto err_attr; + + pci_attr = kmalloc(sizeof(*pci_attr), GFP_KERNEL); + if (!pci_attr) + goto err_pci; + + attributes->type = VME_DMA_PCI; + attributes->private = (void *)pci_attr; + + pci_attr->address = address; + + return attributes; + +err_pci: + kfree(attributes); +err_attr: + return NULL; +} +EXPORT_SYMBOL(vme_dma_pci_attribute); + +/** + * vme_dma_vme_attribute - Create "VME" type VME DMA list attribute. + * @address: VME base address for DMA transfer. + * @aspace: VME address space to use for DMA transfer. + * @cycle: VME bus cycle to use for DMA transfer. + * @dwidth: VME data width to use for DMA transfer. + * + * Create VME DMA list attribute pointing to a location on the VME bus for DMA + * transfers. It is the responsibility of the user to free used attributes + * using vme_dma_free_attribute(). + * + * Return: Pointer to VME DMA attribute, NULL on failure. + */ +struct vme_dma_attr *vme_dma_vme_attribute(unsigned long long address, + u32 aspace, u32 cycle, u32 dwidth) +{ + struct vme_dma_attr *attributes; + struct vme_dma_vme *vme_attr; + + attributes = kmalloc(sizeof(*attributes), GFP_KERNEL); + if (!attributes) + goto err_attr; + + vme_attr = kmalloc(sizeof(*vme_attr), GFP_KERNEL); + if (!vme_attr) + goto err_vme; + + attributes->type = VME_DMA_VME; + attributes->private = (void *)vme_attr; + + vme_attr->address = address; + vme_attr->aspace = aspace; + vme_attr->cycle = cycle; + vme_attr->dwidth = dwidth; + + return attributes; + +err_vme: + kfree(attributes); +err_attr: + return NULL; +} +EXPORT_SYMBOL(vme_dma_vme_attribute); + +/** + * vme_dma_free_attribute - Free DMA list attribute. + * @attributes: Pointer to DMA list attribute. + * + * Free VME DMA list attribute. VME DMA list attributes can be safely freed + * once vme_dma_list_add() has returned. + */ +void vme_dma_free_attribute(struct vme_dma_attr *attributes) +{ + kfree(attributes->private); + kfree(attributes); +} +EXPORT_SYMBOL(vme_dma_free_attribute); + +/** + * vme_dma_list_add - Add enty to a VME DMA list. + * @list: Pointer to VME list. + * @src: Pointer to DMA list attribute to use as source. + * @dest: Pointer to DMA list attribute to use as destination. + * @count: Number of bytes to transfer. + * + * Add an entry to the provided VME DMA list. Entry requires pointers to source + * and destination DMA attributes and a count. + * + * Please note, the attributes supported as source and destinations for + * transfers are hardware dependent. + * + * Return: Zero on success, -EINVAL if operation is not supported on this + * device or if the link list has already been submitted for execution. + * Hardware specific errors also possible. + */ +int vme_dma_list_add(struct vme_dma_list *list, struct vme_dma_attr *src, + struct vme_dma_attr *dest, size_t count) +{ + struct vme_bridge *bridge = list->parent->parent; + int retval; + + if (!bridge->dma_list_add) { + printk(KERN_WARNING "Link List DMA generation not supported\n"); + return -EINVAL; + } + + if (!mutex_trylock(&list->mtx)) { + printk(KERN_ERR "Link List already submitted\n"); + return -EINVAL; + } + + retval = bridge->dma_list_add(list, src, dest, count); + + mutex_unlock(&list->mtx); + + return retval; +} +EXPORT_SYMBOL(vme_dma_list_add); + +/** + * vme_dma_list_exec - Queue a VME DMA list for execution. + * @list: Pointer to VME list. + * + * Queue the provided VME DMA list for execution. The call will return once the + * list has been executed. + * + * Return: Zero on success, -EINVAL if operation is not supported on this + * device. Hardware specific errors also possible. + */ +int vme_dma_list_exec(struct vme_dma_list *list) +{ + struct vme_bridge *bridge = list->parent->parent; + int retval; + + if (!bridge->dma_list_exec) { + printk(KERN_ERR "Link List DMA execution not supported\n"); + return -EINVAL; + } + + mutex_lock(&list->mtx); + + retval = bridge->dma_list_exec(list); + + mutex_unlock(&list->mtx); + + return retval; +} +EXPORT_SYMBOL(vme_dma_list_exec); + +/** + * vme_dma_list_free - Free a VME DMA list. + * @list: Pointer to VME list. + * + * Free the provided DMA list and all its entries. + * + * Return: Zero on success, -EINVAL on invalid VME resource, -EBUSY if resource + * is still in use. Hardware specific errors also possible. + */ +int vme_dma_list_free(struct vme_dma_list *list) +{ + struct vme_bridge *bridge = list->parent->parent; + int retval; + + if (!bridge->dma_list_empty) { + printk(KERN_WARNING "Emptying of Link Lists not supported\n"); + return -EINVAL; + } + + if (!mutex_trylock(&list->mtx)) { + printk(KERN_ERR "Link List in use\n"); + return -EBUSY; + } + + /* + * Empty out all of the entries from the DMA list. We need to go to the + * low level driver as DMA entries are driver specific. + */ + retval = bridge->dma_list_empty(list); + if (retval) { + printk(KERN_ERR "Unable to empty link-list entries\n"); + mutex_unlock(&list->mtx); + return retval; + } + mutex_unlock(&list->mtx); + kfree(list); + + return retval; +} +EXPORT_SYMBOL(vme_dma_list_free); + +/** + * vme_dma_free - Free a VME DMA resource. + * @resource: Pointer to VME DMA resource. + * + * Free the provided DMA resource so that it may be reallocated. + * + * Return: Zero on success, -EINVAL on invalid VME resource, -EBUSY if resource + * is still active. + */ +int vme_dma_free(struct vme_resource *resource) +{ + struct vme_dma_resource *ctrlr; + + if (resource->type != VME_DMA) { + printk(KERN_ERR "Not a DMA resource\n"); + return -EINVAL; + } + + ctrlr = list_entry(resource->entry, struct vme_dma_resource, list); + + if (!mutex_trylock(&ctrlr->mtx)) { + printk(KERN_ERR "Resource busy, can't free\n"); + return -EBUSY; + } + + if (!(list_empty(&ctrlr->pending) && list_empty(&ctrlr->running))) { + printk(KERN_WARNING "Resource still processing transfers\n"); + mutex_unlock(&ctrlr->mtx); + return -EBUSY; + } + + ctrlr->locked = 0; + + mutex_unlock(&ctrlr->mtx); + + kfree(resource); + + return 0; +} +EXPORT_SYMBOL(vme_dma_free); + +void vme_bus_error_handler(struct vme_bridge *bridge, + unsigned long long address, int am) +{ + struct list_head *handler_pos = NULL; + struct vme_error_handler *handler; + int handler_triggered = 0; + u32 aspace = vme_get_aspace(am); + + list_for_each(handler_pos, &bridge->vme_error_handlers) { + handler = list_entry(handler_pos, struct vme_error_handler, + list); + if ((aspace == handler->aspace) && + (address >= handler->start) && + (address < handler->end)) { + if (!handler->num_errors) + handler->first_error = address; + if (handler->num_errors != UINT_MAX) + handler->num_errors++; + handler_triggered = 1; + } + } + + if (!handler_triggered) + dev_err(bridge->parent, + "Unhandled VME access error at address 0x%llx\n", + address); +} +EXPORT_SYMBOL(vme_bus_error_handler); + +struct vme_error_handler *vme_register_error_handler( + struct vme_bridge *bridge, u32 aspace, + unsigned long long address, size_t len) +{ + struct vme_error_handler *handler; + + handler = kmalloc(sizeof(*handler), GFP_ATOMIC); + if (!handler) + return NULL; + + handler->aspace = aspace; + handler->start = address; + handler->end = address + len; + handler->num_errors = 0; + handler->first_error = 0; + list_add_tail(&handler->list, &bridge->vme_error_handlers); + + return handler; +} +EXPORT_SYMBOL(vme_register_error_handler); + +void vme_unregister_error_handler(struct vme_error_handler *handler) +{ + list_del(&handler->list); + kfree(handler); +} +EXPORT_SYMBOL(vme_unregister_error_handler); + +void vme_irq_handler(struct vme_bridge *bridge, int level, int statid) +{ + void (*call)(int, int, void *); + void *priv_data; + + call = bridge->irq[level - 1].callback[statid].func; + priv_data = bridge->irq[level - 1].callback[statid].priv_data; + if (call) + call(level, statid, priv_data); + else + printk(KERN_WARNING "Spurious VME interrupt, level:%x, vector:%x\n", + level, statid); +} +EXPORT_SYMBOL(vme_irq_handler); + +/** + * vme_irq_request - Request a specific VME interrupt. + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * @level: Interrupt priority being requested. + * @statid: Interrupt vector being requested. + * @callback: Pointer to callback function called when VME interrupt/vector + * received. + * @priv_data: Generic pointer that will be passed to the callback function. + * + * Request callback to be attached as a handler for VME interrupts with provided + * level and statid. + * + * Return: Zero on success, -EINVAL on invalid vme device, level or if the + * function is not supported, -EBUSY if the level/statid combination is + * already in use. Hardware specific errors also possible. + */ +int vme_irq_request(struct vme_dev *vdev, int level, int statid, + void (*callback)(int, int, void *), + void *priv_data) +{ + struct vme_bridge *bridge; + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + return -EINVAL; + } + + if ((level < 1) || (level > 7)) { + printk(KERN_ERR "Invalid interrupt level\n"); + return -EINVAL; + } + + if (!bridge->irq_set) { + printk(KERN_ERR "Configuring interrupts not supported\n"); + return -EINVAL; + } + + mutex_lock(&bridge->irq_mtx); + + if (bridge->irq[level - 1].callback[statid].func) { + mutex_unlock(&bridge->irq_mtx); + printk(KERN_WARNING "VME Interrupt already taken\n"); + return -EBUSY; + } + + bridge->irq[level - 1].count++; + bridge->irq[level - 1].callback[statid].priv_data = priv_data; + bridge->irq[level - 1].callback[statid].func = callback; + + /* Enable IRQ level */ + bridge->irq_set(bridge, level, 1, 1); + + mutex_unlock(&bridge->irq_mtx); + + return 0; +} +EXPORT_SYMBOL(vme_irq_request); + +/** + * vme_irq_free - Free a VME interrupt. + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * @level: Interrupt priority of interrupt being freed. + * @statid: Interrupt vector of interrupt being freed. + * + * Remove previously attached callback from VME interrupt priority/vector. + */ +void vme_irq_free(struct vme_dev *vdev, int level, int statid) +{ + struct vme_bridge *bridge; + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + return; + } + + if ((level < 1) || (level > 7)) { + printk(KERN_ERR "Invalid interrupt level\n"); + return; + } + + if (!bridge->irq_set) { + printk(KERN_ERR "Configuring interrupts not supported\n"); + return; + } + + mutex_lock(&bridge->irq_mtx); + + bridge->irq[level - 1].count--; + + /* Disable IRQ level if no more interrupts attached at this level*/ + if (bridge->irq[level - 1].count == 0) + bridge->irq_set(bridge, level, 0, 1); + + bridge->irq[level - 1].callback[statid].func = NULL; + bridge->irq[level - 1].callback[statid].priv_data = NULL; + + mutex_unlock(&bridge->irq_mtx); +} +EXPORT_SYMBOL(vme_irq_free); + +/** + * vme_irq_generate - Generate VME interrupt. + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * @level: Interrupt priority at which to assert the interrupt. + * @statid: Interrupt vector to associate with the interrupt. + * + * Generate a VME interrupt of the provided level and with the provided + * statid. + * + * Return: Zero on success, -EINVAL on invalid vme device, level or if the + * function is not supported. Hardware specific errors also possible. + */ +int vme_irq_generate(struct vme_dev *vdev, int level, int statid) +{ + struct vme_bridge *bridge; + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + return -EINVAL; + } + + if ((level < 1) || (level > 7)) { + printk(KERN_WARNING "Invalid interrupt level\n"); + return -EINVAL; + } + + if (!bridge->irq_generate) { + printk(KERN_WARNING "Interrupt generation not supported\n"); + return -EINVAL; + } + + return bridge->irq_generate(bridge, level, statid); +} +EXPORT_SYMBOL(vme_irq_generate); + +/** + * vme_lm_request - Request a VME location monitor + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * + * Allocate a location monitor resource to the driver. A location monitor + * allows the driver to monitor accesses to a contiguous number of + * addresses on the VME bus. + * + * Return: Pointer to a VME resource on success or NULL on failure. + */ +struct vme_resource *vme_lm_request(struct vme_dev *vdev) +{ + struct vme_bridge *bridge; + struct list_head *lm_pos = NULL; + struct vme_lm_resource *allocated_lm = NULL; + struct vme_lm_resource *lm = NULL; + struct vme_resource *resource = NULL; + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + goto err_bus; + } + + /* Loop through LM resources */ + list_for_each(lm_pos, &bridge->lm_resources) { + lm = list_entry(lm_pos, + struct vme_lm_resource, list); + if (!lm) { + printk(KERN_ERR "Registered NULL Location Monitor resource\n"); + continue; + } + + /* Find an unlocked controller */ + mutex_lock(&lm->mtx); + if (lm->locked == 0) { + lm->locked = 1; + mutex_unlock(&lm->mtx); + allocated_lm = lm; + break; + } + mutex_unlock(&lm->mtx); + } + + /* Check to see if we found a resource */ + if (!allocated_lm) + goto err_lm; + + resource = kmalloc(sizeof(*resource), GFP_KERNEL); + if (!resource) + goto err_alloc; + + resource->type = VME_LM; + resource->entry = &allocated_lm->list; + + return resource; + +err_alloc: + /* Unlock image */ + mutex_lock(&lm->mtx); + lm->locked = 0; + mutex_unlock(&lm->mtx); +err_lm: +err_bus: + return NULL; +} +EXPORT_SYMBOL(vme_lm_request); + +/** + * vme_lm_count - Determine number of VME Addresses monitored + * @resource: Pointer to VME location monitor resource. + * + * The number of contiguous addresses monitored is hardware dependent. + * Return the number of contiguous addresses monitored by the + * location monitor. + * + * Return: Count of addresses monitored or -EINVAL when provided with an + * invalid location monitor resource. + */ +int vme_lm_count(struct vme_resource *resource) +{ + struct vme_lm_resource *lm; + + if (resource->type != VME_LM) { + printk(KERN_ERR "Not a Location Monitor resource\n"); + return -EINVAL; + } + + lm = list_entry(resource->entry, struct vme_lm_resource, list); + + return lm->monitors; +} +EXPORT_SYMBOL(vme_lm_count); + +/** + * vme_lm_set - Configure location monitor + * @resource: Pointer to VME location monitor resource. + * @lm_base: Base address to monitor. + * @aspace: VME address space to monitor. + * @cycle: VME bus cycle type to monitor. + * + * Set the base address, address space and cycle type of accesses to be + * monitored by the location monitor. + * + * Return: Zero on success, -EINVAL when provided with an invalid location + * monitor resource or function is not supported. Hardware specific + * errors may also be returned. + */ +int vme_lm_set(struct vme_resource *resource, unsigned long long lm_base, + u32 aspace, u32 cycle) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_lm_resource *lm; + + if (resource->type != VME_LM) { + printk(KERN_ERR "Not a Location Monitor resource\n"); + return -EINVAL; + } + + lm = list_entry(resource->entry, struct vme_lm_resource, list); + + if (!bridge->lm_set) { + printk(KERN_ERR "vme_lm_set not supported\n"); + return -EINVAL; + } + + return bridge->lm_set(lm, lm_base, aspace, cycle); +} +EXPORT_SYMBOL(vme_lm_set); + +/** + * vme_lm_get - Retrieve location monitor settings + * @resource: Pointer to VME location monitor resource. + * @lm_base: Pointer used to output the base address monitored. + * @aspace: Pointer used to output the address space monitored. + * @cycle: Pointer used to output the VME bus cycle type monitored. + * + * Retrieve the base address, address space and cycle type of accesses to + * be monitored by the location monitor. + * + * Return: Zero on success, -EINVAL when provided with an invalid location + * monitor resource or function is not supported. Hardware specific + * errors may also be returned. + */ +int vme_lm_get(struct vme_resource *resource, unsigned long long *lm_base, + u32 *aspace, u32 *cycle) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_lm_resource *lm; + + if (resource->type != VME_LM) { + printk(KERN_ERR "Not a Location Monitor resource\n"); + return -EINVAL; + } + + lm = list_entry(resource->entry, struct vme_lm_resource, list); + + if (!bridge->lm_get) { + printk(KERN_ERR "vme_lm_get not supported\n"); + return -EINVAL; + } + + return bridge->lm_get(lm, lm_base, aspace, cycle); +} +EXPORT_SYMBOL(vme_lm_get); + +/** + * vme_lm_attach - Provide callback for location monitor address + * @resource: Pointer to VME location monitor resource. + * @monitor: Offset to which callback should be attached. + * @callback: Pointer to callback function called when triggered. + * @data: Generic pointer that will be passed to the callback function. + * + * Attach a callback to the specificed offset into the location monitors + * monitored addresses. A generic pointer is provided to allow data to be + * passed to the callback when called. + * + * Return: Zero on success, -EINVAL when provided with an invalid location + * monitor resource or function is not supported. Hardware specific + * errors may also be returned. + */ +int vme_lm_attach(struct vme_resource *resource, int monitor, + void (*callback)(void *), void *data) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_lm_resource *lm; + + if (resource->type != VME_LM) { + printk(KERN_ERR "Not a Location Monitor resource\n"); + return -EINVAL; + } + + lm = list_entry(resource->entry, struct vme_lm_resource, list); + + if (!bridge->lm_attach) { + printk(KERN_ERR "vme_lm_attach not supported\n"); + return -EINVAL; + } + + return bridge->lm_attach(lm, monitor, callback, data); +} +EXPORT_SYMBOL(vme_lm_attach); + +/** + * vme_lm_detach - Remove callback for location monitor address + * @resource: Pointer to VME location monitor resource. + * @monitor: Offset to which callback should be removed. + * + * Remove the callback associated with the specificed offset into the + * location monitors monitored addresses. + * + * Return: Zero on success, -EINVAL when provided with an invalid location + * monitor resource or function is not supported. Hardware specific + * errors may also be returned. + */ +int vme_lm_detach(struct vme_resource *resource, int monitor) +{ + struct vme_bridge *bridge = find_bridge(resource); + struct vme_lm_resource *lm; + + if (resource->type != VME_LM) { + printk(KERN_ERR "Not a Location Monitor resource\n"); + return -EINVAL; + } + + lm = list_entry(resource->entry, struct vme_lm_resource, list); + + if (!bridge->lm_detach) { + printk(KERN_ERR "vme_lm_detach not supported\n"); + return -EINVAL; + } + + return bridge->lm_detach(lm, monitor); +} +EXPORT_SYMBOL(vme_lm_detach); + +/** + * vme_lm_free - Free allocated VME location monitor + * @resource: Pointer to VME location monitor resource. + * + * Free allocation of a VME location monitor. + * + * WARNING: This function currently expects that any callbacks that have + * been attached to the location monitor have been removed. + * + * Return: Zero on success, -EINVAL when provided with an invalid location + * monitor resource. + */ +void vme_lm_free(struct vme_resource *resource) +{ + struct vme_lm_resource *lm; + + if (resource->type != VME_LM) { + printk(KERN_ERR "Not a Location Monitor resource\n"); + return; + } + + lm = list_entry(resource->entry, struct vme_lm_resource, list); + + mutex_lock(&lm->mtx); + + /* XXX + * Check to see that there aren't any callbacks still attached, if + * there are we should probably be detaching them! + */ + + lm->locked = 0; + + mutex_unlock(&lm->mtx); + + kfree(resource); +} +EXPORT_SYMBOL(vme_lm_free); + +/** + * vme_slot_num - Retrieve slot ID + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * + * Retrieve the slot ID associated with the provided VME device. + * + * Return: The slot ID on success, -EINVAL if VME bridge cannot be determined + * or the function is not supported. Hardware specific errors may also + * be returned. + */ +int vme_slot_num(struct vme_dev *vdev) +{ + struct vme_bridge *bridge; + + bridge = vdev->bridge; + if (!bridge) { + printk(KERN_ERR "Can't find VME bus\n"); + return -EINVAL; + } + + if (!bridge->slot_get) { + printk(KERN_WARNING "vme_slot_num not supported\n"); + return -EINVAL; + } + + return bridge->slot_get(bridge); +} +EXPORT_SYMBOL(vme_slot_num); + +/** + * vme_bus_num - Retrieve bus number + * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. + * + * Retrieve the bus enumeration associated with the provided VME device. + * + * Return: The bus number on success, -EINVAL if VME bridge cannot be + * determined. + */ +int vme_bus_num(struct vme_dev *vdev) +{ + struct vme_bridge *bridge; + + bridge = vdev->bridge; + if (!bridge) { + pr_err("Can't find VME bus\n"); + return -EINVAL; + } + + return bridge->num; +} +EXPORT_SYMBOL(vme_bus_num); + +/* - Bridge Registration --------------------------------------------------- */ + +static void vme_dev_release(struct device *dev) +{ + kfree(dev_to_vme_dev(dev)); +} + +/* Common bridge initialization */ +struct vme_bridge *vme_init_bridge(struct vme_bridge *bridge) +{ + INIT_LIST_HEAD(&bridge->vme_error_handlers); + INIT_LIST_HEAD(&bridge->master_resources); + INIT_LIST_HEAD(&bridge->slave_resources); + INIT_LIST_HEAD(&bridge->dma_resources); + INIT_LIST_HEAD(&bridge->lm_resources); + mutex_init(&bridge->irq_mtx); + + return bridge; +} +EXPORT_SYMBOL(vme_init_bridge); + +int vme_register_bridge(struct vme_bridge *bridge) +{ + int i; + int ret = -1; + + mutex_lock(&vme_buses_lock); + for (i = 0; i < sizeof(vme_bus_numbers) * 8; i++) { + if ((vme_bus_numbers & (1 << i)) == 0) { + vme_bus_numbers |= (1 << i); + bridge->num = i; + INIT_LIST_HEAD(&bridge->devices); + list_add_tail(&bridge->bus_list, &vme_bus_list); + ret = 0; + break; + } + } + mutex_unlock(&vme_buses_lock); + + return ret; +} +EXPORT_SYMBOL(vme_register_bridge); + +void vme_unregister_bridge(struct vme_bridge *bridge) +{ + struct vme_dev *vdev; + struct vme_dev *tmp; + + mutex_lock(&vme_buses_lock); + vme_bus_numbers &= ~(1 << bridge->num); + list_for_each_entry_safe(vdev, tmp, &bridge->devices, bridge_list) { + list_del(&vdev->drv_list); + list_del(&vdev->bridge_list); + device_unregister(&vdev->dev); + } + list_del(&bridge->bus_list); + mutex_unlock(&vme_buses_lock); +} +EXPORT_SYMBOL(vme_unregister_bridge); + +/* - Driver Registration --------------------------------------------------- */ + +static int __vme_register_driver_bus(struct vme_driver *drv, + struct vme_bridge *bridge, unsigned int ndevs) +{ + int err; + unsigned int i; + struct vme_dev *vdev; + struct vme_dev *tmp; + + for (i = 0; i < ndevs; i++) { + vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); + if (!vdev) { + err = -ENOMEM; + goto err_devalloc; + } + vdev->num = i; + vdev->bridge = bridge; + vdev->dev.platform_data = drv; + vdev->dev.release = vme_dev_release; + vdev->dev.parent = bridge->parent; + vdev->dev.bus = &vme_bus_type; + dev_set_name(&vdev->dev, "%s.%u-%u", drv->name, bridge->num, + vdev->num); + + err = device_register(&vdev->dev); + if (err) + goto err_reg; + + if (vdev->dev.platform_data) { + list_add_tail(&vdev->drv_list, &drv->devices); + list_add_tail(&vdev->bridge_list, &bridge->devices); + } else + device_unregister(&vdev->dev); + } + return 0; + +err_reg: + put_device(&vdev->dev); +err_devalloc: + list_for_each_entry_safe(vdev, tmp, &drv->devices, drv_list) { + list_del(&vdev->drv_list); + list_del(&vdev->bridge_list); + device_unregister(&vdev->dev); + } + return err; +} + +static int __vme_register_driver(struct vme_driver *drv, unsigned int ndevs) +{ + struct vme_bridge *bridge; + int err = 0; + + mutex_lock(&vme_buses_lock); + list_for_each_entry(bridge, &vme_bus_list, bus_list) { + /* + * This cannot cause trouble as we already have vme_buses_lock + * and if the bridge is removed, it will have to go through + * vme_unregister_bridge() to do it (which calls remove() on + * the bridge which in turn tries to acquire vme_buses_lock and + * will have to wait). + */ + err = __vme_register_driver_bus(drv, bridge, ndevs); + if (err) + break; + } + mutex_unlock(&vme_buses_lock); + return err; +} + +/** + * vme_register_driver - Register a VME driver + * @drv: Pointer to VME driver structure to register. + * @ndevs: Maximum number of devices to allow to be enumerated. + * + * Register a VME device driver with the VME subsystem. + * + * Return: Zero on success, error value on registration failure. + */ +int vme_register_driver(struct vme_driver *drv, unsigned int ndevs) +{ + int err; + + drv->driver.name = drv->name; + drv->driver.bus = &vme_bus_type; + INIT_LIST_HEAD(&drv->devices); + + err = driver_register(&drv->driver); + if (err) + return err; + + err = __vme_register_driver(drv, ndevs); + if (err) + driver_unregister(&drv->driver); + + return err; +} +EXPORT_SYMBOL(vme_register_driver); + +/** + * vme_unregister_driver - Unregister a VME driver + * @drv: Pointer to VME driver structure to unregister. + * + * Unregister a VME device driver from the VME subsystem. + */ +void vme_unregister_driver(struct vme_driver *drv) +{ + struct vme_dev *dev, *dev_tmp; + + mutex_lock(&vme_buses_lock); + list_for_each_entry_safe(dev, dev_tmp, &drv->devices, drv_list) { + list_del(&dev->drv_list); + list_del(&dev->bridge_list); + device_unregister(&dev->dev); + } + mutex_unlock(&vme_buses_lock); + + driver_unregister(&drv->driver); +} +EXPORT_SYMBOL(vme_unregister_driver); + +/* - Bus Registration ------------------------------------------------------ */ + +static int vme_bus_match(struct device *dev, struct device_driver *drv) +{ + struct vme_driver *vme_drv; + + vme_drv = container_of(drv, struct vme_driver, driver); + + if (dev->platform_data == vme_drv) { + struct vme_dev *vdev = dev_to_vme_dev(dev); + + if (vme_drv->match && vme_drv->match(vdev)) + return 1; + + dev->platform_data = NULL; + } + return 0; +} + +static int vme_bus_probe(struct device *dev) +{ + struct vme_driver *driver; + struct vme_dev *vdev = dev_to_vme_dev(dev); + + driver = dev->platform_data; + if (driver->probe) + return driver->probe(vdev); + + return -ENODEV; +} + +static void vme_bus_remove(struct device *dev) +{ + struct vme_driver *driver; + struct vme_dev *vdev = dev_to_vme_dev(dev); + + driver = dev->platform_data; + if (driver->remove) + driver->remove(vdev); +} + +struct bus_type vme_bus_type = { + .name = "vme", + .match = vme_bus_match, + .probe = vme_bus_probe, + .remove = vme_bus_remove, +}; +EXPORT_SYMBOL(vme_bus_type); + +static int __init vme_init(void) +{ + return bus_register(&vme_bus_type); +} +subsys_initcall(vme_init); diff --git a/drivers/staging/vme_user/vme.h b/drivers/staging/vme_user/vme.h new file mode 100644 index 000000000000..b204a9b4be1b --- /dev/null +++ b/drivers/staging/vme_user/vme.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _VME_H_ +#define _VME_H_ + +/* Resource Type */ +enum vme_resource_type { + VME_MASTER, + VME_SLAVE, + VME_DMA, + VME_LM +}; + +/* VME Address Spaces */ +#define VME_A16 0x1 +#define VME_A24 0x2 +#define VME_A32 0x4 +#define VME_A64 0x8 +#define VME_CRCSR 0x10 +#define VME_USER1 0x20 +#define VME_USER2 0x40 +#define VME_USER3 0x80 +#define VME_USER4 0x100 + +#define VME_A16_MAX 0x10000ULL +#define VME_A24_MAX 0x1000000ULL +#define VME_A32_MAX 0x100000000ULL +#define VME_A64_MAX 0x10000000000000000ULL +#define VME_CRCSR_MAX 0x1000000ULL + + +/* VME Cycle Types */ +#define VME_SCT 0x1 +#define VME_BLT 0x2 +#define VME_MBLT 0x4 +#define VME_2eVME 0x8 +#define VME_2eSST 0x10 +#define VME_2eSSTB 0x20 + +#define VME_2eSST160 0x100 +#define VME_2eSST267 0x200 +#define VME_2eSST320 0x400 + +#define VME_SUPER 0x1000 +#define VME_USER 0x2000 +#define VME_PROG 0x4000 +#define VME_DATA 0x8000 + +/* VME Data Widths */ +#define VME_D8 0x1 +#define VME_D16 0x2 +#define VME_D32 0x4 +#define VME_D64 0x8 + +/* Arbitration Scheduling Modes */ +#define VME_R_ROBIN_MODE 0x1 +#define VME_PRIORITY_MODE 0x2 + +#define VME_DMA_PATTERN (1<<0) +#define VME_DMA_PCI (1<<1) +#define VME_DMA_VME (1<<2) + +#define VME_DMA_PATTERN_BYTE (1<<0) +#define VME_DMA_PATTERN_WORD (1<<1) +#define VME_DMA_PATTERN_INCREMENT (1<<2) + +#define VME_DMA_VME_TO_MEM (1<<0) +#define VME_DMA_MEM_TO_VME (1<<1) +#define VME_DMA_VME_TO_VME (1<<2) +#define VME_DMA_MEM_TO_MEM (1<<3) +#define VME_DMA_PATTERN_TO_VME (1<<4) +#define VME_DMA_PATTERN_TO_MEM (1<<5) + +struct vme_dma_attr { + u32 type; + void *private; +}; + +struct vme_resource { + enum vme_resource_type type; + struct list_head *entry; +}; + +extern struct bus_type vme_bus_type; + +/* Number of VME interrupt vectors */ +#define VME_NUM_STATUSID 256 + +/* VME_MAX_BRIDGES comes from the type of vme_bus_numbers */ +#define VME_MAX_BRIDGES (sizeof(unsigned int)*8) +#define VME_MAX_SLOTS 32 + +#define VME_SLOT_CURRENT -1 +#define VME_SLOT_ALL -2 + +/** + * struct vme_dev - Structure representing a VME device + * @num: The device number + * @bridge: Pointer to the bridge device this device is on + * @dev: Internal device structure + * @drv_list: List of devices (per driver) + * @bridge_list: List of devices (per bridge) + */ +struct vme_dev { + int num; + struct vme_bridge *bridge; + struct device dev; + struct list_head drv_list; + struct list_head bridge_list; +}; + +/** + * struct vme_driver - Structure representing a VME driver + * @name: Driver name, should be unique among VME drivers and usually the same + * as the module name. + * @match: Callback used to determine whether probe should be run. + * @probe: Callback for device binding, called when new device is detected. + * @remove: Callback, called on device removal. + * @driver: Underlying generic device driver structure. + * @devices: List of VME devices (struct vme_dev) associated with this driver. + */ +struct vme_driver { + const char *name; + int (*match)(struct vme_dev *); + int (*probe)(struct vme_dev *); + void (*remove)(struct vme_dev *); + struct device_driver driver; + struct list_head devices; +}; + +void *vme_alloc_consistent(struct vme_resource *, size_t, dma_addr_t *); +void vme_free_consistent(struct vme_resource *, size_t, void *, + dma_addr_t); + +size_t vme_get_size(struct vme_resource *); +int vme_check_window(u32 aspace, unsigned long long vme_base, + unsigned long long size); + +struct vme_resource *vme_slave_request(struct vme_dev *, u32, u32); +int vme_slave_set(struct vme_resource *, int, unsigned long long, + unsigned long long, dma_addr_t, u32, u32); +int vme_slave_get(struct vme_resource *, int *, unsigned long long *, + unsigned long long *, dma_addr_t *, u32 *, u32 *); +void vme_slave_free(struct vme_resource *); + +struct vme_resource *vme_master_request(struct vme_dev *, u32, u32, u32); +int vme_master_set(struct vme_resource *, int, unsigned long long, + unsigned long long, u32, u32, u32); +int vme_master_get(struct vme_resource *, int *, unsigned long long *, + unsigned long long *, u32 *, u32 *, u32 *); +ssize_t vme_master_read(struct vme_resource *, void *, size_t, loff_t); +ssize_t vme_master_write(struct vme_resource *, void *, size_t, loff_t); +unsigned int vme_master_rmw(struct vme_resource *, unsigned int, unsigned int, + unsigned int, loff_t); +int vme_master_mmap(struct vme_resource *resource, struct vm_area_struct *vma); +void vme_master_free(struct vme_resource *); + +struct vme_resource *vme_dma_request(struct vme_dev *, u32); +struct vme_dma_list *vme_new_dma_list(struct vme_resource *); +struct vme_dma_attr *vme_dma_pattern_attribute(u32, u32); +struct vme_dma_attr *vme_dma_pci_attribute(dma_addr_t); +struct vme_dma_attr *vme_dma_vme_attribute(unsigned long long, u32, u32, u32); +void vme_dma_free_attribute(struct vme_dma_attr *); +int vme_dma_list_add(struct vme_dma_list *, struct vme_dma_attr *, + struct vme_dma_attr *, size_t); +int vme_dma_list_exec(struct vme_dma_list *); +int vme_dma_list_free(struct vme_dma_list *); +int vme_dma_free(struct vme_resource *); + +int vme_irq_request(struct vme_dev *, int, int, + void (*callback)(int, int, void *), void *); +void vme_irq_free(struct vme_dev *, int, int); +int vme_irq_generate(struct vme_dev *, int, int); + +struct vme_resource *vme_lm_request(struct vme_dev *); +int vme_lm_count(struct vme_resource *); +int vme_lm_set(struct vme_resource *, unsigned long long, u32, u32); +int vme_lm_get(struct vme_resource *, unsigned long long *, u32 *, u32 *); +int vme_lm_attach(struct vme_resource *, int, void (*callback)(void *), void *); +int vme_lm_detach(struct vme_resource *, int); +void vme_lm_free(struct vme_resource *); + +int vme_slot_num(struct vme_dev *); +int vme_bus_num(struct vme_dev *); + +int vme_register_driver(struct vme_driver *, unsigned int); +void vme_unregister_driver(struct vme_driver *); + + +#endif /* _VME_H_ */ + diff --git a/drivers/staging/vme_user/vme_bridge.h b/drivers/staging/vme_user/vme_bridge.h new file mode 100644 index 000000000000..0bbefe9851d7 --- /dev/null +++ b/drivers/staging/vme_user/vme_bridge.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _VME_BRIDGE_H_ +#define _VME_BRIDGE_H_ + +#include "vme.h" + +#define VME_CRCSR_BUF_SIZE (508*1024) +/* + * Resource structures + */ +struct vme_master_resource { + struct list_head list; + struct vme_bridge *parent; + /* + * We are likely to need to access the VME bus in interrupt context, so + * protect master routines with a spinlock rather than a mutex. + */ + spinlock_t lock; + int locked; + int number; + u32 address_attr; + u32 cycle_attr; + u32 width_attr; + struct resource bus_resource; + void __iomem *kern_base; +}; + +struct vme_slave_resource { + struct list_head list; + struct vme_bridge *parent; + struct mutex mtx; + int locked; + int number; + u32 address_attr; + u32 cycle_attr; +}; + +struct vme_dma_pattern { + u32 pattern; + u32 type; +}; + +struct vme_dma_pci { + dma_addr_t address; +}; + +struct vme_dma_vme { + unsigned long long address; + u32 aspace; + u32 cycle; + u32 dwidth; +}; + +struct vme_dma_list { + struct list_head list; + struct vme_dma_resource *parent; + struct list_head entries; + struct mutex mtx; +}; + +struct vme_dma_resource { + struct list_head list; + struct vme_bridge *parent; + struct mutex mtx; + int locked; + int number; + struct list_head pending; + struct list_head running; + u32 route_attr; +}; + +struct vme_lm_resource { + struct list_head list; + struct vme_bridge *parent; + struct mutex mtx; + int locked; + int number; + int monitors; +}; + +struct vme_error_handler { + struct list_head list; + unsigned long long start; /* Beginning of error window */ + unsigned long long end; /* End of error window */ + unsigned long long first_error; /* Address of the first error */ + u32 aspace; /* Address space of error window*/ + unsigned num_errors; /* Number of errors */ +}; + +struct vme_callback { + void (*func)(int, int, void*); + void *priv_data; +}; + +struct vme_irq { + int count; + struct vme_callback callback[VME_NUM_STATUSID]; +}; + +/* Allow 16 characters for name (including null character) */ +#define VMENAMSIZ 16 + +/* This structure stores all the information about one bridge + * The structure should be dynamically allocated by the driver and one instance + * of the structure should be present for each VME chip present in the system. + */ +struct vme_bridge { + char name[VMENAMSIZ]; + int num; + struct list_head master_resources; + struct list_head slave_resources; + struct list_head dma_resources; + struct list_head lm_resources; + + /* List for registered errors handlers */ + struct list_head vme_error_handlers; + /* List of devices on this bridge */ + struct list_head devices; + + /* Bridge Info - XXX Move to private structure? */ + struct device *parent; /* Parent device (eg. pdev->dev for PCI) */ + void *driver_priv; /* Private pointer for the bridge driver */ + struct list_head bus_list; /* list of VME buses */ + + /* Interrupt callbacks */ + struct vme_irq irq[7]; + /* Locking for VME irq callback configuration */ + struct mutex irq_mtx; + + /* Slave Functions */ + int (*slave_get) (struct vme_slave_resource *, int *, + unsigned long long *, unsigned long long *, dma_addr_t *, + u32 *, u32 *); + int (*slave_set) (struct vme_slave_resource *, int, unsigned long long, + unsigned long long, dma_addr_t, u32, u32); + + /* Master Functions */ + int (*master_get) (struct vme_master_resource *, int *, + unsigned long long *, unsigned long long *, u32 *, u32 *, + u32 *); + int (*master_set) (struct vme_master_resource *, int, + unsigned long long, unsigned long long, u32, u32, u32); + ssize_t (*master_read) (struct vme_master_resource *, void *, size_t, + loff_t); + ssize_t (*master_write) (struct vme_master_resource *, void *, size_t, + loff_t); + unsigned int (*master_rmw) (struct vme_master_resource *, unsigned int, + unsigned int, unsigned int, loff_t); + + /* DMA Functions */ + int (*dma_list_add) (struct vme_dma_list *, struct vme_dma_attr *, + struct vme_dma_attr *, size_t); + int (*dma_list_exec) (struct vme_dma_list *); + int (*dma_list_empty) (struct vme_dma_list *); + + /* Interrupt Functions */ + void (*irq_set) (struct vme_bridge *, int, int, int); + int (*irq_generate) (struct vme_bridge *, int, int); + + /* Location monitor functions */ + int (*lm_set) (struct vme_lm_resource *, unsigned long long, u32, u32); + int (*lm_get) (struct vme_lm_resource *, unsigned long long *, u32 *, + u32 *); + int (*lm_attach)(struct vme_lm_resource *, int, + void (*callback)(void *), void *); + int (*lm_detach) (struct vme_lm_resource *, int); + + /* CR/CSR space functions */ + int (*slot_get) (struct vme_bridge *); + + /* Bridge parent interface */ + void *(*alloc_consistent)(struct device *dev, size_t size, + dma_addr_t *dma); + void (*free_consistent)(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma); +}; + +void vme_bus_error_handler(struct vme_bridge *bridge, + unsigned long long address, int am); +void vme_irq_handler(struct vme_bridge *, int, int); + +struct vme_bridge *vme_init_bridge(struct vme_bridge *); +int vme_register_bridge(struct vme_bridge *); +void vme_unregister_bridge(struct vme_bridge *); +struct vme_error_handler *vme_register_error_handler( + struct vme_bridge *bridge, u32 aspace, + unsigned long long address, size_t len); +void vme_unregister_error_handler(struct vme_error_handler *handler); + +#endif /* _VME_BRIDGE_H_ */ diff --git a/drivers/staging/vme_user/vme_fake.c b/drivers/staging/vme_user/vme_fake.c new file mode 100644 index 000000000000..dd646b0c531d --- /dev/null +++ b/drivers/staging/vme_user/vme_fake.c @@ -0,0 +1,1305 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Fake VME bridge support. + * + * This drive provides a fake VME bridge chip, this enables debugging of the + * VME framework in the absence of a VME system. + * + * This driver has to do a number of things in software that would be driven + * by hardware if it was available, it will also result in extra overhead at + * times when compared with driving actual hardware. + * + * Author: Martyn Welch + * Copyright (c) 2014 Martyn Welch + * + * Based on vme_tsi148.c: + * + * Author: Martyn Welch + * Copyright 2008 GE Intelligent Platforms Embedded Systems, Inc. + * + * Based on work by Tom Armistead and Ajit Prem + * Copyright 2004 Motorola Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vme.h" +#include "vme_bridge.h" + +/* + * Define the number of each that the fake driver supports. + */ +#define FAKE_MAX_MASTER 8 /* Max Master Windows */ +#define FAKE_MAX_SLAVE 8 /* Max Slave Windows */ + +/* Structures to hold information normally held in device registers */ +struct fake_slave_window { + int enabled; + unsigned long long vme_base; + unsigned long long size; + void *buf_base; + u32 aspace; + u32 cycle; +}; + +struct fake_master_window { + int enabled; + unsigned long long vme_base; + unsigned long long size; + u32 aspace; + u32 cycle; + u32 dwidth; +}; + +/* Structure used to hold driver specific information */ +struct fake_driver { + struct vme_bridge *parent; + struct fake_slave_window slaves[FAKE_MAX_SLAVE]; + struct fake_master_window masters[FAKE_MAX_MASTER]; + u32 lm_enabled; + unsigned long long lm_base; + u32 lm_aspace; + u32 lm_cycle; + void (*lm_callback[4])(void *); + void *lm_data[4]; + struct tasklet_struct int_tasklet; + int int_level; + int int_statid; + void *crcsr_kernel; + dma_addr_t crcsr_bus; + /* Only one VME interrupt can be generated at a time, provide locking */ + struct mutex vme_int; +}; + +/* Module parameter */ +static int geoid; + +static const char driver_name[] = "vme_fake"; + +static struct vme_bridge *exit_pointer; + +static struct device *vme_root; + +/* + * Calling VME bus interrupt callback if provided. + */ +static void fake_VIRQ_tasklet(unsigned long data) +{ + struct vme_bridge *fake_bridge; + struct fake_driver *bridge; + + fake_bridge = (struct vme_bridge *) data; + bridge = fake_bridge->driver_priv; + + vme_irq_handler(fake_bridge, bridge->int_level, bridge->int_statid); +} + +/* + * Configure VME interrupt + */ +static void fake_irq_set(struct vme_bridge *fake_bridge, int level, + int state, int sync) +{ + /* Nothing to do */ +} + +static void *fake_pci_to_ptr(dma_addr_t addr) +{ + return (void *)(uintptr_t)addr; +} + +static dma_addr_t fake_ptr_to_pci(void *addr) +{ + return (dma_addr_t)(uintptr_t)addr; +} + +/* + * Generate a VME bus interrupt at the requested level & vector. Wait for + * interrupt to be acked. + */ +static int fake_irq_generate(struct vme_bridge *fake_bridge, int level, + int statid) +{ + struct fake_driver *bridge; + + bridge = fake_bridge->driver_priv; + + mutex_lock(&bridge->vme_int); + + bridge->int_level = level; + + bridge->int_statid = statid; + + /* + * Schedule tasklet to run VME handler to emulate normal VME interrupt + * handler behaviour. + */ + tasklet_schedule(&bridge->int_tasklet); + + mutex_unlock(&bridge->vme_int); + + return 0; +} + +/* + * Initialize a slave window with the requested attributes. + */ +static int fake_slave_set(struct vme_slave_resource *image, int enabled, + unsigned long long vme_base, unsigned long long size, + dma_addr_t buf_base, u32 aspace, u32 cycle) +{ + unsigned int i, granularity = 0; + unsigned long long vme_bound; + struct vme_bridge *fake_bridge; + struct fake_driver *bridge; + + fake_bridge = image->parent; + bridge = fake_bridge->driver_priv; + + i = image->number; + + switch (aspace) { + case VME_A16: + granularity = 0x10; + break; + case VME_A24: + granularity = 0x1000; + break; + case VME_A32: + granularity = 0x10000; + break; + case VME_A64: + granularity = 0x10000; + break; + case VME_CRCSR: + case VME_USER1: + case VME_USER2: + case VME_USER3: + case VME_USER4: + default: + pr_err("Invalid address space\n"); + return -EINVAL; + } + + /* + * Bound address is a valid address for the window, adjust + * accordingly + */ + vme_bound = vme_base + size - granularity; + + if (vme_base & (granularity - 1)) { + pr_err("Invalid VME base alignment\n"); + return -EINVAL; + } + if (vme_bound & (granularity - 1)) { + pr_err("Invalid VME bound alignment\n"); + return -EINVAL; + } + + mutex_lock(&image->mtx); + + bridge->slaves[i].enabled = enabled; + bridge->slaves[i].vme_base = vme_base; + bridge->slaves[i].size = size; + bridge->slaves[i].buf_base = fake_pci_to_ptr(buf_base); + bridge->slaves[i].aspace = aspace; + bridge->slaves[i].cycle = cycle; + + mutex_unlock(&image->mtx); + + return 0; +} + +/* + * Get slave window configuration. + */ +static int fake_slave_get(struct vme_slave_resource *image, int *enabled, + unsigned long long *vme_base, unsigned long long *size, + dma_addr_t *buf_base, u32 *aspace, u32 *cycle) +{ + unsigned int i; + struct fake_driver *bridge; + + bridge = image->parent->driver_priv; + + i = image->number; + + mutex_lock(&image->mtx); + + *enabled = bridge->slaves[i].enabled; + *vme_base = bridge->slaves[i].vme_base; + *size = bridge->slaves[i].size; + *buf_base = fake_ptr_to_pci(bridge->slaves[i].buf_base); + *aspace = bridge->slaves[i].aspace; + *cycle = bridge->slaves[i].cycle; + + mutex_unlock(&image->mtx); + + return 0; +} + +/* + * Set the attributes of an outbound window. + */ +static int fake_master_set(struct vme_master_resource *image, int enabled, + unsigned long long vme_base, unsigned long long size, + u32 aspace, u32 cycle, u32 dwidth) +{ + int retval = 0; + unsigned int i; + struct vme_bridge *fake_bridge; + struct fake_driver *bridge; + + fake_bridge = image->parent; + + bridge = fake_bridge->driver_priv; + + /* Verify input data */ + if (vme_base & 0xFFFF) { + pr_err("Invalid VME Window alignment\n"); + retval = -EINVAL; + goto err_window; + } + + if (size & 0xFFFF) { + pr_err("Invalid size alignment\n"); + retval = -EINVAL; + goto err_window; + } + + if ((size == 0) && (enabled != 0)) { + pr_err("Size must be non-zero for enabled windows\n"); + retval = -EINVAL; + goto err_window; + } + + /* Setup data width */ + switch (dwidth) { + case VME_D8: + case VME_D16: + case VME_D32: + break; + default: + pr_err("Invalid data width\n"); + retval = -EINVAL; + goto err_dwidth; + } + + /* Setup address space */ + switch (aspace) { + case VME_A16: + case VME_A24: + case VME_A32: + case VME_A64: + case VME_CRCSR: + case VME_USER1: + case VME_USER2: + case VME_USER3: + case VME_USER4: + break; + default: + pr_err("Invalid address space\n"); + retval = -EINVAL; + goto err_aspace; + } + + spin_lock(&image->lock); + + i = image->number; + + bridge->masters[i].enabled = enabled; + bridge->masters[i].vme_base = vme_base; + bridge->masters[i].size = size; + bridge->masters[i].aspace = aspace; + bridge->masters[i].cycle = cycle; + bridge->masters[i].dwidth = dwidth; + + spin_unlock(&image->lock); + + return 0; + +err_aspace: +err_dwidth: +err_window: + return retval; + +} + +/* + * Set the attributes of an outbound window. + */ +static int __fake_master_get(struct vme_master_resource *image, int *enabled, + unsigned long long *vme_base, unsigned long long *size, + u32 *aspace, u32 *cycle, u32 *dwidth) +{ + unsigned int i; + struct fake_driver *bridge; + + bridge = image->parent->driver_priv; + + i = image->number; + + *enabled = bridge->masters[i].enabled; + *vme_base = bridge->masters[i].vme_base; + *size = bridge->masters[i].size; + *aspace = bridge->masters[i].aspace; + *cycle = bridge->masters[i].cycle; + *dwidth = bridge->masters[i].dwidth; + + return 0; +} + + +static int fake_master_get(struct vme_master_resource *image, int *enabled, + unsigned long long *vme_base, unsigned long long *size, + u32 *aspace, u32 *cycle, u32 *dwidth) +{ + int retval; + + spin_lock(&image->lock); + + retval = __fake_master_get(image, enabled, vme_base, size, aspace, + cycle, dwidth); + + spin_unlock(&image->lock); + + return retval; +} + + +static void fake_lm_check(struct fake_driver *bridge, unsigned long long addr, + u32 aspace, u32 cycle) +{ + struct vme_bridge *fake_bridge; + unsigned long long lm_base; + u32 lm_aspace, lm_cycle; + int i; + struct vme_lm_resource *lm; + struct list_head *pos = NULL, *n; + + /* Get vme_bridge */ + fake_bridge = bridge->parent; + + /* Loop through each location monitor resource */ + list_for_each_safe(pos, n, &fake_bridge->lm_resources) { + lm = list_entry(pos, struct vme_lm_resource, list); + + /* If disabled, we're done */ + if (bridge->lm_enabled == 0) + return; + + lm_base = bridge->lm_base; + lm_aspace = bridge->lm_aspace; + lm_cycle = bridge->lm_cycle; + + /* First make sure that the cycle and address space match */ + if ((lm_aspace == aspace) && (lm_cycle == cycle)) { + for (i = 0; i < lm->monitors; i++) { + /* Each location monitor covers 8 bytes */ + if (((lm_base + (8 * i)) <= addr) && + ((lm_base + (8 * i) + 8) > addr)) { + if (bridge->lm_callback[i]) + bridge->lm_callback[i]( + bridge->lm_data[i]); + } + } + } + } +} + +static noinline_for_stack u8 fake_vmeread8(struct fake_driver *bridge, + unsigned long long addr, + u32 aspace, u32 cycle) +{ + u8 retval = 0xff; + int i; + unsigned long long start, end, offset; + u8 *loc; + + for (i = 0; i < FAKE_MAX_SLAVE; i++) { + start = bridge->slaves[i].vme_base; + end = bridge->slaves[i].vme_base + bridge->slaves[i].size; + + if (aspace != bridge->slaves[i].aspace) + continue; + + if (cycle != bridge->slaves[i].cycle) + continue; + + if ((addr >= start) && (addr < end)) { + offset = addr - bridge->slaves[i].vme_base; + loc = (u8 *)(bridge->slaves[i].buf_base + offset); + retval = *loc; + + break; + } + } + + fake_lm_check(bridge, addr, aspace, cycle); + + return retval; +} + +static noinline_for_stack u16 fake_vmeread16(struct fake_driver *bridge, + unsigned long long addr, + u32 aspace, u32 cycle) +{ + u16 retval = 0xffff; + int i; + unsigned long long start, end, offset; + u16 *loc; + + for (i = 0; i < FAKE_MAX_SLAVE; i++) { + if (aspace != bridge->slaves[i].aspace) + continue; + + if (cycle != bridge->slaves[i].cycle) + continue; + + start = bridge->slaves[i].vme_base; + end = bridge->slaves[i].vme_base + bridge->slaves[i].size; + + if ((addr >= start) && ((addr + 1) < end)) { + offset = addr - bridge->slaves[i].vme_base; + loc = (u16 *)(bridge->slaves[i].buf_base + offset); + retval = *loc; + + break; + } + } + + fake_lm_check(bridge, addr, aspace, cycle); + + return retval; +} + +static noinline_for_stack u32 fake_vmeread32(struct fake_driver *bridge, + unsigned long long addr, + u32 aspace, u32 cycle) +{ + u32 retval = 0xffffffff; + int i; + unsigned long long start, end, offset; + u32 *loc; + + for (i = 0; i < FAKE_MAX_SLAVE; i++) { + if (aspace != bridge->slaves[i].aspace) + continue; + + if (cycle != bridge->slaves[i].cycle) + continue; + + start = bridge->slaves[i].vme_base; + end = bridge->slaves[i].vme_base + bridge->slaves[i].size; + + if ((addr >= start) && ((addr + 3) < end)) { + offset = addr - bridge->slaves[i].vme_base; + loc = (u32 *)(bridge->slaves[i].buf_base + offset); + retval = *loc; + + break; + } + } + + fake_lm_check(bridge, addr, aspace, cycle); + + return retval; +} + +static ssize_t fake_master_read(struct vme_master_resource *image, void *buf, + size_t count, loff_t offset) +{ + int retval; + u32 aspace, cycle, dwidth; + struct vme_bridge *fake_bridge; + struct fake_driver *priv; + int i; + unsigned long long addr; + unsigned int done = 0; + unsigned int count32; + + fake_bridge = image->parent; + + priv = fake_bridge->driver_priv; + + i = image->number; + + addr = (unsigned long long)priv->masters[i].vme_base + offset; + aspace = priv->masters[i].aspace; + cycle = priv->masters[i].cycle; + dwidth = priv->masters[i].dwidth; + + spin_lock(&image->lock); + + /* The following code handles VME address alignment. We cannot use + * memcpy_xxx here because it may cut data transfers in to 8-bit + * cycles when D16 or D32 cycles are required on the VME bus. + * On the other hand, the bridge itself assures that the maximum data + * cycle configured for the transfer is used and splits it + * automatically for non-aligned addresses, so we don't want the + * overhead of needlessly forcing small transfers for the entire cycle. + */ + if (addr & 0x1) { + *(u8 *)buf = fake_vmeread8(priv, addr, aspace, cycle); + done += 1; + if (done == count) + goto out; + } + if ((dwidth == VME_D16) || (dwidth == VME_D32)) { + if ((addr + done) & 0x2) { + if ((count - done) < 2) { + *(u8 *)(buf + done) = fake_vmeread8(priv, + addr + done, aspace, cycle); + done += 1; + goto out; + } else { + *(u16 *)(buf + done) = fake_vmeread16(priv, + addr + done, aspace, cycle); + done += 2; + } + } + } + + if (dwidth == VME_D32) { + count32 = (count - done) & ~0x3; + while (done < count32) { + *(u32 *)(buf + done) = fake_vmeread32(priv, addr + done, + aspace, cycle); + done += 4; + } + } else if (dwidth == VME_D16) { + count32 = (count - done) & ~0x3; + while (done < count32) { + *(u16 *)(buf + done) = fake_vmeread16(priv, addr + done, + aspace, cycle); + done += 2; + } + } else if (dwidth == VME_D8) { + count32 = (count - done); + while (done < count32) { + *(u8 *)(buf + done) = fake_vmeread8(priv, addr + done, + aspace, cycle); + done += 1; + } + + } + + if ((dwidth == VME_D16) || (dwidth == VME_D32)) { + if ((count - done) & 0x2) { + *(u16 *)(buf + done) = fake_vmeread16(priv, addr + done, + aspace, cycle); + done += 2; + } + } + if ((count - done) & 0x1) { + *(u8 *)(buf + done) = fake_vmeread8(priv, addr + done, aspace, + cycle); + done += 1; + } + +out: + retval = count; + + spin_unlock(&image->lock); + + return retval; +} + +static noinline_for_stack void fake_vmewrite8(struct fake_driver *bridge, + u8 *buf, unsigned long long addr, + u32 aspace, u32 cycle) +{ + int i; + unsigned long long start, end, offset; + u8 *loc; + + for (i = 0; i < FAKE_MAX_SLAVE; i++) { + if (aspace != bridge->slaves[i].aspace) + continue; + + if (cycle != bridge->slaves[i].cycle) + continue; + + start = bridge->slaves[i].vme_base; + end = bridge->slaves[i].vme_base + bridge->slaves[i].size; + + if ((addr >= start) && (addr < end)) { + offset = addr - bridge->slaves[i].vme_base; + loc = (u8 *)((void *)bridge->slaves[i].buf_base + offset); + *loc = *buf; + + break; + } + } + + fake_lm_check(bridge, addr, aspace, cycle); + +} + +static noinline_for_stack void fake_vmewrite16(struct fake_driver *bridge, + u16 *buf, unsigned long long addr, + u32 aspace, u32 cycle) +{ + int i; + unsigned long long start, end, offset; + u16 *loc; + + for (i = 0; i < FAKE_MAX_SLAVE; i++) { + if (aspace != bridge->slaves[i].aspace) + continue; + + if (cycle != bridge->slaves[i].cycle) + continue; + + start = bridge->slaves[i].vme_base; + end = bridge->slaves[i].vme_base + bridge->slaves[i].size; + + if ((addr >= start) && ((addr + 1) < end)) { + offset = addr - bridge->slaves[i].vme_base; + loc = (u16 *)((void *)bridge->slaves[i].buf_base + offset); + *loc = *buf; + + break; + } + } + + fake_lm_check(bridge, addr, aspace, cycle); + +} + +static noinline_for_stack void fake_vmewrite32(struct fake_driver *bridge, + u32 *buf, unsigned long long addr, + u32 aspace, u32 cycle) +{ + int i; + unsigned long long start, end, offset; + u32 *loc; + + for (i = 0; i < FAKE_MAX_SLAVE; i++) { + if (aspace != bridge->slaves[i].aspace) + continue; + + if (cycle != bridge->slaves[i].cycle) + continue; + + start = bridge->slaves[i].vme_base; + end = bridge->slaves[i].vme_base + bridge->slaves[i].size; + + if ((addr >= start) && ((addr + 3) < end)) { + offset = addr - bridge->slaves[i].vme_base; + loc = (u32 *)((void *)bridge->slaves[i].buf_base + offset); + *loc = *buf; + + break; + } + } + + fake_lm_check(bridge, addr, aspace, cycle); + +} + +static ssize_t fake_master_write(struct vme_master_resource *image, void *buf, + size_t count, loff_t offset) +{ + int retval = 0; + u32 aspace, cycle, dwidth; + unsigned long long addr; + int i; + unsigned int done = 0; + unsigned int count32; + + struct vme_bridge *fake_bridge; + struct fake_driver *bridge; + + fake_bridge = image->parent; + + bridge = fake_bridge->driver_priv; + + i = image->number; + + addr = bridge->masters[i].vme_base + offset; + aspace = bridge->masters[i].aspace; + cycle = bridge->masters[i].cycle; + dwidth = bridge->masters[i].dwidth; + + spin_lock(&image->lock); + + /* Here we apply for the same strategy we do in master_read + * function in order to assure the correct cycles. + */ + if (addr & 0x1) { + fake_vmewrite8(bridge, (u8 *)buf, addr, aspace, cycle); + done += 1; + if (done == count) + goto out; + } + + if ((dwidth == VME_D16) || (dwidth == VME_D32)) { + if ((addr + done) & 0x2) { + if ((count - done) < 2) { + fake_vmewrite8(bridge, (u8 *)(buf + done), + addr + done, aspace, cycle); + done += 1; + goto out; + } else { + fake_vmewrite16(bridge, (u16 *)(buf + done), + addr + done, aspace, cycle); + done += 2; + } + } + } + + if (dwidth == VME_D32) { + count32 = (count - done) & ~0x3; + while (done < count32) { + fake_vmewrite32(bridge, (u32 *)(buf + done), + addr + done, aspace, cycle); + done += 4; + } + } else if (dwidth == VME_D16) { + count32 = (count - done) & ~0x3; + while (done < count32) { + fake_vmewrite16(bridge, (u16 *)(buf + done), + addr + done, aspace, cycle); + done += 2; + } + } else if (dwidth == VME_D8) { + count32 = (count - done); + while (done < count32) { + fake_vmewrite8(bridge, (u8 *)(buf + done), addr + done, + aspace, cycle); + done += 1; + } + + } + + if ((dwidth == VME_D16) || (dwidth == VME_D32)) { + if ((count - done) & 0x2) { + fake_vmewrite16(bridge, (u16 *)(buf + done), + addr + done, aspace, cycle); + done += 2; + } + } + + if ((count - done) & 0x1) { + fake_vmewrite8(bridge, (u8 *)(buf + done), addr + done, aspace, + cycle); + done += 1; + } + +out: + retval = count; + + spin_unlock(&image->lock); + + return retval; +} + +/* + * Perform an RMW cycle on the VME bus. + * + * Requires a previously configured master window, returns final value. + */ +static unsigned int fake_master_rmw(struct vme_master_resource *image, + unsigned int mask, unsigned int compare, unsigned int swap, + loff_t offset) +{ + u32 tmp, base; + u32 aspace, cycle; + int i; + struct fake_driver *bridge; + + bridge = image->parent->driver_priv; + + /* Find the PCI address that maps to the desired VME address */ + i = image->number; + + base = bridge->masters[i].vme_base; + aspace = bridge->masters[i].aspace; + cycle = bridge->masters[i].cycle; + + /* Lock image */ + spin_lock(&image->lock); + + /* Read existing value */ + tmp = fake_vmeread32(bridge, base + offset, aspace, cycle); + + /* Perform check */ + if ((tmp && mask) == (compare && mask)) { + tmp = tmp | (mask | swap); + tmp = tmp & (~mask | swap); + + /* Write back */ + fake_vmewrite32(bridge, &tmp, base + offset, aspace, cycle); + } + + /* Unlock image */ + spin_unlock(&image->lock); + + return tmp; +} + +/* + * All 4 location monitors reside at the same base - this is therefore a + * system wide configuration. + * + * This does not enable the LM monitor - that should be done when the first + * callback is attached and disabled when the last callback is removed. + */ +static int fake_lm_set(struct vme_lm_resource *lm, unsigned long long lm_base, + u32 aspace, u32 cycle) +{ + int i; + struct vme_bridge *fake_bridge; + struct fake_driver *bridge; + + fake_bridge = lm->parent; + + bridge = fake_bridge->driver_priv; + + mutex_lock(&lm->mtx); + + /* If we already have a callback attached, we can't move it! */ + for (i = 0; i < lm->monitors; i++) { + if (bridge->lm_callback[i]) { + mutex_unlock(&lm->mtx); + pr_err("Location monitor callback attached, can't reset\n"); + return -EBUSY; + } + } + + switch (aspace) { + case VME_A16: + case VME_A24: + case VME_A32: + case VME_A64: + break; + default: + mutex_unlock(&lm->mtx); + pr_err("Invalid address space\n"); + return -EINVAL; + } + + bridge->lm_base = lm_base; + bridge->lm_aspace = aspace; + bridge->lm_cycle = cycle; + + mutex_unlock(&lm->mtx); + + return 0; +} + +/* Get configuration of the callback monitor and return whether it is enabled + * or disabled. + */ +static int fake_lm_get(struct vme_lm_resource *lm, + unsigned long long *lm_base, u32 *aspace, u32 *cycle) +{ + struct fake_driver *bridge; + + bridge = lm->parent->driver_priv; + + mutex_lock(&lm->mtx); + + *lm_base = bridge->lm_base; + *aspace = bridge->lm_aspace; + *cycle = bridge->lm_cycle; + + mutex_unlock(&lm->mtx); + + return bridge->lm_enabled; +} + +/* + * Attach a callback to a specific location monitor. + * + * Callback will be passed the monitor triggered. + */ +static int fake_lm_attach(struct vme_lm_resource *lm, int monitor, + void (*callback)(void *), void *data) +{ + struct vme_bridge *fake_bridge; + struct fake_driver *bridge; + + fake_bridge = lm->parent; + + bridge = fake_bridge->driver_priv; + + mutex_lock(&lm->mtx); + + /* Ensure that the location monitor is configured - need PGM or DATA */ + if (bridge->lm_cycle == 0) { + mutex_unlock(&lm->mtx); + pr_err("Location monitor not properly configured\n"); + return -EINVAL; + } + + /* Check that a callback isn't already attached */ + if (bridge->lm_callback[monitor]) { + mutex_unlock(&lm->mtx); + pr_err("Existing callback attached\n"); + return -EBUSY; + } + + /* Attach callback */ + bridge->lm_callback[monitor] = callback; + bridge->lm_data[monitor] = data; + + /* Ensure that global Location Monitor Enable set */ + bridge->lm_enabled = 1; + + mutex_unlock(&lm->mtx); + + return 0; +} + +/* + * Detach a callback function forn a specific location monitor. + */ +static int fake_lm_detach(struct vme_lm_resource *lm, int monitor) +{ + u32 tmp; + int i; + struct fake_driver *bridge; + + bridge = lm->parent->driver_priv; + + mutex_lock(&lm->mtx); + + /* Detach callback */ + bridge->lm_callback[monitor] = NULL; + bridge->lm_data[monitor] = NULL; + + /* If all location monitors disabled, disable global Location Monitor */ + tmp = 0; + for (i = 0; i < lm->monitors; i++) { + if (bridge->lm_callback[i]) + tmp = 1; + } + + if (tmp == 0) + bridge->lm_enabled = 0; + + mutex_unlock(&lm->mtx); + + return 0; +} + +/* + * Determine Geographical Addressing + */ +static int fake_slot_get(struct vme_bridge *fake_bridge) +{ + return geoid; +} + +static void *fake_alloc_consistent(struct device *parent, size_t size, + dma_addr_t *dma) +{ + void *alloc = kmalloc(size, GFP_KERNEL); + + if (alloc) + *dma = fake_ptr_to_pci(alloc); + + return alloc; +} + +static void fake_free_consistent(struct device *parent, size_t size, + void *vaddr, dma_addr_t dma) +{ + kfree(vaddr); +/* + dma_free_coherent(parent, size, vaddr, dma); +*/ +} + +/* + * Configure CR/CSR space + * + * Access to the CR/CSR can be configured at power-up. The location of the + * CR/CSR registers in the CR/CSR address space is determined by the boards + * Geographic address. + * + * Each board has a 512kB window, with the highest 4kB being used for the + * boards registers, this means there is a fix length 508kB window which must + * be mapped onto PCI memory. + */ +static int fake_crcsr_init(struct vme_bridge *fake_bridge) +{ + u32 vstat; + struct fake_driver *bridge; + + bridge = fake_bridge->driver_priv; + + /* Allocate mem for CR/CSR image */ + bridge->crcsr_kernel = kzalloc(VME_CRCSR_BUF_SIZE, GFP_KERNEL); + bridge->crcsr_bus = fake_ptr_to_pci(bridge->crcsr_kernel); + if (!bridge->crcsr_kernel) + return -ENOMEM; + + vstat = fake_slot_get(fake_bridge); + + pr_info("CR/CSR Offset: %d\n", vstat); + + return 0; +} + +static void fake_crcsr_exit(struct vme_bridge *fake_bridge) +{ + struct fake_driver *bridge; + + bridge = fake_bridge->driver_priv; + + kfree(bridge->crcsr_kernel); +} + + +static int __init fake_init(void) +{ + int retval, i; + struct list_head *pos = NULL, *n; + struct vme_bridge *fake_bridge; + struct fake_driver *fake_device; + struct vme_master_resource *master_image; + struct vme_slave_resource *slave_image; + struct vme_lm_resource *lm; + + /* We need a fake parent device */ + vme_root = __root_device_register("vme", THIS_MODULE); + + /* If we want to support more than one bridge at some point, we need to + * dynamically allocate this so we get one per device. + */ + fake_bridge = kzalloc(sizeof(*fake_bridge), GFP_KERNEL); + if (!fake_bridge) { + retval = -ENOMEM; + goto err_struct; + } + + fake_device = kzalloc(sizeof(*fake_device), GFP_KERNEL); + if (!fake_device) { + retval = -ENOMEM; + goto err_driver; + } + + fake_bridge->driver_priv = fake_device; + + fake_bridge->parent = vme_root; + + fake_device->parent = fake_bridge; + + /* Initialize wait queues & mutual exclusion flags */ + mutex_init(&fake_device->vme_int); + mutex_init(&fake_bridge->irq_mtx); + tasklet_init(&fake_device->int_tasklet, fake_VIRQ_tasklet, + (unsigned long) fake_bridge); + + strcpy(fake_bridge->name, driver_name); + + /* Add master windows to list */ + INIT_LIST_HEAD(&fake_bridge->master_resources); + for (i = 0; i < FAKE_MAX_MASTER; i++) { + master_image = kmalloc(sizeof(*master_image), GFP_KERNEL); + if (!master_image) { + retval = -ENOMEM; + goto err_master; + } + master_image->parent = fake_bridge; + spin_lock_init(&master_image->lock); + master_image->locked = 0; + master_image->number = i; + master_image->address_attr = VME_A16 | VME_A24 | VME_A32 | + VME_A64; + master_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | + VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | + VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | + VME_PROG | VME_DATA; + master_image->width_attr = VME_D16 | VME_D32; + memset(&master_image->bus_resource, 0, + sizeof(struct resource)); + master_image->kern_base = NULL; + list_add_tail(&master_image->list, + &fake_bridge->master_resources); + } + + /* Add slave windows to list */ + INIT_LIST_HEAD(&fake_bridge->slave_resources); + for (i = 0; i < FAKE_MAX_SLAVE; i++) { + slave_image = kmalloc(sizeof(*slave_image), GFP_KERNEL); + if (!slave_image) { + retval = -ENOMEM; + goto err_slave; + } + slave_image->parent = fake_bridge; + mutex_init(&slave_image->mtx); + slave_image->locked = 0; + slave_image->number = i; + slave_image->address_attr = VME_A16 | VME_A24 | VME_A32 | + VME_A64 | VME_CRCSR | VME_USER1 | VME_USER2 | + VME_USER3 | VME_USER4; + slave_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | + VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | + VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | + VME_PROG | VME_DATA; + list_add_tail(&slave_image->list, + &fake_bridge->slave_resources); + } + + /* Add location monitor to list */ + INIT_LIST_HEAD(&fake_bridge->lm_resources); + lm = kmalloc(sizeof(*lm), GFP_KERNEL); + if (!lm) { + retval = -ENOMEM; + goto err_lm; + } + lm->parent = fake_bridge; + mutex_init(&lm->mtx); + lm->locked = 0; + lm->number = 1; + lm->monitors = 4; + list_add_tail(&lm->list, &fake_bridge->lm_resources); + + fake_bridge->slave_get = fake_slave_get; + fake_bridge->slave_set = fake_slave_set; + fake_bridge->master_get = fake_master_get; + fake_bridge->master_set = fake_master_set; + fake_bridge->master_read = fake_master_read; + fake_bridge->master_write = fake_master_write; + fake_bridge->master_rmw = fake_master_rmw; + fake_bridge->irq_set = fake_irq_set; + fake_bridge->irq_generate = fake_irq_generate; + fake_bridge->lm_set = fake_lm_set; + fake_bridge->lm_get = fake_lm_get; + fake_bridge->lm_attach = fake_lm_attach; + fake_bridge->lm_detach = fake_lm_detach; + fake_bridge->slot_get = fake_slot_get; + fake_bridge->alloc_consistent = fake_alloc_consistent; + fake_bridge->free_consistent = fake_free_consistent; + + pr_info("Board is%s the VME system controller\n", + (geoid == 1) ? "" : " not"); + + pr_info("VME geographical address is set to %d\n", geoid); + + retval = fake_crcsr_init(fake_bridge); + if (retval) { + pr_err("CR/CSR configuration failed.\n"); + goto err_crcsr; + } + + retval = vme_register_bridge(fake_bridge); + if (retval != 0) { + pr_err("Chip Registration failed.\n"); + goto err_reg; + } + + exit_pointer = fake_bridge; + + return 0; + +err_reg: + fake_crcsr_exit(fake_bridge); +err_crcsr: +err_lm: + /* resources are stored in link list */ + list_for_each_safe(pos, n, &fake_bridge->lm_resources) { + lm = list_entry(pos, struct vme_lm_resource, list); + list_del(pos); + kfree(lm); + } +err_slave: + /* resources are stored in link list */ + list_for_each_safe(pos, n, &fake_bridge->slave_resources) { + slave_image = list_entry(pos, struct vme_slave_resource, list); + list_del(pos); + kfree(slave_image); + } +err_master: + /* resources are stored in link list */ + list_for_each_safe(pos, n, &fake_bridge->master_resources) { + master_image = list_entry(pos, struct vme_master_resource, + list); + list_del(pos); + kfree(master_image); + } + + kfree(fake_device); +err_driver: + kfree(fake_bridge); +err_struct: + return retval; + +} + + +static void __exit fake_exit(void) +{ + struct list_head *pos = NULL; + struct list_head *tmplist; + struct vme_master_resource *master_image; + struct vme_slave_resource *slave_image; + int i; + struct vme_bridge *fake_bridge; + struct fake_driver *bridge; + + fake_bridge = exit_pointer; + + bridge = fake_bridge->driver_priv; + + pr_debug("Driver is being unloaded.\n"); + + /* + * Shutdown all inbound and outbound windows. + */ + for (i = 0; i < FAKE_MAX_MASTER; i++) + bridge->masters[i].enabled = 0; + + for (i = 0; i < FAKE_MAX_SLAVE; i++) + bridge->slaves[i].enabled = 0; + + /* + * Shutdown Location monitor. + */ + bridge->lm_enabled = 0; + + vme_unregister_bridge(fake_bridge); + + fake_crcsr_exit(fake_bridge); + /* resources are stored in link list */ + list_for_each_safe(pos, tmplist, &fake_bridge->slave_resources) { + slave_image = list_entry(pos, struct vme_slave_resource, list); + list_del(pos); + kfree(slave_image); + } + + /* resources are stored in link list */ + list_for_each_safe(pos, tmplist, &fake_bridge->master_resources) { + master_image = list_entry(pos, struct vme_master_resource, + list); + list_del(pos); + kfree(master_image); + } + + kfree(fake_bridge->driver_priv); + + kfree(fake_bridge); + + root_device_unregister(vme_root); +} + + +MODULE_PARM_DESC(geoid, "Set geographical addressing"); +module_param(geoid, int, 0); + +MODULE_DESCRIPTION("Fake VME bridge driver"); +MODULE_LICENSE("GPL"); + +module_init(fake_init); +module_exit(fake_exit); diff --git a/drivers/staging/vme_user/vme_tsi148.c b/drivers/staging/vme_user/vme_tsi148.c new file mode 100644 index 000000000000..956476213241 --- /dev/null +++ b/drivers/staging/vme_user/vme_tsi148.c @@ -0,0 +1,2661 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Support for the Tundra TSI148 VME-PCI Bridge Chip + * + * Author: Martyn Welch + * Copyright 2008 GE Intelligent Platforms Embedded Systems, Inc. + * + * Based on work by Tom Armistead and Ajit Prem + * Copyright 2004 Motorola Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vme.h" +#include "vme_bridge.h" +#include "vme_tsi148.h" + +static int tsi148_probe(struct pci_dev *, const struct pci_device_id *); +static void tsi148_remove(struct pci_dev *); + + +/* Module parameter */ +static bool err_chk; +static int geoid; + +static const char driver_name[] = "vme_tsi148"; + +static const struct pci_device_id tsi148_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_TUNDRA, PCI_DEVICE_ID_TUNDRA_TSI148) }, + { }, +}; + +MODULE_DEVICE_TABLE(pci, tsi148_ids); + +static struct pci_driver tsi148_driver = { + .name = driver_name, + .id_table = tsi148_ids, + .probe = tsi148_probe, + .remove = tsi148_remove, +}; + +static void reg_join(unsigned int high, unsigned int low, + unsigned long long *variable) +{ + *variable = (unsigned long long)high << 32; + *variable |= (unsigned long long)low; +} + +static void reg_split(unsigned long long variable, unsigned int *high, + unsigned int *low) +{ + *low = (unsigned int)variable & 0xFFFFFFFF; + *high = (unsigned int)(variable >> 32); +} + +/* + * Wakes up DMA queue. + */ +static u32 tsi148_DMA_irqhandler(struct tsi148_driver *bridge, + int channel_mask) +{ + u32 serviced = 0; + + if (channel_mask & TSI148_LCSR_INTS_DMA0S) { + wake_up(&bridge->dma_queue[0]); + serviced |= TSI148_LCSR_INTC_DMA0C; + } + if (channel_mask & TSI148_LCSR_INTS_DMA1S) { + wake_up(&bridge->dma_queue[1]); + serviced |= TSI148_LCSR_INTC_DMA1C; + } + + return serviced; +} + +/* + * Wake up location monitor queue + */ +static u32 tsi148_LM_irqhandler(struct tsi148_driver *bridge, u32 stat) +{ + int i; + u32 serviced = 0; + + for (i = 0; i < 4; i++) { + if (stat & TSI148_LCSR_INTS_LMS[i]) { + /* We only enable interrupts if the callback is set */ + bridge->lm_callback[i](bridge->lm_data[i]); + serviced |= TSI148_LCSR_INTC_LMC[i]; + } + } + + return serviced; +} + +/* + * Wake up mail box queue. + * + * XXX This functionality is not exposed up though API. + */ +static u32 tsi148_MB_irqhandler(struct vme_bridge *tsi148_bridge, u32 stat) +{ + int i; + u32 val; + u32 serviced = 0; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + for (i = 0; i < 4; i++) { + if (stat & TSI148_LCSR_INTS_MBS[i]) { + val = ioread32be(bridge->base + TSI148_GCSR_MBOX[i]); + dev_err(tsi148_bridge->parent, "VME Mailbox %d received" + ": 0x%x\n", i, val); + serviced |= TSI148_LCSR_INTC_MBC[i]; + } + } + + return serviced; +} + +/* + * Display error & status message when PERR (PCI) exception interrupt occurs. + */ +static u32 tsi148_PERR_irqhandler(struct vme_bridge *tsi148_bridge) +{ + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + dev_err(tsi148_bridge->parent, "PCI Exception at address: 0x%08x:%08x, " + "attributes: %08x\n", + ioread32be(bridge->base + TSI148_LCSR_EDPAU), + ioread32be(bridge->base + TSI148_LCSR_EDPAL), + ioread32be(bridge->base + TSI148_LCSR_EDPAT)); + + dev_err(tsi148_bridge->parent, "PCI-X attribute reg: %08x, PCI-X split " + "completion reg: %08x\n", + ioread32be(bridge->base + TSI148_LCSR_EDPXA), + ioread32be(bridge->base + TSI148_LCSR_EDPXS)); + + iowrite32be(TSI148_LCSR_EDPAT_EDPCL, bridge->base + TSI148_LCSR_EDPAT); + + return TSI148_LCSR_INTC_PERRC; +} + +/* + * Save address and status when VME error interrupt occurs. + */ +static u32 tsi148_VERR_irqhandler(struct vme_bridge *tsi148_bridge) +{ + unsigned int error_addr_high, error_addr_low; + unsigned long long error_addr; + u32 error_attrib; + int error_am; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + error_addr_high = ioread32be(bridge->base + TSI148_LCSR_VEAU); + error_addr_low = ioread32be(bridge->base + TSI148_LCSR_VEAL); + error_attrib = ioread32be(bridge->base + TSI148_LCSR_VEAT); + error_am = (error_attrib & TSI148_LCSR_VEAT_AM_M) >> 8; + + reg_join(error_addr_high, error_addr_low, &error_addr); + + /* Check for exception register overflow (we have lost error data) */ + if (error_attrib & TSI148_LCSR_VEAT_VEOF) { + dev_err(tsi148_bridge->parent, "VME Bus Exception Overflow " + "Occurred\n"); + } + + if (err_chk) + vme_bus_error_handler(tsi148_bridge, error_addr, error_am); + else + dev_err(tsi148_bridge->parent, + "VME Bus Error at address: 0x%llx, attributes: %08x\n", + error_addr, error_attrib); + + /* Clear Status */ + iowrite32be(TSI148_LCSR_VEAT_VESCL, bridge->base + TSI148_LCSR_VEAT); + + return TSI148_LCSR_INTC_VERRC; +} + +/* + * Wake up IACK queue. + */ +static u32 tsi148_IACK_irqhandler(struct tsi148_driver *bridge) +{ + wake_up(&bridge->iack_queue); + + return TSI148_LCSR_INTC_IACKC; +} + +/* + * Calling VME bus interrupt callback if provided. + */ +static u32 tsi148_VIRQ_irqhandler(struct vme_bridge *tsi148_bridge, + u32 stat) +{ + int vec, i, serviced = 0; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + for (i = 7; i > 0; i--) { + if (stat & (1 << i)) { + /* + * Note: Even though the registers are defined as + * 32-bits in the spec, we only want to issue 8-bit + * IACK cycles on the bus, read from offset 3. + */ + vec = ioread8(bridge->base + TSI148_LCSR_VIACK[i] + 3); + + vme_irq_handler(tsi148_bridge, i, vec); + + serviced |= (1 << i); + } + } + + return serviced; +} + +/* + * Top level interrupt handler. Clears appropriate interrupt status bits and + * then calls appropriate sub handler(s). + */ +static irqreturn_t tsi148_irqhandler(int irq, void *ptr) +{ + u32 stat, enable, serviced = 0; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *bridge; + + tsi148_bridge = ptr; + + bridge = tsi148_bridge->driver_priv; + + /* Determine which interrupts are unmasked and set */ + enable = ioread32be(bridge->base + TSI148_LCSR_INTEO); + stat = ioread32be(bridge->base + TSI148_LCSR_INTS); + + /* Only look at unmasked interrupts */ + stat &= enable; + + if (unlikely(!stat)) + return IRQ_NONE; + + /* Call subhandlers as appropriate */ + /* DMA irqs */ + if (stat & (TSI148_LCSR_INTS_DMA1S | TSI148_LCSR_INTS_DMA0S)) + serviced |= tsi148_DMA_irqhandler(bridge, stat); + + /* Location monitor irqs */ + if (stat & (TSI148_LCSR_INTS_LM3S | TSI148_LCSR_INTS_LM2S | + TSI148_LCSR_INTS_LM1S | TSI148_LCSR_INTS_LM0S)) + serviced |= tsi148_LM_irqhandler(bridge, stat); + + /* Mail box irqs */ + if (stat & (TSI148_LCSR_INTS_MB3S | TSI148_LCSR_INTS_MB2S | + TSI148_LCSR_INTS_MB1S | TSI148_LCSR_INTS_MB0S)) + serviced |= tsi148_MB_irqhandler(tsi148_bridge, stat); + + /* PCI bus error */ + if (stat & TSI148_LCSR_INTS_PERRS) + serviced |= tsi148_PERR_irqhandler(tsi148_bridge); + + /* VME bus error */ + if (stat & TSI148_LCSR_INTS_VERRS) + serviced |= tsi148_VERR_irqhandler(tsi148_bridge); + + /* IACK irq */ + if (stat & TSI148_LCSR_INTS_IACKS) + serviced |= tsi148_IACK_irqhandler(bridge); + + /* VME bus irqs */ + if (stat & (TSI148_LCSR_INTS_IRQ7S | TSI148_LCSR_INTS_IRQ6S | + TSI148_LCSR_INTS_IRQ5S | TSI148_LCSR_INTS_IRQ4S | + TSI148_LCSR_INTS_IRQ3S | TSI148_LCSR_INTS_IRQ2S | + TSI148_LCSR_INTS_IRQ1S)) + serviced |= tsi148_VIRQ_irqhandler(tsi148_bridge, stat); + + /* Clear serviced interrupts */ + iowrite32be(serviced, bridge->base + TSI148_LCSR_INTC); + + return IRQ_HANDLED; +} + +static int tsi148_irq_init(struct vme_bridge *tsi148_bridge) +{ + int result; + unsigned int tmp; + struct pci_dev *pdev; + struct tsi148_driver *bridge; + + pdev = to_pci_dev(tsi148_bridge->parent); + + bridge = tsi148_bridge->driver_priv; + + result = request_irq(pdev->irq, + tsi148_irqhandler, + IRQF_SHARED, + driver_name, tsi148_bridge); + if (result) { + dev_err(tsi148_bridge->parent, "Can't get assigned pci irq " + "vector %02X\n", pdev->irq); + return result; + } + + /* Enable and unmask interrupts */ + tmp = TSI148_LCSR_INTEO_DMA1EO | TSI148_LCSR_INTEO_DMA0EO | + TSI148_LCSR_INTEO_MB3EO | TSI148_LCSR_INTEO_MB2EO | + TSI148_LCSR_INTEO_MB1EO | TSI148_LCSR_INTEO_MB0EO | + TSI148_LCSR_INTEO_PERREO | TSI148_LCSR_INTEO_VERREO | + TSI148_LCSR_INTEO_IACKEO; + + /* This leaves the following interrupts masked. + * TSI148_LCSR_INTEO_VIEEO + * TSI148_LCSR_INTEO_SYSFLEO + * TSI148_LCSR_INTEO_ACFLEO + */ + + /* Don't enable Location Monitor interrupts here - they will be + * enabled when the location monitors are properly configured and + * a callback has been attached. + * TSI148_LCSR_INTEO_LM0EO + * TSI148_LCSR_INTEO_LM1EO + * TSI148_LCSR_INTEO_LM2EO + * TSI148_LCSR_INTEO_LM3EO + */ + + /* Don't enable VME interrupts until we add a handler, else the board + * will respond to it and we don't want that unless it knows how to + * properly deal with it. + * TSI148_LCSR_INTEO_IRQ7EO + * TSI148_LCSR_INTEO_IRQ6EO + * TSI148_LCSR_INTEO_IRQ5EO + * TSI148_LCSR_INTEO_IRQ4EO + * TSI148_LCSR_INTEO_IRQ3EO + * TSI148_LCSR_INTEO_IRQ2EO + * TSI148_LCSR_INTEO_IRQ1EO + */ + + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); + + return 0; +} + +static void tsi148_irq_exit(struct vme_bridge *tsi148_bridge, + struct pci_dev *pdev) +{ + struct tsi148_driver *bridge = tsi148_bridge->driver_priv; + + /* Turn off interrupts */ + iowrite32be(0x0, bridge->base + TSI148_LCSR_INTEO); + iowrite32be(0x0, bridge->base + TSI148_LCSR_INTEN); + + /* Clear all interrupts */ + iowrite32be(0xFFFFFFFF, bridge->base + TSI148_LCSR_INTC); + + /* Detach interrupt handler */ + free_irq(pdev->irq, tsi148_bridge); +} + +/* + * Check to see if an IACk has been received, return true (1) or false (0). + */ +static int tsi148_iack_received(struct tsi148_driver *bridge) +{ + u32 tmp; + + tmp = ioread32be(bridge->base + TSI148_LCSR_VICR); + + if (tmp & TSI148_LCSR_VICR_IRQS) + return 0; + else + return 1; +} + +/* + * Configure VME interrupt + */ +static void tsi148_irq_set(struct vme_bridge *tsi148_bridge, int level, + int state, int sync) +{ + struct pci_dev *pdev; + u32 tmp; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + /* We need to do the ordering differently for enabling and disabling */ + if (state == 0) { + tmp = ioread32be(bridge->base + TSI148_LCSR_INTEN); + tmp &= ~TSI148_LCSR_INTEN_IRQEN[level - 1]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); + + tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); + tmp &= ~TSI148_LCSR_INTEO_IRQEO[level - 1]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); + + if (sync != 0) { + pdev = to_pci_dev(tsi148_bridge->parent); + synchronize_irq(pdev->irq); + } + } else { + tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); + tmp |= TSI148_LCSR_INTEO_IRQEO[level - 1]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); + + tmp = ioread32be(bridge->base + TSI148_LCSR_INTEN); + tmp |= TSI148_LCSR_INTEN_IRQEN[level - 1]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); + } +} + +/* + * Generate a VME bus interrupt at the requested level & vector. Wait for + * interrupt to be acked. + */ +static int tsi148_irq_generate(struct vme_bridge *tsi148_bridge, int level, + int statid) +{ + u32 tmp; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + mutex_lock(&bridge->vme_int); + + /* Read VICR register */ + tmp = ioread32be(bridge->base + TSI148_LCSR_VICR); + + /* Set Status/ID */ + tmp = (tmp & ~TSI148_LCSR_VICR_STID_M) | + (statid & TSI148_LCSR_VICR_STID_M); + iowrite32be(tmp, bridge->base + TSI148_LCSR_VICR); + + /* Assert VMEbus IRQ */ + tmp = tmp | TSI148_LCSR_VICR_IRQL[level]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_VICR); + + /* XXX Consider implementing a timeout? */ + wait_event_interruptible(bridge->iack_queue, + tsi148_iack_received(bridge)); + + mutex_unlock(&bridge->vme_int); + + return 0; +} + +/* + * Initialize a slave window with the requested attributes. + */ +static int tsi148_slave_set(struct vme_slave_resource *image, int enabled, + unsigned long long vme_base, unsigned long long size, + dma_addr_t pci_base, u32 aspace, u32 cycle) +{ + unsigned int i, addr = 0, granularity = 0; + unsigned int temp_ctl = 0; + unsigned int vme_base_low, vme_base_high; + unsigned int vme_bound_low, vme_bound_high; + unsigned int pci_offset_low, pci_offset_high; + unsigned long long vme_bound, pci_offset; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *bridge; + + tsi148_bridge = image->parent; + bridge = tsi148_bridge->driver_priv; + + i = image->number; + + switch (aspace) { + case VME_A16: + granularity = 0x10; + addr |= TSI148_LCSR_ITAT_AS_A16; + break; + case VME_A24: + granularity = 0x1000; + addr |= TSI148_LCSR_ITAT_AS_A24; + break; + case VME_A32: + granularity = 0x10000; + addr |= TSI148_LCSR_ITAT_AS_A32; + break; + case VME_A64: + granularity = 0x10000; + addr |= TSI148_LCSR_ITAT_AS_A64; + break; + default: + dev_err(tsi148_bridge->parent, "Invalid address space\n"); + return -EINVAL; + } + + /* Convert 64-bit variables to 2x 32-bit variables */ + reg_split(vme_base, &vme_base_high, &vme_base_low); + + /* + * Bound address is a valid address for the window, adjust + * accordingly + */ + vme_bound = vme_base + size - granularity; + reg_split(vme_bound, &vme_bound_high, &vme_bound_low); + pci_offset = (unsigned long long)pci_base - vme_base; + reg_split(pci_offset, &pci_offset_high, &pci_offset_low); + + if (vme_base_low & (granularity - 1)) { + dev_err(tsi148_bridge->parent, "Invalid VME base alignment\n"); + return -EINVAL; + } + if (vme_bound_low & (granularity - 1)) { + dev_err(tsi148_bridge->parent, "Invalid VME bound alignment\n"); + return -EINVAL; + } + if (pci_offset_low & (granularity - 1)) { + dev_err(tsi148_bridge->parent, "Invalid PCI Offset " + "alignment\n"); + return -EINVAL; + } + + /* Disable while we are mucking around */ + temp_ctl = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITAT); + temp_ctl &= ~TSI148_LCSR_ITAT_EN; + iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITAT); + + /* Setup mapping */ + iowrite32be(vme_base_high, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITSAU); + iowrite32be(vme_base_low, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITSAL); + iowrite32be(vme_bound_high, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITEAU); + iowrite32be(vme_bound_low, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITEAL); + iowrite32be(pci_offset_high, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITOFU); + iowrite32be(pci_offset_low, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITOFL); + + /* Setup 2eSST speeds */ + temp_ctl &= ~TSI148_LCSR_ITAT_2eSSTM_M; + switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { + case VME_2eSST160: + temp_ctl |= TSI148_LCSR_ITAT_2eSSTM_160; + break; + case VME_2eSST267: + temp_ctl |= TSI148_LCSR_ITAT_2eSSTM_267; + break; + case VME_2eSST320: + temp_ctl |= TSI148_LCSR_ITAT_2eSSTM_320; + break; + } + + /* Setup cycle types */ + temp_ctl &= ~(0x1F << 7); + if (cycle & VME_BLT) + temp_ctl |= TSI148_LCSR_ITAT_BLT; + if (cycle & VME_MBLT) + temp_ctl |= TSI148_LCSR_ITAT_MBLT; + if (cycle & VME_2eVME) + temp_ctl |= TSI148_LCSR_ITAT_2eVME; + if (cycle & VME_2eSST) + temp_ctl |= TSI148_LCSR_ITAT_2eSST; + if (cycle & VME_2eSSTB) + temp_ctl |= TSI148_LCSR_ITAT_2eSSTB; + + /* Setup address space */ + temp_ctl &= ~TSI148_LCSR_ITAT_AS_M; + temp_ctl |= addr; + + temp_ctl &= ~0xF; + if (cycle & VME_SUPER) + temp_ctl |= TSI148_LCSR_ITAT_SUPR ; + if (cycle & VME_USER) + temp_ctl |= TSI148_LCSR_ITAT_NPRIV; + if (cycle & VME_PROG) + temp_ctl |= TSI148_LCSR_ITAT_PGM; + if (cycle & VME_DATA) + temp_ctl |= TSI148_LCSR_ITAT_DATA; + + /* Write ctl reg without enable */ + iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITAT); + + if (enabled) + temp_ctl |= TSI148_LCSR_ITAT_EN; + + iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITAT); + + return 0; +} + +/* + * Get slave window configuration. + */ +static int tsi148_slave_get(struct vme_slave_resource *image, int *enabled, + unsigned long long *vme_base, unsigned long long *size, + dma_addr_t *pci_base, u32 *aspace, u32 *cycle) +{ + unsigned int i, granularity = 0, ctl = 0; + unsigned int vme_base_low, vme_base_high; + unsigned int vme_bound_low, vme_bound_high; + unsigned int pci_offset_low, pci_offset_high; + unsigned long long vme_bound, pci_offset; + struct tsi148_driver *bridge; + + bridge = image->parent->driver_priv; + + i = image->number; + + /* Read registers */ + ctl = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITAT); + + vme_base_high = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITSAU); + vme_base_low = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITSAL); + vme_bound_high = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITEAU); + vme_bound_low = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITEAL); + pci_offset_high = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITOFU); + pci_offset_low = ioread32be(bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITOFL); + + /* Convert 64-bit variables to 2x 32-bit variables */ + reg_join(vme_base_high, vme_base_low, vme_base); + reg_join(vme_bound_high, vme_bound_low, &vme_bound); + reg_join(pci_offset_high, pci_offset_low, &pci_offset); + + *pci_base = (dma_addr_t)(*vme_base + pci_offset); + + *enabled = 0; + *aspace = 0; + *cycle = 0; + + if (ctl & TSI148_LCSR_ITAT_EN) + *enabled = 1; + + if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A16) { + granularity = 0x10; + *aspace |= VME_A16; + } + if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A24) { + granularity = 0x1000; + *aspace |= VME_A24; + } + if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A32) { + granularity = 0x10000; + *aspace |= VME_A32; + } + if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A64) { + granularity = 0x10000; + *aspace |= VME_A64; + } + + /* Need granularity before we set the size */ + *size = (unsigned long long)((vme_bound - *vme_base) + granularity); + + + if ((ctl & TSI148_LCSR_ITAT_2eSSTM_M) == TSI148_LCSR_ITAT_2eSSTM_160) + *cycle |= VME_2eSST160; + if ((ctl & TSI148_LCSR_ITAT_2eSSTM_M) == TSI148_LCSR_ITAT_2eSSTM_267) + *cycle |= VME_2eSST267; + if ((ctl & TSI148_LCSR_ITAT_2eSSTM_M) == TSI148_LCSR_ITAT_2eSSTM_320) + *cycle |= VME_2eSST320; + + if (ctl & TSI148_LCSR_ITAT_BLT) + *cycle |= VME_BLT; + if (ctl & TSI148_LCSR_ITAT_MBLT) + *cycle |= VME_MBLT; + if (ctl & TSI148_LCSR_ITAT_2eVME) + *cycle |= VME_2eVME; + if (ctl & TSI148_LCSR_ITAT_2eSST) + *cycle |= VME_2eSST; + if (ctl & TSI148_LCSR_ITAT_2eSSTB) + *cycle |= VME_2eSSTB; + + if (ctl & TSI148_LCSR_ITAT_SUPR) + *cycle |= VME_SUPER; + if (ctl & TSI148_LCSR_ITAT_NPRIV) + *cycle |= VME_USER; + if (ctl & TSI148_LCSR_ITAT_PGM) + *cycle |= VME_PROG; + if (ctl & TSI148_LCSR_ITAT_DATA) + *cycle |= VME_DATA; + + return 0; +} + +/* + * Allocate and map PCI Resource + */ +static int tsi148_alloc_resource(struct vme_master_resource *image, + unsigned long long size) +{ + unsigned long long existing_size; + int retval = 0; + struct pci_dev *pdev; + struct vme_bridge *tsi148_bridge; + + tsi148_bridge = image->parent; + + pdev = to_pci_dev(tsi148_bridge->parent); + + existing_size = (unsigned long long)(image->bus_resource.end - + image->bus_resource.start); + + /* If the existing size is OK, return */ + if ((size != 0) && (existing_size == (size - 1))) + return 0; + + if (existing_size != 0) { + iounmap(image->kern_base); + image->kern_base = NULL; + kfree(image->bus_resource.name); + release_resource(&image->bus_resource); + memset(&image->bus_resource, 0, sizeof(image->bus_resource)); + } + + /* Exit here if size is zero */ + if (size == 0) + return 0; + + if (!image->bus_resource.name) { + image->bus_resource.name = kmalloc(VMENAMSIZ+3, GFP_ATOMIC); + if (!image->bus_resource.name) { + retval = -ENOMEM; + goto err_name; + } + } + + sprintf((char *)image->bus_resource.name, "%s.%d", tsi148_bridge->name, + image->number); + + image->bus_resource.start = 0; + image->bus_resource.end = (unsigned long)size; + image->bus_resource.flags = IORESOURCE_MEM; + + retval = pci_bus_alloc_resource(pdev->bus, + &image->bus_resource, size, 0x10000, PCIBIOS_MIN_MEM, + 0, NULL, NULL); + if (retval) { + dev_err(tsi148_bridge->parent, "Failed to allocate mem " + "resource for window %d size 0x%lx start 0x%lx\n", + image->number, (unsigned long)size, + (unsigned long)image->bus_resource.start); + goto err_resource; + } + + image->kern_base = ioremap( + image->bus_resource.start, size); + if (!image->kern_base) { + dev_err(tsi148_bridge->parent, "Failed to remap resource\n"); + retval = -ENOMEM; + goto err_remap; + } + + return 0; + +err_remap: + release_resource(&image->bus_resource); +err_resource: + kfree(image->bus_resource.name); + memset(&image->bus_resource, 0, sizeof(image->bus_resource)); +err_name: + return retval; +} + +/* + * Free and unmap PCI Resource + */ +static void tsi148_free_resource(struct vme_master_resource *image) +{ + iounmap(image->kern_base); + image->kern_base = NULL; + release_resource(&image->bus_resource); + kfree(image->bus_resource.name); + memset(&image->bus_resource, 0, sizeof(image->bus_resource)); +} + +/* + * Set the attributes of an outbound window. + */ +static int tsi148_master_set(struct vme_master_resource *image, int enabled, + unsigned long long vme_base, unsigned long long size, u32 aspace, + u32 cycle, u32 dwidth) +{ + int retval = 0; + unsigned int i; + unsigned int temp_ctl = 0; + unsigned int pci_base_low, pci_base_high; + unsigned int pci_bound_low, pci_bound_high; + unsigned int vme_offset_low, vme_offset_high; + unsigned long long pci_bound, vme_offset, pci_base; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *bridge; + struct pci_bus_region region; + struct pci_dev *pdev; + + tsi148_bridge = image->parent; + + bridge = tsi148_bridge->driver_priv; + + pdev = to_pci_dev(tsi148_bridge->parent); + + /* Verify input data */ + if (vme_base & 0xFFFF) { + dev_err(tsi148_bridge->parent, "Invalid VME Window " + "alignment\n"); + retval = -EINVAL; + goto err_window; + } + + if ((size == 0) && (enabled != 0)) { + dev_err(tsi148_bridge->parent, "Size must be non-zero for " + "enabled windows\n"); + retval = -EINVAL; + goto err_window; + } + + spin_lock(&image->lock); + + /* Let's allocate the resource here rather than further up the stack as + * it avoids pushing loads of bus dependent stuff up the stack. If size + * is zero, any existing resource will be freed. + */ + retval = tsi148_alloc_resource(image, size); + if (retval) { + spin_unlock(&image->lock); + dev_err(tsi148_bridge->parent, "Unable to allocate memory for " + "resource\n"); + goto err_res; + } + + if (size == 0) { + pci_base = 0; + pci_bound = 0; + vme_offset = 0; + } else { + pcibios_resource_to_bus(pdev->bus, ®ion, + &image->bus_resource); + pci_base = region.start; + + /* + * Bound address is a valid address for the window, adjust + * according to window granularity. + */ + pci_bound = pci_base + (size - 0x10000); + vme_offset = vme_base - pci_base; + } + + /* Convert 64-bit variables to 2x 32-bit variables */ + reg_split(pci_base, &pci_base_high, &pci_base_low); + reg_split(pci_bound, &pci_bound_high, &pci_bound_low); + reg_split(vme_offset, &vme_offset_high, &vme_offset_low); + + if (pci_base_low & 0xFFFF) { + spin_unlock(&image->lock); + dev_err(tsi148_bridge->parent, "Invalid PCI base alignment\n"); + retval = -EINVAL; + goto err_gran; + } + if (pci_bound_low & 0xFFFF) { + spin_unlock(&image->lock); + dev_err(tsi148_bridge->parent, "Invalid PCI bound alignment\n"); + retval = -EINVAL; + goto err_gran; + } + if (vme_offset_low & 0xFFFF) { + spin_unlock(&image->lock); + dev_err(tsi148_bridge->parent, "Invalid VME Offset " + "alignment\n"); + retval = -EINVAL; + goto err_gran; + } + + i = image->number; + + /* Disable while we are mucking around */ + temp_ctl = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTAT); + temp_ctl &= ~TSI148_LCSR_OTAT_EN; + iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTAT); + + /* Setup 2eSST speeds */ + temp_ctl &= ~TSI148_LCSR_OTAT_2eSSTM_M; + switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { + case VME_2eSST160: + temp_ctl |= TSI148_LCSR_OTAT_2eSSTM_160; + break; + case VME_2eSST267: + temp_ctl |= TSI148_LCSR_OTAT_2eSSTM_267; + break; + case VME_2eSST320: + temp_ctl |= TSI148_LCSR_OTAT_2eSSTM_320; + break; + } + + /* Setup cycle types */ + if (cycle & VME_BLT) { + temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; + temp_ctl |= TSI148_LCSR_OTAT_TM_BLT; + } + if (cycle & VME_MBLT) { + temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; + temp_ctl |= TSI148_LCSR_OTAT_TM_MBLT; + } + if (cycle & VME_2eVME) { + temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; + temp_ctl |= TSI148_LCSR_OTAT_TM_2eVME; + } + if (cycle & VME_2eSST) { + temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; + temp_ctl |= TSI148_LCSR_OTAT_TM_2eSST; + } + if (cycle & VME_2eSSTB) { + dev_warn(tsi148_bridge->parent, "Currently not setting " + "Broadcast Select Registers\n"); + temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; + temp_ctl |= TSI148_LCSR_OTAT_TM_2eSSTB; + } + + /* Setup data width */ + temp_ctl &= ~TSI148_LCSR_OTAT_DBW_M; + switch (dwidth) { + case VME_D16: + temp_ctl |= TSI148_LCSR_OTAT_DBW_16; + break; + case VME_D32: + temp_ctl |= TSI148_LCSR_OTAT_DBW_32; + break; + default: + spin_unlock(&image->lock); + dev_err(tsi148_bridge->parent, "Invalid data width\n"); + retval = -EINVAL; + goto err_dwidth; + } + + /* Setup address space */ + temp_ctl &= ~TSI148_LCSR_OTAT_AMODE_M; + switch (aspace) { + case VME_A16: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_A16; + break; + case VME_A24: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_A24; + break; + case VME_A32: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_A32; + break; + case VME_A64: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_A64; + break; + case VME_CRCSR: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_CRCSR; + break; + case VME_USER1: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER1; + break; + case VME_USER2: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER2; + break; + case VME_USER3: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER3; + break; + case VME_USER4: + temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER4; + break; + default: + spin_unlock(&image->lock); + dev_err(tsi148_bridge->parent, "Invalid address space\n"); + retval = -EINVAL; + goto err_aspace; + } + + temp_ctl &= ~(3<<4); + if (cycle & VME_SUPER) + temp_ctl |= TSI148_LCSR_OTAT_SUP; + if (cycle & VME_PROG) + temp_ctl |= TSI148_LCSR_OTAT_PGM; + + /* Setup mapping */ + iowrite32be(pci_base_high, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTSAU); + iowrite32be(pci_base_low, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTSAL); + iowrite32be(pci_bound_high, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTEAU); + iowrite32be(pci_bound_low, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTEAL); + iowrite32be(vme_offset_high, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTOFU); + iowrite32be(vme_offset_low, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTOFL); + + /* Write ctl reg without enable */ + iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTAT); + + if (enabled) + temp_ctl |= TSI148_LCSR_OTAT_EN; + + iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTAT); + + spin_unlock(&image->lock); + return 0; + +err_aspace: +err_dwidth: +err_gran: + tsi148_free_resource(image); +err_res: +err_window: + return retval; + +} + +/* + * Set the attributes of an outbound window. + * + * XXX Not parsing prefetch information. + */ +static int __tsi148_master_get(struct vme_master_resource *image, int *enabled, + unsigned long long *vme_base, unsigned long long *size, u32 *aspace, + u32 *cycle, u32 *dwidth) +{ + unsigned int i, ctl; + unsigned int pci_base_low, pci_base_high; + unsigned int pci_bound_low, pci_bound_high; + unsigned int vme_offset_low, vme_offset_high; + + unsigned long long pci_base, pci_bound, vme_offset; + struct tsi148_driver *bridge; + + bridge = image->parent->driver_priv; + + i = image->number; + + ctl = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTAT); + + pci_base_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTSAU); + pci_base_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTSAL); + pci_bound_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTEAU); + pci_bound_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTEAL); + vme_offset_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTOFU); + vme_offset_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTOFL); + + /* Convert 64-bit variables to 2x 32-bit variables */ + reg_join(pci_base_high, pci_base_low, &pci_base); + reg_join(pci_bound_high, pci_bound_low, &pci_bound); + reg_join(vme_offset_high, vme_offset_low, &vme_offset); + + *vme_base = pci_base + vme_offset; + *size = (unsigned long long)(pci_bound - pci_base) + 0x10000; + + *enabled = 0; + *aspace = 0; + *cycle = 0; + *dwidth = 0; + + if (ctl & TSI148_LCSR_OTAT_EN) + *enabled = 1; + + /* Setup address space */ + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A16) + *aspace |= VME_A16; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A24) + *aspace |= VME_A24; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A32) + *aspace |= VME_A32; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A64) + *aspace |= VME_A64; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_CRCSR) + *aspace |= VME_CRCSR; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER1) + *aspace |= VME_USER1; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER2) + *aspace |= VME_USER2; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER3) + *aspace |= VME_USER3; + if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER4) + *aspace |= VME_USER4; + + /* Setup 2eSST speeds */ + if ((ctl & TSI148_LCSR_OTAT_2eSSTM_M) == TSI148_LCSR_OTAT_2eSSTM_160) + *cycle |= VME_2eSST160; + if ((ctl & TSI148_LCSR_OTAT_2eSSTM_M) == TSI148_LCSR_OTAT_2eSSTM_267) + *cycle |= VME_2eSST267; + if ((ctl & TSI148_LCSR_OTAT_2eSSTM_M) == TSI148_LCSR_OTAT_2eSSTM_320) + *cycle |= VME_2eSST320; + + /* Setup cycle types */ + if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_SCT) + *cycle |= VME_SCT; + if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_BLT) + *cycle |= VME_BLT; + if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_MBLT) + *cycle |= VME_MBLT; + if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_2eVME) + *cycle |= VME_2eVME; + if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_2eSST) + *cycle |= VME_2eSST; + if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_2eSSTB) + *cycle |= VME_2eSSTB; + + if (ctl & TSI148_LCSR_OTAT_SUP) + *cycle |= VME_SUPER; + else + *cycle |= VME_USER; + + if (ctl & TSI148_LCSR_OTAT_PGM) + *cycle |= VME_PROG; + else + *cycle |= VME_DATA; + + /* Setup data width */ + if ((ctl & TSI148_LCSR_OTAT_DBW_M) == TSI148_LCSR_OTAT_DBW_16) + *dwidth = VME_D16; + if ((ctl & TSI148_LCSR_OTAT_DBW_M) == TSI148_LCSR_OTAT_DBW_32) + *dwidth = VME_D32; + + return 0; +} + + +static int tsi148_master_get(struct vme_master_resource *image, int *enabled, + unsigned long long *vme_base, unsigned long long *size, u32 *aspace, + u32 *cycle, u32 *dwidth) +{ + int retval; + + spin_lock(&image->lock); + + retval = __tsi148_master_get(image, enabled, vme_base, size, aspace, + cycle, dwidth); + + spin_unlock(&image->lock); + + return retval; +} + +static ssize_t tsi148_master_read(struct vme_master_resource *image, void *buf, + size_t count, loff_t offset) +{ + int retval, enabled; + unsigned long long vme_base, size; + u32 aspace, cycle, dwidth; + struct vme_error_handler *handler = NULL; + struct vme_bridge *tsi148_bridge; + void __iomem *addr = image->kern_base + offset; + unsigned int done = 0; + unsigned int count32; + + tsi148_bridge = image->parent; + + spin_lock(&image->lock); + + if (err_chk) { + __tsi148_master_get(image, &enabled, &vme_base, &size, &aspace, + &cycle, &dwidth); + handler = vme_register_error_handler(tsi148_bridge, aspace, + vme_base + offset, count); + if (!handler) { + spin_unlock(&image->lock); + return -ENOMEM; + } + } + + /* The following code handles VME address alignment. We cannot use + * memcpy_xxx here because it may cut data transfers in to 8-bit + * cycles when D16 or D32 cycles are required on the VME bus. + * On the other hand, the bridge itself assures that the maximum data + * cycle configured for the transfer is used and splits it + * automatically for non-aligned addresses, so we don't want the + * overhead of needlessly forcing small transfers for the entire cycle. + */ + if ((uintptr_t)addr & 0x1) { + *(u8 *)buf = ioread8(addr); + done += 1; + if (done == count) + goto out; + } + if ((uintptr_t)(addr + done) & 0x2) { + if ((count - done) < 2) { + *(u8 *)(buf + done) = ioread8(addr + done); + done += 1; + goto out; + } else { + *(u16 *)(buf + done) = ioread16(addr + done); + done += 2; + } + } + + count32 = (count - done) & ~0x3; + while (done < count32) { + *(u32 *)(buf + done) = ioread32(addr + done); + done += 4; + } + + if ((count - done) & 0x2) { + *(u16 *)(buf + done) = ioread16(addr + done); + done += 2; + } + if ((count - done) & 0x1) { + *(u8 *)(buf + done) = ioread8(addr + done); + done += 1; + } + +out: + retval = count; + + if (err_chk) { + if (handler->num_errors) { + dev_err(image->parent->parent, + "First VME read error detected an at address 0x%llx\n", + handler->first_error); + retval = handler->first_error - (vme_base + offset); + } + vme_unregister_error_handler(handler); + } + + spin_unlock(&image->lock); + + return retval; +} + + +static ssize_t tsi148_master_write(struct vme_master_resource *image, void *buf, + size_t count, loff_t offset) +{ + int retval = 0, enabled; + unsigned long long vme_base, size; + u32 aspace, cycle, dwidth; + void __iomem *addr = image->kern_base + offset; + unsigned int done = 0; + unsigned int count32; + + struct vme_error_handler *handler = NULL; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *bridge; + + tsi148_bridge = image->parent; + + bridge = tsi148_bridge->driver_priv; + + spin_lock(&image->lock); + + if (err_chk) { + __tsi148_master_get(image, &enabled, &vme_base, &size, &aspace, + &cycle, &dwidth); + handler = vme_register_error_handler(tsi148_bridge, aspace, + vme_base + offset, count); + if (!handler) { + spin_unlock(&image->lock); + return -ENOMEM; + } + } + + /* Here we apply for the same strategy we do in master_read + * function in order to assure the correct cycles. + */ + if ((uintptr_t)addr & 0x1) { + iowrite8(*(u8 *)buf, addr); + done += 1; + if (done == count) + goto out; + } + if ((uintptr_t)(addr + done) & 0x2) { + if ((count - done) < 2) { + iowrite8(*(u8 *)(buf + done), addr + done); + done += 1; + goto out; + } else { + iowrite16(*(u16 *)(buf + done), addr + done); + done += 2; + } + } + + count32 = (count - done) & ~0x3; + while (done < count32) { + iowrite32(*(u32 *)(buf + done), addr + done); + done += 4; + } + + if ((count - done) & 0x2) { + iowrite16(*(u16 *)(buf + done), addr + done); + done += 2; + } + if ((count - done) & 0x1) { + iowrite8(*(u8 *)(buf + done), addr + done); + done += 1; + } + +out: + retval = count; + + /* + * Writes are posted. We need to do a read on the VME bus to flush out + * all of the writes before we check for errors. We can't guarantee + * that reading the data we have just written is safe. It is believed + * that there isn't any read, write re-ordering, so we can read any + * location in VME space, so lets read the Device ID from the tsi148's + * own registers as mapped into CR/CSR space. + * + * We check for saved errors in the written address range/space. + */ + + if (err_chk) { + ioread16(bridge->flush_image->kern_base + 0x7F000); + + if (handler->num_errors) { + dev_warn(tsi148_bridge->parent, + "First VME write error detected an at address 0x%llx\n", + handler->first_error); + retval = handler->first_error - (vme_base + offset); + } + vme_unregister_error_handler(handler); + } + + spin_unlock(&image->lock); + + return retval; +} + +/* + * Perform an RMW cycle on the VME bus. + * + * Requires a previously configured master window, returns final value. + */ +static unsigned int tsi148_master_rmw(struct vme_master_resource *image, + unsigned int mask, unsigned int compare, unsigned int swap, + loff_t offset) +{ + unsigned long long pci_addr; + unsigned int pci_addr_high, pci_addr_low; + u32 tmp, result; + int i; + struct tsi148_driver *bridge; + + bridge = image->parent->driver_priv; + + /* Find the PCI address that maps to the desired VME address */ + i = image->number; + + /* Locking as we can only do one of these at a time */ + mutex_lock(&bridge->vme_rmw); + + /* Lock image */ + spin_lock(&image->lock); + + pci_addr_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTSAU); + pci_addr_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTSAL); + + reg_join(pci_addr_high, pci_addr_low, &pci_addr); + reg_split(pci_addr + offset, &pci_addr_high, &pci_addr_low); + + /* Configure registers */ + iowrite32be(mask, bridge->base + TSI148_LCSR_RMWEN); + iowrite32be(compare, bridge->base + TSI148_LCSR_RMWC); + iowrite32be(swap, bridge->base + TSI148_LCSR_RMWS); + iowrite32be(pci_addr_high, bridge->base + TSI148_LCSR_RMWAU); + iowrite32be(pci_addr_low, bridge->base + TSI148_LCSR_RMWAL); + + /* Enable RMW */ + tmp = ioread32be(bridge->base + TSI148_LCSR_VMCTRL); + tmp |= TSI148_LCSR_VMCTRL_RMWEN; + iowrite32be(tmp, bridge->base + TSI148_LCSR_VMCTRL); + + /* Kick process off with a read to the required address. */ + result = ioread32be(image->kern_base + offset); + + /* Disable RMW */ + tmp = ioread32be(bridge->base + TSI148_LCSR_VMCTRL); + tmp &= ~TSI148_LCSR_VMCTRL_RMWEN; + iowrite32be(tmp, bridge->base + TSI148_LCSR_VMCTRL); + + spin_unlock(&image->lock); + + mutex_unlock(&bridge->vme_rmw); + + return result; +} + +static int tsi148_dma_set_vme_src_attributes(struct device *dev, __be32 *attr, + u32 aspace, u32 cycle, u32 dwidth) +{ + u32 val; + + val = be32_to_cpu(*attr); + + /* Setup 2eSST speeds */ + switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { + case VME_2eSST160: + val |= TSI148_LCSR_DSAT_2eSSTM_160; + break; + case VME_2eSST267: + val |= TSI148_LCSR_DSAT_2eSSTM_267; + break; + case VME_2eSST320: + val |= TSI148_LCSR_DSAT_2eSSTM_320; + break; + } + + /* Setup cycle types */ + if (cycle & VME_SCT) + val |= TSI148_LCSR_DSAT_TM_SCT; + + if (cycle & VME_BLT) + val |= TSI148_LCSR_DSAT_TM_BLT; + + if (cycle & VME_MBLT) + val |= TSI148_LCSR_DSAT_TM_MBLT; + + if (cycle & VME_2eVME) + val |= TSI148_LCSR_DSAT_TM_2eVME; + + if (cycle & VME_2eSST) + val |= TSI148_LCSR_DSAT_TM_2eSST; + + if (cycle & VME_2eSSTB) { + dev_err(dev, "Currently not setting Broadcast Select " + "Registers\n"); + val |= TSI148_LCSR_DSAT_TM_2eSSTB; + } + + /* Setup data width */ + switch (dwidth) { + case VME_D16: + val |= TSI148_LCSR_DSAT_DBW_16; + break; + case VME_D32: + val |= TSI148_LCSR_DSAT_DBW_32; + break; + default: + dev_err(dev, "Invalid data width\n"); + return -EINVAL; + } + + /* Setup address space */ + switch (aspace) { + case VME_A16: + val |= TSI148_LCSR_DSAT_AMODE_A16; + break; + case VME_A24: + val |= TSI148_LCSR_DSAT_AMODE_A24; + break; + case VME_A32: + val |= TSI148_LCSR_DSAT_AMODE_A32; + break; + case VME_A64: + val |= TSI148_LCSR_DSAT_AMODE_A64; + break; + case VME_CRCSR: + val |= TSI148_LCSR_DSAT_AMODE_CRCSR; + break; + case VME_USER1: + val |= TSI148_LCSR_DSAT_AMODE_USER1; + break; + case VME_USER2: + val |= TSI148_LCSR_DSAT_AMODE_USER2; + break; + case VME_USER3: + val |= TSI148_LCSR_DSAT_AMODE_USER3; + break; + case VME_USER4: + val |= TSI148_LCSR_DSAT_AMODE_USER4; + break; + default: + dev_err(dev, "Invalid address space\n"); + return -EINVAL; + } + + if (cycle & VME_SUPER) + val |= TSI148_LCSR_DSAT_SUP; + if (cycle & VME_PROG) + val |= TSI148_LCSR_DSAT_PGM; + + *attr = cpu_to_be32(val); + + return 0; +} + +static int tsi148_dma_set_vme_dest_attributes(struct device *dev, __be32 *attr, + u32 aspace, u32 cycle, u32 dwidth) +{ + u32 val; + + val = be32_to_cpu(*attr); + + /* Setup 2eSST speeds */ + switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { + case VME_2eSST160: + val |= TSI148_LCSR_DDAT_2eSSTM_160; + break; + case VME_2eSST267: + val |= TSI148_LCSR_DDAT_2eSSTM_267; + break; + case VME_2eSST320: + val |= TSI148_LCSR_DDAT_2eSSTM_320; + break; + } + + /* Setup cycle types */ + if (cycle & VME_SCT) + val |= TSI148_LCSR_DDAT_TM_SCT; + + if (cycle & VME_BLT) + val |= TSI148_LCSR_DDAT_TM_BLT; + + if (cycle & VME_MBLT) + val |= TSI148_LCSR_DDAT_TM_MBLT; + + if (cycle & VME_2eVME) + val |= TSI148_LCSR_DDAT_TM_2eVME; + + if (cycle & VME_2eSST) + val |= TSI148_LCSR_DDAT_TM_2eSST; + + if (cycle & VME_2eSSTB) { + dev_err(dev, "Currently not setting Broadcast Select " + "Registers\n"); + val |= TSI148_LCSR_DDAT_TM_2eSSTB; + } + + /* Setup data width */ + switch (dwidth) { + case VME_D16: + val |= TSI148_LCSR_DDAT_DBW_16; + break; + case VME_D32: + val |= TSI148_LCSR_DDAT_DBW_32; + break; + default: + dev_err(dev, "Invalid data width\n"); + return -EINVAL; + } + + /* Setup address space */ + switch (aspace) { + case VME_A16: + val |= TSI148_LCSR_DDAT_AMODE_A16; + break; + case VME_A24: + val |= TSI148_LCSR_DDAT_AMODE_A24; + break; + case VME_A32: + val |= TSI148_LCSR_DDAT_AMODE_A32; + break; + case VME_A64: + val |= TSI148_LCSR_DDAT_AMODE_A64; + break; + case VME_CRCSR: + val |= TSI148_LCSR_DDAT_AMODE_CRCSR; + break; + case VME_USER1: + val |= TSI148_LCSR_DDAT_AMODE_USER1; + break; + case VME_USER2: + val |= TSI148_LCSR_DDAT_AMODE_USER2; + break; + case VME_USER3: + val |= TSI148_LCSR_DDAT_AMODE_USER3; + break; + case VME_USER4: + val |= TSI148_LCSR_DDAT_AMODE_USER4; + break; + default: + dev_err(dev, "Invalid address space\n"); + return -EINVAL; + } + + if (cycle & VME_SUPER) + val |= TSI148_LCSR_DDAT_SUP; + if (cycle & VME_PROG) + val |= TSI148_LCSR_DDAT_PGM; + + *attr = cpu_to_be32(val); + + return 0; +} + +/* + * Add a link list descriptor to the list + * + * Note: DMA engine expects the DMA descriptor to be big endian. + */ +static int tsi148_dma_list_add(struct vme_dma_list *list, + struct vme_dma_attr *src, struct vme_dma_attr *dest, size_t count) +{ + struct tsi148_dma_entry *entry, *prev; + u32 address_high, address_low, val; + struct vme_dma_pattern *pattern_attr; + struct vme_dma_pci *pci_attr; + struct vme_dma_vme *vme_attr; + int retval = 0; + struct vme_bridge *tsi148_bridge; + + tsi148_bridge = list->parent->parent; + + /* Descriptor must be aligned on 64-bit boundaries */ + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + retval = -ENOMEM; + goto err_mem; + } + + /* Test descriptor alignment */ + if ((unsigned long)&entry->descriptor & 0x7) { + dev_err(tsi148_bridge->parent, "Descriptor not aligned to 8 " + "byte boundary as required: %p\n", + &entry->descriptor); + retval = -EINVAL; + goto err_align; + } + + /* Given we are going to fill out the structure, we probably don't + * need to zero it, but better safe than sorry for now. + */ + memset(&entry->descriptor, 0, sizeof(entry->descriptor)); + + /* Fill out source part */ + switch (src->type) { + case VME_DMA_PATTERN: + pattern_attr = src->private; + + entry->descriptor.dsal = cpu_to_be32(pattern_attr->pattern); + + val = TSI148_LCSR_DSAT_TYP_PAT; + + /* Default behaviour is 32 bit pattern */ + if (pattern_attr->type & VME_DMA_PATTERN_BYTE) + val |= TSI148_LCSR_DSAT_PSZ; + + /* It seems that the default behaviour is to increment */ + if ((pattern_attr->type & VME_DMA_PATTERN_INCREMENT) == 0) + val |= TSI148_LCSR_DSAT_NIN; + entry->descriptor.dsat = cpu_to_be32(val); + break; + case VME_DMA_PCI: + pci_attr = src->private; + + reg_split((unsigned long long)pci_attr->address, &address_high, + &address_low); + entry->descriptor.dsau = cpu_to_be32(address_high); + entry->descriptor.dsal = cpu_to_be32(address_low); + entry->descriptor.dsat = cpu_to_be32(TSI148_LCSR_DSAT_TYP_PCI); + break; + case VME_DMA_VME: + vme_attr = src->private; + + reg_split((unsigned long long)vme_attr->address, &address_high, + &address_low); + entry->descriptor.dsau = cpu_to_be32(address_high); + entry->descriptor.dsal = cpu_to_be32(address_low); + entry->descriptor.dsat = cpu_to_be32(TSI148_LCSR_DSAT_TYP_VME); + + retval = tsi148_dma_set_vme_src_attributes( + tsi148_bridge->parent, &entry->descriptor.dsat, + vme_attr->aspace, vme_attr->cycle, vme_attr->dwidth); + if (retval < 0) + goto err_source; + break; + default: + dev_err(tsi148_bridge->parent, "Invalid source type\n"); + retval = -EINVAL; + goto err_source; + } + + /* Assume last link - this will be over-written by adding another */ + entry->descriptor.dnlau = cpu_to_be32(0); + entry->descriptor.dnlal = cpu_to_be32(TSI148_LCSR_DNLAL_LLA); + + /* Fill out destination part */ + switch (dest->type) { + case VME_DMA_PCI: + pci_attr = dest->private; + + reg_split((unsigned long long)pci_attr->address, &address_high, + &address_low); + entry->descriptor.ddau = cpu_to_be32(address_high); + entry->descriptor.ddal = cpu_to_be32(address_low); + entry->descriptor.ddat = cpu_to_be32(TSI148_LCSR_DDAT_TYP_PCI); + break; + case VME_DMA_VME: + vme_attr = dest->private; + + reg_split((unsigned long long)vme_attr->address, &address_high, + &address_low); + entry->descriptor.ddau = cpu_to_be32(address_high); + entry->descriptor.ddal = cpu_to_be32(address_low); + entry->descriptor.ddat = cpu_to_be32(TSI148_LCSR_DDAT_TYP_VME); + + retval = tsi148_dma_set_vme_dest_attributes( + tsi148_bridge->parent, &entry->descriptor.ddat, + vme_attr->aspace, vme_attr->cycle, vme_attr->dwidth); + if (retval < 0) + goto err_dest; + break; + default: + dev_err(tsi148_bridge->parent, "Invalid destination type\n"); + retval = -EINVAL; + goto err_dest; + } + + /* Fill out count */ + entry->descriptor.dcnt = cpu_to_be32((u32)count); + + /* Add to list */ + list_add_tail(&entry->list, &list->entries); + + entry->dma_handle = dma_map_single(tsi148_bridge->parent, + &entry->descriptor, + sizeof(entry->descriptor), + DMA_TO_DEVICE); + if (dma_mapping_error(tsi148_bridge->parent, entry->dma_handle)) { + dev_err(tsi148_bridge->parent, "DMA mapping error\n"); + retval = -EINVAL; + goto err_dma; + } + + /* Fill out previous descriptors "Next Address" */ + if (entry->list.prev != &list->entries) { + reg_split((unsigned long long)entry->dma_handle, &address_high, + &address_low); + prev = list_entry(entry->list.prev, struct tsi148_dma_entry, + list); + prev->descriptor.dnlau = cpu_to_be32(address_high); + prev->descriptor.dnlal = cpu_to_be32(address_low); + + } + + return 0; + +err_dma: +err_dest: +err_source: +err_align: + kfree(entry); +err_mem: + return retval; +} + +/* + * Check to see if the provided DMA channel is busy. + */ +static int tsi148_dma_busy(struct vme_bridge *tsi148_bridge, int channel) +{ + u32 tmp; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + tmp = ioread32be(bridge->base + TSI148_LCSR_DMA[channel] + + TSI148_LCSR_OFFSET_DSTA); + + if (tmp & TSI148_LCSR_DSTA_BSY) + return 0; + else + return 1; + +} + +/* + * Execute a previously generated link list + * + * XXX Need to provide control register configuration. + */ +static int tsi148_dma_list_exec(struct vme_dma_list *list) +{ + struct vme_dma_resource *ctrlr; + int channel, retval; + struct tsi148_dma_entry *entry; + u32 bus_addr_high, bus_addr_low; + u32 val, dctlreg = 0; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *bridge; + + ctrlr = list->parent; + + tsi148_bridge = ctrlr->parent; + + bridge = tsi148_bridge->driver_priv; + + mutex_lock(&ctrlr->mtx); + + channel = ctrlr->number; + + if (!list_empty(&ctrlr->running)) { + /* + * XXX We have an active DMA transfer and currently haven't + * sorted out the mechanism for "pending" DMA transfers. + * Return busy. + */ + /* Need to add to pending here */ + mutex_unlock(&ctrlr->mtx); + return -EBUSY; + } else { + list_add(&list->list, &ctrlr->running); + } + + /* Get first bus address and write into registers */ + entry = list_first_entry(&list->entries, struct tsi148_dma_entry, + list); + + mutex_unlock(&ctrlr->mtx); + + reg_split(entry->dma_handle, &bus_addr_high, &bus_addr_low); + + iowrite32be(bus_addr_high, bridge->base + + TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DNLAU); + iowrite32be(bus_addr_low, bridge->base + + TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DNLAL); + + dctlreg = ioread32be(bridge->base + TSI148_LCSR_DMA[channel] + + TSI148_LCSR_OFFSET_DCTL); + + /* Start the operation */ + iowrite32be(dctlreg | TSI148_LCSR_DCTL_DGO, bridge->base + + TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DCTL); + + retval = wait_event_interruptible(bridge->dma_queue[channel], + tsi148_dma_busy(ctrlr->parent, channel)); + + if (retval) { + iowrite32be(dctlreg | TSI148_LCSR_DCTL_ABT, bridge->base + + TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DCTL); + /* Wait for the operation to abort */ + wait_event(bridge->dma_queue[channel], + tsi148_dma_busy(ctrlr->parent, channel)); + retval = -EINTR; + goto exit; + } + + /* + * Read status register, this register is valid until we kick off a + * new transfer. + */ + val = ioread32be(bridge->base + TSI148_LCSR_DMA[channel] + + TSI148_LCSR_OFFSET_DSTA); + + if (val & TSI148_LCSR_DSTA_VBE) { + dev_err(tsi148_bridge->parent, "DMA Error. DSTA=%08X\n", val); + retval = -EIO; + } + +exit: + /* Remove list from running list */ + mutex_lock(&ctrlr->mtx); + list_del(&list->list); + mutex_unlock(&ctrlr->mtx); + + return retval; +} + +/* + * Clean up a previously generated link list + * + * We have a separate function, don't assume that the chain can't be reused. + */ +static int tsi148_dma_list_empty(struct vme_dma_list *list) +{ + struct list_head *pos, *temp; + struct tsi148_dma_entry *entry; + + struct vme_bridge *tsi148_bridge = list->parent->parent; + + /* detach and free each entry */ + list_for_each_safe(pos, temp, &list->entries) { + list_del(pos); + entry = list_entry(pos, struct tsi148_dma_entry, list); + + dma_unmap_single(tsi148_bridge->parent, entry->dma_handle, + sizeof(struct tsi148_dma_descriptor), DMA_TO_DEVICE); + kfree(entry); + } + + return 0; +} + +/* + * All 4 location monitors reside at the same base - this is therefore a + * system wide configuration. + * + * This does not enable the LM monitor - that should be done when the first + * callback is attached and disabled when the last callback is removed. + */ +static int tsi148_lm_set(struct vme_lm_resource *lm, unsigned long long lm_base, + u32 aspace, u32 cycle) +{ + u32 lm_base_high, lm_base_low, lm_ctl = 0; + int i; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *bridge; + + tsi148_bridge = lm->parent; + + bridge = tsi148_bridge->driver_priv; + + mutex_lock(&lm->mtx); + + /* If we already have a callback attached, we can't move it! */ + for (i = 0; i < lm->monitors; i++) { + if (bridge->lm_callback[i]) { + mutex_unlock(&lm->mtx); + dev_err(tsi148_bridge->parent, "Location monitor " + "callback attached, can't reset\n"); + return -EBUSY; + } + } + + switch (aspace) { + case VME_A16: + lm_ctl |= TSI148_LCSR_LMAT_AS_A16; + break; + case VME_A24: + lm_ctl |= TSI148_LCSR_LMAT_AS_A24; + break; + case VME_A32: + lm_ctl |= TSI148_LCSR_LMAT_AS_A32; + break; + case VME_A64: + lm_ctl |= TSI148_LCSR_LMAT_AS_A64; + break; + default: + mutex_unlock(&lm->mtx); + dev_err(tsi148_bridge->parent, "Invalid address space\n"); + return -EINVAL; + } + + if (cycle & VME_SUPER) + lm_ctl |= TSI148_LCSR_LMAT_SUPR ; + if (cycle & VME_USER) + lm_ctl |= TSI148_LCSR_LMAT_NPRIV; + if (cycle & VME_PROG) + lm_ctl |= TSI148_LCSR_LMAT_PGM; + if (cycle & VME_DATA) + lm_ctl |= TSI148_LCSR_LMAT_DATA; + + reg_split(lm_base, &lm_base_high, &lm_base_low); + + iowrite32be(lm_base_high, bridge->base + TSI148_LCSR_LMBAU); + iowrite32be(lm_base_low, bridge->base + TSI148_LCSR_LMBAL); + iowrite32be(lm_ctl, bridge->base + TSI148_LCSR_LMAT); + + mutex_unlock(&lm->mtx); + + return 0; +} + +/* Get configuration of the callback monitor and return whether it is enabled + * or disabled. + */ +static int tsi148_lm_get(struct vme_lm_resource *lm, + unsigned long long *lm_base, u32 *aspace, u32 *cycle) +{ + u32 lm_base_high, lm_base_low, lm_ctl, enabled = 0; + struct tsi148_driver *bridge; + + bridge = lm->parent->driver_priv; + + mutex_lock(&lm->mtx); + + lm_base_high = ioread32be(bridge->base + TSI148_LCSR_LMBAU); + lm_base_low = ioread32be(bridge->base + TSI148_LCSR_LMBAL); + lm_ctl = ioread32be(bridge->base + TSI148_LCSR_LMAT); + + reg_join(lm_base_high, lm_base_low, lm_base); + + if (lm_ctl & TSI148_LCSR_LMAT_EN) + enabled = 1; + + if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A16) + *aspace |= VME_A16; + + if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A24) + *aspace |= VME_A24; + + if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A32) + *aspace |= VME_A32; + + if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A64) + *aspace |= VME_A64; + + + if (lm_ctl & TSI148_LCSR_LMAT_SUPR) + *cycle |= VME_SUPER; + if (lm_ctl & TSI148_LCSR_LMAT_NPRIV) + *cycle |= VME_USER; + if (lm_ctl & TSI148_LCSR_LMAT_PGM) + *cycle |= VME_PROG; + if (lm_ctl & TSI148_LCSR_LMAT_DATA) + *cycle |= VME_DATA; + + mutex_unlock(&lm->mtx); + + return enabled; +} + +/* + * Attach a callback to a specific location monitor. + * + * Callback will be passed the monitor triggered. + */ +static int tsi148_lm_attach(struct vme_lm_resource *lm, int monitor, + void (*callback)(void *), void *data) +{ + u32 lm_ctl, tmp; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *bridge; + + tsi148_bridge = lm->parent; + + bridge = tsi148_bridge->driver_priv; + + mutex_lock(&lm->mtx); + + /* Ensure that the location monitor is configured - need PGM or DATA */ + lm_ctl = ioread32be(bridge->base + TSI148_LCSR_LMAT); + if ((lm_ctl & (TSI148_LCSR_LMAT_PGM | TSI148_LCSR_LMAT_DATA)) == 0) { + mutex_unlock(&lm->mtx); + dev_err(tsi148_bridge->parent, "Location monitor not properly " + "configured\n"); + return -EINVAL; + } + + /* Check that a callback isn't already attached */ + if (bridge->lm_callback[monitor]) { + mutex_unlock(&lm->mtx); + dev_err(tsi148_bridge->parent, "Existing callback attached\n"); + return -EBUSY; + } + + /* Attach callback */ + bridge->lm_callback[monitor] = callback; + bridge->lm_data[monitor] = data; + + /* Enable Location Monitor interrupt */ + tmp = ioread32be(bridge->base + TSI148_LCSR_INTEN); + tmp |= TSI148_LCSR_INTEN_LMEN[monitor]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); + + tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); + tmp |= TSI148_LCSR_INTEO_LMEO[monitor]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); + + /* Ensure that global Location Monitor Enable set */ + if ((lm_ctl & TSI148_LCSR_LMAT_EN) == 0) { + lm_ctl |= TSI148_LCSR_LMAT_EN; + iowrite32be(lm_ctl, bridge->base + TSI148_LCSR_LMAT); + } + + mutex_unlock(&lm->mtx); + + return 0; +} + +/* + * Detach a callback function forn a specific location monitor. + */ +static int tsi148_lm_detach(struct vme_lm_resource *lm, int monitor) +{ + u32 lm_en, tmp; + struct tsi148_driver *bridge; + + bridge = lm->parent->driver_priv; + + mutex_lock(&lm->mtx); + + /* Disable Location Monitor and ensure previous interrupts are clear */ + lm_en = ioread32be(bridge->base + TSI148_LCSR_INTEN); + lm_en &= ~TSI148_LCSR_INTEN_LMEN[monitor]; + iowrite32be(lm_en, bridge->base + TSI148_LCSR_INTEN); + + tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); + tmp &= ~TSI148_LCSR_INTEO_LMEO[monitor]; + iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); + + iowrite32be(TSI148_LCSR_INTC_LMC[monitor], + bridge->base + TSI148_LCSR_INTC); + + /* Detach callback */ + bridge->lm_callback[monitor] = NULL; + bridge->lm_data[monitor] = NULL; + + /* If all location monitors disabled, disable global Location Monitor */ + if ((lm_en & (TSI148_LCSR_INTS_LM0S | TSI148_LCSR_INTS_LM1S | + TSI148_LCSR_INTS_LM2S | TSI148_LCSR_INTS_LM3S)) == 0) { + tmp = ioread32be(bridge->base + TSI148_LCSR_LMAT); + tmp &= ~TSI148_LCSR_LMAT_EN; + iowrite32be(tmp, bridge->base + TSI148_LCSR_LMAT); + } + + mutex_unlock(&lm->mtx); + + return 0; +} + +/* + * Determine Geographical Addressing + */ +static int tsi148_slot_get(struct vme_bridge *tsi148_bridge) +{ + u32 slot = 0; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + if (!geoid) { + slot = ioread32be(bridge->base + TSI148_LCSR_VSTAT); + slot = slot & TSI148_LCSR_VSTAT_GA_M; + } else + slot = geoid; + + return (int)slot; +} + +static void *tsi148_alloc_consistent(struct device *parent, size_t size, + dma_addr_t *dma) +{ + struct pci_dev *pdev; + + /* Find pci_dev container of dev */ + pdev = to_pci_dev(parent); + + return dma_alloc_coherent(&pdev->dev, size, dma, GFP_KERNEL); +} + +static void tsi148_free_consistent(struct device *parent, size_t size, + void *vaddr, dma_addr_t dma) +{ + struct pci_dev *pdev; + + /* Find pci_dev container of dev */ + pdev = to_pci_dev(parent); + + dma_free_coherent(&pdev->dev, size, vaddr, dma); +} + +/* + * Configure CR/CSR space + * + * Access to the CR/CSR can be configured at power-up. The location of the + * CR/CSR registers in the CR/CSR address space is determined by the boards + * Auto-ID or Geographic address. This function ensures that the window is + * enabled at an offset consistent with the boards geopgraphic address. + * + * Each board has a 512kB window, with the highest 4kB being used for the + * boards registers, this means there is a fix length 508kB window which must + * be mapped onto PCI memory. + */ +static int tsi148_crcsr_init(struct vme_bridge *tsi148_bridge, + struct pci_dev *pdev) +{ + u32 cbar, crat, vstat; + u32 crcsr_bus_high, crcsr_bus_low; + int retval; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + /* Allocate mem for CR/CSR image */ + bridge->crcsr_kernel = dma_alloc_coherent(&pdev->dev, + VME_CRCSR_BUF_SIZE, + &bridge->crcsr_bus, GFP_KERNEL); + if (!bridge->crcsr_kernel) { + dev_err(tsi148_bridge->parent, "Failed to allocate memory for " + "CR/CSR image\n"); + return -ENOMEM; + } + + reg_split(bridge->crcsr_bus, &crcsr_bus_high, &crcsr_bus_low); + + iowrite32be(crcsr_bus_high, bridge->base + TSI148_LCSR_CROU); + iowrite32be(crcsr_bus_low, bridge->base + TSI148_LCSR_CROL); + + /* Ensure that the CR/CSR is configured at the correct offset */ + cbar = ioread32be(bridge->base + TSI148_CBAR); + cbar = (cbar & TSI148_CRCSR_CBAR_M)>>3; + + vstat = tsi148_slot_get(tsi148_bridge); + + if (cbar != vstat) { + cbar = vstat; + dev_info(tsi148_bridge->parent, "Setting CR/CSR offset\n"); + iowrite32be(cbar<<3, bridge->base + TSI148_CBAR); + } + dev_info(tsi148_bridge->parent, "CR/CSR Offset: %d\n", cbar); + + crat = ioread32be(bridge->base + TSI148_LCSR_CRAT); + if (crat & TSI148_LCSR_CRAT_EN) + dev_info(tsi148_bridge->parent, "CR/CSR already enabled\n"); + else { + dev_info(tsi148_bridge->parent, "Enabling CR/CSR space\n"); + iowrite32be(crat | TSI148_LCSR_CRAT_EN, + bridge->base + TSI148_LCSR_CRAT); + } + + /* If we want flushed, error-checked writes, set up a window + * over the CR/CSR registers. We read from here to safely flush + * through VME writes. + */ + if (err_chk) { + retval = tsi148_master_set(bridge->flush_image, 1, + (vstat * 0x80000), 0x80000, VME_CRCSR, VME_SCT, + VME_D16); + if (retval) + dev_err(tsi148_bridge->parent, "Configuring flush image" + " failed\n"); + } + + return 0; + +} + +static void tsi148_crcsr_exit(struct vme_bridge *tsi148_bridge, + struct pci_dev *pdev) +{ + u32 crat; + struct tsi148_driver *bridge; + + bridge = tsi148_bridge->driver_priv; + + /* Turn off CR/CSR space */ + crat = ioread32be(bridge->base + TSI148_LCSR_CRAT); + iowrite32be(crat & ~TSI148_LCSR_CRAT_EN, + bridge->base + TSI148_LCSR_CRAT); + + /* Free image */ + iowrite32be(0, bridge->base + TSI148_LCSR_CROU); + iowrite32be(0, bridge->base + TSI148_LCSR_CROL); + + dma_free_coherent(&pdev->dev, VME_CRCSR_BUF_SIZE, + bridge->crcsr_kernel, bridge->crcsr_bus); +} + +static int tsi148_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + int retval, i, master_num; + u32 data; + struct list_head *pos = NULL, *n; + struct vme_bridge *tsi148_bridge; + struct tsi148_driver *tsi148_device; + struct vme_master_resource *master_image; + struct vme_slave_resource *slave_image; + struct vme_dma_resource *dma_ctrlr; + struct vme_lm_resource *lm; + + /* If we want to support more than one of each bridge, we need to + * dynamically generate this so we get one per device + */ + tsi148_bridge = kzalloc(sizeof(*tsi148_bridge), GFP_KERNEL); + if (!tsi148_bridge) { + retval = -ENOMEM; + goto err_struct; + } + vme_init_bridge(tsi148_bridge); + + tsi148_device = kzalloc(sizeof(*tsi148_device), GFP_KERNEL); + if (!tsi148_device) { + retval = -ENOMEM; + goto err_driver; + } + + tsi148_bridge->driver_priv = tsi148_device; + + /* Enable the device */ + retval = pci_enable_device(pdev); + if (retval) { + dev_err(&pdev->dev, "Unable to enable device\n"); + goto err_enable; + } + + /* Map Registers */ + retval = pci_request_regions(pdev, driver_name); + if (retval) { + dev_err(&pdev->dev, "Unable to reserve resources\n"); + goto err_resource; + } + + /* map registers in BAR 0 */ + tsi148_device->base = ioremap(pci_resource_start(pdev, 0), + 4096); + if (!tsi148_device->base) { + dev_err(&pdev->dev, "Unable to remap CRG region\n"); + retval = -EIO; + goto err_remap; + } + + /* Check to see if the mapping worked out */ + data = ioread32(tsi148_device->base + TSI148_PCFS_ID) & 0x0000FFFF; + if (data != PCI_VENDOR_ID_TUNDRA) { + dev_err(&pdev->dev, "CRG region check failed\n"); + retval = -EIO; + goto err_test; + } + + /* Initialize wait queues & mutual exclusion flags */ + init_waitqueue_head(&tsi148_device->dma_queue[0]); + init_waitqueue_head(&tsi148_device->dma_queue[1]); + init_waitqueue_head(&tsi148_device->iack_queue); + mutex_init(&tsi148_device->vme_int); + mutex_init(&tsi148_device->vme_rmw); + + tsi148_bridge->parent = &pdev->dev; + strcpy(tsi148_bridge->name, driver_name); + + /* Setup IRQ */ + retval = tsi148_irq_init(tsi148_bridge); + if (retval != 0) { + dev_err(&pdev->dev, "Chip Initialization failed.\n"); + goto err_irq; + } + + /* If we are going to flush writes, we need to read from the VME bus. + * We need to do this safely, thus we read the devices own CR/CSR + * register. To do this we must set up a window in CR/CSR space and + * hence have one less master window resource available. + */ + master_num = TSI148_MAX_MASTER; + if (err_chk) { + master_num--; + + tsi148_device->flush_image = + kmalloc(sizeof(*tsi148_device->flush_image), + GFP_KERNEL); + if (!tsi148_device->flush_image) { + retval = -ENOMEM; + goto err_master; + } + tsi148_device->flush_image->parent = tsi148_bridge; + spin_lock_init(&tsi148_device->flush_image->lock); + tsi148_device->flush_image->locked = 1; + tsi148_device->flush_image->number = master_num; + memset(&tsi148_device->flush_image->bus_resource, 0, + sizeof(tsi148_device->flush_image->bus_resource)); + tsi148_device->flush_image->kern_base = NULL; + } + + /* Add master windows to list */ + for (i = 0; i < master_num; i++) { + master_image = kmalloc(sizeof(*master_image), GFP_KERNEL); + if (!master_image) { + retval = -ENOMEM; + goto err_master; + } + master_image->parent = tsi148_bridge; + spin_lock_init(&master_image->lock); + master_image->locked = 0; + master_image->number = i; + master_image->address_attr = VME_A16 | VME_A24 | VME_A32 | + VME_A64 | VME_CRCSR | VME_USER1 | VME_USER2 | + VME_USER3 | VME_USER4; + master_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | + VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | + VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | + VME_PROG | VME_DATA; + master_image->width_attr = VME_D16 | VME_D32; + memset(&master_image->bus_resource, 0, + sizeof(master_image->bus_resource)); + master_image->kern_base = NULL; + list_add_tail(&master_image->list, + &tsi148_bridge->master_resources); + } + + /* Add slave windows to list */ + for (i = 0; i < TSI148_MAX_SLAVE; i++) { + slave_image = kmalloc(sizeof(*slave_image), GFP_KERNEL); + if (!slave_image) { + retval = -ENOMEM; + goto err_slave; + } + slave_image->parent = tsi148_bridge; + mutex_init(&slave_image->mtx); + slave_image->locked = 0; + slave_image->number = i; + slave_image->address_attr = VME_A16 | VME_A24 | VME_A32 | + VME_A64; + slave_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | + VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | + VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | + VME_PROG | VME_DATA; + list_add_tail(&slave_image->list, + &tsi148_bridge->slave_resources); + } + + /* Add dma engines to list */ + for (i = 0; i < TSI148_MAX_DMA; i++) { + dma_ctrlr = kmalloc(sizeof(*dma_ctrlr), GFP_KERNEL); + if (!dma_ctrlr) { + retval = -ENOMEM; + goto err_dma; + } + dma_ctrlr->parent = tsi148_bridge; + mutex_init(&dma_ctrlr->mtx); + dma_ctrlr->locked = 0; + dma_ctrlr->number = i; + dma_ctrlr->route_attr = VME_DMA_VME_TO_MEM | + VME_DMA_MEM_TO_VME | VME_DMA_VME_TO_VME | + VME_DMA_MEM_TO_MEM | VME_DMA_PATTERN_TO_VME | + VME_DMA_PATTERN_TO_MEM; + INIT_LIST_HEAD(&dma_ctrlr->pending); + INIT_LIST_HEAD(&dma_ctrlr->running); + list_add_tail(&dma_ctrlr->list, + &tsi148_bridge->dma_resources); + } + + /* Add location monitor to list */ + lm = kmalloc(sizeof(*lm), GFP_KERNEL); + if (!lm) { + retval = -ENOMEM; + goto err_lm; + } + lm->parent = tsi148_bridge; + mutex_init(&lm->mtx); + lm->locked = 0; + lm->number = 1; + lm->monitors = 4; + list_add_tail(&lm->list, &tsi148_bridge->lm_resources); + + tsi148_bridge->slave_get = tsi148_slave_get; + tsi148_bridge->slave_set = tsi148_slave_set; + tsi148_bridge->master_get = tsi148_master_get; + tsi148_bridge->master_set = tsi148_master_set; + tsi148_bridge->master_read = tsi148_master_read; + tsi148_bridge->master_write = tsi148_master_write; + tsi148_bridge->master_rmw = tsi148_master_rmw; + tsi148_bridge->dma_list_add = tsi148_dma_list_add; + tsi148_bridge->dma_list_exec = tsi148_dma_list_exec; + tsi148_bridge->dma_list_empty = tsi148_dma_list_empty; + tsi148_bridge->irq_set = tsi148_irq_set; + tsi148_bridge->irq_generate = tsi148_irq_generate; + tsi148_bridge->lm_set = tsi148_lm_set; + tsi148_bridge->lm_get = tsi148_lm_get; + tsi148_bridge->lm_attach = tsi148_lm_attach; + tsi148_bridge->lm_detach = tsi148_lm_detach; + tsi148_bridge->slot_get = tsi148_slot_get; + tsi148_bridge->alloc_consistent = tsi148_alloc_consistent; + tsi148_bridge->free_consistent = tsi148_free_consistent; + + data = ioread32be(tsi148_device->base + TSI148_LCSR_VSTAT); + dev_info(&pdev->dev, "Board is%s the VME system controller\n", + (data & TSI148_LCSR_VSTAT_SCONS) ? "" : " not"); + if (!geoid) + dev_info(&pdev->dev, "VME geographical address is %d\n", + data & TSI148_LCSR_VSTAT_GA_M); + else + dev_info(&pdev->dev, "VME geographical address is set to %d\n", + geoid); + + dev_info(&pdev->dev, "VME Write and flush and error check is %s\n", + err_chk ? "enabled" : "disabled"); + + retval = tsi148_crcsr_init(tsi148_bridge, pdev); + if (retval) { + dev_err(&pdev->dev, "CR/CSR configuration failed.\n"); + goto err_crcsr; + } + + retval = vme_register_bridge(tsi148_bridge); + if (retval != 0) { + dev_err(&pdev->dev, "Chip Registration failed.\n"); + goto err_reg; + } + + pci_set_drvdata(pdev, tsi148_bridge); + + /* Clear VME bus "board fail", and "power-up reset" lines */ + data = ioread32be(tsi148_device->base + TSI148_LCSR_VSTAT); + data &= ~TSI148_LCSR_VSTAT_BRDFL; + data |= TSI148_LCSR_VSTAT_CPURST; + iowrite32be(data, tsi148_device->base + TSI148_LCSR_VSTAT); + + return 0; + +err_reg: + tsi148_crcsr_exit(tsi148_bridge, pdev); +err_crcsr: +err_lm: + /* resources are stored in link list */ + list_for_each_safe(pos, n, &tsi148_bridge->lm_resources) { + lm = list_entry(pos, struct vme_lm_resource, list); + list_del(pos); + kfree(lm); + } +err_dma: + /* resources are stored in link list */ + list_for_each_safe(pos, n, &tsi148_bridge->dma_resources) { + dma_ctrlr = list_entry(pos, struct vme_dma_resource, list); + list_del(pos); + kfree(dma_ctrlr); + } +err_slave: + /* resources are stored in link list */ + list_for_each_safe(pos, n, &tsi148_bridge->slave_resources) { + slave_image = list_entry(pos, struct vme_slave_resource, list); + list_del(pos); + kfree(slave_image); + } +err_master: + /* resources are stored in link list */ + list_for_each_safe(pos, n, &tsi148_bridge->master_resources) { + master_image = list_entry(pos, struct vme_master_resource, + list); + list_del(pos); + kfree(master_image); + } + + tsi148_irq_exit(tsi148_bridge, pdev); +err_irq: +err_test: + iounmap(tsi148_device->base); +err_remap: + pci_release_regions(pdev); +err_resource: + pci_disable_device(pdev); +err_enable: + kfree(tsi148_device); +err_driver: + kfree(tsi148_bridge); +err_struct: + return retval; + +} + +static void tsi148_remove(struct pci_dev *pdev) +{ + struct list_head *pos = NULL; + struct list_head *tmplist; + struct vme_master_resource *master_image; + struct vme_slave_resource *slave_image; + struct vme_dma_resource *dma_ctrlr; + int i; + struct tsi148_driver *bridge; + struct vme_bridge *tsi148_bridge = pci_get_drvdata(pdev); + + bridge = tsi148_bridge->driver_priv; + + + dev_dbg(&pdev->dev, "Driver is being unloaded.\n"); + + /* + * Shutdown all inbound and outbound windows. + */ + for (i = 0; i < 8; i++) { + iowrite32be(0, bridge->base + TSI148_LCSR_IT[i] + + TSI148_LCSR_OFFSET_ITAT); + iowrite32be(0, bridge->base + TSI148_LCSR_OT[i] + + TSI148_LCSR_OFFSET_OTAT); + } + + /* + * Shutdown Location monitor. + */ + iowrite32be(0, bridge->base + TSI148_LCSR_LMAT); + + /* + * Shutdown CRG map. + */ + iowrite32be(0, bridge->base + TSI148_LCSR_CSRAT); + + /* + * Clear error status. + */ + iowrite32be(0xFFFFFFFF, bridge->base + TSI148_LCSR_EDPAT); + iowrite32be(0xFFFFFFFF, bridge->base + TSI148_LCSR_VEAT); + iowrite32be(0x07000700, bridge->base + TSI148_LCSR_PSTAT); + + /* + * Remove VIRQ interrupt (if any) + */ + if (ioread32be(bridge->base + TSI148_LCSR_VICR) & 0x800) + iowrite32be(0x8000, bridge->base + TSI148_LCSR_VICR); + + /* + * Map all Interrupts to PCI INTA + */ + iowrite32be(0x0, bridge->base + TSI148_LCSR_INTM1); + iowrite32be(0x0, bridge->base + TSI148_LCSR_INTM2); + + tsi148_irq_exit(tsi148_bridge, pdev); + + vme_unregister_bridge(tsi148_bridge); + + tsi148_crcsr_exit(tsi148_bridge, pdev); + + /* resources are stored in link list */ + list_for_each_safe(pos, tmplist, &tsi148_bridge->dma_resources) { + dma_ctrlr = list_entry(pos, struct vme_dma_resource, list); + list_del(pos); + kfree(dma_ctrlr); + } + + /* resources are stored in link list */ + list_for_each_safe(pos, tmplist, &tsi148_bridge->slave_resources) { + slave_image = list_entry(pos, struct vme_slave_resource, list); + list_del(pos); + kfree(slave_image); + } + + /* resources are stored in link list */ + list_for_each_safe(pos, tmplist, &tsi148_bridge->master_resources) { + master_image = list_entry(pos, struct vme_master_resource, + list); + list_del(pos); + kfree(master_image); + } + + iounmap(bridge->base); + + pci_release_regions(pdev); + + pci_disable_device(pdev); + + kfree(tsi148_bridge->driver_priv); + + kfree(tsi148_bridge); +} + +module_pci_driver(tsi148_driver); + +MODULE_PARM_DESC(err_chk, "Check for VME errors on reads and writes"); +module_param(err_chk, bool, 0); + +MODULE_PARM_DESC(geoid, "Override geographical addressing"); +module_param(geoid, int, 0); + +MODULE_DESCRIPTION("VME driver for the Tundra Tempe VME bridge"); +MODULE_LICENSE("GPL"); diff --git a/drivers/staging/vme_user/vme_tsi148.h b/drivers/staging/vme_user/vme_tsi148.h new file mode 100644 index 000000000000..226fedc6f167 --- /dev/null +++ b/drivers/staging/vme_user/vme_tsi148.h @@ -0,0 +1,1407 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * tsi148.h + * + * Support for the Tundra TSI148 VME Bridge chip + * + * Author: Tom Armistead + * Updated and maintained by Ajit Prem + * Copyright 2004 Motorola Inc. + */ + +#ifndef TSI148_H +#define TSI148_H + +#ifndef PCI_VENDOR_ID_TUNDRA +#define PCI_VENDOR_ID_TUNDRA 0x10e3 +#endif + +#ifndef PCI_DEVICE_ID_TUNDRA_TSI148 +#define PCI_DEVICE_ID_TUNDRA_TSI148 0x148 +#endif + +/* + * Define the number of each that the Tsi148 supports. + */ +#define TSI148_MAX_MASTER 8 /* Max Master Windows */ +#define TSI148_MAX_SLAVE 8 /* Max Slave Windows */ +#define TSI148_MAX_DMA 2 /* Max DMA Controllers */ +#define TSI148_MAX_MAILBOX 4 /* Max Mail Box registers */ +#define TSI148_MAX_SEMAPHORE 8 /* Max Semaphores */ + +/* Structure used to hold driver specific information */ +struct tsi148_driver { + void __iomem *base; /* Base Address of device registers */ + wait_queue_head_t dma_queue[2]; + wait_queue_head_t iack_queue; + void (*lm_callback[4])(void *); /* Called in interrupt handler */ + void *lm_data[4]; + void *crcsr_kernel; + dma_addr_t crcsr_bus; + struct vme_master_resource *flush_image; + struct mutex vme_rmw; /* Only one RMW cycle at a time */ + struct mutex vme_int; /* + * Only one VME interrupt can be + * generated at a time, provide locking + */ +}; + +/* + * Layout of a DMAC Linked-List Descriptor + * + * Note: This structure is accessed via the chip and therefore must be + * correctly laid out - It must also be aligned on 64-bit boundaries. + */ +struct tsi148_dma_descriptor { + __be32 dsau; /* Source Address */ + __be32 dsal; + __be32 ddau; /* Destination Address */ + __be32 ddal; + __be32 dsat; /* Source attributes */ + __be32 ddat; /* Destination attributes */ + __be32 dnlau; /* Next link address */ + __be32 dnlal; + __be32 dcnt; /* Byte count */ + __be32 ddbs; /* 2eSST Broadcast select */ +}; + +struct tsi148_dma_entry { + /* + * The descriptor needs to be aligned on a 64-bit boundary, we increase + * the chance of this by putting it first in the structure. + */ + struct tsi148_dma_descriptor descriptor; + struct list_head list; + dma_addr_t dma_handle; +}; + +/* + * TSI148 ASIC register structure overlays and bit field definitions. + * + * Note: Tsi148 Register Group (CRG) consists of the following + * combination of registers: + * PCFS - PCI Configuration Space Registers + * LCSR - Local Control and Status Registers + * GCSR - Global Control and Status Registers + * CR/CSR - Subset of Configuration ROM / + * Control and Status Registers + */ + + +/* + * Command/Status Registers (CRG + $004) + */ +#define TSI148_PCFS_ID 0x0 +#define TSI148_PCFS_CSR 0x4 +#define TSI148_PCFS_CLASS 0x8 +#define TSI148_PCFS_MISC0 0xC +#define TSI148_PCFS_MBARL 0x10 +#define TSI148_PCFS_MBARU 0x14 + +#define TSI148_PCFS_SUBID 0x28 + +#define TSI148_PCFS_CAPP 0x34 + +#define TSI148_PCFS_MISC1 0x3C + +#define TSI148_PCFS_XCAPP 0x40 +#define TSI148_PCFS_XSTAT 0x44 + +/* + * LCSR definitions + */ + +/* + * Outbound Translations + */ +#define TSI148_LCSR_OT0_OTSAU 0x100 +#define TSI148_LCSR_OT0_OTSAL 0x104 +#define TSI148_LCSR_OT0_OTEAU 0x108 +#define TSI148_LCSR_OT0_OTEAL 0x10C +#define TSI148_LCSR_OT0_OTOFU 0x110 +#define TSI148_LCSR_OT0_OTOFL 0x114 +#define TSI148_LCSR_OT0_OTBS 0x118 +#define TSI148_LCSR_OT0_OTAT 0x11C + +#define TSI148_LCSR_OT1_OTSAU 0x120 +#define TSI148_LCSR_OT1_OTSAL 0x124 +#define TSI148_LCSR_OT1_OTEAU 0x128 +#define TSI148_LCSR_OT1_OTEAL 0x12C +#define TSI148_LCSR_OT1_OTOFU 0x130 +#define TSI148_LCSR_OT1_OTOFL 0x134 +#define TSI148_LCSR_OT1_OTBS 0x138 +#define TSI148_LCSR_OT1_OTAT 0x13C + +#define TSI148_LCSR_OT2_OTSAU 0x140 +#define TSI148_LCSR_OT2_OTSAL 0x144 +#define TSI148_LCSR_OT2_OTEAU 0x148 +#define TSI148_LCSR_OT2_OTEAL 0x14C +#define TSI148_LCSR_OT2_OTOFU 0x150 +#define TSI148_LCSR_OT2_OTOFL 0x154 +#define TSI148_LCSR_OT2_OTBS 0x158 +#define TSI148_LCSR_OT2_OTAT 0x15C + +#define TSI148_LCSR_OT3_OTSAU 0x160 +#define TSI148_LCSR_OT3_OTSAL 0x164 +#define TSI148_LCSR_OT3_OTEAU 0x168 +#define TSI148_LCSR_OT3_OTEAL 0x16C +#define TSI148_LCSR_OT3_OTOFU 0x170 +#define TSI148_LCSR_OT3_OTOFL 0x174 +#define TSI148_LCSR_OT3_OTBS 0x178 +#define TSI148_LCSR_OT3_OTAT 0x17C + +#define TSI148_LCSR_OT4_OTSAU 0x180 +#define TSI148_LCSR_OT4_OTSAL 0x184 +#define TSI148_LCSR_OT4_OTEAU 0x188 +#define TSI148_LCSR_OT4_OTEAL 0x18C +#define TSI148_LCSR_OT4_OTOFU 0x190 +#define TSI148_LCSR_OT4_OTOFL 0x194 +#define TSI148_LCSR_OT4_OTBS 0x198 +#define TSI148_LCSR_OT4_OTAT 0x19C + +#define TSI148_LCSR_OT5_OTSAU 0x1A0 +#define TSI148_LCSR_OT5_OTSAL 0x1A4 +#define TSI148_LCSR_OT5_OTEAU 0x1A8 +#define TSI148_LCSR_OT5_OTEAL 0x1AC +#define TSI148_LCSR_OT5_OTOFU 0x1B0 +#define TSI148_LCSR_OT5_OTOFL 0x1B4 +#define TSI148_LCSR_OT5_OTBS 0x1B8 +#define TSI148_LCSR_OT5_OTAT 0x1BC + +#define TSI148_LCSR_OT6_OTSAU 0x1C0 +#define TSI148_LCSR_OT6_OTSAL 0x1C4 +#define TSI148_LCSR_OT6_OTEAU 0x1C8 +#define TSI148_LCSR_OT6_OTEAL 0x1CC +#define TSI148_LCSR_OT6_OTOFU 0x1D0 +#define TSI148_LCSR_OT6_OTOFL 0x1D4 +#define TSI148_LCSR_OT6_OTBS 0x1D8 +#define TSI148_LCSR_OT6_OTAT 0x1DC + +#define TSI148_LCSR_OT7_OTSAU 0x1E0 +#define TSI148_LCSR_OT7_OTSAL 0x1E4 +#define TSI148_LCSR_OT7_OTEAU 0x1E8 +#define TSI148_LCSR_OT7_OTEAL 0x1EC +#define TSI148_LCSR_OT7_OTOFU 0x1F0 +#define TSI148_LCSR_OT7_OTOFL 0x1F4 +#define TSI148_LCSR_OT7_OTBS 0x1F8 +#define TSI148_LCSR_OT7_OTAT 0x1FC + +#define TSI148_LCSR_OT0 0x100 +#define TSI148_LCSR_OT1 0x120 +#define TSI148_LCSR_OT2 0x140 +#define TSI148_LCSR_OT3 0x160 +#define TSI148_LCSR_OT4 0x180 +#define TSI148_LCSR_OT5 0x1A0 +#define TSI148_LCSR_OT6 0x1C0 +#define TSI148_LCSR_OT7 0x1E0 + +static const int TSI148_LCSR_OT[8] = { TSI148_LCSR_OT0, TSI148_LCSR_OT1, + TSI148_LCSR_OT2, TSI148_LCSR_OT3, + TSI148_LCSR_OT4, TSI148_LCSR_OT5, + TSI148_LCSR_OT6, TSI148_LCSR_OT7 }; + +#define TSI148_LCSR_OFFSET_OTSAU 0x0 +#define TSI148_LCSR_OFFSET_OTSAL 0x4 +#define TSI148_LCSR_OFFSET_OTEAU 0x8 +#define TSI148_LCSR_OFFSET_OTEAL 0xC +#define TSI148_LCSR_OFFSET_OTOFU 0x10 +#define TSI148_LCSR_OFFSET_OTOFL 0x14 +#define TSI148_LCSR_OFFSET_OTBS 0x18 +#define TSI148_LCSR_OFFSET_OTAT 0x1C + +/* + * VMEbus interrupt ack + * offset 200 + */ +#define TSI148_LCSR_VIACK1 0x204 +#define TSI148_LCSR_VIACK2 0x208 +#define TSI148_LCSR_VIACK3 0x20C +#define TSI148_LCSR_VIACK4 0x210 +#define TSI148_LCSR_VIACK5 0x214 +#define TSI148_LCSR_VIACK6 0x218 +#define TSI148_LCSR_VIACK7 0x21C + +static const int TSI148_LCSR_VIACK[8] = { 0, TSI148_LCSR_VIACK1, + TSI148_LCSR_VIACK2, TSI148_LCSR_VIACK3, + TSI148_LCSR_VIACK4, TSI148_LCSR_VIACK5, + TSI148_LCSR_VIACK6, TSI148_LCSR_VIACK7 }; + +/* + * RMW + * offset 220 + */ +#define TSI148_LCSR_RMWAU 0x220 +#define TSI148_LCSR_RMWAL 0x224 +#define TSI148_LCSR_RMWEN 0x228 +#define TSI148_LCSR_RMWC 0x22C +#define TSI148_LCSR_RMWS 0x230 + +/* + * VMEbus control + * offset 234 + */ +#define TSI148_LCSR_VMCTRL 0x234 +#define TSI148_LCSR_VCTRL 0x238 +#define TSI148_LCSR_VSTAT 0x23C + +/* + * PCI status + * offset 240 + */ +#define TSI148_LCSR_PSTAT 0x240 + +/* + * VME filter. + * offset 250 + */ +#define TSI148_LCSR_VMEFL 0x250 + + /* + * VME exception. + * offset 260 + */ +#define TSI148_LCSR_VEAU 0x260 +#define TSI148_LCSR_VEAL 0x264 +#define TSI148_LCSR_VEAT 0x268 + + /* + * PCI error + * offset 270 + */ +#define TSI148_LCSR_EDPAU 0x270 +#define TSI148_LCSR_EDPAL 0x274 +#define TSI148_LCSR_EDPXA 0x278 +#define TSI148_LCSR_EDPXS 0x27C +#define TSI148_LCSR_EDPAT 0x280 + + /* + * Inbound Translations + * offset 300 + */ +#define TSI148_LCSR_IT0_ITSAU 0x300 +#define TSI148_LCSR_IT0_ITSAL 0x304 +#define TSI148_LCSR_IT0_ITEAU 0x308 +#define TSI148_LCSR_IT0_ITEAL 0x30C +#define TSI148_LCSR_IT0_ITOFU 0x310 +#define TSI148_LCSR_IT0_ITOFL 0x314 +#define TSI148_LCSR_IT0_ITAT 0x318 + +#define TSI148_LCSR_IT1_ITSAU 0x320 +#define TSI148_LCSR_IT1_ITSAL 0x324 +#define TSI148_LCSR_IT1_ITEAU 0x328 +#define TSI148_LCSR_IT1_ITEAL 0x32C +#define TSI148_LCSR_IT1_ITOFU 0x330 +#define TSI148_LCSR_IT1_ITOFL 0x334 +#define TSI148_LCSR_IT1_ITAT 0x338 + +#define TSI148_LCSR_IT2_ITSAU 0x340 +#define TSI148_LCSR_IT2_ITSAL 0x344 +#define TSI148_LCSR_IT2_ITEAU 0x348 +#define TSI148_LCSR_IT2_ITEAL 0x34C +#define TSI148_LCSR_IT2_ITOFU 0x350 +#define TSI148_LCSR_IT2_ITOFL 0x354 +#define TSI148_LCSR_IT2_ITAT 0x358 + +#define TSI148_LCSR_IT3_ITSAU 0x360 +#define TSI148_LCSR_IT3_ITSAL 0x364 +#define TSI148_LCSR_IT3_ITEAU 0x368 +#define TSI148_LCSR_IT3_ITEAL 0x36C +#define TSI148_LCSR_IT3_ITOFU 0x370 +#define TSI148_LCSR_IT3_ITOFL 0x374 +#define TSI148_LCSR_IT3_ITAT 0x378 + +#define TSI148_LCSR_IT4_ITSAU 0x380 +#define TSI148_LCSR_IT4_ITSAL 0x384 +#define TSI148_LCSR_IT4_ITEAU 0x388 +#define TSI148_LCSR_IT4_ITEAL 0x38C +#define TSI148_LCSR_IT4_ITOFU 0x390 +#define TSI148_LCSR_IT4_ITOFL 0x394 +#define TSI148_LCSR_IT4_ITAT 0x398 + +#define TSI148_LCSR_IT5_ITSAU 0x3A0 +#define TSI148_LCSR_IT5_ITSAL 0x3A4 +#define TSI148_LCSR_IT5_ITEAU 0x3A8 +#define TSI148_LCSR_IT5_ITEAL 0x3AC +#define TSI148_LCSR_IT5_ITOFU 0x3B0 +#define TSI148_LCSR_IT5_ITOFL 0x3B4 +#define TSI148_LCSR_IT5_ITAT 0x3B8 + +#define TSI148_LCSR_IT6_ITSAU 0x3C0 +#define TSI148_LCSR_IT6_ITSAL 0x3C4 +#define TSI148_LCSR_IT6_ITEAU 0x3C8 +#define TSI148_LCSR_IT6_ITEAL 0x3CC +#define TSI148_LCSR_IT6_ITOFU 0x3D0 +#define TSI148_LCSR_IT6_ITOFL 0x3D4 +#define TSI148_LCSR_IT6_ITAT 0x3D8 + +#define TSI148_LCSR_IT7_ITSAU 0x3E0 +#define TSI148_LCSR_IT7_ITSAL 0x3E4 +#define TSI148_LCSR_IT7_ITEAU 0x3E8 +#define TSI148_LCSR_IT7_ITEAL 0x3EC +#define TSI148_LCSR_IT7_ITOFU 0x3F0 +#define TSI148_LCSR_IT7_ITOFL 0x3F4 +#define TSI148_LCSR_IT7_ITAT 0x3F8 + + +#define TSI148_LCSR_IT0 0x300 +#define TSI148_LCSR_IT1 0x320 +#define TSI148_LCSR_IT2 0x340 +#define TSI148_LCSR_IT3 0x360 +#define TSI148_LCSR_IT4 0x380 +#define TSI148_LCSR_IT5 0x3A0 +#define TSI148_LCSR_IT6 0x3C0 +#define TSI148_LCSR_IT7 0x3E0 + +static const int TSI148_LCSR_IT[8] = { TSI148_LCSR_IT0, TSI148_LCSR_IT1, + TSI148_LCSR_IT2, TSI148_LCSR_IT3, + TSI148_LCSR_IT4, TSI148_LCSR_IT5, + TSI148_LCSR_IT6, TSI148_LCSR_IT7 }; + +#define TSI148_LCSR_OFFSET_ITSAU 0x0 +#define TSI148_LCSR_OFFSET_ITSAL 0x4 +#define TSI148_LCSR_OFFSET_ITEAU 0x8 +#define TSI148_LCSR_OFFSET_ITEAL 0xC +#define TSI148_LCSR_OFFSET_ITOFU 0x10 +#define TSI148_LCSR_OFFSET_ITOFL 0x14 +#define TSI148_LCSR_OFFSET_ITAT 0x18 + + /* + * Inbound Translation GCSR + * offset 400 + */ +#define TSI148_LCSR_GBAU 0x400 +#define TSI148_LCSR_GBAL 0x404 +#define TSI148_LCSR_GCSRAT 0x408 + + /* + * Inbound Translation CRG + * offset 40C + */ +#define TSI148_LCSR_CBAU 0x40C +#define TSI148_LCSR_CBAL 0x410 +#define TSI148_LCSR_CSRAT 0x414 + + /* + * Inbound Translation CR/CSR + * CRG + * offset 418 + */ +#define TSI148_LCSR_CROU 0x418 +#define TSI148_LCSR_CROL 0x41C +#define TSI148_LCSR_CRAT 0x420 + + /* + * Inbound Translation Location Monitor + * offset 424 + */ +#define TSI148_LCSR_LMBAU 0x424 +#define TSI148_LCSR_LMBAL 0x428 +#define TSI148_LCSR_LMAT 0x42C + + /* + * VMEbus Interrupt Control. + * offset 430 + */ +#define TSI148_LCSR_BCU 0x430 +#define TSI148_LCSR_BCL 0x434 +#define TSI148_LCSR_BPGTR 0x438 +#define TSI148_LCSR_BPCTR 0x43C +#define TSI148_LCSR_VICR 0x440 + + /* + * Local Bus Interrupt Control. + * offset 448 + */ +#define TSI148_LCSR_INTEN 0x448 +#define TSI148_LCSR_INTEO 0x44C +#define TSI148_LCSR_INTS 0x450 +#define TSI148_LCSR_INTC 0x454 +#define TSI148_LCSR_INTM1 0x458 +#define TSI148_LCSR_INTM2 0x45C + + /* + * DMA Controllers + * offset 500 + */ +#define TSI148_LCSR_DCTL0 0x500 +#define TSI148_LCSR_DSTA0 0x504 +#define TSI148_LCSR_DCSAU0 0x508 +#define TSI148_LCSR_DCSAL0 0x50C +#define TSI148_LCSR_DCDAU0 0x510 +#define TSI148_LCSR_DCDAL0 0x514 +#define TSI148_LCSR_DCLAU0 0x518 +#define TSI148_LCSR_DCLAL0 0x51C +#define TSI148_LCSR_DSAU0 0x520 +#define TSI148_LCSR_DSAL0 0x524 +#define TSI148_LCSR_DDAU0 0x528 +#define TSI148_LCSR_DDAL0 0x52C +#define TSI148_LCSR_DSAT0 0x530 +#define TSI148_LCSR_DDAT0 0x534 +#define TSI148_LCSR_DNLAU0 0x538 +#define TSI148_LCSR_DNLAL0 0x53C +#define TSI148_LCSR_DCNT0 0x540 +#define TSI148_LCSR_DDBS0 0x544 + +#define TSI148_LCSR_DCTL1 0x580 +#define TSI148_LCSR_DSTA1 0x584 +#define TSI148_LCSR_DCSAU1 0x588 +#define TSI148_LCSR_DCSAL1 0x58C +#define TSI148_LCSR_DCDAU1 0x590 +#define TSI148_LCSR_DCDAL1 0x594 +#define TSI148_LCSR_DCLAU1 0x598 +#define TSI148_LCSR_DCLAL1 0x59C +#define TSI148_LCSR_DSAU1 0x5A0 +#define TSI148_LCSR_DSAL1 0x5A4 +#define TSI148_LCSR_DDAU1 0x5A8 +#define TSI148_LCSR_DDAL1 0x5AC +#define TSI148_LCSR_DSAT1 0x5B0 +#define TSI148_LCSR_DDAT1 0x5B4 +#define TSI148_LCSR_DNLAU1 0x5B8 +#define TSI148_LCSR_DNLAL1 0x5BC +#define TSI148_LCSR_DCNT1 0x5C0 +#define TSI148_LCSR_DDBS1 0x5C4 + +#define TSI148_LCSR_DMA0 0x500 +#define TSI148_LCSR_DMA1 0x580 + + +static const int TSI148_LCSR_DMA[TSI148_MAX_DMA] = { TSI148_LCSR_DMA0, + TSI148_LCSR_DMA1 }; + +#define TSI148_LCSR_OFFSET_DCTL 0x0 +#define TSI148_LCSR_OFFSET_DSTA 0x4 +#define TSI148_LCSR_OFFSET_DCSAU 0x8 +#define TSI148_LCSR_OFFSET_DCSAL 0xC +#define TSI148_LCSR_OFFSET_DCDAU 0x10 +#define TSI148_LCSR_OFFSET_DCDAL 0x14 +#define TSI148_LCSR_OFFSET_DCLAU 0x18 +#define TSI148_LCSR_OFFSET_DCLAL 0x1C +#define TSI148_LCSR_OFFSET_DSAU 0x20 +#define TSI148_LCSR_OFFSET_DSAL 0x24 +#define TSI148_LCSR_OFFSET_DDAU 0x28 +#define TSI148_LCSR_OFFSET_DDAL 0x2C +#define TSI148_LCSR_OFFSET_DSAT 0x30 +#define TSI148_LCSR_OFFSET_DDAT 0x34 +#define TSI148_LCSR_OFFSET_DNLAU 0x38 +#define TSI148_LCSR_OFFSET_DNLAL 0x3C +#define TSI148_LCSR_OFFSET_DCNT 0x40 +#define TSI148_LCSR_OFFSET_DDBS 0x44 + + /* + * GCSR Register Group + */ + + /* + * GCSR CRG + * offset 00 600 - DEVI/VENI + * offset 04 604 - CTRL/GA/REVID + * offset 08 608 - Semaphore3/2/1/0 + * offset 0C 60C - Seamphore7/6/5/4 + */ +#define TSI148_GCSR_ID 0x600 +#define TSI148_GCSR_CSR 0x604 +#define TSI148_GCSR_SEMA0 0x608 +#define TSI148_GCSR_SEMA1 0x60C + + /* + * Mail Box + * GCSR CRG + * offset 10 610 - Mailbox0 + */ +#define TSI148_GCSR_MBOX0 0x610 +#define TSI148_GCSR_MBOX1 0x614 +#define TSI148_GCSR_MBOX2 0x618 +#define TSI148_GCSR_MBOX3 0x61C + +static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0, + TSI148_GCSR_MBOX1, + TSI148_GCSR_MBOX2, + TSI148_GCSR_MBOX3 }; + + /* + * CR/CSR + */ + + /* + * CR/CSR CRG + * offset 7FFF4 FF4 - CSRBCR + * offset 7FFF8 FF8 - CSRBSR + * offset 7FFFC FFC - CBAR + */ +#define TSI148_CSRBCR 0xFF4 +#define TSI148_CSRBSR 0xFF8 +#define TSI148_CBAR 0xFFC + + + + + /* + * TSI148 Register Bit Definitions + */ + + /* + * PFCS Register Set + */ +#define TSI148_PCFS_CMMD_SERR (1<<8) /* SERR_L out pin ssys err */ +#define TSI148_PCFS_CMMD_PERR (1<<6) /* PERR_L out pin parity */ +#define TSI148_PCFS_CMMD_MSTR (1<<2) /* PCI bus master */ +#define TSI148_PCFS_CMMD_MEMSP (1<<1) /* PCI mem space access */ +#define TSI148_PCFS_CMMD_IOSP (1<<0) /* PCI I/O space enable */ + +#define TSI148_PCFS_STAT_RCPVE (1<<15) /* Detected Parity Error */ +#define TSI148_PCFS_STAT_SIGSE (1<<14) /* Signalled System Error */ +#define TSI148_PCFS_STAT_RCVMA (1<<13) /* Received Master Abort */ +#define TSI148_PCFS_STAT_RCVTA (1<<12) /* Received Target Abort */ +#define TSI148_PCFS_STAT_SIGTA (1<<11) /* Signalled Target Abort */ +#define TSI148_PCFS_STAT_SELTIM (3<<9) /* DELSEL Timing */ +#define TSI148_PCFS_STAT_DPAR (1<<8) /* Data Parity Err Reported */ +#define TSI148_PCFS_STAT_FAST (1<<7) /* Fast back-to-back Cap */ +#define TSI148_PCFS_STAT_P66M (1<<5) /* 66 MHz Capable */ +#define TSI148_PCFS_STAT_CAPL (1<<4) /* Capab List - address $34 */ + +/* + * Revision ID/Class Code Registers (CRG +$008) + */ +#define TSI148_PCFS_CLAS_M (0xFF<<24) /* Class ID */ +#define TSI148_PCFS_SUBCLAS_M (0xFF<<16) /* Sub-Class ID */ +#define TSI148_PCFS_PROGIF_M (0xFF<<8) /* Sub-Class ID */ +#define TSI148_PCFS_REVID_M (0xFF<<0) /* Rev ID */ + +/* + * Cache Line Size/ Master Latency Timer/ Header Type Registers (CRG + $00C) + */ +#define TSI148_PCFS_HEAD_M (0xFF<<16) /* Master Lat Timer */ +#define TSI148_PCFS_MLAT_M (0xFF<<8) /* Master Lat Timer */ +#define TSI148_PCFS_CLSZ_M (0xFF<<0) /* Cache Line Size */ + +/* + * Memory Base Address Lower Reg (CRG + $010) + */ +#define TSI148_PCFS_MBARL_BASEL_M (0xFFFFF<<12) /* Base Addr Lower Mask */ +#define TSI148_PCFS_MBARL_PRE (1<<3) /* Prefetch */ +#define TSI148_PCFS_MBARL_MTYPE_M (3<<1) /* Memory Type Mask */ +#define TSI148_PCFS_MBARL_IOMEM (1<<0) /* I/O Space Indicator */ + +/* + * Message Signaled Interrupt Capabilities Register (CRG + $040) + */ +#define TSI148_PCFS_MSICAP_64BAC (1<<7) /* 64-bit Address Capable */ +#define TSI148_PCFS_MSICAP_MME_M (7<<4) /* Multiple Msg Enable Mask */ +#define TSI148_PCFS_MSICAP_MMC_M (7<<1) /* Multiple Msg Capable Mask */ +#define TSI148_PCFS_MSICAP_MSIEN (1<<0) /* Msg signaled INT Enable */ + +/* + * Message Address Lower Register (CRG +$044) + */ +#define TSI148_PCFS_MSIAL_M (0x3FFFFFFF<<2) /* Mask */ + +/* + * Message Data Register (CRG + 4C) + */ +#define TSI148_PCFS_MSIMD_M (0xFFFF<<0) /* Mask */ + +/* + * PCI-X Capabilities Register (CRG + $050) + */ +#define TSI148_PCFS_PCIXCAP_MOST_M (7<<4) /* Max outstanding Split Tran */ +#define TSI148_PCFS_PCIXCAP_MMRBC_M (3<<2) /* Max Mem Read byte cnt */ +#define TSI148_PCFS_PCIXCAP_ERO (1<<1) /* Enable Relaxed Ordering */ +#define TSI148_PCFS_PCIXCAP_DPERE (1<<0) /* Data Parity Recover Enable */ + +/* + * PCI-X Status Register (CRG +$054) + */ +#define TSI148_PCFS_PCIXSTAT_RSCEM (1<<29) /* Received Split Comp Error */ +#define TSI148_PCFS_PCIXSTAT_DMCRS_M (7<<26) /* max Cumulative Read Size */ +#define TSI148_PCFS_PCIXSTAT_DMOST_M (7<<23) /* max outstanding Split Trans + */ +#define TSI148_PCFS_PCIXSTAT_DMMRC_M (3<<21) /* max mem read byte count */ +#define TSI148_PCFS_PCIXSTAT_DC (1<<20) /* Device Complexity */ +#define TSI148_PCFS_PCIXSTAT_USC (1<<19) /* Unexpected Split comp */ +#define TSI148_PCFS_PCIXSTAT_SCD (1<<18) /* Split completion discard */ +#define TSI148_PCFS_PCIXSTAT_133C (1<<17) /* 133MHz capable */ +#define TSI148_PCFS_PCIXSTAT_64D (1<<16) /* 64 bit device */ +#define TSI148_PCFS_PCIXSTAT_BN_M (0xFF<<8) /* Bus number */ +#define TSI148_PCFS_PCIXSTAT_DN_M (0x1F<<3) /* Device number */ +#define TSI148_PCFS_PCIXSTAT_FN_M (7<<0) /* Function Number */ + +/* + * LCSR Registers + */ + +/* + * Outbound Translation Starting Address Lower + */ +#define TSI148_LCSR_OTSAL_M (0xFFFF<<16) /* Mask */ + +/* + * Outbound Translation Ending Address Lower + */ +#define TSI148_LCSR_OTEAL_M (0xFFFF<<16) /* Mask */ + +/* + * Outbound Translation Offset Lower + */ +#define TSI148_LCSR_OTOFFL_M (0xFFFF<<16) /* Mask */ + +/* + * Outbound Translation 2eSST Broadcast Select + */ +#define TSI148_LCSR_OTBS_M (0xFFFFF<<0) /* Mask */ + +/* + * Outbound Translation Attribute + */ +#define TSI148_LCSR_OTAT_EN (1<<31) /* Window Enable */ +#define TSI148_LCSR_OTAT_MRPFD (1<<18) /* Prefetch Disable */ + +#define TSI148_LCSR_OTAT_PFS_M (3<<16) /* Prefetch Size Mask */ +#define TSI148_LCSR_OTAT_PFS_2 (0<<16) /* 2 Cache Lines P Size */ +#define TSI148_LCSR_OTAT_PFS_4 (1<<16) /* 4 Cache Lines P Size */ +#define TSI148_LCSR_OTAT_PFS_8 (2<<16) /* 8 Cache Lines P Size */ +#define TSI148_LCSR_OTAT_PFS_16 (3<<16) /* 16 Cache Lines P Size */ + +#define TSI148_LCSR_OTAT_2eSSTM_M (7<<11) /* 2eSST Xfer Rate Mask */ +#define TSI148_LCSR_OTAT_2eSSTM_160 (0<<11) /* 160MB/s 2eSST Xfer Rate */ +#define TSI148_LCSR_OTAT_2eSSTM_267 (1<<11) /* 267MB/s 2eSST Xfer Rate */ +#define TSI148_LCSR_OTAT_2eSSTM_320 (2<<11) /* 320MB/s 2eSST Xfer Rate */ + +#define TSI148_LCSR_OTAT_TM_M (7<<8) /* Xfer Protocol Mask */ +#define TSI148_LCSR_OTAT_TM_SCT (0<<8) /* SCT Xfer Protocol */ +#define TSI148_LCSR_OTAT_TM_BLT (1<<8) /* BLT Xfer Protocol */ +#define TSI148_LCSR_OTAT_TM_MBLT (2<<8) /* MBLT Xfer Protocol */ +#define TSI148_LCSR_OTAT_TM_2eVME (3<<8) /* 2eVME Xfer Protocol */ +#define TSI148_LCSR_OTAT_TM_2eSST (4<<8) /* 2eSST Xfer Protocol */ +#define TSI148_LCSR_OTAT_TM_2eSSTB (5<<8) /* 2eSST Bcast Xfer Protocol */ + +#define TSI148_LCSR_OTAT_DBW_M (3<<6) /* Max Data Width */ +#define TSI148_LCSR_OTAT_DBW_16 (0<<6) /* 16-bit Data Width */ +#define TSI148_LCSR_OTAT_DBW_32 (1<<6) /* 32-bit Data Width */ + +#define TSI148_LCSR_OTAT_SUP (1<<5) /* Supervisory Access */ +#define TSI148_LCSR_OTAT_PGM (1<<4) /* Program Access */ + +#define TSI148_LCSR_OTAT_AMODE_M (0xf<<0) /* Address Mode Mask */ +#define TSI148_LCSR_OTAT_AMODE_A16 (0<<0) /* A16 Address Space */ +#define TSI148_LCSR_OTAT_AMODE_A24 (1<<0) /* A24 Address Space */ +#define TSI148_LCSR_OTAT_AMODE_A32 (2<<0) /* A32 Address Space */ +#define TSI148_LCSR_OTAT_AMODE_A64 (4<<0) /* A32 Address Space */ +#define TSI148_LCSR_OTAT_AMODE_CRCSR (5<<0) /* CR/CSR Address Space */ +#define TSI148_LCSR_OTAT_AMODE_USER1 (8<<0) /* User1 Address Space */ +#define TSI148_LCSR_OTAT_AMODE_USER2 (9<<0) /* User2 Address Space */ +#define TSI148_LCSR_OTAT_AMODE_USER3 (10<<0) /* User3 Address Space */ +#define TSI148_LCSR_OTAT_AMODE_USER4 (11<<0) /* User4 Address Space */ + +/* + * VME Master Control Register CRG+$234 + */ +#define TSI148_LCSR_VMCTRL_VSA (1<<27) /* VMEbus Stop Ack */ +#define TSI148_LCSR_VMCTRL_VS (1<<26) /* VMEbus Stop */ +#define TSI148_LCSR_VMCTRL_DHB (1<<25) /* Device Has Bus */ +#define TSI148_LCSR_VMCTRL_DWB (1<<24) /* Device Wants Bus */ + +#define TSI148_LCSR_VMCTRL_RMWEN (1<<20) /* RMW Enable */ + +#define TSI148_LCSR_VMCTRL_ATO_M (7<<16) /* Master Access Time-out Mask + */ +#define TSI148_LCSR_VMCTRL_ATO_32 (0<<16) /* 32 us */ +#define TSI148_LCSR_VMCTRL_ATO_128 (1<<16) /* 128 us */ +#define TSI148_LCSR_VMCTRL_ATO_512 (2<<16) /* 512 us */ +#define TSI148_LCSR_VMCTRL_ATO_2M (3<<16) /* 2 ms */ +#define TSI148_LCSR_VMCTRL_ATO_8M (4<<16) /* 8 ms */ +#define TSI148_LCSR_VMCTRL_ATO_32M (5<<16) /* 32 ms */ +#define TSI148_LCSR_VMCTRL_ATO_128M (6<<16) /* 128 ms */ +#define TSI148_LCSR_VMCTRL_ATO_DIS (7<<16) /* Disabled */ + +#define TSI148_LCSR_VMCTRL_VTOFF_M (7<<12) /* VMEbus Master Time off */ +#define TSI148_LCSR_VMCTRL_VTOFF_0 (0<<12) /* 0us */ +#define TSI148_LCSR_VMCTRL_VTOFF_1 (1<<12) /* 1us */ +#define TSI148_LCSR_VMCTRL_VTOFF_2 (2<<12) /* 2us */ +#define TSI148_LCSR_VMCTRL_VTOFF_4 (3<<12) /* 4us */ +#define TSI148_LCSR_VMCTRL_VTOFF_8 (4<<12) /* 8us */ +#define TSI148_LCSR_VMCTRL_VTOFF_16 (5<<12) /* 16us */ +#define TSI148_LCSR_VMCTRL_VTOFF_32 (6<<12) /* 32us */ +#define TSI148_LCSR_VMCTRL_VTOFF_64 (7<<12) /* 64us */ + +#define TSI148_LCSR_VMCTRL_VTON_M (7<<8) /* VMEbus Master Time On */ +#define TSI148_LCSR_VMCTRL_VTON_4 (0<<8) /* 8us */ +#define TSI148_LCSR_VMCTRL_VTON_8 (1<<8) /* 8us */ +#define TSI148_LCSR_VMCTRL_VTON_16 (2<<8) /* 16us */ +#define TSI148_LCSR_VMCTRL_VTON_32 (3<<8) /* 32us */ +#define TSI148_LCSR_VMCTRL_VTON_64 (4<<8) /* 64us */ +#define TSI148_LCSR_VMCTRL_VTON_128 (5<<8) /* 128us */ +#define TSI148_LCSR_VMCTRL_VTON_256 (6<<8) /* 256us */ +#define TSI148_LCSR_VMCTRL_VTON_512 (7<<8) /* 512us */ + +#define TSI148_LCSR_VMCTRL_VREL_M (3<<3) /* VMEbus Master Rel Mode Mask + */ +#define TSI148_LCSR_VMCTRL_VREL_T_D (0<<3) /* Time on or Done */ +#define TSI148_LCSR_VMCTRL_VREL_T_R_D (1<<3) /* Time on and REQ or Done */ +#define TSI148_LCSR_VMCTRL_VREL_T_B_D (2<<3) /* Time on and BCLR or Done */ +#define TSI148_LCSR_VMCTRL_VREL_T_D_R (3<<3) /* Time on or Done and REQ */ + +#define TSI148_LCSR_VMCTRL_VFAIR (1<<2) /* VMEbus Master Fair Mode */ +#define TSI148_LCSR_VMCTRL_VREQL_M (3<<0) /* VMEbus Master Req Level Mask + */ + +/* + * VMEbus Control Register CRG+$238 + */ +#define TSI148_LCSR_VCTRL_LRE (1<<31) /* Late Retry Enable */ + +#define TSI148_LCSR_VCTRL_DLT_M (0xF<<24) /* Deadlock Timer */ +#define TSI148_LCSR_VCTRL_DLT_OFF (0<<24) /* Deadlock Timer Off */ +#define TSI148_LCSR_VCTRL_DLT_16 (1<<24) /* 16 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_32 (2<<24) /* 32 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_64 (3<<24) /* 64 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_128 (4<<24) /* 128 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_256 (5<<24) /* 256 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_512 (6<<24) /* 512 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_1024 (7<<24) /* 1024 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_2048 (8<<24) /* 2048 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_4096 (9<<24) /* 4096 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_8192 (0xA<<24) /* 8192 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_16384 (0xB<<24) /* 16384 VCLKS */ +#define TSI148_LCSR_VCTRL_DLT_32768 (0xC<<24) /* 32768 VCLKS */ + +#define TSI148_LCSR_VCTRL_NERBB (1<<20) /* No Early Release of Bus Busy + */ + +#define TSI148_LCSR_VCTRL_SRESET (1<<17) /* System Reset */ +#define TSI148_LCSR_VCTRL_LRESET (1<<16) /* Local Reset */ + +#define TSI148_LCSR_VCTRL_SFAILAI (1<<15) /* SYSFAIL Auto Slot ID */ +#define TSI148_LCSR_VCTRL_BID_M (0x1F<<8) /* Broadcast ID Mask */ + +#define TSI148_LCSR_VCTRL_ATOEN (1<<7) /* Arbiter Time-out Enable */ +#define TSI148_LCSR_VCTRL_ROBIN (1<<6) /* VMEbus Round Robin */ + +#define TSI148_LCSR_VCTRL_GTO_M (7<<0) /* VMEbus Global Time-out Mask + */ +#define TSI148_LCSR_VCTRL_GTO_8 (0<<0) /* 8 us */ +#define TSI148_LCSR_VCTRL_GTO_16 (1<<0) /* 16 us */ +#define TSI148_LCSR_VCTRL_GTO_32 (2<<0) /* 32 us */ +#define TSI148_LCSR_VCTRL_GTO_64 (3<<0) /* 64 us */ +#define TSI148_LCSR_VCTRL_GTO_128 (4<<0) /* 128 us */ +#define TSI148_LCSR_VCTRL_GTO_256 (5<<0) /* 256 us */ +#define TSI148_LCSR_VCTRL_GTO_512 (6<<0) /* 512 us */ +#define TSI148_LCSR_VCTRL_GTO_DIS (7<<0) /* Disabled */ + +/* + * VMEbus Status Register CRG + $23C + */ +#define TSI148_LCSR_VSTAT_CPURST (1<<15) /* Clear power up reset */ +#define TSI148_LCSR_VSTAT_BRDFL (1<<14) /* Board fail */ +#define TSI148_LCSR_VSTAT_PURSTS (1<<12) /* Power up reset status */ +#define TSI148_LCSR_VSTAT_BDFAILS (1<<11) /* Board Fail Status */ +#define TSI148_LCSR_VSTAT_SYSFAILS (1<<10) /* System Fail Status */ +#define TSI148_LCSR_VSTAT_ACFAILS (1<<9) /* AC fail status */ +#define TSI148_LCSR_VSTAT_SCONS (1<<8) /* System Cont Status */ +#define TSI148_LCSR_VSTAT_GAP (1<<5) /* Geographic Addr Parity */ +#define TSI148_LCSR_VSTAT_GA_M (0x1F<<0) /* Geographic Addr Mask */ + +/* + * PCI Configuration Status Register CRG+$240 + */ +#define TSI148_LCSR_PSTAT_REQ64S (1<<6) /* Request 64 status set */ +#define TSI148_LCSR_PSTAT_M66ENS (1<<5) /* M66ENS 66Mhz enable */ +#define TSI148_LCSR_PSTAT_FRAMES (1<<4) /* Frame Status */ +#define TSI148_LCSR_PSTAT_IRDYS (1<<3) /* IRDY status */ +#define TSI148_LCSR_PSTAT_DEVSELS (1<<2) /* DEVL status */ +#define TSI148_LCSR_PSTAT_STOPS (1<<1) /* STOP status */ +#define TSI148_LCSR_PSTAT_TRDYS (1<<0) /* TRDY status */ + +/* + * VMEbus Exception Attributes Register CRG + $268 + */ +#define TSI148_LCSR_VEAT_VES (1<<31) /* Status */ +#define TSI148_LCSR_VEAT_VEOF (1<<30) /* Overflow */ +#define TSI148_LCSR_VEAT_VESCL (1<<29) /* Status Clear */ +#define TSI148_LCSR_VEAT_2EOT (1<<21) /* 2e Odd Termination */ +#define TSI148_LCSR_VEAT_2EST (1<<20) /* 2e Slave terminated */ +#define TSI148_LCSR_VEAT_BERR (1<<19) /* Bus Error */ +#define TSI148_LCSR_VEAT_LWORD (1<<18) /* LWORD_ signal state */ +#define TSI148_LCSR_VEAT_WRITE (1<<17) /* WRITE_ signal state */ +#define TSI148_LCSR_VEAT_IACK (1<<16) /* IACK_ signal state */ +#define TSI148_LCSR_VEAT_DS1 (1<<15) /* DS1_ signal state */ +#define TSI148_LCSR_VEAT_DS0 (1<<14) /* DS0_ signal state */ +#define TSI148_LCSR_VEAT_AM_M (0x3F<<8) /* Address Mode Mask */ +#define TSI148_LCSR_VEAT_XAM_M (0xFF<<0) /* Master AMode Mask */ + + +/* + * VMEbus PCI Error Diagnostics PCI/X Attributes Register CRG + $280 + */ +#define TSI148_LCSR_EDPAT_EDPCL (1<<29) + +/* + * Inbound Translation Starting Address Lower + */ +#define TSI148_LCSR_ITSAL6432_M (0xFFFF<<16) /* Mask */ +#define TSI148_LCSR_ITSAL24_M (0x00FFF<<12) /* Mask */ +#define TSI148_LCSR_ITSAL16_M (0x0000FFF<<4) /* Mask */ + +/* + * Inbound Translation Ending Address Lower + */ +#define TSI148_LCSR_ITEAL6432_M (0xFFFF<<16) /* Mask */ +#define TSI148_LCSR_ITEAL24_M (0x00FFF<<12) /* Mask */ +#define TSI148_LCSR_ITEAL16_M (0x0000FFF<<4) /* Mask */ + +/* + * Inbound Translation Offset Lower + */ +#define TSI148_LCSR_ITOFFL6432_M (0xFFFF<<16) /* Mask */ +#define TSI148_LCSR_ITOFFL24_M (0xFFFFF<<12) /* Mask */ +#define TSI148_LCSR_ITOFFL16_M (0xFFFFFFF<<4) /* Mask */ + +/* + * Inbound Translation Attribute + */ +#define TSI148_LCSR_ITAT_EN (1<<31) /* Window Enable */ +#define TSI148_LCSR_ITAT_TH (1<<18) /* Prefetch Threshold */ + +#define TSI148_LCSR_ITAT_VFS_M (3<<16) /* Virtual FIFO Size Mask */ +#define TSI148_LCSR_ITAT_VFS_64 (0<<16) /* 64 bytes Virtual FIFO Size */ +#define TSI148_LCSR_ITAT_VFS_128 (1<<16) /* 128 bytes Virtual FIFO Sz */ +#define TSI148_LCSR_ITAT_VFS_256 (2<<16) /* 256 bytes Virtual FIFO Sz */ +#define TSI148_LCSR_ITAT_VFS_512 (3<<16) /* 512 bytes Virtual FIFO Sz */ + +#define TSI148_LCSR_ITAT_2eSSTM_M (7<<12) /* 2eSST Xfer Rate Mask */ +#define TSI148_LCSR_ITAT_2eSSTM_160 (0<<12) /* 160MB/s 2eSST Xfer Rate */ +#define TSI148_LCSR_ITAT_2eSSTM_267 (1<<12) /* 267MB/s 2eSST Xfer Rate */ +#define TSI148_LCSR_ITAT_2eSSTM_320 (2<<12) /* 320MB/s 2eSST Xfer Rate */ + +#define TSI148_LCSR_ITAT_2eSSTB (1<<11) /* 2eSST Bcast Xfer Protocol */ +#define TSI148_LCSR_ITAT_2eSST (1<<10) /* 2eSST Xfer Protocol */ +#define TSI148_LCSR_ITAT_2eVME (1<<9) /* 2eVME Xfer Protocol */ +#define TSI148_LCSR_ITAT_MBLT (1<<8) /* MBLT Xfer Protocol */ +#define TSI148_LCSR_ITAT_BLT (1<<7) /* BLT Xfer Protocol */ + +#define TSI148_LCSR_ITAT_AS_M (7<<4) /* Address Space Mask */ +#define TSI148_LCSR_ITAT_AS_A16 (0<<4) /* A16 Address Space */ +#define TSI148_LCSR_ITAT_AS_A24 (1<<4) /* A24 Address Space */ +#define TSI148_LCSR_ITAT_AS_A32 (2<<4) /* A32 Address Space */ +#define TSI148_LCSR_ITAT_AS_A64 (4<<4) /* A64 Address Space */ + +#define TSI148_LCSR_ITAT_SUPR (1<<3) /* Supervisor Access */ +#define TSI148_LCSR_ITAT_NPRIV (1<<2) /* Non-Priv (User) Access */ +#define TSI148_LCSR_ITAT_PGM (1<<1) /* Program Access */ +#define TSI148_LCSR_ITAT_DATA (1<<0) /* Data Access */ + +/* + * GCSR Base Address Lower Address CRG +$404 + */ +#define TSI148_LCSR_GBAL_M (0x7FFFFFF<<5) /* Mask */ + +/* + * GCSR Attribute Register CRG + $408 + */ +#define TSI148_LCSR_GCSRAT_EN (1<<7) /* Enable access to GCSR */ + +#define TSI148_LCSR_GCSRAT_AS_M (7<<4) /* Address Space Mask */ +#define TSI148_LCSR_GCSRAT_AS_A16 (0<<4) /* Address Space 16 */ +#define TSI148_LCSR_GCSRAT_AS_A24 (1<<4) /* Address Space 24 */ +#define TSI148_LCSR_GCSRAT_AS_A32 (2<<4) /* Address Space 32 */ +#define TSI148_LCSR_GCSRAT_AS_A64 (4<<4) /* Address Space 64 */ + +#define TSI148_LCSR_GCSRAT_SUPR (1<<3) /* Sup set -GCSR decoder */ +#define TSI148_LCSR_GCSRAT_NPRIV (1<<2) /* Non-Privliged set - CGSR */ +#define TSI148_LCSR_GCSRAT_PGM (1<<1) /* Program set - GCSR decoder */ +#define TSI148_LCSR_GCSRAT_DATA (1<<0) /* DATA set GCSR decoder */ + +/* + * CRG Base Address Lower Address CRG + $410 + */ +#define TSI148_LCSR_CBAL_M (0xFFFFF<<12) + +/* + * CRG Attribute Register CRG + $414 + */ +#define TSI148_LCSR_CRGAT_EN (1<<7) /* Enable PRG Access */ + +#define TSI148_LCSR_CRGAT_AS_M (7<<4) /* Address Space */ +#define TSI148_LCSR_CRGAT_AS_A16 (0<<4) /* Address Space 16 */ +#define TSI148_LCSR_CRGAT_AS_A24 (1<<4) /* Address Space 24 */ +#define TSI148_LCSR_CRGAT_AS_A32 (2<<4) /* Address Space 32 */ +#define TSI148_LCSR_CRGAT_AS_A64 (4<<4) /* Address Space 64 */ + +#define TSI148_LCSR_CRGAT_SUPR (1<<3) /* Supervisor Access */ +#define TSI148_LCSR_CRGAT_NPRIV (1<<2) /* Non-Privliged(User) Access */ +#define TSI148_LCSR_CRGAT_PGM (1<<1) /* Program Access */ +#define TSI148_LCSR_CRGAT_DATA (1<<0) /* Data Access */ + +/* + * CR/CSR Offset Lower Register CRG + $41C + */ +#define TSI148_LCSR_CROL_M (0x1FFF<<19) /* Mask */ + +/* + * CR/CSR Attribute register CRG + $420 + */ +#define TSI148_LCSR_CRAT_EN (1<<7) /* Enable access to CR/CSR */ + +/* + * Location Monitor base address lower register CRG + $428 + */ +#define TSI148_LCSR_LMBAL_M (0x7FFFFFF<<5) /* Mask */ + +/* + * Location Monitor Attribute Register CRG + $42C + */ +#define TSI148_LCSR_LMAT_EN (1<<7) /* Enable Location Monitor */ + +#define TSI148_LCSR_LMAT_AS_M (7<<4) /* Address Space MASK */ +#define TSI148_LCSR_LMAT_AS_A16 (0<<4) /* A16 */ +#define TSI148_LCSR_LMAT_AS_A24 (1<<4) /* A24 */ +#define TSI148_LCSR_LMAT_AS_A32 (2<<4) /* A32 */ +#define TSI148_LCSR_LMAT_AS_A64 (4<<4) /* A64 */ + +#define TSI148_LCSR_LMAT_SUPR (1<<3) /* Supervisor Access */ +#define TSI148_LCSR_LMAT_NPRIV (1<<2) /* Non-Priv (User) Access */ +#define TSI148_LCSR_LMAT_PGM (1<<1) /* Program Access */ +#define TSI148_LCSR_LMAT_DATA (1<<0) /* Data Access */ + +/* + * Broadcast Pulse Generator Timer Register CRG + $438 + */ +#define TSI148_LCSR_BPGTR_BPGT_M (0xFFFF<<0) /* Mask */ + +/* + * Broadcast Programmable Clock Timer Register CRG + $43C + */ +#define TSI148_LCSR_BPCTR_BPCT_M (0xFFFFFF<<0) /* Mask */ + +/* + * VMEbus Interrupt Control Register CRG + $43C + */ +#define TSI148_LCSR_VICR_CNTS_M (3<<22) /* Cntr Source MASK */ +#define TSI148_LCSR_VICR_CNTS_DIS (1<<22) /* Cntr Disable */ +#define TSI148_LCSR_VICR_CNTS_IRQ1 (2<<22) /* IRQ1 to Cntr */ +#define TSI148_LCSR_VICR_CNTS_IRQ2 (3<<22) /* IRQ2 to Cntr */ + +#define TSI148_LCSR_VICR_EDGIS_M (3<<20) /* Edge interrupt MASK */ +#define TSI148_LCSR_VICR_EDGIS_DIS (1<<20) /* Edge interrupt Disable */ +#define TSI148_LCSR_VICR_EDGIS_IRQ1 (2<<20) /* IRQ1 to Edge */ +#define TSI148_LCSR_VICR_EDGIS_IRQ2 (3<<20) /* IRQ2 to Edge */ + +#define TSI148_LCSR_VICR_IRQIF_M (3<<18) /* IRQ1* Function MASK */ +#define TSI148_LCSR_VICR_IRQIF_NORM (1<<18) /* Normal */ +#define TSI148_LCSR_VICR_IRQIF_PULSE (2<<18) /* Pulse Generator */ +#define TSI148_LCSR_VICR_IRQIF_PROG (3<<18) /* Programmable Clock */ +#define TSI148_LCSR_VICR_IRQIF_1U (4<<18) /* 1us Clock */ + +#define TSI148_LCSR_VICR_IRQ2F_M (3<<16) /* IRQ2* Function MASK */ +#define TSI148_LCSR_VICR_IRQ2F_NORM (1<<16) /* Normal */ +#define TSI148_LCSR_VICR_IRQ2F_PULSE (2<<16) /* Pulse Generator */ +#define TSI148_LCSR_VICR_IRQ2F_PROG (3<<16) /* Programmable Clock */ +#define TSI148_LCSR_VICR_IRQ2F_1U (4<<16) /* 1us Clock */ + +#define TSI148_LCSR_VICR_BIP (1<<15) /* Broadcast Interrupt Pulse */ + +#define TSI148_LCSR_VICR_IRQC (1<<12) /* VMEbus IRQ Clear */ +#define TSI148_LCSR_VICR_IRQS (1<<11) /* VMEbus IRQ Status */ + +#define TSI148_LCSR_VICR_IRQL_M (7<<8) /* VMEbus SW IRQ Level Mask */ +#define TSI148_LCSR_VICR_IRQL_1 (1<<8) /* VMEbus SW IRQ Level 1 */ +#define TSI148_LCSR_VICR_IRQL_2 (2<<8) /* VMEbus SW IRQ Level 2 */ +#define TSI148_LCSR_VICR_IRQL_3 (3<<8) /* VMEbus SW IRQ Level 3 */ +#define TSI148_LCSR_VICR_IRQL_4 (4<<8) /* VMEbus SW IRQ Level 4 */ +#define TSI148_LCSR_VICR_IRQL_5 (5<<8) /* VMEbus SW IRQ Level 5 */ +#define TSI148_LCSR_VICR_IRQL_6 (6<<8) /* VMEbus SW IRQ Level 6 */ +#define TSI148_LCSR_VICR_IRQL_7 (7<<8) /* VMEbus SW IRQ Level 7 */ + +static const int TSI148_LCSR_VICR_IRQL[8] = { 0, TSI148_LCSR_VICR_IRQL_1, + TSI148_LCSR_VICR_IRQL_2, TSI148_LCSR_VICR_IRQL_3, + TSI148_LCSR_VICR_IRQL_4, TSI148_LCSR_VICR_IRQL_5, + TSI148_LCSR_VICR_IRQL_6, TSI148_LCSR_VICR_IRQL_7 }; + +#define TSI148_LCSR_VICR_STID_M (0xFF<<0) /* Status/ID Mask */ + +/* + * Interrupt Enable Register CRG + $440 + */ +#define TSI148_LCSR_INTEN_DMA1EN (1<<25) /* DMAC 1 */ +#define TSI148_LCSR_INTEN_DMA0EN (1<<24) /* DMAC 0 */ +#define TSI148_LCSR_INTEN_LM3EN (1<<23) /* Location Monitor 3 */ +#define TSI148_LCSR_INTEN_LM2EN (1<<22) /* Location Monitor 2 */ +#define TSI148_LCSR_INTEN_LM1EN (1<<21) /* Location Monitor 1 */ +#define TSI148_LCSR_INTEN_LM0EN (1<<20) /* Location Monitor 0 */ +#define TSI148_LCSR_INTEN_MB3EN (1<<19) /* Mail Box 3 */ +#define TSI148_LCSR_INTEN_MB2EN (1<<18) /* Mail Box 2 */ +#define TSI148_LCSR_INTEN_MB1EN (1<<17) /* Mail Box 1 */ +#define TSI148_LCSR_INTEN_MB0EN (1<<16) /* Mail Box 0 */ +#define TSI148_LCSR_INTEN_PERREN (1<<13) /* PCI/X Error */ +#define TSI148_LCSR_INTEN_VERREN (1<<12) /* VMEbus Error */ +#define TSI148_LCSR_INTEN_VIEEN (1<<11) /* VMEbus IRQ Edge */ +#define TSI148_LCSR_INTEN_IACKEN (1<<10) /* IACK */ +#define TSI148_LCSR_INTEN_SYSFLEN (1<<9) /* System Fail */ +#define TSI148_LCSR_INTEN_ACFLEN (1<<8) /* AC Fail */ +#define TSI148_LCSR_INTEN_IRQ7EN (1<<7) /* IRQ7 */ +#define TSI148_LCSR_INTEN_IRQ6EN (1<<6) /* IRQ6 */ +#define TSI148_LCSR_INTEN_IRQ5EN (1<<5) /* IRQ5 */ +#define TSI148_LCSR_INTEN_IRQ4EN (1<<4) /* IRQ4 */ +#define TSI148_LCSR_INTEN_IRQ3EN (1<<3) /* IRQ3 */ +#define TSI148_LCSR_INTEN_IRQ2EN (1<<2) /* IRQ2 */ +#define TSI148_LCSR_INTEN_IRQ1EN (1<<1) /* IRQ1 */ + +static const int TSI148_LCSR_INTEN_LMEN[4] = { TSI148_LCSR_INTEN_LM0EN, + TSI148_LCSR_INTEN_LM1EN, + TSI148_LCSR_INTEN_LM2EN, + TSI148_LCSR_INTEN_LM3EN }; + +static const int TSI148_LCSR_INTEN_IRQEN[7] = { TSI148_LCSR_INTEN_IRQ1EN, + TSI148_LCSR_INTEN_IRQ2EN, + TSI148_LCSR_INTEN_IRQ3EN, + TSI148_LCSR_INTEN_IRQ4EN, + TSI148_LCSR_INTEN_IRQ5EN, + TSI148_LCSR_INTEN_IRQ6EN, + TSI148_LCSR_INTEN_IRQ7EN }; + +/* + * Interrupt Enable Out Register CRG + $444 + */ +#define TSI148_LCSR_INTEO_DMA1EO (1<<25) /* DMAC 1 */ +#define TSI148_LCSR_INTEO_DMA0EO (1<<24) /* DMAC 0 */ +#define TSI148_LCSR_INTEO_LM3EO (1<<23) /* Loc Monitor 3 */ +#define TSI148_LCSR_INTEO_LM2EO (1<<22) /* Loc Monitor 2 */ +#define TSI148_LCSR_INTEO_LM1EO (1<<21) /* Loc Monitor 1 */ +#define TSI148_LCSR_INTEO_LM0EO (1<<20) /* Location Monitor 0 */ +#define TSI148_LCSR_INTEO_MB3EO (1<<19) /* Mail Box 3 */ +#define TSI148_LCSR_INTEO_MB2EO (1<<18) /* Mail Box 2 */ +#define TSI148_LCSR_INTEO_MB1EO (1<<17) /* Mail Box 1 */ +#define TSI148_LCSR_INTEO_MB0EO (1<<16) /* Mail Box 0 */ +#define TSI148_LCSR_INTEO_PERREO (1<<13) /* PCI/X Error */ +#define TSI148_LCSR_INTEO_VERREO (1<<12) /* VMEbus Error */ +#define TSI148_LCSR_INTEO_VIEEO (1<<11) /* VMEbus IRQ Edge */ +#define TSI148_LCSR_INTEO_IACKEO (1<<10) /* IACK */ +#define TSI148_LCSR_INTEO_SYSFLEO (1<<9) /* System Fail */ +#define TSI148_LCSR_INTEO_ACFLEO (1<<8) /* AC Fail */ +#define TSI148_LCSR_INTEO_IRQ7EO (1<<7) /* IRQ7 */ +#define TSI148_LCSR_INTEO_IRQ6EO (1<<6) /* IRQ6 */ +#define TSI148_LCSR_INTEO_IRQ5EO (1<<5) /* IRQ5 */ +#define TSI148_LCSR_INTEO_IRQ4EO (1<<4) /* IRQ4 */ +#define TSI148_LCSR_INTEO_IRQ3EO (1<<3) /* IRQ3 */ +#define TSI148_LCSR_INTEO_IRQ2EO (1<<2) /* IRQ2 */ +#define TSI148_LCSR_INTEO_IRQ1EO (1<<1) /* IRQ1 */ + +static const int TSI148_LCSR_INTEO_LMEO[4] = { TSI148_LCSR_INTEO_LM0EO, + TSI148_LCSR_INTEO_LM1EO, + TSI148_LCSR_INTEO_LM2EO, + TSI148_LCSR_INTEO_LM3EO }; + +static const int TSI148_LCSR_INTEO_IRQEO[7] = { TSI148_LCSR_INTEO_IRQ1EO, + TSI148_LCSR_INTEO_IRQ2EO, + TSI148_LCSR_INTEO_IRQ3EO, + TSI148_LCSR_INTEO_IRQ4EO, + TSI148_LCSR_INTEO_IRQ5EO, + TSI148_LCSR_INTEO_IRQ6EO, + TSI148_LCSR_INTEO_IRQ7EO }; + +/* + * Interrupt Status Register CRG + $448 + */ +#define TSI148_LCSR_INTS_DMA1S (1<<25) /* DMA 1 */ +#define TSI148_LCSR_INTS_DMA0S (1<<24) /* DMA 0 */ +#define TSI148_LCSR_INTS_LM3S (1<<23) /* Location Monitor 3 */ +#define TSI148_LCSR_INTS_LM2S (1<<22) /* Location Monitor 2 */ +#define TSI148_LCSR_INTS_LM1S (1<<21) /* Location Monitor 1 */ +#define TSI148_LCSR_INTS_LM0S (1<<20) /* Location Monitor 0 */ +#define TSI148_LCSR_INTS_MB3S (1<<19) /* Mail Box 3 */ +#define TSI148_LCSR_INTS_MB2S (1<<18) /* Mail Box 2 */ +#define TSI148_LCSR_INTS_MB1S (1<<17) /* Mail Box 1 */ +#define TSI148_LCSR_INTS_MB0S (1<<16) /* Mail Box 0 */ +#define TSI148_LCSR_INTS_PERRS (1<<13) /* PCI/X Error */ +#define TSI148_LCSR_INTS_VERRS (1<<12) /* VMEbus Error */ +#define TSI148_LCSR_INTS_VIES (1<<11) /* VMEbus IRQ Edge */ +#define TSI148_LCSR_INTS_IACKS (1<<10) /* IACK */ +#define TSI148_LCSR_INTS_SYSFLS (1<<9) /* System Fail */ +#define TSI148_LCSR_INTS_ACFLS (1<<8) /* AC Fail */ +#define TSI148_LCSR_INTS_IRQ7S (1<<7) /* IRQ7 */ +#define TSI148_LCSR_INTS_IRQ6S (1<<6) /* IRQ6 */ +#define TSI148_LCSR_INTS_IRQ5S (1<<5) /* IRQ5 */ +#define TSI148_LCSR_INTS_IRQ4S (1<<4) /* IRQ4 */ +#define TSI148_LCSR_INTS_IRQ3S (1<<3) /* IRQ3 */ +#define TSI148_LCSR_INTS_IRQ2S (1<<2) /* IRQ2 */ +#define TSI148_LCSR_INTS_IRQ1S (1<<1) /* IRQ1 */ + +static const int TSI148_LCSR_INTS_LMS[4] = { TSI148_LCSR_INTS_LM0S, + TSI148_LCSR_INTS_LM1S, + TSI148_LCSR_INTS_LM2S, + TSI148_LCSR_INTS_LM3S }; + +static const int TSI148_LCSR_INTS_MBS[4] = { TSI148_LCSR_INTS_MB0S, + TSI148_LCSR_INTS_MB1S, + TSI148_LCSR_INTS_MB2S, + TSI148_LCSR_INTS_MB3S }; + +/* + * Interrupt Clear Register CRG + $44C + */ +#define TSI148_LCSR_INTC_DMA1C (1<<25) /* DMA 1 */ +#define TSI148_LCSR_INTC_DMA0C (1<<24) /* DMA 0 */ +#define TSI148_LCSR_INTC_LM3C (1<<23) /* Location Monitor 3 */ +#define TSI148_LCSR_INTC_LM2C (1<<22) /* Location Monitor 2 */ +#define TSI148_LCSR_INTC_LM1C (1<<21) /* Location Monitor 1 */ +#define TSI148_LCSR_INTC_LM0C (1<<20) /* Location Monitor 0 */ +#define TSI148_LCSR_INTC_MB3C (1<<19) /* Mail Box 3 */ +#define TSI148_LCSR_INTC_MB2C (1<<18) /* Mail Box 2 */ +#define TSI148_LCSR_INTC_MB1C (1<<17) /* Mail Box 1 */ +#define TSI148_LCSR_INTC_MB0C (1<<16) /* Mail Box 0 */ +#define TSI148_LCSR_INTC_PERRC (1<<13) /* VMEbus Error */ +#define TSI148_LCSR_INTC_VERRC (1<<12) /* VMEbus Access Time-out */ +#define TSI148_LCSR_INTC_VIEC (1<<11) /* VMEbus IRQ Edge */ +#define TSI148_LCSR_INTC_IACKC (1<<10) /* IACK */ +#define TSI148_LCSR_INTC_SYSFLC (1<<9) /* System Fail */ +#define TSI148_LCSR_INTC_ACFLC (1<<8) /* AC Fail */ + +static const int TSI148_LCSR_INTC_LMC[4] = { TSI148_LCSR_INTC_LM0C, + TSI148_LCSR_INTC_LM1C, + TSI148_LCSR_INTC_LM2C, + TSI148_LCSR_INTC_LM3C }; + +static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C, + TSI148_LCSR_INTC_MB1C, + TSI148_LCSR_INTC_MB2C, + TSI148_LCSR_INTC_MB3C }; + +/* + * Interrupt Map Register 1 CRG + $458 + */ +#define TSI148_LCSR_INTM1_DMA1M_M (3<<18) /* DMA 1 */ +#define TSI148_LCSR_INTM1_DMA0M_M (3<<16) /* DMA 0 */ +#define TSI148_LCSR_INTM1_LM3M_M (3<<14) /* Location Monitor 3 */ +#define TSI148_LCSR_INTM1_LM2M_M (3<<12) /* Location Monitor 2 */ +#define TSI148_LCSR_INTM1_LM1M_M (3<<10) /* Location Monitor 1 */ +#define TSI148_LCSR_INTM1_LM0M_M (3<<8) /* Location Monitor 0 */ +#define TSI148_LCSR_INTM1_MB3M_M (3<<6) /* Mail Box 3 */ +#define TSI148_LCSR_INTM1_MB2M_M (3<<4) /* Mail Box 2 */ +#define TSI148_LCSR_INTM1_MB1M_M (3<<2) /* Mail Box 1 */ +#define TSI148_LCSR_INTM1_MB0M_M (3<<0) /* Mail Box 0 */ + +/* + * Interrupt Map Register 2 CRG + $45C + */ +#define TSI148_LCSR_INTM2_PERRM_M (3<<26) /* PCI Bus Error */ +#define TSI148_LCSR_INTM2_VERRM_M (3<<24) /* VMEbus Error */ +#define TSI148_LCSR_INTM2_VIEM_M (3<<22) /* VMEbus IRQ Edge */ +#define TSI148_LCSR_INTM2_IACKM_M (3<<20) /* IACK */ +#define TSI148_LCSR_INTM2_SYSFLM_M (3<<18) /* System Fail */ +#define TSI148_LCSR_INTM2_ACFLM_M (3<<16) /* AC Fail */ +#define TSI148_LCSR_INTM2_IRQ7M_M (3<<14) /* IRQ7 */ +#define TSI148_LCSR_INTM2_IRQ6M_M (3<<12) /* IRQ6 */ +#define TSI148_LCSR_INTM2_IRQ5M_M (3<<10) /* IRQ5 */ +#define TSI148_LCSR_INTM2_IRQ4M_M (3<<8) /* IRQ4 */ +#define TSI148_LCSR_INTM2_IRQ3M_M (3<<6) /* IRQ3 */ +#define TSI148_LCSR_INTM2_IRQ2M_M (3<<4) /* IRQ2 */ +#define TSI148_LCSR_INTM2_IRQ1M_M (3<<2) /* IRQ1 */ + +/* + * DMA Control (0-1) Registers CRG + $500 + */ +#define TSI148_LCSR_DCTL_ABT (1<<27) /* Abort */ +#define TSI148_LCSR_DCTL_PAU (1<<26) /* Pause */ +#define TSI148_LCSR_DCTL_DGO (1<<25) /* DMA Go */ + +#define TSI148_LCSR_DCTL_MOD (1<<23) /* Mode */ + +#define TSI148_LCSR_DCTL_VBKS_M (7<<12) /* VMEbus block Size MASK */ +#define TSI148_LCSR_DCTL_VBKS_32 (0<<12) /* VMEbus block Size 32 */ +#define TSI148_LCSR_DCTL_VBKS_64 (1<<12) /* VMEbus block Size 64 */ +#define TSI148_LCSR_DCTL_VBKS_128 (2<<12) /* VMEbus block Size 128 */ +#define TSI148_LCSR_DCTL_VBKS_256 (3<<12) /* VMEbus block Size 256 */ +#define TSI148_LCSR_DCTL_VBKS_512 (4<<12) /* VMEbus block Size 512 */ +#define TSI148_LCSR_DCTL_VBKS_1024 (5<<12) /* VMEbus block Size 1024 */ +#define TSI148_LCSR_DCTL_VBKS_2048 (6<<12) /* VMEbus block Size 2048 */ +#define TSI148_LCSR_DCTL_VBKS_4096 (7<<12) /* VMEbus block Size 4096 */ + +#define TSI148_LCSR_DCTL_VBOT_M (7<<8) /* VMEbus back-off MASK */ +#define TSI148_LCSR_DCTL_VBOT_0 (0<<8) /* VMEbus back-off 0us */ +#define TSI148_LCSR_DCTL_VBOT_1 (1<<8) /* VMEbus back-off 1us */ +#define TSI148_LCSR_DCTL_VBOT_2 (2<<8) /* VMEbus back-off 2us */ +#define TSI148_LCSR_DCTL_VBOT_4 (3<<8) /* VMEbus back-off 4us */ +#define TSI148_LCSR_DCTL_VBOT_8 (4<<8) /* VMEbus back-off 8us */ +#define TSI148_LCSR_DCTL_VBOT_16 (5<<8) /* VMEbus back-off 16us */ +#define TSI148_LCSR_DCTL_VBOT_32 (6<<8) /* VMEbus back-off 32us */ +#define TSI148_LCSR_DCTL_VBOT_64 (7<<8) /* VMEbus back-off 64us */ + +#define TSI148_LCSR_DCTL_PBKS_M (7<<4) /* PCI block size MASK */ +#define TSI148_LCSR_DCTL_PBKS_32 (0<<4) /* PCI block size 32 bytes */ +#define TSI148_LCSR_DCTL_PBKS_64 (1<<4) /* PCI block size 64 bytes */ +#define TSI148_LCSR_DCTL_PBKS_128 (2<<4) /* PCI block size 128 bytes */ +#define TSI148_LCSR_DCTL_PBKS_256 (3<<4) /* PCI block size 256 bytes */ +#define TSI148_LCSR_DCTL_PBKS_512 (4<<4) /* PCI block size 512 bytes */ +#define TSI148_LCSR_DCTL_PBKS_1024 (5<<4) /* PCI block size 1024 bytes */ +#define TSI148_LCSR_DCTL_PBKS_2048 (6<<4) /* PCI block size 2048 bytes */ +#define TSI148_LCSR_DCTL_PBKS_4096 (7<<4) /* PCI block size 4096 bytes */ + +#define TSI148_LCSR_DCTL_PBOT_M (7<<0) /* PCI back off MASK */ +#define TSI148_LCSR_DCTL_PBOT_0 (0<<0) /* PCI back off 0us */ +#define TSI148_LCSR_DCTL_PBOT_1 (1<<0) /* PCI back off 1us */ +#define TSI148_LCSR_DCTL_PBOT_2 (2<<0) /* PCI back off 2us */ +#define TSI148_LCSR_DCTL_PBOT_4 (3<<0) /* PCI back off 3us */ +#define TSI148_LCSR_DCTL_PBOT_8 (4<<0) /* PCI back off 4us */ +#define TSI148_LCSR_DCTL_PBOT_16 (5<<0) /* PCI back off 8us */ +#define TSI148_LCSR_DCTL_PBOT_32 (6<<0) /* PCI back off 16us */ +#define TSI148_LCSR_DCTL_PBOT_64 (7<<0) /* PCI back off 32us */ + +/* + * DMA Status Registers (0-1) CRG + $504 + */ +#define TSI148_LCSR_DSTA_SMA (1<<31) /* PCI Signalled Master Abt */ +#define TSI148_LCSR_DSTA_RTA (1<<30) /* PCI Received Target Abt */ +#define TSI148_LCSR_DSTA_MRC (1<<29) /* PCI Max Retry Count */ +#define TSI148_LCSR_DSTA_VBE (1<<28) /* VMEbus error */ +#define TSI148_LCSR_DSTA_ABT (1<<27) /* Abort */ +#define TSI148_LCSR_DSTA_PAU (1<<26) /* Pause */ +#define TSI148_LCSR_DSTA_DON (1<<25) /* Done */ +#define TSI148_LCSR_DSTA_BSY (1<<24) /* Busy */ + +/* + * DMA Current Link Address Lower (0-1) + */ +#define TSI148_LCSR_DCLAL_M (0x3FFFFFF<<6) /* Mask */ + +/* + * DMA Source Attribute (0-1) Reg + */ +#define TSI148_LCSR_DSAT_TYP_M (3<<28) /* Source Bus Type */ +#define TSI148_LCSR_DSAT_TYP_PCI (0<<28) /* PCI Bus */ +#define TSI148_LCSR_DSAT_TYP_VME (1<<28) /* VMEbus */ +#define TSI148_LCSR_DSAT_TYP_PAT (2<<28) /* Data Pattern */ + +#define TSI148_LCSR_DSAT_PSZ (1<<25) /* Pattern Size */ +#define TSI148_LCSR_DSAT_NIN (1<<24) /* No Increment */ + +#define TSI148_LCSR_DSAT_2eSSTM_M (3<<11) /* 2eSST Trans Rate Mask */ +#define TSI148_LCSR_DSAT_2eSSTM_160 (0<<11) /* 160 MB/s */ +#define TSI148_LCSR_DSAT_2eSSTM_267 (1<<11) /* 267 MB/s */ +#define TSI148_LCSR_DSAT_2eSSTM_320 (2<<11) /* 320 MB/s */ + +#define TSI148_LCSR_DSAT_TM_M (7<<8) /* Bus Transfer Protocol Mask */ +#define TSI148_LCSR_DSAT_TM_SCT (0<<8) /* SCT */ +#define TSI148_LCSR_DSAT_TM_BLT (1<<8) /* BLT */ +#define TSI148_LCSR_DSAT_TM_MBLT (2<<8) /* MBLT */ +#define TSI148_LCSR_DSAT_TM_2eVME (3<<8) /* 2eVME */ +#define TSI148_LCSR_DSAT_TM_2eSST (4<<8) /* 2eSST */ +#define TSI148_LCSR_DSAT_TM_2eSSTB (5<<8) /* 2eSST Broadcast */ + +#define TSI148_LCSR_DSAT_DBW_M (3<<6) /* Max Data Width MASK */ +#define TSI148_LCSR_DSAT_DBW_16 (0<<6) /* 16 Bits */ +#define TSI148_LCSR_DSAT_DBW_32 (1<<6) /* 32 Bits */ + +#define TSI148_LCSR_DSAT_SUP (1<<5) /* Supervisory Mode */ +#define TSI148_LCSR_DSAT_PGM (1<<4) /* Program Mode */ + +#define TSI148_LCSR_DSAT_AMODE_M (0xf<<0) /* Address Space Mask */ +#define TSI148_LCSR_DSAT_AMODE_A16 (0<<0) /* A16 */ +#define TSI148_LCSR_DSAT_AMODE_A24 (1<<0) /* A24 */ +#define TSI148_LCSR_DSAT_AMODE_A32 (2<<0) /* A32 */ +#define TSI148_LCSR_DSAT_AMODE_A64 (4<<0) /* A64 */ +#define TSI148_LCSR_DSAT_AMODE_CRCSR (5<<0) /* CR/CSR */ +#define TSI148_LCSR_DSAT_AMODE_USER1 (8<<0) /* User1 */ +#define TSI148_LCSR_DSAT_AMODE_USER2 (9<<0) /* User2 */ +#define TSI148_LCSR_DSAT_AMODE_USER3 (0xa<<0) /* User3 */ +#define TSI148_LCSR_DSAT_AMODE_USER4 (0xb<<0) /* User4 */ + +/* + * DMA Destination Attribute Registers (0-1) + */ +#define TSI148_LCSR_DDAT_TYP_PCI (0<<28) /* Destination PCI Bus */ +#define TSI148_LCSR_DDAT_TYP_VME (1<<28) /* Destination VMEbus */ + +#define TSI148_LCSR_DDAT_2eSSTM_M (3<<11) /* 2eSST Transfer Rate Mask */ +#define TSI148_LCSR_DDAT_2eSSTM_160 (0<<11) /* 160 MB/s */ +#define TSI148_LCSR_DDAT_2eSSTM_267 (1<<11) /* 267 MB/s */ +#define TSI148_LCSR_DDAT_2eSSTM_320 (2<<11) /* 320 MB/s */ + +#define TSI148_LCSR_DDAT_TM_M (7<<8) /* Bus Transfer Protocol Mask */ +#define TSI148_LCSR_DDAT_TM_SCT (0<<8) /* SCT */ +#define TSI148_LCSR_DDAT_TM_BLT (1<<8) /* BLT */ +#define TSI148_LCSR_DDAT_TM_MBLT (2<<8) /* MBLT */ +#define TSI148_LCSR_DDAT_TM_2eVME (3<<8) /* 2eVME */ +#define TSI148_LCSR_DDAT_TM_2eSST (4<<8) /* 2eSST */ +#define TSI148_LCSR_DDAT_TM_2eSSTB (5<<8) /* 2eSST Broadcast */ + +#define TSI148_LCSR_DDAT_DBW_M (3<<6) /* Max Data Width MASK */ +#define TSI148_LCSR_DDAT_DBW_16 (0<<6) /* 16 Bits */ +#define TSI148_LCSR_DDAT_DBW_32 (1<<6) /* 32 Bits */ + +#define TSI148_LCSR_DDAT_SUP (1<<5) /* Supervisory/User Access */ +#define TSI148_LCSR_DDAT_PGM (1<<4) /* Program/Data Access */ + +#define TSI148_LCSR_DDAT_AMODE_M (0xf<<0) /* Address Space Mask */ +#define TSI148_LCSR_DDAT_AMODE_A16 (0<<0) /* A16 */ +#define TSI148_LCSR_DDAT_AMODE_A24 (1<<0) /* A24 */ +#define TSI148_LCSR_DDAT_AMODE_A32 (2<<0) /* A32 */ +#define TSI148_LCSR_DDAT_AMODE_A64 (4<<0) /* A64 */ +#define TSI148_LCSR_DDAT_AMODE_CRCSR (5<<0) /* CRC/SR */ +#define TSI148_LCSR_DDAT_AMODE_USER1 (8<<0) /* User1 */ +#define TSI148_LCSR_DDAT_AMODE_USER2 (9<<0) /* User2 */ +#define TSI148_LCSR_DDAT_AMODE_USER3 (0xa<<0) /* User3 */ +#define TSI148_LCSR_DDAT_AMODE_USER4 (0xb<<0) /* User4 */ + +/* + * DMA Next Link Address Lower + */ +#define TSI148_LCSR_DNLAL_DNLAL_M (0x3FFFFFF<<6) /* Address Mask */ +#define TSI148_LCSR_DNLAL_LLA (1<<0) /* Last Link Address Indicator */ + +/* + * DMA 2eSST Broadcast Select + */ +#define TSI148_LCSR_DBS_M (0x1FFFFF<<0) /* Mask */ + +/* + * GCSR Register Group + */ + +/* + * GCSR Control and Status Register CRG + $604 + */ +#define TSI148_GCSR_GCTRL_LRST (1<<15) /* Local Reset */ +#define TSI148_GCSR_GCTRL_SFAILEN (1<<14) /* System Fail enable */ +#define TSI148_GCSR_GCTRL_BDFAILS (1<<13) /* Board Fail Status */ +#define TSI148_GCSR_GCTRL_SCON (1<<12) /* System Copntroller */ +#define TSI148_GCSR_GCTRL_MEN (1<<11) /* Module Enable (READY) */ + +#define TSI148_GCSR_GCTRL_LMI3S (1<<7) /* Loc Monitor 3 Int Status */ +#define TSI148_GCSR_GCTRL_LMI2S (1<<6) /* Loc Monitor 2 Int Status */ +#define TSI148_GCSR_GCTRL_LMI1S (1<<5) /* Loc Monitor 1 Int Status */ +#define TSI148_GCSR_GCTRL_LMI0S (1<<4) /* Loc Monitor 0 Int Status */ +#define TSI148_GCSR_GCTRL_MBI3S (1<<3) /* Mail box 3 Int Status */ +#define TSI148_GCSR_GCTRL_MBI2S (1<<2) /* Mail box 2 Int Status */ +#define TSI148_GCSR_GCTRL_MBI1S (1<<1) /* Mail box 1 Int Status */ +#define TSI148_GCSR_GCTRL_MBI0S (1<<0) /* Mail box 0 Int Status */ + +#define TSI148_GCSR_GAP (1<<5) /* Geographic Addr Parity */ +#define TSI148_GCSR_GA_M (0x1F<<0) /* Geographic Address Mask */ + +/* + * CR/CSR Register Group + */ + +/* + * CR/CSR Bit Clear Register CRG + $FF4 + */ +#define TSI148_CRCSR_CSRBCR_LRSTC (1<<7) /* Local Reset Clear */ +#define TSI148_CRCSR_CSRBCR_SFAILC (1<<6) /* System Fail Enable Clear */ +#define TSI148_CRCSR_CSRBCR_BDFAILS (1<<5) /* Board Fail Status */ +#define TSI148_CRCSR_CSRBCR_MENC (1<<4) /* Module Enable Clear */ +#define TSI148_CRCSR_CSRBCR_BERRSC (1<<3) /* Bus Error Status Clear */ + +/* + * CR/CSR Bit Set Register CRG+$FF8 + */ +#define TSI148_CRCSR_CSRBSR_LISTS (1<<7) /* Local Reset Clear */ +#define TSI148_CRCSR_CSRBSR_SFAILS (1<<6) /* System Fail Enable Clear */ +#define TSI148_CRCSR_CSRBSR_BDFAILS (1<<5) /* Board Fail Status */ +#define TSI148_CRCSR_CSRBSR_MENS (1<<4) /* Module Enable Clear */ +#define TSI148_CRCSR_CSRBSR_BERRS (1<<3) /* Bus Error Status Clear */ + +/* + * CR/CSR Base Address Register CRG + FFC + */ +#define TSI148_CRCSR_CBAR_M (0x1F<<3) /* Mask */ + +#endif /* TSI148_H */ diff --git a/drivers/staging/vme_user/vme_user.c b/drivers/staging/vme_user/vme_user.c index 859af797630c..4e533c0bfe6d 100644 --- a/drivers/staging/vme_user/vme_user.c +++ b/drivers/staging/vme_user/vme_user.c @@ -33,8 +33,8 @@ #include #include -#include +#include "vme.h" #include "vme_user.h" static const char driver_name[] = "vme_user"; diff --git a/drivers/vme/Kconfig b/drivers/vme/Kconfig deleted file mode 100644 index 26feabba19d2..000000000000 --- a/drivers/vme/Kconfig +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# VME configuration. -# - -menuconfig VME_BUS - bool "VME bridge support" - depends on PCI - help - If you say Y here you get support for the VME bridge Framework. - -if VME_BUS - -source "drivers/vme/bridges/Kconfig" - -endif # VME diff --git a/drivers/vme/Makefile b/drivers/vme/Makefile deleted file mode 100644 index 2dfb929a23de..000000000000 --- a/drivers/vme/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for the VME bridge device drivers. -# -obj-$(CONFIG_VME_BUS) += vme.o - -obj-y += bridges/ diff --git a/drivers/vme/bridges/Kconfig b/drivers/vme/bridges/Kconfig deleted file mode 100644 index 9493b22b5276..000000000000 --- a/drivers/vme/bridges/Kconfig +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -comment "VME Bridge Drivers" - -config VME_TSI148 - tristate "Tempe" - depends on HAS_DMA - help - If you say Y here you get support for the Tundra TSI148 VME bridge - chip. - -config VME_FAKE - tristate "Fake" - help - If you say Y here you get support for the fake VME bridge. This - provides a virtualised VME Bus for devices with no VME bridge. This - is mainly useful for VME development (in the absence of VME - hardware). diff --git a/drivers/vme/bridges/Makefile b/drivers/vme/bridges/Makefile deleted file mode 100644 index 043f9cd7a510..000000000000 --- a/drivers/vme/bridges/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_VME_TSI148) += vme_tsi148.o -obj-$(CONFIG_VME_FAKE) += vme_fake.o diff --git a/drivers/vme/bridges/vme_fake.c b/drivers/vme/bridges/vme_fake.c deleted file mode 100644 index 6a1bc284f297..000000000000 --- a/drivers/vme/bridges/vme_fake.c +++ /dev/null @@ -1,1305 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Fake VME bridge support. - * - * This drive provides a fake VME bridge chip, this enables debugging of the - * VME framework in the absence of a VME system. - * - * This driver has to do a number of things in software that would be driven - * by hardware if it was available, it will also result in extra overhead at - * times when compared with driving actual hardware. - * - * Author: Martyn Welch - * Copyright (c) 2014 Martyn Welch - * - * Based on vme_tsi148.c: - * - * Author: Martyn Welch - * Copyright 2008 GE Intelligent Platforms Embedded Systems, Inc. - * - * Based on work by Tom Armistead and Ajit Prem - * Copyright 2004 Motorola Inc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../vme_bridge.h" - -/* - * Define the number of each that the fake driver supports. - */ -#define FAKE_MAX_MASTER 8 /* Max Master Windows */ -#define FAKE_MAX_SLAVE 8 /* Max Slave Windows */ - -/* Structures to hold information normally held in device registers */ -struct fake_slave_window { - int enabled; - unsigned long long vme_base; - unsigned long long size; - void *buf_base; - u32 aspace; - u32 cycle; -}; - -struct fake_master_window { - int enabled; - unsigned long long vme_base; - unsigned long long size; - u32 aspace; - u32 cycle; - u32 dwidth; -}; - -/* Structure used to hold driver specific information */ -struct fake_driver { - struct vme_bridge *parent; - struct fake_slave_window slaves[FAKE_MAX_SLAVE]; - struct fake_master_window masters[FAKE_MAX_MASTER]; - u32 lm_enabled; - unsigned long long lm_base; - u32 lm_aspace; - u32 lm_cycle; - void (*lm_callback[4])(void *); - void *lm_data[4]; - struct tasklet_struct int_tasklet; - int int_level; - int int_statid; - void *crcsr_kernel; - dma_addr_t crcsr_bus; - /* Only one VME interrupt can be generated at a time, provide locking */ - struct mutex vme_int; -}; - -/* Module parameter */ -static int geoid; - -static const char driver_name[] = "vme_fake"; - -static struct vme_bridge *exit_pointer; - -static struct device *vme_root; - -/* - * Calling VME bus interrupt callback if provided. - */ -static void fake_VIRQ_tasklet(unsigned long data) -{ - struct vme_bridge *fake_bridge; - struct fake_driver *bridge; - - fake_bridge = (struct vme_bridge *) data; - bridge = fake_bridge->driver_priv; - - vme_irq_handler(fake_bridge, bridge->int_level, bridge->int_statid); -} - -/* - * Configure VME interrupt - */ -static void fake_irq_set(struct vme_bridge *fake_bridge, int level, - int state, int sync) -{ - /* Nothing to do */ -} - -static void *fake_pci_to_ptr(dma_addr_t addr) -{ - return (void *)(uintptr_t)addr; -} - -static dma_addr_t fake_ptr_to_pci(void *addr) -{ - return (dma_addr_t)(uintptr_t)addr; -} - -/* - * Generate a VME bus interrupt at the requested level & vector. Wait for - * interrupt to be acked. - */ -static int fake_irq_generate(struct vme_bridge *fake_bridge, int level, - int statid) -{ - struct fake_driver *bridge; - - bridge = fake_bridge->driver_priv; - - mutex_lock(&bridge->vme_int); - - bridge->int_level = level; - - bridge->int_statid = statid; - - /* - * Schedule tasklet to run VME handler to emulate normal VME interrupt - * handler behaviour. - */ - tasklet_schedule(&bridge->int_tasklet); - - mutex_unlock(&bridge->vme_int); - - return 0; -} - -/* - * Initialize a slave window with the requested attributes. - */ -static int fake_slave_set(struct vme_slave_resource *image, int enabled, - unsigned long long vme_base, unsigned long long size, - dma_addr_t buf_base, u32 aspace, u32 cycle) -{ - unsigned int i, granularity = 0; - unsigned long long vme_bound; - struct vme_bridge *fake_bridge; - struct fake_driver *bridge; - - fake_bridge = image->parent; - bridge = fake_bridge->driver_priv; - - i = image->number; - - switch (aspace) { - case VME_A16: - granularity = 0x10; - break; - case VME_A24: - granularity = 0x1000; - break; - case VME_A32: - granularity = 0x10000; - break; - case VME_A64: - granularity = 0x10000; - break; - case VME_CRCSR: - case VME_USER1: - case VME_USER2: - case VME_USER3: - case VME_USER4: - default: - pr_err("Invalid address space\n"); - return -EINVAL; - } - - /* - * Bound address is a valid address for the window, adjust - * accordingly - */ - vme_bound = vme_base + size - granularity; - - if (vme_base & (granularity - 1)) { - pr_err("Invalid VME base alignment\n"); - return -EINVAL; - } - if (vme_bound & (granularity - 1)) { - pr_err("Invalid VME bound alignment\n"); - return -EINVAL; - } - - mutex_lock(&image->mtx); - - bridge->slaves[i].enabled = enabled; - bridge->slaves[i].vme_base = vme_base; - bridge->slaves[i].size = size; - bridge->slaves[i].buf_base = fake_pci_to_ptr(buf_base); - bridge->slaves[i].aspace = aspace; - bridge->slaves[i].cycle = cycle; - - mutex_unlock(&image->mtx); - - return 0; -} - -/* - * Get slave window configuration. - */ -static int fake_slave_get(struct vme_slave_resource *image, int *enabled, - unsigned long long *vme_base, unsigned long long *size, - dma_addr_t *buf_base, u32 *aspace, u32 *cycle) -{ - unsigned int i; - struct fake_driver *bridge; - - bridge = image->parent->driver_priv; - - i = image->number; - - mutex_lock(&image->mtx); - - *enabled = bridge->slaves[i].enabled; - *vme_base = bridge->slaves[i].vme_base; - *size = bridge->slaves[i].size; - *buf_base = fake_ptr_to_pci(bridge->slaves[i].buf_base); - *aspace = bridge->slaves[i].aspace; - *cycle = bridge->slaves[i].cycle; - - mutex_unlock(&image->mtx); - - return 0; -} - -/* - * Set the attributes of an outbound window. - */ -static int fake_master_set(struct vme_master_resource *image, int enabled, - unsigned long long vme_base, unsigned long long size, - u32 aspace, u32 cycle, u32 dwidth) -{ - int retval = 0; - unsigned int i; - struct vme_bridge *fake_bridge; - struct fake_driver *bridge; - - fake_bridge = image->parent; - - bridge = fake_bridge->driver_priv; - - /* Verify input data */ - if (vme_base & 0xFFFF) { - pr_err("Invalid VME Window alignment\n"); - retval = -EINVAL; - goto err_window; - } - - if (size & 0xFFFF) { - pr_err("Invalid size alignment\n"); - retval = -EINVAL; - goto err_window; - } - - if ((size == 0) && (enabled != 0)) { - pr_err("Size must be non-zero for enabled windows\n"); - retval = -EINVAL; - goto err_window; - } - - /* Setup data width */ - switch (dwidth) { - case VME_D8: - case VME_D16: - case VME_D32: - break; - default: - pr_err("Invalid data width\n"); - retval = -EINVAL; - goto err_dwidth; - } - - /* Setup address space */ - switch (aspace) { - case VME_A16: - case VME_A24: - case VME_A32: - case VME_A64: - case VME_CRCSR: - case VME_USER1: - case VME_USER2: - case VME_USER3: - case VME_USER4: - break; - default: - pr_err("Invalid address space\n"); - retval = -EINVAL; - goto err_aspace; - } - - spin_lock(&image->lock); - - i = image->number; - - bridge->masters[i].enabled = enabled; - bridge->masters[i].vme_base = vme_base; - bridge->masters[i].size = size; - bridge->masters[i].aspace = aspace; - bridge->masters[i].cycle = cycle; - bridge->masters[i].dwidth = dwidth; - - spin_unlock(&image->lock); - - return 0; - -err_aspace: -err_dwidth: -err_window: - return retval; - -} - -/* - * Set the attributes of an outbound window. - */ -static int __fake_master_get(struct vme_master_resource *image, int *enabled, - unsigned long long *vme_base, unsigned long long *size, - u32 *aspace, u32 *cycle, u32 *dwidth) -{ - unsigned int i; - struct fake_driver *bridge; - - bridge = image->parent->driver_priv; - - i = image->number; - - *enabled = bridge->masters[i].enabled; - *vme_base = bridge->masters[i].vme_base; - *size = bridge->masters[i].size; - *aspace = bridge->masters[i].aspace; - *cycle = bridge->masters[i].cycle; - *dwidth = bridge->masters[i].dwidth; - - return 0; -} - - -static int fake_master_get(struct vme_master_resource *image, int *enabled, - unsigned long long *vme_base, unsigned long long *size, - u32 *aspace, u32 *cycle, u32 *dwidth) -{ - int retval; - - spin_lock(&image->lock); - - retval = __fake_master_get(image, enabled, vme_base, size, aspace, - cycle, dwidth); - - spin_unlock(&image->lock); - - return retval; -} - - -static void fake_lm_check(struct fake_driver *bridge, unsigned long long addr, - u32 aspace, u32 cycle) -{ - struct vme_bridge *fake_bridge; - unsigned long long lm_base; - u32 lm_aspace, lm_cycle; - int i; - struct vme_lm_resource *lm; - struct list_head *pos = NULL, *n; - - /* Get vme_bridge */ - fake_bridge = bridge->parent; - - /* Loop through each location monitor resource */ - list_for_each_safe(pos, n, &fake_bridge->lm_resources) { - lm = list_entry(pos, struct vme_lm_resource, list); - - /* If disabled, we're done */ - if (bridge->lm_enabled == 0) - return; - - lm_base = bridge->lm_base; - lm_aspace = bridge->lm_aspace; - lm_cycle = bridge->lm_cycle; - - /* First make sure that the cycle and address space match */ - if ((lm_aspace == aspace) && (lm_cycle == cycle)) { - for (i = 0; i < lm->monitors; i++) { - /* Each location monitor covers 8 bytes */ - if (((lm_base + (8 * i)) <= addr) && - ((lm_base + (8 * i) + 8) > addr)) { - if (bridge->lm_callback[i]) - bridge->lm_callback[i]( - bridge->lm_data[i]); - } - } - } - } -} - -static noinline_for_stack u8 fake_vmeread8(struct fake_driver *bridge, - unsigned long long addr, - u32 aspace, u32 cycle) -{ - u8 retval = 0xff; - int i; - unsigned long long start, end, offset; - u8 *loc; - - for (i = 0; i < FAKE_MAX_SLAVE; i++) { - start = bridge->slaves[i].vme_base; - end = bridge->slaves[i].vme_base + bridge->slaves[i].size; - - if (aspace != bridge->slaves[i].aspace) - continue; - - if (cycle != bridge->slaves[i].cycle) - continue; - - if ((addr >= start) && (addr < end)) { - offset = addr - bridge->slaves[i].vme_base; - loc = (u8 *)(bridge->slaves[i].buf_base + offset); - retval = *loc; - - break; - } - } - - fake_lm_check(bridge, addr, aspace, cycle); - - return retval; -} - -static noinline_for_stack u16 fake_vmeread16(struct fake_driver *bridge, - unsigned long long addr, - u32 aspace, u32 cycle) -{ - u16 retval = 0xffff; - int i; - unsigned long long start, end, offset; - u16 *loc; - - for (i = 0; i < FAKE_MAX_SLAVE; i++) { - if (aspace != bridge->slaves[i].aspace) - continue; - - if (cycle != bridge->slaves[i].cycle) - continue; - - start = bridge->slaves[i].vme_base; - end = bridge->slaves[i].vme_base + bridge->slaves[i].size; - - if ((addr >= start) && ((addr + 1) < end)) { - offset = addr - bridge->slaves[i].vme_base; - loc = (u16 *)(bridge->slaves[i].buf_base + offset); - retval = *loc; - - break; - } - } - - fake_lm_check(bridge, addr, aspace, cycle); - - return retval; -} - -static noinline_for_stack u32 fake_vmeread32(struct fake_driver *bridge, - unsigned long long addr, - u32 aspace, u32 cycle) -{ - u32 retval = 0xffffffff; - int i; - unsigned long long start, end, offset; - u32 *loc; - - for (i = 0; i < FAKE_MAX_SLAVE; i++) { - if (aspace != bridge->slaves[i].aspace) - continue; - - if (cycle != bridge->slaves[i].cycle) - continue; - - start = bridge->slaves[i].vme_base; - end = bridge->slaves[i].vme_base + bridge->slaves[i].size; - - if ((addr >= start) && ((addr + 3) < end)) { - offset = addr - bridge->slaves[i].vme_base; - loc = (u32 *)(bridge->slaves[i].buf_base + offset); - retval = *loc; - - break; - } - } - - fake_lm_check(bridge, addr, aspace, cycle); - - return retval; -} - -static ssize_t fake_master_read(struct vme_master_resource *image, void *buf, - size_t count, loff_t offset) -{ - int retval; - u32 aspace, cycle, dwidth; - struct vme_bridge *fake_bridge; - struct fake_driver *priv; - int i; - unsigned long long addr; - unsigned int done = 0; - unsigned int count32; - - fake_bridge = image->parent; - - priv = fake_bridge->driver_priv; - - i = image->number; - - addr = (unsigned long long)priv->masters[i].vme_base + offset; - aspace = priv->masters[i].aspace; - cycle = priv->masters[i].cycle; - dwidth = priv->masters[i].dwidth; - - spin_lock(&image->lock); - - /* The following code handles VME address alignment. We cannot use - * memcpy_xxx here because it may cut data transfers in to 8-bit - * cycles when D16 or D32 cycles are required on the VME bus. - * On the other hand, the bridge itself assures that the maximum data - * cycle configured for the transfer is used and splits it - * automatically for non-aligned addresses, so we don't want the - * overhead of needlessly forcing small transfers for the entire cycle. - */ - if (addr & 0x1) { - *(u8 *)buf = fake_vmeread8(priv, addr, aspace, cycle); - done += 1; - if (done == count) - goto out; - } - if ((dwidth == VME_D16) || (dwidth == VME_D32)) { - if ((addr + done) & 0x2) { - if ((count - done) < 2) { - *(u8 *)(buf + done) = fake_vmeread8(priv, - addr + done, aspace, cycle); - done += 1; - goto out; - } else { - *(u16 *)(buf + done) = fake_vmeread16(priv, - addr + done, aspace, cycle); - done += 2; - } - } - } - - if (dwidth == VME_D32) { - count32 = (count - done) & ~0x3; - while (done < count32) { - *(u32 *)(buf + done) = fake_vmeread32(priv, addr + done, - aspace, cycle); - done += 4; - } - } else if (dwidth == VME_D16) { - count32 = (count - done) & ~0x3; - while (done < count32) { - *(u16 *)(buf + done) = fake_vmeread16(priv, addr + done, - aspace, cycle); - done += 2; - } - } else if (dwidth == VME_D8) { - count32 = (count - done); - while (done < count32) { - *(u8 *)(buf + done) = fake_vmeread8(priv, addr + done, - aspace, cycle); - done += 1; - } - - } - - if ((dwidth == VME_D16) || (dwidth == VME_D32)) { - if ((count - done) & 0x2) { - *(u16 *)(buf + done) = fake_vmeread16(priv, addr + done, - aspace, cycle); - done += 2; - } - } - if ((count - done) & 0x1) { - *(u8 *)(buf + done) = fake_vmeread8(priv, addr + done, aspace, - cycle); - done += 1; - } - -out: - retval = count; - - spin_unlock(&image->lock); - - return retval; -} - -static noinline_for_stack void fake_vmewrite8(struct fake_driver *bridge, - u8 *buf, unsigned long long addr, - u32 aspace, u32 cycle) -{ - int i; - unsigned long long start, end, offset; - u8 *loc; - - for (i = 0; i < FAKE_MAX_SLAVE; i++) { - if (aspace != bridge->slaves[i].aspace) - continue; - - if (cycle != bridge->slaves[i].cycle) - continue; - - start = bridge->slaves[i].vme_base; - end = bridge->slaves[i].vme_base + bridge->slaves[i].size; - - if ((addr >= start) && (addr < end)) { - offset = addr - bridge->slaves[i].vme_base; - loc = (u8 *)((void *)bridge->slaves[i].buf_base + offset); - *loc = *buf; - - break; - } - } - - fake_lm_check(bridge, addr, aspace, cycle); - -} - -static noinline_for_stack void fake_vmewrite16(struct fake_driver *bridge, - u16 *buf, unsigned long long addr, - u32 aspace, u32 cycle) -{ - int i; - unsigned long long start, end, offset; - u16 *loc; - - for (i = 0; i < FAKE_MAX_SLAVE; i++) { - if (aspace != bridge->slaves[i].aspace) - continue; - - if (cycle != bridge->slaves[i].cycle) - continue; - - start = bridge->slaves[i].vme_base; - end = bridge->slaves[i].vme_base + bridge->slaves[i].size; - - if ((addr >= start) && ((addr + 1) < end)) { - offset = addr - bridge->slaves[i].vme_base; - loc = (u16 *)((void *)bridge->slaves[i].buf_base + offset); - *loc = *buf; - - break; - } - } - - fake_lm_check(bridge, addr, aspace, cycle); - -} - -static noinline_for_stack void fake_vmewrite32(struct fake_driver *bridge, - u32 *buf, unsigned long long addr, - u32 aspace, u32 cycle) -{ - int i; - unsigned long long start, end, offset; - u32 *loc; - - for (i = 0; i < FAKE_MAX_SLAVE; i++) { - if (aspace != bridge->slaves[i].aspace) - continue; - - if (cycle != bridge->slaves[i].cycle) - continue; - - start = bridge->slaves[i].vme_base; - end = bridge->slaves[i].vme_base + bridge->slaves[i].size; - - if ((addr >= start) && ((addr + 3) < end)) { - offset = addr - bridge->slaves[i].vme_base; - loc = (u32 *)((void *)bridge->slaves[i].buf_base + offset); - *loc = *buf; - - break; - } - } - - fake_lm_check(bridge, addr, aspace, cycle); - -} - -static ssize_t fake_master_write(struct vme_master_resource *image, void *buf, - size_t count, loff_t offset) -{ - int retval = 0; - u32 aspace, cycle, dwidth; - unsigned long long addr; - int i; - unsigned int done = 0; - unsigned int count32; - - struct vme_bridge *fake_bridge; - struct fake_driver *bridge; - - fake_bridge = image->parent; - - bridge = fake_bridge->driver_priv; - - i = image->number; - - addr = bridge->masters[i].vme_base + offset; - aspace = bridge->masters[i].aspace; - cycle = bridge->masters[i].cycle; - dwidth = bridge->masters[i].dwidth; - - spin_lock(&image->lock); - - /* Here we apply for the same strategy we do in master_read - * function in order to assure the correct cycles. - */ - if (addr & 0x1) { - fake_vmewrite8(bridge, (u8 *)buf, addr, aspace, cycle); - done += 1; - if (done == count) - goto out; - } - - if ((dwidth == VME_D16) || (dwidth == VME_D32)) { - if ((addr + done) & 0x2) { - if ((count - done) < 2) { - fake_vmewrite8(bridge, (u8 *)(buf + done), - addr + done, aspace, cycle); - done += 1; - goto out; - } else { - fake_vmewrite16(bridge, (u16 *)(buf + done), - addr + done, aspace, cycle); - done += 2; - } - } - } - - if (dwidth == VME_D32) { - count32 = (count - done) & ~0x3; - while (done < count32) { - fake_vmewrite32(bridge, (u32 *)(buf + done), - addr + done, aspace, cycle); - done += 4; - } - } else if (dwidth == VME_D16) { - count32 = (count - done) & ~0x3; - while (done < count32) { - fake_vmewrite16(bridge, (u16 *)(buf + done), - addr + done, aspace, cycle); - done += 2; - } - } else if (dwidth == VME_D8) { - count32 = (count - done); - while (done < count32) { - fake_vmewrite8(bridge, (u8 *)(buf + done), addr + done, - aspace, cycle); - done += 1; - } - - } - - if ((dwidth == VME_D16) || (dwidth == VME_D32)) { - if ((count - done) & 0x2) { - fake_vmewrite16(bridge, (u16 *)(buf + done), - addr + done, aspace, cycle); - done += 2; - } - } - - if ((count - done) & 0x1) { - fake_vmewrite8(bridge, (u8 *)(buf + done), addr + done, aspace, - cycle); - done += 1; - } - -out: - retval = count; - - spin_unlock(&image->lock); - - return retval; -} - -/* - * Perform an RMW cycle on the VME bus. - * - * Requires a previously configured master window, returns final value. - */ -static unsigned int fake_master_rmw(struct vme_master_resource *image, - unsigned int mask, unsigned int compare, unsigned int swap, - loff_t offset) -{ - u32 tmp, base; - u32 aspace, cycle; - int i; - struct fake_driver *bridge; - - bridge = image->parent->driver_priv; - - /* Find the PCI address that maps to the desired VME address */ - i = image->number; - - base = bridge->masters[i].vme_base; - aspace = bridge->masters[i].aspace; - cycle = bridge->masters[i].cycle; - - /* Lock image */ - spin_lock(&image->lock); - - /* Read existing value */ - tmp = fake_vmeread32(bridge, base + offset, aspace, cycle); - - /* Perform check */ - if ((tmp && mask) == (compare && mask)) { - tmp = tmp | (mask | swap); - tmp = tmp & (~mask | swap); - - /* Write back */ - fake_vmewrite32(bridge, &tmp, base + offset, aspace, cycle); - } - - /* Unlock image */ - spin_unlock(&image->lock); - - return tmp; -} - -/* - * All 4 location monitors reside at the same base - this is therefore a - * system wide configuration. - * - * This does not enable the LM monitor - that should be done when the first - * callback is attached and disabled when the last callback is removed. - */ -static int fake_lm_set(struct vme_lm_resource *lm, unsigned long long lm_base, - u32 aspace, u32 cycle) -{ - int i; - struct vme_bridge *fake_bridge; - struct fake_driver *bridge; - - fake_bridge = lm->parent; - - bridge = fake_bridge->driver_priv; - - mutex_lock(&lm->mtx); - - /* If we already have a callback attached, we can't move it! */ - for (i = 0; i < lm->monitors; i++) { - if (bridge->lm_callback[i]) { - mutex_unlock(&lm->mtx); - pr_err("Location monitor callback attached, can't reset\n"); - return -EBUSY; - } - } - - switch (aspace) { - case VME_A16: - case VME_A24: - case VME_A32: - case VME_A64: - break; - default: - mutex_unlock(&lm->mtx); - pr_err("Invalid address space\n"); - return -EINVAL; - } - - bridge->lm_base = lm_base; - bridge->lm_aspace = aspace; - bridge->lm_cycle = cycle; - - mutex_unlock(&lm->mtx); - - return 0; -} - -/* Get configuration of the callback monitor and return whether it is enabled - * or disabled. - */ -static int fake_lm_get(struct vme_lm_resource *lm, - unsigned long long *lm_base, u32 *aspace, u32 *cycle) -{ - struct fake_driver *bridge; - - bridge = lm->parent->driver_priv; - - mutex_lock(&lm->mtx); - - *lm_base = bridge->lm_base; - *aspace = bridge->lm_aspace; - *cycle = bridge->lm_cycle; - - mutex_unlock(&lm->mtx); - - return bridge->lm_enabled; -} - -/* - * Attach a callback to a specific location monitor. - * - * Callback will be passed the monitor triggered. - */ -static int fake_lm_attach(struct vme_lm_resource *lm, int monitor, - void (*callback)(void *), void *data) -{ - struct vme_bridge *fake_bridge; - struct fake_driver *bridge; - - fake_bridge = lm->parent; - - bridge = fake_bridge->driver_priv; - - mutex_lock(&lm->mtx); - - /* Ensure that the location monitor is configured - need PGM or DATA */ - if (bridge->lm_cycle == 0) { - mutex_unlock(&lm->mtx); - pr_err("Location monitor not properly configured\n"); - return -EINVAL; - } - - /* Check that a callback isn't already attached */ - if (bridge->lm_callback[monitor]) { - mutex_unlock(&lm->mtx); - pr_err("Existing callback attached\n"); - return -EBUSY; - } - - /* Attach callback */ - bridge->lm_callback[monitor] = callback; - bridge->lm_data[monitor] = data; - - /* Ensure that global Location Monitor Enable set */ - bridge->lm_enabled = 1; - - mutex_unlock(&lm->mtx); - - return 0; -} - -/* - * Detach a callback function forn a specific location monitor. - */ -static int fake_lm_detach(struct vme_lm_resource *lm, int monitor) -{ - u32 tmp; - int i; - struct fake_driver *bridge; - - bridge = lm->parent->driver_priv; - - mutex_lock(&lm->mtx); - - /* Detach callback */ - bridge->lm_callback[monitor] = NULL; - bridge->lm_data[monitor] = NULL; - - /* If all location monitors disabled, disable global Location Monitor */ - tmp = 0; - for (i = 0; i < lm->monitors; i++) { - if (bridge->lm_callback[i]) - tmp = 1; - } - - if (tmp == 0) - bridge->lm_enabled = 0; - - mutex_unlock(&lm->mtx); - - return 0; -} - -/* - * Determine Geographical Addressing - */ -static int fake_slot_get(struct vme_bridge *fake_bridge) -{ - return geoid; -} - -static void *fake_alloc_consistent(struct device *parent, size_t size, - dma_addr_t *dma) -{ - void *alloc = kmalloc(size, GFP_KERNEL); - - if (alloc) - *dma = fake_ptr_to_pci(alloc); - - return alloc; -} - -static void fake_free_consistent(struct device *parent, size_t size, - void *vaddr, dma_addr_t dma) -{ - kfree(vaddr); -/* - dma_free_coherent(parent, size, vaddr, dma); -*/ -} - -/* - * Configure CR/CSR space - * - * Access to the CR/CSR can be configured at power-up. The location of the - * CR/CSR registers in the CR/CSR address space is determined by the boards - * Geographic address. - * - * Each board has a 512kB window, with the highest 4kB being used for the - * boards registers, this means there is a fix length 508kB window which must - * be mapped onto PCI memory. - */ -static int fake_crcsr_init(struct vme_bridge *fake_bridge) -{ - u32 vstat; - struct fake_driver *bridge; - - bridge = fake_bridge->driver_priv; - - /* Allocate mem for CR/CSR image */ - bridge->crcsr_kernel = kzalloc(VME_CRCSR_BUF_SIZE, GFP_KERNEL); - bridge->crcsr_bus = fake_ptr_to_pci(bridge->crcsr_kernel); - if (!bridge->crcsr_kernel) - return -ENOMEM; - - vstat = fake_slot_get(fake_bridge); - - pr_info("CR/CSR Offset: %d\n", vstat); - - return 0; -} - -static void fake_crcsr_exit(struct vme_bridge *fake_bridge) -{ - struct fake_driver *bridge; - - bridge = fake_bridge->driver_priv; - - kfree(bridge->crcsr_kernel); -} - - -static int __init fake_init(void) -{ - int retval, i; - struct list_head *pos = NULL, *n; - struct vme_bridge *fake_bridge; - struct fake_driver *fake_device; - struct vme_master_resource *master_image; - struct vme_slave_resource *slave_image; - struct vme_lm_resource *lm; - - /* We need a fake parent device */ - vme_root = __root_device_register("vme", THIS_MODULE); - - /* If we want to support more than one bridge at some point, we need to - * dynamically allocate this so we get one per device. - */ - fake_bridge = kzalloc(sizeof(*fake_bridge), GFP_KERNEL); - if (!fake_bridge) { - retval = -ENOMEM; - goto err_struct; - } - - fake_device = kzalloc(sizeof(*fake_device), GFP_KERNEL); - if (!fake_device) { - retval = -ENOMEM; - goto err_driver; - } - - fake_bridge->driver_priv = fake_device; - - fake_bridge->parent = vme_root; - - fake_device->parent = fake_bridge; - - /* Initialize wait queues & mutual exclusion flags */ - mutex_init(&fake_device->vme_int); - mutex_init(&fake_bridge->irq_mtx); - tasklet_init(&fake_device->int_tasklet, fake_VIRQ_tasklet, - (unsigned long) fake_bridge); - - strcpy(fake_bridge->name, driver_name); - - /* Add master windows to list */ - INIT_LIST_HEAD(&fake_bridge->master_resources); - for (i = 0; i < FAKE_MAX_MASTER; i++) { - master_image = kmalloc(sizeof(*master_image), GFP_KERNEL); - if (!master_image) { - retval = -ENOMEM; - goto err_master; - } - master_image->parent = fake_bridge; - spin_lock_init(&master_image->lock); - master_image->locked = 0; - master_image->number = i; - master_image->address_attr = VME_A16 | VME_A24 | VME_A32 | - VME_A64; - master_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | - VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | - VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | - VME_PROG | VME_DATA; - master_image->width_attr = VME_D16 | VME_D32; - memset(&master_image->bus_resource, 0, - sizeof(struct resource)); - master_image->kern_base = NULL; - list_add_tail(&master_image->list, - &fake_bridge->master_resources); - } - - /* Add slave windows to list */ - INIT_LIST_HEAD(&fake_bridge->slave_resources); - for (i = 0; i < FAKE_MAX_SLAVE; i++) { - slave_image = kmalloc(sizeof(*slave_image), GFP_KERNEL); - if (!slave_image) { - retval = -ENOMEM; - goto err_slave; - } - slave_image->parent = fake_bridge; - mutex_init(&slave_image->mtx); - slave_image->locked = 0; - slave_image->number = i; - slave_image->address_attr = VME_A16 | VME_A24 | VME_A32 | - VME_A64 | VME_CRCSR | VME_USER1 | VME_USER2 | - VME_USER3 | VME_USER4; - slave_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | - VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | - VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | - VME_PROG | VME_DATA; - list_add_tail(&slave_image->list, - &fake_bridge->slave_resources); - } - - /* Add location monitor to list */ - INIT_LIST_HEAD(&fake_bridge->lm_resources); - lm = kmalloc(sizeof(*lm), GFP_KERNEL); - if (!lm) { - retval = -ENOMEM; - goto err_lm; - } - lm->parent = fake_bridge; - mutex_init(&lm->mtx); - lm->locked = 0; - lm->number = 1; - lm->monitors = 4; - list_add_tail(&lm->list, &fake_bridge->lm_resources); - - fake_bridge->slave_get = fake_slave_get; - fake_bridge->slave_set = fake_slave_set; - fake_bridge->master_get = fake_master_get; - fake_bridge->master_set = fake_master_set; - fake_bridge->master_read = fake_master_read; - fake_bridge->master_write = fake_master_write; - fake_bridge->master_rmw = fake_master_rmw; - fake_bridge->irq_set = fake_irq_set; - fake_bridge->irq_generate = fake_irq_generate; - fake_bridge->lm_set = fake_lm_set; - fake_bridge->lm_get = fake_lm_get; - fake_bridge->lm_attach = fake_lm_attach; - fake_bridge->lm_detach = fake_lm_detach; - fake_bridge->slot_get = fake_slot_get; - fake_bridge->alloc_consistent = fake_alloc_consistent; - fake_bridge->free_consistent = fake_free_consistent; - - pr_info("Board is%s the VME system controller\n", - (geoid == 1) ? "" : " not"); - - pr_info("VME geographical address is set to %d\n", geoid); - - retval = fake_crcsr_init(fake_bridge); - if (retval) { - pr_err("CR/CSR configuration failed.\n"); - goto err_crcsr; - } - - retval = vme_register_bridge(fake_bridge); - if (retval != 0) { - pr_err("Chip Registration failed.\n"); - goto err_reg; - } - - exit_pointer = fake_bridge; - - return 0; - -err_reg: - fake_crcsr_exit(fake_bridge); -err_crcsr: -err_lm: - /* resources are stored in link list */ - list_for_each_safe(pos, n, &fake_bridge->lm_resources) { - lm = list_entry(pos, struct vme_lm_resource, list); - list_del(pos); - kfree(lm); - } -err_slave: - /* resources are stored in link list */ - list_for_each_safe(pos, n, &fake_bridge->slave_resources) { - slave_image = list_entry(pos, struct vme_slave_resource, list); - list_del(pos); - kfree(slave_image); - } -err_master: - /* resources are stored in link list */ - list_for_each_safe(pos, n, &fake_bridge->master_resources) { - master_image = list_entry(pos, struct vme_master_resource, - list); - list_del(pos); - kfree(master_image); - } - - kfree(fake_device); -err_driver: - kfree(fake_bridge); -err_struct: - return retval; - -} - - -static void __exit fake_exit(void) -{ - struct list_head *pos = NULL; - struct list_head *tmplist; - struct vme_master_resource *master_image; - struct vme_slave_resource *slave_image; - int i; - struct vme_bridge *fake_bridge; - struct fake_driver *bridge; - - fake_bridge = exit_pointer; - - bridge = fake_bridge->driver_priv; - - pr_debug("Driver is being unloaded.\n"); - - /* - * Shutdown all inbound and outbound windows. - */ - for (i = 0; i < FAKE_MAX_MASTER; i++) - bridge->masters[i].enabled = 0; - - for (i = 0; i < FAKE_MAX_SLAVE; i++) - bridge->slaves[i].enabled = 0; - - /* - * Shutdown Location monitor. - */ - bridge->lm_enabled = 0; - - vme_unregister_bridge(fake_bridge); - - fake_crcsr_exit(fake_bridge); - /* resources are stored in link list */ - list_for_each_safe(pos, tmplist, &fake_bridge->slave_resources) { - slave_image = list_entry(pos, struct vme_slave_resource, list); - list_del(pos); - kfree(slave_image); - } - - /* resources are stored in link list */ - list_for_each_safe(pos, tmplist, &fake_bridge->master_resources) { - master_image = list_entry(pos, struct vme_master_resource, - list); - list_del(pos); - kfree(master_image); - } - - kfree(fake_bridge->driver_priv); - - kfree(fake_bridge); - - root_device_unregister(vme_root); -} - - -MODULE_PARM_DESC(geoid, "Set geographical addressing"); -module_param(geoid, int, 0); - -MODULE_DESCRIPTION("Fake VME bridge driver"); -MODULE_LICENSE("GPL"); - -module_init(fake_init); -module_exit(fake_exit); diff --git a/drivers/vme/bridges/vme_tsi148.c b/drivers/vme/bridges/vme_tsi148.c deleted file mode 100644 index be9051b02f24..000000000000 --- a/drivers/vme/bridges/vme_tsi148.c +++ /dev/null @@ -1,2661 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Support for the Tundra TSI148 VME-PCI Bridge Chip - * - * Author: Martyn Welch - * Copyright 2008 GE Intelligent Platforms Embedded Systems, Inc. - * - * Based on work by Tom Armistead and Ajit Prem - * Copyright 2004 Motorola Inc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../vme_bridge.h" -#include "vme_tsi148.h" - -static int tsi148_probe(struct pci_dev *, const struct pci_device_id *); -static void tsi148_remove(struct pci_dev *); - - -/* Module parameter */ -static bool err_chk; -static int geoid; - -static const char driver_name[] = "vme_tsi148"; - -static const struct pci_device_id tsi148_ids[] = { - { PCI_DEVICE(PCI_VENDOR_ID_TUNDRA, PCI_DEVICE_ID_TUNDRA_TSI148) }, - { }, -}; - -MODULE_DEVICE_TABLE(pci, tsi148_ids); - -static struct pci_driver tsi148_driver = { - .name = driver_name, - .id_table = tsi148_ids, - .probe = tsi148_probe, - .remove = tsi148_remove, -}; - -static void reg_join(unsigned int high, unsigned int low, - unsigned long long *variable) -{ - *variable = (unsigned long long)high << 32; - *variable |= (unsigned long long)low; -} - -static void reg_split(unsigned long long variable, unsigned int *high, - unsigned int *low) -{ - *low = (unsigned int)variable & 0xFFFFFFFF; - *high = (unsigned int)(variable >> 32); -} - -/* - * Wakes up DMA queue. - */ -static u32 tsi148_DMA_irqhandler(struct tsi148_driver *bridge, - int channel_mask) -{ - u32 serviced = 0; - - if (channel_mask & TSI148_LCSR_INTS_DMA0S) { - wake_up(&bridge->dma_queue[0]); - serviced |= TSI148_LCSR_INTC_DMA0C; - } - if (channel_mask & TSI148_LCSR_INTS_DMA1S) { - wake_up(&bridge->dma_queue[1]); - serviced |= TSI148_LCSR_INTC_DMA1C; - } - - return serviced; -} - -/* - * Wake up location monitor queue - */ -static u32 tsi148_LM_irqhandler(struct tsi148_driver *bridge, u32 stat) -{ - int i; - u32 serviced = 0; - - for (i = 0; i < 4; i++) { - if (stat & TSI148_LCSR_INTS_LMS[i]) { - /* We only enable interrupts if the callback is set */ - bridge->lm_callback[i](bridge->lm_data[i]); - serviced |= TSI148_LCSR_INTC_LMC[i]; - } - } - - return serviced; -} - -/* - * Wake up mail box queue. - * - * XXX This functionality is not exposed up though API. - */ -static u32 tsi148_MB_irqhandler(struct vme_bridge *tsi148_bridge, u32 stat) -{ - int i; - u32 val; - u32 serviced = 0; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - for (i = 0; i < 4; i++) { - if (stat & TSI148_LCSR_INTS_MBS[i]) { - val = ioread32be(bridge->base + TSI148_GCSR_MBOX[i]); - dev_err(tsi148_bridge->parent, "VME Mailbox %d received" - ": 0x%x\n", i, val); - serviced |= TSI148_LCSR_INTC_MBC[i]; - } - } - - return serviced; -} - -/* - * Display error & status message when PERR (PCI) exception interrupt occurs. - */ -static u32 tsi148_PERR_irqhandler(struct vme_bridge *tsi148_bridge) -{ - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - dev_err(tsi148_bridge->parent, "PCI Exception at address: 0x%08x:%08x, " - "attributes: %08x\n", - ioread32be(bridge->base + TSI148_LCSR_EDPAU), - ioread32be(bridge->base + TSI148_LCSR_EDPAL), - ioread32be(bridge->base + TSI148_LCSR_EDPAT)); - - dev_err(tsi148_bridge->parent, "PCI-X attribute reg: %08x, PCI-X split " - "completion reg: %08x\n", - ioread32be(bridge->base + TSI148_LCSR_EDPXA), - ioread32be(bridge->base + TSI148_LCSR_EDPXS)); - - iowrite32be(TSI148_LCSR_EDPAT_EDPCL, bridge->base + TSI148_LCSR_EDPAT); - - return TSI148_LCSR_INTC_PERRC; -} - -/* - * Save address and status when VME error interrupt occurs. - */ -static u32 tsi148_VERR_irqhandler(struct vme_bridge *tsi148_bridge) -{ - unsigned int error_addr_high, error_addr_low; - unsigned long long error_addr; - u32 error_attrib; - int error_am; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - error_addr_high = ioread32be(bridge->base + TSI148_LCSR_VEAU); - error_addr_low = ioread32be(bridge->base + TSI148_LCSR_VEAL); - error_attrib = ioread32be(bridge->base + TSI148_LCSR_VEAT); - error_am = (error_attrib & TSI148_LCSR_VEAT_AM_M) >> 8; - - reg_join(error_addr_high, error_addr_low, &error_addr); - - /* Check for exception register overflow (we have lost error data) */ - if (error_attrib & TSI148_LCSR_VEAT_VEOF) { - dev_err(tsi148_bridge->parent, "VME Bus Exception Overflow " - "Occurred\n"); - } - - if (err_chk) - vme_bus_error_handler(tsi148_bridge, error_addr, error_am); - else - dev_err(tsi148_bridge->parent, - "VME Bus Error at address: 0x%llx, attributes: %08x\n", - error_addr, error_attrib); - - /* Clear Status */ - iowrite32be(TSI148_LCSR_VEAT_VESCL, bridge->base + TSI148_LCSR_VEAT); - - return TSI148_LCSR_INTC_VERRC; -} - -/* - * Wake up IACK queue. - */ -static u32 tsi148_IACK_irqhandler(struct tsi148_driver *bridge) -{ - wake_up(&bridge->iack_queue); - - return TSI148_LCSR_INTC_IACKC; -} - -/* - * Calling VME bus interrupt callback if provided. - */ -static u32 tsi148_VIRQ_irqhandler(struct vme_bridge *tsi148_bridge, - u32 stat) -{ - int vec, i, serviced = 0; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - for (i = 7; i > 0; i--) { - if (stat & (1 << i)) { - /* - * Note: Even though the registers are defined as - * 32-bits in the spec, we only want to issue 8-bit - * IACK cycles on the bus, read from offset 3. - */ - vec = ioread8(bridge->base + TSI148_LCSR_VIACK[i] + 3); - - vme_irq_handler(tsi148_bridge, i, vec); - - serviced |= (1 << i); - } - } - - return serviced; -} - -/* - * Top level interrupt handler. Clears appropriate interrupt status bits and - * then calls appropriate sub handler(s). - */ -static irqreturn_t tsi148_irqhandler(int irq, void *ptr) -{ - u32 stat, enable, serviced = 0; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *bridge; - - tsi148_bridge = ptr; - - bridge = tsi148_bridge->driver_priv; - - /* Determine which interrupts are unmasked and set */ - enable = ioread32be(bridge->base + TSI148_LCSR_INTEO); - stat = ioread32be(bridge->base + TSI148_LCSR_INTS); - - /* Only look at unmasked interrupts */ - stat &= enable; - - if (unlikely(!stat)) - return IRQ_NONE; - - /* Call subhandlers as appropriate */ - /* DMA irqs */ - if (stat & (TSI148_LCSR_INTS_DMA1S | TSI148_LCSR_INTS_DMA0S)) - serviced |= tsi148_DMA_irqhandler(bridge, stat); - - /* Location monitor irqs */ - if (stat & (TSI148_LCSR_INTS_LM3S | TSI148_LCSR_INTS_LM2S | - TSI148_LCSR_INTS_LM1S | TSI148_LCSR_INTS_LM0S)) - serviced |= tsi148_LM_irqhandler(bridge, stat); - - /* Mail box irqs */ - if (stat & (TSI148_LCSR_INTS_MB3S | TSI148_LCSR_INTS_MB2S | - TSI148_LCSR_INTS_MB1S | TSI148_LCSR_INTS_MB0S)) - serviced |= tsi148_MB_irqhandler(tsi148_bridge, stat); - - /* PCI bus error */ - if (stat & TSI148_LCSR_INTS_PERRS) - serviced |= tsi148_PERR_irqhandler(tsi148_bridge); - - /* VME bus error */ - if (stat & TSI148_LCSR_INTS_VERRS) - serviced |= tsi148_VERR_irqhandler(tsi148_bridge); - - /* IACK irq */ - if (stat & TSI148_LCSR_INTS_IACKS) - serviced |= tsi148_IACK_irqhandler(bridge); - - /* VME bus irqs */ - if (stat & (TSI148_LCSR_INTS_IRQ7S | TSI148_LCSR_INTS_IRQ6S | - TSI148_LCSR_INTS_IRQ5S | TSI148_LCSR_INTS_IRQ4S | - TSI148_LCSR_INTS_IRQ3S | TSI148_LCSR_INTS_IRQ2S | - TSI148_LCSR_INTS_IRQ1S)) - serviced |= tsi148_VIRQ_irqhandler(tsi148_bridge, stat); - - /* Clear serviced interrupts */ - iowrite32be(serviced, bridge->base + TSI148_LCSR_INTC); - - return IRQ_HANDLED; -} - -static int tsi148_irq_init(struct vme_bridge *tsi148_bridge) -{ - int result; - unsigned int tmp; - struct pci_dev *pdev; - struct tsi148_driver *bridge; - - pdev = to_pci_dev(tsi148_bridge->parent); - - bridge = tsi148_bridge->driver_priv; - - result = request_irq(pdev->irq, - tsi148_irqhandler, - IRQF_SHARED, - driver_name, tsi148_bridge); - if (result) { - dev_err(tsi148_bridge->parent, "Can't get assigned pci irq " - "vector %02X\n", pdev->irq); - return result; - } - - /* Enable and unmask interrupts */ - tmp = TSI148_LCSR_INTEO_DMA1EO | TSI148_LCSR_INTEO_DMA0EO | - TSI148_LCSR_INTEO_MB3EO | TSI148_LCSR_INTEO_MB2EO | - TSI148_LCSR_INTEO_MB1EO | TSI148_LCSR_INTEO_MB0EO | - TSI148_LCSR_INTEO_PERREO | TSI148_LCSR_INTEO_VERREO | - TSI148_LCSR_INTEO_IACKEO; - - /* This leaves the following interrupts masked. - * TSI148_LCSR_INTEO_VIEEO - * TSI148_LCSR_INTEO_SYSFLEO - * TSI148_LCSR_INTEO_ACFLEO - */ - - /* Don't enable Location Monitor interrupts here - they will be - * enabled when the location monitors are properly configured and - * a callback has been attached. - * TSI148_LCSR_INTEO_LM0EO - * TSI148_LCSR_INTEO_LM1EO - * TSI148_LCSR_INTEO_LM2EO - * TSI148_LCSR_INTEO_LM3EO - */ - - /* Don't enable VME interrupts until we add a handler, else the board - * will respond to it and we don't want that unless it knows how to - * properly deal with it. - * TSI148_LCSR_INTEO_IRQ7EO - * TSI148_LCSR_INTEO_IRQ6EO - * TSI148_LCSR_INTEO_IRQ5EO - * TSI148_LCSR_INTEO_IRQ4EO - * TSI148_LCSR_INTEO_IRQ3EO - * TSI148_LCSR_INTEO_IRQ2EO - * TSI148_LCSR_INTEO_IRQ1EO - */ - - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); - - return 0; -} - -static void tsi148_irq_exit(struct vme_bridge *tsi148_bridge, - struct pci_dev *pdev) -{ - struct tsi148_driver *bridge = tsi148_bridge->driver_priv; - - /* Turn off interrupts */ - iowrite32be(0x0, bridge->base + TSI148_LCSR_INTEO); - iowrite32be(0x0, bridge->base + TSI148_LCSR_INTEN); - - /* Clear all interrupts */ - iowrite32be(0xFFFFFFFF, bridge->base + TSI148_LCSR_INTC); - - /* Detach interrupt handler */ - free_irq(pdev->irq, tsi148_bridge); -} - -/* - * Check to see if an IACk has been received, return true (1) or false (0). - */ -static int tsi148_iack_received(struct tsi148_driver *bridge) -{ - u32 tmp; - - tmp = ioread32be(bridge->base + TSI148_LCSR_VICR); - - if (tmp & TSI148_LCSR_VICR_IRQS) - return 0; - else - return 1; -} - -/* - * Configure VME interrupt - */ -static void tsi148_irq_set(struct vme_bridge *tsi148_bridge, int level, - int state, int sync) -{ - struct pci_dev *pdev; - u32 tmp; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - /* We need to do the ordering differently for enabling and disabling */ - if (state == 0) { - tmp = ioread32be(bridge->base + TSI148_LCSR_INTEN); - tmp &= ~TSI148_LCSR_INTEN_IRQEN[level - 1]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); - - tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); - tmp &= ~TSI148_LCSR_INTEO_IRQEO[level - 1]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); - - if (sync != 0) { - pdev = to_pci_dev(tsi148_bridge->parent); - synchronize_irq(pdev->irq); - } - } else { - tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); - tmp |= TSI148_LCSR_INTEO_IRQEO[level - 1]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); - - tmp = ioread32be(bridge->base + TSI148_LCSR_INTEN); - tmp |= TSI148_LCSR_INTEN_IRQEN[level - 1]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); - } -} - -/* - * Generate a VME bus interrupt at the requested level & vector. Wait for - * interrupt to be acked. - */ -static int tsi148_irq_generate(struct vme_bridge *tsi148_bridge, int level, - int statid) -{ - u32 tmp; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - mutex_lock(&bridge->vme_int); - - /* Read VICR register */ - tmp = ioread32be(bridge->base + TSI148_LCSR_VICR); - - /* Set Status/ID */ - tmp = (tmp & ~TSI148_LCSR_VICR_STID_M) | - (statid & TSI148_LCSR_VICR_STID_M); - iowrite32be(tmp, bridge->base + TSI148_LCSR_VICR); - - /* Assert VMEbus IRQ */ - tmp = tmp | TSI148_LCSR_VICR_IRQL[level]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_VICR); - - /* XXX Consider implementing a timeout? */ - wait_event_interruptible(bridge->iack_queue, - tsi148_iack_received(bridge)); - - mutex_unlock(&bridge->vme_int); - - return 0; -} - -/* - * Initialize a slave window with the requested attributes. - */ -static int tsi148_slave_set(struct vme_slave_resource *image, int enabled, - unsigned long long vme_base, unsigned long long size, - dma_addr_t pci_base, u32 aspace, u32 cycle) -{ - unsigned int i, addr = 0, granularity = 0; - unsigned int temp_ctl = 0; - unsigned int vme_base_low, vme_base_high; - unsigned int vme_bound_low, vme_bound_high; - unsigned int pci_offset_low, pci_offset_high; - unsigned long long vme_bound, pci_offset; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *bridge; - - tsi148_bridge = image->parent; - bridge = tsi148_bridge->driver_priv; - - i = image->number; - - switch (aspace) { - case VME_A16: - granularity = 0x10; - addr |= TSI148_LCSR_ITAT_AS_A16; - break; - case VME_A24: - granularity = 0x1000; - addr |= TSI148_LCSR_ITAT_AS_A24; - break; - case VME_A32: - granularity = 0x10000; - addr |= TSI148_LCSR_ITAT_AS_A32; - break; - case VME_A64: - granularity = 0x10000; - addr |= TSI148_LCSR_ITAT_AS_A64; - break; - default: - dev_err(tsi148_bridge->parent, "Invalid address space\n"); - return -EINVAL; - } - - /* Convert 64-bit variables to 2x 32-bit variables */ - reg_split(vme_base, &vme_base_high, &vme_base_low); - - /* - * Bound address is a valid address for the window, adjust - * accordingly - */ - vme_bound = vme_base + size - granularity; - reg_split(vme_bound, &vme_bound_high, &vme_bound_low); - pci_offset = (unsigned long long)pci_base - vme_base; - reg_split(pci_offset, &pci_offset_high, &pci_offset_low); - - if (vme_base_low & (granularity - 1)) { - dev_err(tsi148_bridge->parent, "Invalid VME base alignment\n"); - return -EINVAL; - } - if (vme_bound_low & (granularity - 1)) { - dev_err(tsi148_bridge->parent, "Invalid VME bound alignment\n"); - return -EINVAL; - } - if (pci_offset_low & (granularity - 1)) { - dev_err(tsi148_bridge->parent, "Invalid PCI Offset " - "alignment\n"); - return -EINVAL; - } - - /* Disable while we are mucking around */ - temp_ctl = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITAT); - temp_ctl &= ~TSI148_LCSR_ITAT_EN; - iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITAT); - - /* Setup mapping */ - iowrite32be(vme_base_high, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITSAU); - iowrite32be(vme_base_low, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITSAL); - iowrite32be(vme_bound_high, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITEAU); - iowrite32be(vme_bound_low, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITEAL); - iowrite32be(pci_offset_high, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITOFU); - iowrite32be(pci_offset_low, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITOFL); - - /* Setup 2eSST speeds */ - temp_ctl &= ~TSI148_LCSR_ITAT_2eSSTM_M; - switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { - case VME_2eSST160: - temp_ctl |= TSI148_LCSR_ITAT_2eSSTM_160; - break; - case VME_2eSST267: - temp_ctl |= TSI148_LCSR_ITAT_2eSSTM_267; - break; - case VME_2eSST320: - temp_ctl |= TSI148_LCSR_ITAT_2eSSTM_320; - break; - } - - /* Setup cycle types */ - temp_ctl &= ~(0x1F << 7); - if (cycle & VME_BLT) - temp_ctl |= TSI148_LCSR_ITAT_BLT; - if (cycle & VME_MBLT) - temp_ctl |= TSI148_LCSR_ITAT_MBLT; - if (cycle & VME_2eVME) - temp_ctl |= TSI148_LCSR_ITAT_2eVME; - if (cycle & VME_2eSST) - temp_ctl |= TSI148_LCSR_ITAT_2eSST; - if (cycle & VME_2eSSTB) - temp_ctl |= TSI148_LCSR_ITAT_2eSSTB; - - /* Setup address space */ - temp_ctl &= ~TSI148_LCSR_ITAT_AS_M; - temp_ctl |= addr; - - temp_ctl &= ~0xF; - if (cycle & VME_SUPER) - temp_ctl |= TSI148_LCSR_ITAT_SUPR ; - if (cycle & VME_USER) - temp_ctl |= TSI148_LCSR_ITAT_NPRIV; - if (cycle & VME_PROG) - temp_ctl |= TSI148_LCSR_ITAT_PGM; - if (cycle & VME_DATA) - temp_ctl |= TSI148_LCSR_ITAT_DATA; - - /* Write ctl reg without enable */ - iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITAT); - - if (enabled) - temp_ctl |= TSI148_LCSR_ITAT_EN; - - iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITAT); - - return 0; -} - -/* - * Get slave window configuration. - */ -static int tsi148_slave_get(struct vme_slave_resource *image, int *enabled, - unsigned long long *vme_base, unsigned long long *size, - dma_addr_t *pci_base, u32 *aspace, u32 *cycle) -{ - unsigned int i, granularity = 0, ctl = 0; - unsigned int vme_base_low, vme_base_high; - unsigned int vme_bound_low, vme_bound_high; - unsigned int pci_offset_low, pci_offset_high; - unsigned long long vme_bound, pci_offset; - struct tsi148_driver *bridge; - - bridge = image->parent->driver_priv; - - i = image->number; - - /* Read registers */ - ctl = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITAT); - - vme_base_high = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITSAU); - vme_base_low = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITSAL); - vme_bound_high = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITEAU); - vme_bound_low = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITEAL); - pci_offset_high = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITOFU); - pci_offset_low = ioread32be(bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITOFL); - - /* Convert 64-bit variables to 2x 32-bit variables */ - reg_join(vme_base_high, vme_base_low, vme_base); - reg_join(vme_bound_high, vme_bound_low, &vme_bound); - reg_join(pci_offset_high, pci_offset_low, &pci_offset); - - *pci_base = (dma_addr_t)(*vme_base + pci_offset); - - *enabled = 0; - *aspace = 0; - *cycle = 0; - - if (ctl & TSI148_LCSR_ITAT_EN) - *enabled = 1; - - if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A16) { - granularity = 0x10; - *aspace |= VME_A16; - } - if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A24) { - granularity = 0x1000; - *aspace |= VME_A24; - } - if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A32) { - granularity = 0x10000; - *aspace |= VME_A32; - } - if ((ctl & TSI148_LCSR_ITAT_AS_M) == TSI148_LCSR_ITAT_AS_A64) { - granularity = 0x10000; - *aspace |= VME_A64; - } - - /* Need granularity before we set the size */ - *size = (unsigned long long)((vme_bound - *vme_base) + granularity); - - - if ((ctl & TSI148_LCSR_ITAT_2eSSTM_M) == TSI148_LCSR_ITAT_2eSSTM_160) - *cycle |= VME_2eSST160; - if ((ctl & TSI148_LCSR_ITAT_2eSSTM_M) == TSI148_LCSR_ITAT_2eSSTM_267) - *cycle |= VME_2eSST267; - if ((ctl & TSI148_LCSR_ITAT_2eSSTM_M) == TSI148_LCSR_ITAT_2eSSTM_320) - *cycle |= VME_2eSST320; - - if (ctl & TSI148_LCSR_ITAT_BLT) - *cycle |= VME_BLT; - if (ctl & TSI148_LCSR_ITAT_MBLT) - *cycle |= VME_MBLT; - if (ctl & TSI148_LCSR_ITAT_2eVME) - *cycle |= VME_2eVME; - if (ctl & TSI148_LCSR_ITAT_2eSST) - *cycle |= VME_2eSST; - if (ctl & TSI148_LCSR_ITAT_2eSSTB) - *cycle |= VME_2eSSTB; - - if (ctl & TSI148_LCSR_ITAT_SUPR) - *cycle |= VME_SUPER; - if (ctl & TSI148_LCSR_ITAT_NPRIV) - *cycle |= VME_USER; - if (ctl & TSI148_LCSR_ITAT_PGM) - *cycle |= VME_PROG; - if (ctl & TSI148_LCSR_ITAT_DATA) - *cycle |= VME_DATA; - - return 0; -} - -/* - * Allocate and map PCI Resource - */ -static int tsi148_alloc_resource(struct vme_master_resource *image, - unsigned long long size) -{ - unsigned long long existing_size; - int retval = 0; - struct pci_dev *pdev; - struct vme_bridge *tsi148_bridge; - - tsi148_bridge = image->parent; - - pdev = to_pci_dev(tsi148_bridge->parent); - - existing_size = (unsigned long long)(image->bus_resource.end - - image->bus_resource.start); - - /* If the existing size is OK, return */ - if ((size != 0) && (existing_size == (size - 1))) - return 0; - - if (existing_size != 0) { - iounmap(image->kern_base); - image->kern_base = NULL; - kfree(image->bus_resource.name); - release_resource(&image->bus_resource); - memset(&image->bus_resource, 0, sizeof(image->bus_resource)); - } - - /* Exit here if size is zero */ - if (size == 0) - return 0; - - if (!image->bus_resource.name) { - image->bus_resource.name = kmalloc(VMENAMSIZ+3, GFP_ATOMIC); - if (!image->bus_resource.name) { - retval = -ENOMEM; - goto err_name; - } - } - - sprintf((char *)image->bus_resource.name, "%s.%d", tsi148_bridge->name, - image->number); - - image->bus_resource.start = 0; - image->bus_resource.end = (unsigned long)size; - image->bus_resource.flags = IORESOURCE_MEM; - - retval = pci_bus_alloc_resource(pdev->bus, - &image->bus_resource, size, 0x10000, PCIBIOS_MIN_MEM, - 0, NULL, NULL); - if (retval) { - dev_err(tsi148_bridge->parent, "Failed to allocate mem " - "resource for window %d size 0x%lx start 0x%lx\n", - image->number, (unsigned long)size, - (unsigned long)image->bus_resource.start); - goto err_resource; - } - - image->kern_base = ioremap( - image->bus_resource.start, size); - if (!image->kern_base) { - dev_err(tsi148_bridge->parent, "Failed to remap resource\n"); - retval = -ENOMEM; - goto err_remap; - } - - return 0; - -err_remap: - release_resource(&image->bus_resource); -err_resource: - kfree(image->bus_resource.name); - memset(&image->bus_resource, 0, sizeof(image->bus_resource)); -err_name: - return retval; -} - -/* - * Free and unmap PCI Resource - */ -static void tsi148_free_resource(struct vme_master_resource *image) -{ - iounmap(image->kern_base); - image->kern_base = NULL; - release_resource(&image->bus_resource); - kfree(image->bus_resource.name); - memset(&image->bus_resource, 0, sizeof(image->bus_resource)); -} - -/* - * Set the attributes of an outbound window. - */ -static int tsi148_master_set(struct vme_master_resource *image, int enabled, - unsigned long long vme_base, unsigned long long size, u32 aspace, - u32 cycle, u32 dwidth) -{ - int retval = 0; - unsigned int i; - unsigned int temp_ctl = 0; - unsigned int pci_base_low, pci_base_high; - unsigned int pci_bound_low, pci_bound_high; - unsigned int vme_offset_low, vme_offset_high; - unsigned long long pci_bound, vme_offset, pci_base; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *bridge; - struct pci_bus_region region; - struct pci_dev *pdev; - - tsi148_bridge = image->parent; - - bridge = tsi148_bridge->driver_priv; - - pdev = to_pci_dev(tsi148_bridge->parent); - - /* Verify input data */ - if (vme_base & 0xFFFF) { - dev_err(tsi148_bridge->parent, "Invalid VME Window " - "alignment\n"); - retval = -EINVAL; - goto err_window; - } - - if ((size == 0) && (enabled != 0)) { - dev_err(tsi148_bridge->parent, "Size must be non-zero for " - "enabled windows\n"); - retval = -EINVAL; - goto err_window; - } - - spin_lock(&image->lock); - - /* Let's allocate the resource here rather than further up the stack as - * it avoids pushing loads of bus dependent stuff up the stack. If size - * is zero, any existing resource will be freed. - */ - retval = tsi148_alloc_resource(image, size); - if (retval) { - spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Unable to allocate memory for " - "resource\n"); - goto err_res; - } - - if (size == 0) { - pci_base = 0; - pci_bound = 0; - vme_offset = 0; - } else { - pcibios_resource_to_bus(pdev->bus, ®ion, - &image->bus_resource); - pci_base = region.start; - - /* - * Bound address is a valid address for the window, adjust - * according to window granularity. - */ - pci_bound = pci_base + (size - 0x10000); - vme_offset = vme_base - pci_base; - } - - /* Convert 64-bit variables to 2x 32-bit variables */ - reg_split(pci_base, &pci_base_high, &pci_base_low); - reg_split(pci_bound, &pci_bound_high, &pci_bound_low); - reg_split(vme_offset, &vme_offset_high, &vme_offset_low); - - if (pci_base_low & 0xFFFF) { - spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Invalid PCI base alignment\n"); - retval = -EINVAL; - goto err_gran; - } - if (pci_bound_low & 0xFFFF) { - spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Invalid PCI bound alignment\n"); - retval = -EINVAL; - goto err_gran; - } - if (vme_offset_low & 0xFFFF) { - spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Invalid VME Offset " - "alignment\n"); - retval = -EINVAL; - goto err_gran; - } - - i = image->number; - - /* Disable while we are mucking around */ - temp_ctl = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTAT); - temp_ctl &= ~TSI148_LCSR_OTAT_EN; - iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTAT); - - /* Setup 2eSST speeds */ - temp_ctl &= ~TSI148_LCSR_OTAT_2eSSTM_M; - switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { - case VME_2eSST160: - temp_ctl |= TSI148_LCSR_OTAT_2eSSTM_160; - break; - case VME_2eSST267: - temp_ctl |= TSI148_LCSR_OTAT_2eSSTM_267; - break; - case VME_2eSST320: - temp_ctl |= TSI148_LCSR_OTAT_2eSSTM_320; - break; - } - - /* Setup cycle types */ - if (cycle & VME_BLT) { - temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; - temp_ctl |= TSI148_LCSR_OTAT_TM_BLT; - } - if (cycle & VME_MBLT) { - temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; - temp_ctl |= TSI148_LCSR_OTAT_TM_MBLT; - } - if (cycle & VME_2eVME) { - temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; - temp_ctl |= TSI148_LCSR_OTAT_TM_2eVME; - } - if (cycle & VME_2eSST) { - temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; - temp_ctl |= TSI148_LCSR_OTAT_TM_2eSST; - } - if (cycle & VME_2eSSTB) { - dev_warn(tsi148_bridge->parent, "Currently not setting " - "Broadcast Select Registers\n"); - temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; - temp_ctl |= TSI148_LCSR_OTAT_TM_2eSSTB; - } - - /* Setup data width */ - temp_ctl &= ~TSI148_LCSR_OTAT_DBW_M; - switch (dwidth) { - case VME_D16: - temp_ctl |= TSI148_LCSR_OTAT_DBW_16; - break; - case VME_D32: - temp_ctl |= TSI148_LCSR_OTAT_DBW_32; - break; - default: - spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Invalid data width\n"); - retval = -EINVAL; - goto err_dwidth; - } - - /* Setup address space */ - temp_ctl &= ~TSI148_LCSR_OTAT_AMODE_M; - switch (aspace) { - case VME_A16: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_A16; - break; - case VME_A24: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_A24; - break; - case VME_A32: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_A32; - break; - case VME_A64: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_A64; - break; - case VME_CRCSR: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_CRCSR; - break; - case VME_USER1: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER1; - break; - case VME_USER2: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER2; - break; - case VME_USER3: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER3; - break; - case VME_USER4: - temp_ctl |= TSI148_LCSR_OTAT_AMODE_USER4; - break; - default: - spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Invalid address space\n"); - retval = -EINVAL; - goto err_aspace; - } - - temp_ctl &= ~(3<<4); - if (cycle & VME_SUPER) - temp_ctl |= TSI148_LCSR_OTAT_SUP; - if (cycle & VME_PROG) - temp_ctl |= TSI148_LCSR_OTAT_PGM; - - /* Setup mapping */ - iowrite32be(pci_base_high, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTSAU); - iowrite32be(pci_base_low, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTSAL); - iowrite32be(pci_bound_high, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTEAU); - iowrite32be(pci_bound_low, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTEAL); - iowrite32be(vme_offset_high, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTOFU); - iowrite32be(vme_offset_low, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTOFL); - - /* Write ctl reg without enable */ - iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTAT); - - if (enabled) - temp_ctl |= TSI148_LCSR_OTAT_EN; - - iowrite32be(temp_ctl, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTAT); - - spin_unlock(&image->lock); - return 0; - -err_aspace: -err_dwidth: -err_gran: - tsi148_free_resource(image); -err_res: -err_window: - return retval; - -} - -/* - * Set the attributes of an outbound window. - * - * XXX Not parsing prefetch information. - */ -static int __tsi148_master_get(struct vme_master_resource *image, int *enabled, - unsigned long long *vme_base, unsigned long long *size, u32 *aspace, - u32 *cycle, u32 *dwidth) -{ - unsigned int i, ctl; - unsigned int pci_base_low, pci_base_high; - unsigned int pci_bound_low, pci_bound_high; - unsigned int vme_offset_low, vme_offset_high; - - unsigned long long pci_base, pci_bound, vme_offset; - struct tsi148_driver *bridge; - - bridge = image->parent->driver_priv; - - i = image->number; - - ctl = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTAT); - - pci_base_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTSAU); - pci_base_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTSAL); - pci_bound_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTEAU); - pci_bound_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTEAL); - vme_offset_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTOFU); - vme_offset_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTOFL); - - /* Convert 64-bit variables to 2x 32-bit variables */ - reg_join(pci_base_high, pci_base_low, &pci_base); - reg_join(pci_bound_high, pci_bound_low, &pci_bound); - reg_join(vme_offset_high, vme_offset_low, &vme_offset); - - *vme_base = pci_base + vme_offset; - *size = (unsigned long long)(pci_bound - pci_base) + 0x10000; - - *enabled = 0; - *aspace = 0; - *cycle = 0; - *dwidth = 0; - - if (ctl & TSI148_LCSR_OTAT_EN) - *enabled = 1; - - /* Setup address space */ - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A16) - *aspace |= VME_A16; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A24) - *aspace |= VME_A24; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A32) - *aspace |= VME_A32; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_A64) - *aspace |= VME_A64; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_CRCSR) - *aspace |= VME_CRCSR; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER1) - *aspace |= VME_USER1; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER2) - *aspace |= VME_USER2; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER3) - *aspace |= VME_USER3; - if ((ctl & TSI148_LCSR_OTAT_AMODE_M) == TSI148_LCSR_OTAT_AMODE_USER4) - *aspace |= VME_USER4; - - /* Setup 2eSST speeds */ - if ((ctl & TSI148_LCSR_OTAT_2eSSTM_M) == TSI148_LCSR_OTAT_2eSSTM_160) - *cycle |= VME_2eSST160; - if ((ctl & TSI148_LCSR_OTAT_2eSSTM_M) == TSI148_LCSR_OTAT_2eSSTM_267) - *cycle |= VME_2eSST267; - if ((ctl & TSI148_LCSR_OTAT_2eSSTM_M) == TSI148_LCSR_OTAT_2eSSTM_320) - *cycle |= VME_2eSST320; - - /* Setup cycle types */ - if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_SCT) - *cycle |= VME_SCT; - if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_BLT) - *cycle |= VME_BLT; - if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_MBLT) - *cycle |= VME_MBLT; - if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_2eVME) - *cycle |= VME_2eVME; - if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_2eSST) - *cycle |= VME_2eSST; - if ((ctl & TSI148_LCSR_OTAT_TM_M) == TSI148_LCSR_OTAT_TM_2eSSTB) - *cycle |= VME_2eSSTB; - - if (ctl & TSI148_LCSR_OTAT_SUP) - *cycle |= VME_SUPER; - else - *cycle |= VME_USER; - - if (ctl & TSI148_LCSR_OTAT_PGM) - *cycle |= VME_PROG; - else - *cycle |= VME_DATA; - - /* Setup data width */ - if ((ctl & TSI148_LCSR_OTAT_DBW_M) == TSI148_LCSR_OTAT_DBW_16) - *dwidth = VME_D16; - if ((ctl & TSI148_LCSR_OTAT_DBW_M) == TSI148_LCSR_OTAT_DBW_32) - *dwidth = VME_D32; - - return 0; -} - - -static int tsi148_master_get(struct vme_master_resource *image, int *enabled, - unsigned long long *vme_base, unsigned long long *size, u32 *aspace, - u32 *cycle, u32 *dwidth) -{ - int retval; - - spin_lock(&image->lock); - - retval = __tsi148_master_get(image, enabled, vme_base, size, aspace, - cycle, dwidth); - - spin_unlock(&image->lock); - - return retval; -} - -static ssize_t tsi148_master_read(struct vme_master_resource *image, void *buf, - size_t count, loff_t offset) -{ - int retval, enabled; - unsigned long long vme_base, size; - u32 aspace, cycle, dwidth; - struct vme_error_handler *handler = NULL; - struct vme_bridge *tsi148_bridge; - void __iomem *addr = image->kern_base + offset; - unsigned int done = 0; - unsigned int count32; - - tsi148_bridge = image->parent; - - spin_lock(&image->lock); - - if (err_chk) { - __tsi148_master_get(image, &enabled, &vme_base, &size, &aspace, - &cycle, &dwidth); - handler = vme_register_error_handler(tsi148_bridge, aspace, - vme_base + offset, count); - if (!handler) { - spin_unlock(&image->lock); - return -ENOMEM; - } - } - - /* The following code handles VME address alignment. We cannot use - * memcpy_xxx here because it may cut data transfers in to 8-bit - * cycles when D16 or D32 cycles are required on the VME bus. - * On the other hand, the bridge itself assures that the maximum data - * cycle configured for the transfer is used and splits it - * automatically for non-aligned addresses, so we don't want the - * overhead of needlessly forcing small transfers for the entire cycle. - */ - if ((uintptr_t)addr & 0x1) { - *(u8 *)buf = ioread8(addr); - done += 1; - if (done == count) - goto out; - } - if ((uintptr_t)(addr + done) & 0x2) { - if ((count - done) < 2) { - *(u8 *)(buf + done) = ioread8(addr + done); - done += 1; - goto out; - } else { - *(u16 *)(buf + done) = ioread16(addr + done); - done += 2; - } - } - - count32 = (count - done) & ~0x3; - while (done < count32) { - *(u32 *)(buf + done) = ioread32(addr + done); - done += 4; - } - - if ((count - done) & 0x2) { - *(u16 *)(buf + done) = ioread16(addr + done); - done += 2; - } - if ((count - done) & 0x1) { - *(u8 *)(buf + done) = ioread8(addr + done); - done += 1; - } - -out: - retval = count; - - if (err_chk) { - if (handler->num_errors) { - dev_err(image->parent->parent, - "First VME read error detected an at address 0x%llx\n", - handler->first_error); - retval = handler->first_error - (vme_base + offset); - } - vme_unregister_error_handler(handler); - } - - spin_unlock(&image->lock); - - return retval; -} - - -static ssize_t tsi148_master_write(struct vme_master_resource *image, void *buf, - size_t count, loff_t offset) -{ - int retval = 0, enabled; - unsigned long long vme_base, size; - u32 aspace, cycle, dwidth; - void __iomem *addr = image->kern_base + offset; - unsigned int done = 0; - unsigned int count32; - - struct vme_error_handler *handler = NULL; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *bridge; - - tsi148_bridge = image->parent; - - bridge = tsi148_bridge->driver_priv; - - spin_lock(&image->lock); - - if (err_chk) { - __tsi148_master_get(image, &enabled, &vme_base, &size, &aspace, - &cycle, &dwidth); - handler = vme_register_error_handler(tsi148_bridge, aspace, - vme_base + offset, count); - if (!handler) { - spin_unlock(&image->lock); - return -ENOMEM; - } - } - - /* Here we apply for the same strategy we do in master_read - * function in order to assure the correct cycles. - */ - if ((uintptr_t)addr & 0x1) { - iowrite8(*(u8 *)buf, addr); - done += 1; - if (done == count) - goto out; - } - if ((uintptr_t)(addr + done) & 0x2) { - if ((count - done) < 2) { - iowrite8(*(u8 *)(buf + done), addr + done); - done += 1; - goto out; - } else { - iowrite16(*(u16 *)(buf + done), addr + done); - done += 2; - } - } - - count32 = (count - done) & ~0x3; - while (done < count32) { - iowrite32(*(u32 *)(buf + done), addr + done); - done += 4; - } - - if ((count - done) & 0x2) { - iowrite16(*(u16 *)(buf + done), addr + done); - done += 2; - } - if ((count - done) & 0x1) { - iowrite8(*(u8 *)(buf + done), addr + done); - done += 1; - } - -out: - retval = count; - - /* - * Writes are posted. We need to do a read on the VME bus to flush out - * all of the writes before we check for errors. We can't guarantee - * that reading the data we have just written is safe. It is believed - * that there isn't any read, write re-ordering, so we can read any - * location in VME space, so lets read the Device ID from the tsi148's - * own registers as mapped into CR/CSR space. - * - * We check for saved errors in the written address range/space. - */ - - if (err_chk) { - ioread16(bridge->flush_image->kern_base + 0x7F000); - - if (handler->num_errors) { - dev_warn(tsi148_bridge->parent, - "First VME write error detected an at address 0x%llx\n", - handler->first_error); - retval = handler->first_error - (vme_base + offset); - } - vme_unregister_error_handler(handler); - } - - spin_unlock(&image->lock); - - return retval; -} - -/* - * Perform an RMW cycle on the VME bus. - * - * Requires a previously configured master window, returns final value. - */ -static unsigned int tsi148_master_rmw(struct vme_master_resource *image, - unsigned int mask, unsigned int compare, unsigned int swap, - loff_t offset) -{ - unsigned long long pci_addr; - unsigned int pci_addr_high, pci_addr_low; - u32 tmp, result; - int i; - struct tsi148_driver *bridge; - - bridge = image->parent->driver_priv; - - /* Find the PCI address that maps to the desired VME address */ - i = image->number; - - /* Locking as we can only do one of these at a time */ - mutex_lock(&bridge->vme_rmw); - - /* Lock image */ - spin_lock(&image->lock); - - pci_addr_high = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTSAU); - pci_addr_low = ioread32be(bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTSAL); - - reg_join(pci_addr_high, pci_addr_low, &pci_addr); - reg_split(pci_addr + offset, &pci_addr_high, &pci_addr_low); - - /* Configure registers */ - iowrite32be(mask, bridge->base + TSI148_LCSR_RMWEN); - iowrite32be(compare, bridge->base + TSI148_LCSR_RMWC); - iowrite32be(swap, bridge->base + TSI148_LCSR_RMWS); - iowrite32be(pci_addr_high, bridge->base + TSI148_LCSR_RMWAU); - iowrite32be(pci_addr_low, bridge->base + TSI148_LCSR_RMWAL); - - /* Enable RMW */ - tmp = ioread32be(bridge->base + TSI148_LCSR_VMCTRL); - tmp |= TSI148_LCSR_VMCTRL_RMWEN; - iowrite32be(tmp, bridge->base + TSI148_LCSR_VMCTRL); - - /* Kick process off with a read to the required address. */ - result = ioread32be(image->kern_base + offset); - - /* Disable RMW */ - tmp = ioread32be(bridge->base + TSI148_LCSR_VMCTRL); - tmp &= ~TSI148_LCSR_VMCTRL_RMWEN; - iowrite32be(tmp, bridge->base + TSI148_LCSR_VMCTRL); - - spin_unlock(&image->lock); - - mutex_unlock(&bridge->vme_rmw); - - return result; -} - -static int tsi148_dma_set_vme_src_attributes(struct device *dev, __be32 *attr, - u32 aspace, u32 cycle, u32 dwidth) -{ - u32 val; - - val = be32_to_cpu(*attr); - - /* Setup 2eSST speeds */ - switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { - case VME_2eSST160: - val |= TSI148_LCSR_DSAT_2eSSTM_160; - break; - case VME_2eSST267: - val |= TSI148_LCSR_DSAT_2eSSTM_267; - break; - case VME_2eSST320: - val |= TSI148_LCSR_DSAT_2eSSTM_320; - break; - } - - /* Setup cycle types */ - if (cycle & VME_SCT) - val |= TSI148_LCSR_DSAT_TM_SCT; - - if (cycle & VME_BLT) - val |= TSI148_LCSR_DSAT_TM_BLT; - - if (cycle & VME_MBLT) - val |= TSI148_LCSR_DSAT_TM_MBLT; - - if (cycle & VME_2eVME) - val |= TSI148_LCSR_DSAT_TM_2eVME; - - if (cycle & VME_2eSST) - val |= TSI148_LCSR_DSAT_TM_2eSST; - - if (cycle & VME_2eSSTB) { - dev_err(dev, "Currently not setting Broadcast Select " - "Registers\n"); - val |= TSI148_LCSR_DSAT_TM_2eSSTB; - } - - /* Setup data width */ - switch (dwidth) { - case VME_D16: - val |= TSI148_LCSR_DSAT_DBW_16; - break; - case VME_D32: - val |= TSI148_LCSR_DSAT_DBW_32; - break; - default: - dev_err(dev, "Invalid data width\n"); - return -EINVAL; - } - - /* Setup address space */ - switch (aspace) { - case VME_A16: - val |= TSI148_LCSR_DSAT_AMODE_A16; - break; - case VME_A24: - val |= TSI148_LCSR_DSAT_AMODE_A24; - break; - case VME_A32: - val |= TSI148_LCSR_DSAT_AMODE_A32; - break; - case VME_A64: - val |= TSI148_LCSR_DSAT_AMODE_A64; - break; - case VME_CRCSR: - val |= TSI148_LCSR_DSAT_AMODE_CRCSR; - break; - case VME_USER1: - val |= TSI148_LCSR_DSAT_AMODE_USER1; - break; - case VME_USER2: - val |= TSI148_LCSR_DSAT_AMODE_USER2; - break; - case VME_USER3: - val |= TSI148_LCSR_DSAT_AMODE_USER3; - break; - case VME_USER4: - val |= TSI148_LCSR_DSAT_AMODE_USER4; - break; - default: - dev_err(dev, "Invalid address space\n"); - return -EINVAL; - } - - if (cycle & VME_SUPER) - val |= TSI148_LCSR_DSAT_SUP; - if (cycle & VME_PROG) - val |= TSI148_LCSR_DSAT_PGM; - - *attr = cpu_to_be32(val); - - return 0; -} - -static int tsi148_dma_set_vme_dest_attributes(struct device *dev, __be32 *attr, - u32 aspace, u32 cycle, u32 dwidth) -{ - u32 val; - - val = be32_to_cpu(*attr); - - /* Setup 2eSST speeds */ - switch (cycle & (VME_2eSST160 | VME_2eSST267 | VME_2eSST320)) { - case VME_2eSST160: - val |= TSI148_LCSR_DDAT_2eSSTM_160; - break; - case VME_2eSST267: - val |= TSI148_LCSR_DDAT_2eSSTM_267; - break; - case VME_2eSST320: - val |= TSI148_LCSR_DDAT_2eSSTM_320; - break; - } - - /* Setup cycle types */ - if (cycle & VME_SCT) - val |= TSI148_LCSR_DDAT_TM_SCT; - - if (cycle & VME_BLT) - val |= TSI148_LCSR_DDAT_TM_BLT; - - if (cycle & VME_MBLT) - val |= TSI148_LCSR_DDAT_TM_MBLT; - - if (cycle & VME_2eVME) - val |= TSI148_LCSR_DDAT_TM_2eVME; - - if (cycle & VME_2eSST) - val |= TSI148_LCSR_DDAT_TM_2eSST; - - if (cycle & VME_2eSSTB) { - dev_err(dev, "Currently not setting Broadcast Select " - "Registers\n"); - val |= TSI148_LCSR_DDAT_TM_2eSSTB; - } - - /* Setup data width */ - switch (dwidth) { - case VME_D16: - val |= TSI148_LCSR_DDAT_DBW_16; - break; - case VME_D32: - val |= TSI148_LCSR_DDAT_DBW_32; - break; - default: - dev_err(dev, "Invalid data width\n"); - return -EINVAL; - } - - /* Setup address space */ - switch (aspace) { - case VME_A16: - val |= TSI148_LCSR_DDAT_AMODE_A16; - break; - case VME_A24: - val |= TSI148_LCSR_DDAT_AMODE_A24; - break; - case VME_A32: - val |= TSI148_LCSR_DDAT_AMODE_A32; - break; - case VME_A64: - val |= TSI148_LCSR_DDAT_AMODE_A64; - break; - case VME_CRCSR: - val |= TSI148_LCSR_DDAT_AMODE_CRCSR; - break; - case VME_USER1: - val |= TSI148_LCSR_DDAT_AMODE_USER1; - break; - case VME_USER2: - val |= TSI148_LCSR_DDAT_AMODE_USER2; - break; - case VME_USER3: - val |= TSI148_LCSR_DDAT_AMODE_USER3; - break; - case VME_USER4: - val |= TSI148_LCSR_DDAT_AMODE_USER4; - break; - default: - dev_err(dev, "Invalid address space\n"); - return -EINVAL; - } - - if (cycle & VME_SUPER) - val |= TSI148_LCSR_DDAT_SUP; - if (cycle & VME_PROG) - val |= TSI148_LCSR_DDAT_PGM; - - *attr = cpu_to_be32(val); - - return 0; -} - -/* - * Add a link list descriptor to the list - * - * Note: DMA engine expects the DMA descriptor to be big endian. - */ -static int tsi148_dma_list_add(struct vme_dma_list *list, - struct vme_dma_attr *src, struct vme_dma_attr *dest, size_t count) -{ - struct tsi148_dma_entry *entry, *prev; - u32 address_high, address_low, val; - struct vme_dma_pattern *pattern_attr; - struct vme_dma_pci *pci_attr; - struct vme_dma_vme *vme_attr; - int retval = 0; - struct vme_bridge *tsi148_bridge; - - tsi148_bridge = list->parent->parent; - - /* Descriptor must be aligned on 64-bit boundaries */ - entry = kmalloc(sizeof(*entry), GFP_KERNEL); - if (!entry) { - retval = -ENOMEM; - goto err_mem; - } - - /* Test descriptor alignment */ - if ((unsigned long)&entry->descriptor & 0x7) { - dev_err(tsi148_bridge->parent, "Descriptor not aligned to 8 " - "byte boundary as required: %p\n", - &entry->descriptor); - retval = -EINVAL; - goto err_align; - } - - /* Given we are going to fill out the structure, we probably don't - * need to zero it, but better safe than sorry for now. - */ - memset(&entry->descriptor, 0, sizeof(entry->descriptor)); - - /* Fill out source part */ - switch (src->type) { - case VME_DMA_PATTERN: - pattern_attr = src->private; - - entry->descriptor.dsal = cpu_to_be32(pattern_attr->pattern); - - val = TSI148_LCSR_DSAT_TYP_PAT; - - /* Default behaviour is 32 bit pattern */ - if (pattern_attr->type & VME_DMA_PATTERN_BYTE) - val |= TSI148_LCSR_DSAT_PSZ; - - /* It seems that the default behaviour is to increment */ - if ((pattern_attr->type & VME_DMA_PATTERN_INCREMENT) == 0) - val |= TSI148_LCSR_DSAT_NIN; - entry->descriptor.dsat = cpu_to_be32(val); - break; - case VME_DMA_PCI: - pci_attr = src->private; - - reg_split((unsigned long long)pci_attr->address, &address_high, - &address_low); - entry->descriptor.dsau = cpu_to_be32(address_high); - entry->descriptor.dsal = cpu_to_be32(address_low); - entry->descriptor.dsat = cpu_to_be32(TSI148_LCSR_DSAT_TYP_PCI); - break; - case VME_DMA_VME: - vme_attr = src->private; - - reg_split((unsigned long long)vme_attr->address, &address_high, - &address_low); - entry->descriptor.dsau = cpu_to_be32(address_high); - entry->descriptor.dsal = cpu_to_be32(address_low); - entry->descriptor.dsat = cpu_to_be32(TSI148_LCSR_DSAT_TYP_VME); - - retval = tsi148_dma_set_vme_src_attributes( - tsi148_bridge->parent, &entry->descriptor.dsat, - vme_attr->aspace, vme_attr->cycle, vme_attr->dwidth); - if (retval < 0) - goto err_source; - break; - default: - dev_err(tsi148_bridge->parent, "Invalid source type\n"); - retval = -EINVAL; - goto err_source; - } - - /* Assume last link - this will be over-written by adding another */ - entry->descriptor.dnlau = cpu_to_be32(0); - entry->descriptor.dnlal = cpu_to_be32(TSI148_LCSR_DNLAL_LLA); - - /* Fill out destination part */ - switch (dest->type) { - case VME_DMA_PCI: - pci_attr = dest->private; - - reg_split((unsigned long long)pci_attr->address, &address_high, - &address_low); - entry->descriptor.ddau = cpu_to_be32(address_high); - entry->descriptor.ddal = cpu_to_be32(address_low); - entry->descriptor.ddat = cpu_to_be32(TSI148_LCSR_DDAT_TYP_PCI); - break; - case VME_DMA_VME: - vme_attr = dest->private; - - reg_split((unsigned long long)vme_attr->address, &address_high, - &address_low); - entry->descriptor.ddau = cpu_to_be32(address_high); - entry->descriptor.ddal = cpu_to_be32(address_low); - entry->descriptor.ddat = cpu_to_be32(TSI148_LCSR_DDAT_TYP_VME); - - retval = tsi148_dma_set_vme_dest_attributes( - tsi148_bridge->parent, &entry->descriptor.ddat, - vme_attr->aspace, vme_attr->cycle, vme_attr->dwidth); - if (retval < 0) - goto err_dest; - break; - default: - dev_err(tsi148_bridge->parent, "Invalid destination type\n"); - retval = -EINVAL; - goto err_dest; - } - - /* Fill out count */ - entry->descriptor.dcnt = cpu_to_be32((u32)count); - - /* Add to list */ - list_add_tail(&entry->list, &list->entries); - - entry->dma_handle = dma_map_single(tsi148_bridge->parent, - &entry->descriptor, - sizeof(entry->descriptor), - DMA_TO_DEVICE); - if (dma_mapping_error(tsi148_bridge->parent, entry->dma_handle)) { - dev_err(tsi148_bridge->parent, "DMA mapping error\n"); - retval = -EINVAL; - goto err_dma; - } - - /* Fill out previous descriptors "Next Address" */ - if (entry->list.prev != &list->entries) { - reg_split((unsigned long long)entry->dma_handle, &address_high, - &address_low); - prev = list_entry(entry->list.prev, struct tsi148_dma_entry, - list); - prev->descriptor.dnlau = cpu_to_be32(address_high); - prev->descriptor.dnlal = cpu_to_be32(address_low); - - } - - return 0; - -err_dma: -err_dest: -err_source: -err_align: - kfree(entry); -err_mem: - return retval; -} - -/* - * Check to see if the provided DMA channel is busy. - */ -static int tsi148_dma_busy(struct vme_bridge *tsi148_bridge, int channel) -{ - u32 tmp; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - tmp = ioread32be(bridge->base + TSI148_LCSR_DMA[channel] + - TSI148_LCSR_OFFSET_DSTA); - - if (tmp & TSI148_LCSR_DSTA_BSY) - return 0; - else - return 1; - -} - -/* - * Execute a previously generated link list - * - * XXX Need to provide control register configuration. - */ -static int tsi148_dma_list_exec(struct vme_dma_list *list) -{ - struct vme_dma_resource *ctrlr; - int channel, retval; - struct tsi148_dma_entry *entry; - u32 bus_addr_high, bus_addr_low; - u32 val, dctlreg = 0; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *bridge; - - ctrlr = list->parent; - - tsi148_bridge = ctrlr->parent; - - bridge = tsi148_bridge->driver_priv; - - mutex_lock(&ctrlr->mtx); - - channel = ctrlr->number; - - if (!list_empty(&ctrlr->running)) { - /* - * XXX We have an active DMA transfer and currently haven't - * sorted out the mechanism for "pending" DMA transfers. - * Return busy. - */ - /* Need to add to pending here */ - mutex_unlock(&ctrlr->mtx); - return -EBUSY; - } else { - list_add(&list->list, &ctrlr->running); - } - - /* Get first bus address and write into registers */ - entry = list_first_entry(&list->entries, struct tsi148_dma_entry, - list); - - mutex_unlock(&ctrlr->mtx); - - reg_split(entry->dma_handle, &bus_addr_high, &bus_addr_low); - - iowrite32be(bus_addr_high, bridge->base + - TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DNLAU); - iowrite32be(bus_addr_low, bridge->base + - TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DNLAL); - - dctlreg = ioread32be(bridge->base + TSI148_LCSR_DMA[channel] + - TSI148_LCSR_OFFSET_DCTL); - - /* Start the operation */ - iowrite32be(dctlreg | TSI148_LCSR_DCTL_DGO, bridge->base + - TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DCTL); - - retval = wait_event_interruptible(bridge->dma_queue[channel], - tsi148_dma_busy(ctrlr->parent, channel)); - - if (retval) { - iowrite32be(dctlreg | TSI148_LCSR_DCTL_ABT, bridge->base + - TSI148_LCSR_DMA[channel] + TSI148_LCSR_OFFSET_DCTL); - /* Wait for the operation to abort */ - wait_event(bridge->dma_queue[channel], - tsi148_dma_busy(ctrlr->parent, channel)); - retval = -EINTR; - goto exit; - } - - /* - * Read status register, this register is valid until we kick off a - * new transfer. - */ - val = ioread32be(bridge->base + TSI148_LCSR_DMA[channel] + - TSI148_LCSR_OFFSET_DSTA); - - if (val & TSI148_LCSR_DSTA_VBE) { - dev_err(tsi148_bridge->parent, "DMA Error. DSTA=%08X\n", val); - retval = -EIO; - } - -exit: - /* Remove list from running list */ - mutex_lock(&ctrlr->mtx); - list_del(&list->list); - mutex_unlock(&ctrlr->mtx); - - return retval; -} - -/* - * Clean up a previously generated link list - * - * We have a separate function, don't assume that the chain can't be reused. - */ -static int tsi148_dma_list_empty(struct vme_dma_list *list) -{ - struct list_head *pos, *temp; - struct tsi148_dma_entry *entry; - - struct vme_bridge *tsi148_bridge = list->parent->parent; - - /* detach and free each entry */ - list_for_each_safe(pos, temp, &list->entries) { - list_del(pos); - entry = list_entry(pos, struct tsi148_dma_entry, list); - - dma_unmap_single(tsi148_bridge->parent, entry->dma_handle, - sizeof(struct tsi148_dma_descriptor), DMA_TO_DEVICE); - kfree(entry); - } - - return 0; -} - -/* - * All 4 location monitors reside at the same base - this is therefore a - * system wide configuration. - * - * This does not enable the LM monitor - that should be done when the first - * callback is attached and disabled when the last callback is removed. - */ -static int tsi148_lm_set(struct vme_lm_resource *lm, unsigned long long lm_base, - u32 aspace, u32 cycle) -{ - u32 lm_base_high, lm_base_low, lm_ctl = 0; - int i; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *bridge; - - tsi148_bridge = lm->parent; - - bridge = tsi148_bridge->driver_priv; - - mutex_lock(&lm->mtx); - - /* If we already have a callback attached, we can't move it! */ - for (i = 0; i < lm->monitors; i++) { - if (bridge->lm_callback[i]) { - mutex_unlock(&lm->mtx); - dev_err(tsi148_bridge->parent, "Location monitor " - "callback attached, can't reset\n"); - return -EBUSY; - } - } - - switch (aspace) { - case VME_A16: - lm_ctl |= TSI148_LCSR_LMAT_AS_A16; - break; - case VME_A24: - lm_ctl |= TSI148_LCSR_LMAT_AS_A24; - break; - case VME_A32: - lm_ctl |= TSI148_LCSR_LMAT_AS_A32; - break; - case VME_A64: - lm_ctl |= TSI148_LCSR_LMAT_AS_A64; - break; - default: - mutex_unlock(&lm->mtx); - dev_err(tsi148_bridge->parent, "Invalid address space\n"); - return -EINVAL; - } - - if (cycle & VME_SUPER) - lm_ctl |= TSI148_LCSR_LMAT_SUPR ; - if (cycle & VME_USER) - lm_ctl |= TSI148_LCSR_LMAT_NPRIV; - if (cycle & VME_PROG) - lm_ctl |= TSI148_LCSR_LMAT_PGM; - if (cycle & VME_DATA) - lm_ctl |= TSI148_LCSR_LMAT_DATA; - - reg_split(lm_base, &lm_base_high, &lm_base_low); - - iowrite32be(lm_base_high, bridge->base + TSI148_LCSR_LMBAU); - iowrite32be(lm_base_low, bridge->base + TSI148_LCSR_LMBAL); - iowrite32be(lm_ctl, bridge->base + TSI148_LCSR_LMAT); - - mutex_unlock(&lm->mtx); - - return 0; -} - -/* Get configuration of the callback monitor and return whether it is enabled - * or disabled. - */ -static int tsi148_lm_get(struct vme_lm_resource *lm, - unsigned long long *lm_base, u32 *aspace, u32 *cycle) -{ - u32 lm_base_high, lm_base_low, lm_ctl, enabled = 0; - struct tsi148_driver *bridge; - - bridge = lm->parent->driver_priv; - - mutex_lock(&lm->mtx); - - lm_base_high = ioread32be(bridge->base + TSI148_LCSR_LMBAU); - lm_base_low = ioread32be(bridge->base + TSI148_LCSR_LMBAL); - lm_ctl = ioread32be(bridge->base + TSI148_LCSR_LMAT); - - reg_join(lm_base_high, lm_base_low, lm_base); - - if (lm_ctl & TSI148_LCSR_LMAT_EN) - enabled = 1; - - if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A16) - *aspace |= VME_A16; - - if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A24) - *aspace |= VME_A24; - - if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A32) - *aspace |= VME_A32; - - if ((lm_ctl & TSI148_LCSR_LMAT_AS_M) == TSI148_LCSR_LMAT_AS_A64) - *aspace |= VME_A64; - - - if (lm_ctl & TSI148_LCSR_LMAT_SUPR) - *cycle |= VME_SUPER; - if (lm_ctl & TSI148_LCSR_LMAT_NPRIV) - *cycle |= VME_USER; - if (lm_ctl & TSI148_LCSR_LMAT_PGM) - *cycle |= VME_PROG; - if (lm_ctl & TSI148_LCSR_LMAT_DATA) - *cycle |= VME_DATA; - - mutex_unlock(&lm->mtx); - - return enabled; -} - -/* - * Attach a callback to a specific location monitor. - * - * Callback will be passed the monitor triggered. - */ -static int tsi148_lm_attach(struct vme_lm_resource *lm, int monitor, - void (*callback)(void *), void *data) -{ - u32 lm_ctl, tmp; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *bridge; - - tsi148_bridge = lm->parent; - - bridge = tsi148_bridge->driver_priv; - - mutex_lock(&lm->mtx); - - /* Ensure that the location monitor is configured - need PGM or DATA */ - lm_ctl = ioread32be(bridge->base + TSI148_LCSR_LMAT); - if ((lm_ctl & (TSI148_LCSR_LMAT_PGM | TSI148_LCSR_LMAT_DATA)) == 0) { - mutex_unlock(&lm->mtx); - dev_err(tsi148_bridge->parent, "Location monitor not properly " - "configured\n"); - return -EINVAL; - } - - /* Check that a callback isn't already attached */ - if (bridge->lm_callback[monitor]) { - mutex_unlock(&lm->mtx); - dev_err(tsi148_bridge->parent, "Existing callback attached\n"); - return -EBUSY; - } - - /* Attach callback */ - bridge->lm_callback[monitor] = callback; - bridge->lm_data[monitor] = data; - - /* Enable Location Monitor interrupt */ - tmp = ioread32be(bridge->base + TSI148_LCSR_INTEN); - tmp |= TSI148_LCSR_INTEN_LMEN[monitor]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEN); - - tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); - tmp |= TSI148_LCSR_INTEO_LMEO[monitor]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); - - /* Ensure that global Location Monitor Enable set */ - if ((lm_ctl & TSI148_LCSR_LMAT_EN) == 0) { - lm_ctl |= TSI148_LCSR_LMAT_EN; - iowrite32be(lm_ctl, bridge->base + TSI148_LCSR_LMAT); - } - - mutex_unlock(&lm->mtx); - - return 0; -} - -/* - * Detach a callback function forn a specific location monitor. - */ -static int tsi148_lm_detach(struct vme_lm_resource *lm, int monitor) -{ - u32 lm_en, tmp; - struct tsi148_driver *bridge; - - bridge = lm->parent->driver_priv; - - mutex_lock(&lm->mtx); - - /* Disable Location Monitor and ensure previous interrupts are clear */ - lm_en = ioread32be(bridge->base + TSI148_LCSR_INTEN); - lm_en &= ~TSI148_LCSR_INTEN_LMEN[monitor]; - iowrite32be(lm_en, bridge->base + TSI148_LCSR_INTEN); - - tmp = ioread32be(bridge->base + TSI148_LCSR_INTEO); - tmp &= ~TSI148_LCSR_INTEO_LMEO[monitor]; - iowrite32be(tmp, bridge->base + TSI148_LCSR_INTEO); - - iowrite32be(TSI148_LCSR_INTC_LMC[monitor], - bridge->base + TSI148_LCSR_INTC); - - /* Detach callback */ - bridge->lm_callback[monitor] = NULL; - bridge->lm_data[monitor] = NULL; - - /* If all location monitors disabled, disable global Location Monitor */ - if ((lm_en & (TSI148_LCSR_INTS_LM0S | TSI148_LCSR_INTS_LM1S | - TSI148_LCSR_INTS_LM2S | TSI148_LCSR_INTS_LM3S)) == 0) { - tmp = ioread32be(bridge->base + TSI148_LCSR_LMAT); - tmp &= ~TSI148_LCSR_LMAT_EN; - iowrite32be(tmp, bridge->base + TSI148_LCSR_LMAT); - } - - mutex_unlock(&lm->mtx); - - return 0; -} - -/* - * Determine Geographical Addressing - */ -static int tsi148_slot_get(struct vme_bridge *tsi148_bridge) -{ - u32 slot = 0; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - if (!geoid) { - slot = ioread32be(bridge->base + TSI148_LCSR_VSTAT); - slot = slot & TSI148_LCSR_VSTAT_GA_M; - } else - slot = geoid; - - return (int)slot; -} - -static void *tsi148_alloc_consistent(struct device *parent, size_t size, - dma_addr_t *dma) -{ - struct pci_dev *pdev; - - /* Find pci_dev container of dev */ - pdev = to_pci_dev(parent); - - return dma_alloc_coherent(&pdev->dev, size, dma, GFP_KERNEL); -} - -static void tsi148_free_consistent(struct device *parent, size_t size, - void *vaddr, dma_addr_t dma) -{ - struct pci_dev *pdev; - - /* Find pci_dev container of dev */ - pdev = to_pci_dev(parent); - - dma_free_coherent(&pdev->dev, size, vaddr, dma); -} - -/* - * Configure CR/CSR space - * - * Access to the CR/CSR can be configured at power-up. The location of the - * CR/CSR registers in the CR/CSR address space is determined by the boards - * Auto-ID or Geographic address. This function ensures that the window is - * enabled at an offset consistent with the boards geopgraphic address. - * - * Each board has a 512kB window, with the highest 4kB being used for the - * boards registers, this means there is a fix length 508kB window which must - * be mapped onto PCI memory. - */ -static int tsi148_crcsr_init(struct vme_bridge *tsi148_bridge, - struct pci_dev *pdev) -{ - u32 cbar, crat, vstat; - u32 crcsr_bus_high, crcsr_bus_low; - int retval; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - /* Allocate mem for CR/CSR image */ - bridge->crcsr_kernel = dma_alloc_coherent(&pdev->dev, - VME_CRCSR_BUF_SIZE, - &bridge->crcsr_bus, GFP_KERNEL); - if (!bridge->crcsr_kernel) { - dev_err(tsi148_bridge->parent, "Failed to allocate memory for " - "CR/CSR image\n"); - return -ENOMEM; - } - - reg_split(bridge->crcsr_bus, &crcsr_bus_high, &crcsr_bus_low); - - iowrite32be(crcsr_bus_high, bridge->base + TSI148_LCSR_CROU); - iowrite32be(crcsr_bus_low, bridge->base + TSI148_LCSR_CROL); - - /* Ensure that the CR/CSR is configured at the correct offset */ - cbar = ioread32be(bridge->base + TSI148_CBAR); - cbar = (cbar & TSI148_CRCSR_CBAR_M)>>3; - - vstat = tsi148_slot_get(tsi148_bridge); - - if (cbar != vstat) { - cbar = vstat; - dev_info(tsi148_bridge->parent, "Setting CR/CSR offset\n"); - iowrite32be(cbar<<3, bridge->base + TSI148_CBAR); - } - dev_info(tsi148_bridge->parent, "CR/CSR Offset: %d\n", cbar); - - crat = ioread32be(bridge->base + TSI148_LCSR_CRAT); - if (crat & TSI148_LCSR_CRAT_EN) - dev_info(tsi148_bridge->parent, "CR/CSR already enabled\n"); - else { - dev_info(tsi148_bridge->parent, "Enabling CR/CSR space\n"); - iowrite32be(crat | TSI148_LCSR_CRAT_EN, - bridge->base + TSI148_LCSR_CRAT); - } - - /* If we want flushed, error-checked writes, set up a window - * over the CR/CSR registers. We read from here to safely flush - * through VME writes. - */ - if (err_chk) { - retval = tsi148_master_set(bridge->flush_image, 1, - (vstat * 0x80000), 0x80000, VME_CRCSR, VME_SCT, - VME_D16); - if (retval) - dev_err(tsi148_bridge->parent, "Configuring flush image" - " failed\n"); - } - - return 0; - -} - -static void tsi148_crcsr_exit(struct vme_bridge *tsi148_bridge, - struct pci_dev *pdev) -{ - u32 crat; - struct tsi148_driver *bridge; - - bridge = tsi148_bridge->driver_priv; - - /* Turn off CR/CSR space */ - crat = ioread32be(bridge->base + TSI148_LCSR_CRAT); - iowrite32be(crat & ~TSI148_LCSR_CRAT_EN, - bridge->base + TSI148_LCSR_CRAT); - - /* Free image */ - iowrite32be(0, bridge->base + TSI148_LCSR_CROU); - iowrite32be(0, bridge->base + TSI148_LCSR_CROL); - - dma_free_coherent(&pdev->dev, VME_CRCSR_BUF_SIZE, - bridge->crcsr_kernel, bridge->crcsr_bus); -} - -static int tsi148_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - int retval, i, master_num; - u32 data; - struct list_head *pos = NULL, *n; - struct vme_bridge *tsi148_bridge; - struct tsi148_driver *tsi148_device; - struct vme_master_resource *master_image; - struct vme_slave_resource *slave_image; - struct vme_dma_resource *dma_ctrlr; - struct vme_lm_resource *lm; - - /* If we want to support more than one of each bridge, we need to - * dynamically generate this so we get one per device - */ - tsi148_bridge = kzalloc(sizeof(*tsi148_bridge), GFP_KERNEL); - if (!tsi148_bridge) { - retval = -ENOMEM; - goto err_struct; - } - vme_init_bridge(tsi148_bridge); - - tsi148_device = kzalloc(sizeof(*tsi148_device), GFP_KERNEL); - if (!tsi148_device) { - retval = -ENOMEM; - goto err_driver; - } - - tsi148_bridge->driver_priv = tsi148_device; - - /* Enable the device */ - retval = pci_enable_device(pdev); - if (retval) { - dev_err(&pdev->dev, "Unable to enable device\n"); - goto err_enable; - } - - /* Map Registers */ - retval = pci_request_regions(pdev, driver_name); - if (retval) { - dev_err(&pdev->dev, "Unable to reserve resources\n"); - goto err_resource; - } - - /* map registers in BAR 0 */ - tsi148_device->base = ioremap(pci_resource_start(pdev, 0), - 4096); - if (!tsi148_device->base) { - dev_err(&pdev->dev, "Unable to remap CRG region\n"); - retval = -EIO; - goto err_remap; - } - - /* Check to see if the mapping worked out */ - data = ioread32(tsi148_device->base + TSI148_PCFS_ID) & 0x0000FFFF; - if (data != PCI_VENDOR_ID_TUNDRA) { - dev_err(&pdev->dev, "CRG region check failed\n"); - retval = -EIO; - goto err_test; - } - - /* Initialize wait queues & mutual exclusion flags */ - init_waitqueue_head(&tsi148_device->dma_queue[0]); - init_waitqueue_head(&tsi148_device->dma_queue[1]); - init_waitqueue_head(&tsi148_device->iack_queue); - mutex_init(&tsi148_device->vme_int); - mutex_init(&tsi148_device->vme_rmw); - - tsi148_bridge->parent = &pdev->dev; - strcpy(tsi148_bridge->name, driver_name); - - /* Setup IRQ */ - retval = tsi148_irq_init(tsi148_bridge); - if (retval != 0) { - dev_err(&pdev->dev, "Chip Initialization failed.\n"); - goto err_irq; - } - - /* If we are going to flush writes, we need to read from the VME bus. - * We need to do this safely, thus we read the devices own CR/CSR - * register. To do this we must set up a window in CR/CSR space and - * hence have one less master window resource available. - */ - master_num = TSI148_MAX_MASTER; - if (err_chk) { - master_num--; - - tsi148_device->flush_image = - kmalloc(sizeof(*tsi148_device->flush_image), - GFP_KERNEL); - if (!tsi148_device->flush_image) { - retval = -ENOMEM; - goto err_master; - } - tsi148_device->flush_image->parent = tsi148_bridge; - spin_lock_init(&tsi148_device->flush_image->lock); - tsi148_device->flush_image->locked = 1; - tsi148_device->flush_image->number = master_num; - memset(&tsi148_device->flush_image->bus_resource, 0, - sizeof(tsi148_device->flush_image->bus_resource)); - tsi148_device->flush_image->kern_base = NULL; - } - - /* Add master windows to list */ - for (i = 0; i < master_num; i++) { - master_image = kmalloc(sizeof(*master_image), GFP_KERNEL); - if (!master_image) { - retval = -ENOMEM; - goto err_master; - } - master_image->parent = tsi148_bridge; - spin_lock_init(&master_image->lock); - master_image->locked = 0; - master_image->number = i; - master_image->address_attr = VME_A16 | VME_A24 | VME_A32 | - VME_A64 | VME_CRCSR | VME_USER1 | VME_USER2 | - VME_USER3 | VME_USER4; - master_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | - VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | - VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | - VME_PROG | VME_DATA; - master_image->width_attr = VME_D16 | VME_D32; - memset(&master_image->bus_resource, 0, - sizeof(master_image->bus_resource)); - master_image->kern_base = NULL; - list_add_tail(&master_image->list, - &tsi148_bridge->master_resources); - } - - /* Add slave windows to list */ - for (i = 0; i < TSI148_MAX_SLAVE; i++) { - slave_image = kmalloc(sizeof(*slave_image), GFP_KERNEL); - if (!slave_image) { - retval = -ENOMEM; - goto err_slave; - } - slave_image->parent = tsi148_bridge; - mutex_init(&slave_image->mtx); - slave_image->locked = 0; - slave_image->number = i; - slave_image->address_attr = VME_A16 | VME_A24 | VME_A32 | - VME_A64; - slave_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT | - VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 | - VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER | - VME_PROG | VME_DATA; - list_add_tail(&slave_image->list, - &tsi148_bridge->slave_resources); - } - - /* Add dma engines to list */ - for (i = 0; i < TSI148_MAX_DMA; i++) { - dma_ctrlr = kmalloc(sizeof(*dma_ctrlr), GFP_KERNEL); - if (!dma_ctrlr) { - retval = -ENOMEM; - goto err_dma; - } - dma_ctrlr->parent = tsi148_bridge; - mutex_init(&dma_ctrlr->mtx); - dma_ctrlr->locked = 0; - dma_ctrlr->number = i; - dma_ctrlr->route_attr = VME_DMA_VME_TO_MEM | - VME_DMA_MEM_TO_VME | VME_DMA_VME_TO_VME | - VME_DMA_MEM_TO_MEM | VME_DMA_PATTERN_TO_VME | - VME_DMA_PATTERN_TO_MEM; - INIT_LIST_HEAD(&dma_ctrlr->pending); - INIT_LIST_HEAD(&dma_ctrlr->running); - list_add_tail(&dma_ctrlr->list, - &tsi148_bridge->dma_resources); - } - - /* Add location monitor to list */ - lm = kmalloc(sizeof(*lm), GFP_KERNEL); - if (!lm) { - retval = -ENOMEM; - goto err_lm; - } - lm->parent = tsi148_bridge; - mutex_init(&lm->mtx); - lm->locked = 0; - lm->number = 1; - lm->monitors = 4; - list_add_tail(&lm->list, &tsi148_bridge->lm_resources); - - tsi148_bridge->slave_get = tsi148_slave_get; - tsi148_bridge->slave_set = tsi148_slave_set; - tsi148_bridge->master_get = tsi148_master_get; - tsi148_bridge->master_set = tsi148_master_set; - tsi148_bridge->master_read = tsi148_master_read; - tsi148_bridge->master_write = tsi148_master_write; - tsi148_bridge->master_rmw = tsi148_master_rmw; - tsi148_bridge->dma_list_add = tsi148_dma_list_add; - tsi148_bridge->dma_list_exec = tsi148_dma_list_exec; - tsi148_bridge->dma_list_empty = tsi148_dma_list_empty; - tsi148_bridge->irq_set = tsi148_irq_set; - tsi148_bridge->irq_generate = tsi148_irq_generate; - tsi148_bridge->lm_set = tsi148_lm_set; - tsi148_bridge->lm_get = tsi148_lm_get; - tsi148_bridge->lm_attach = tsi148_lm_attach; - tsi148_bridge->lm_detach = tsi148_lm_detach; - tsi148_bridge->slot_get = tsi148_slot_get; - tsi148_bridge->alloc_consistent = tsi148_alloc_consistent; - tsi148_bridge->free_consistent = tsi148_free_consistent; - - data = ioread32be(tsi148_device->base + TSI148_LCSR_VSTAT); - dev_info(&pdev->dev, "Board is%s the VME system controller\n", - (data & TSI148_LCSR_VSTAT_SCONS) ? "" : " not"); - if (!geoid) - dev_info(&pdev->dev, "VME geographical address is %d\n", - data & TSI148_LCSR_VSTAT_GA_M); - else - dev_info(&pdev->dev, "VME geographical address is set to %d\n", - geoid); - - dev_info(&pdev->dev, "VME Write and flush and error check is %s\n", - err_chk ? "enabled" : "disabled"); - - retval = tsi148_crcsr_init(tsi148_bridge, pdev); - if (retval) { - dev_err(&pdev->dev, "CR/CSR configuration failed.\n"); - goto err_crcsr; - } - - retval = vme_register_bridge(tsi148_bridge); - if (retval != 0) { - dev_err(&pdev->dev, "Chip Registration failed.\n"); - goto err_reg; - } - - pci_set_drvdata(pdev, tsi148_bridge); - - /* Clear VME bus "board fail", and "power-up reset" lines */ - data = ioread32be(tsi148_device->base + TSI148_LCSR_VSTAT); - data &= ~TSI148_LCSR_VSTAT_BRDFL; - data |= TSI148_LCSR_VSTAT_CPURST; - iowrite32be(data, tsi148_device->base + TSI148_LCSR_VSTAT); - - return 0; - -err_reg: - tsi148_crcsr_exit(tsi148_bridge, pdev); -err_crcsr: -err_lm: - /* resources are stored in link list */ - list_for_each_safe(pos, n, &tsi148_bridge->lm_resources) { - lm = list_entry(pos, struct vme_lm_resource, list); - list_del(pos); - kfree(lm); - } -err_dma: - /* resources are stored in link list */ - list_for_each_safe(pos, n, &tsi148_bridge->dma_resources) { - dma_ctrlr = list_entry(pos, struct vme_dma_resource, list); - list_del(pos); - kfree(dma_ctrlr); - } -err_slave: - /* resources are stored in link list */ - list_for_each_safe(pos, n, &tsi148_bridge->slave_resources) { - slave_image = list_entry(pos, struct vme_slave_resource, list); - list_del(pos); - kfree(slave_image); - } -err_master: - /* resources are stored in link list */ - list_for_each_safe(pos, n, &tsi148_bridge->master_resources) { - master_image = list_entry(pos, struct vme_master_resource, - list); - list_del(pos); - kfree(master_image); - } - - tsi148_irq_exit(tsi148_bridge, pdev); -err_irq: -err_test: - iounmap(tsi148_device->base); -err_remap: - pci_release_regions(pdev); -err_resource: - pci_disable_device(pdev); -err_enable: - kfree(tsi148_device); -err_driver: - kfree(tsi148_bridge); -err_struct: - return retval; - -} - -static void tsi148_remove(struct pci_dev *pdev) -{ - struct list_head *pos = NULL; - struct list_head *tmplist; - struct vme_master_resource *master_image; - struct vme_slave_resource *slave_image; - struct vme_dma_resource *dma_ctrlr; - int i; - struct tsi148_driver *bridge; - struct vme_bridge *tsi148_bridge = pci_get_drvdata(pdev); - - bridge = tsi148_bridge->driver_priv; - - - dev_dbg(&pdev->dev, "Driver is being unloaded.\n"); - - /* - * Shutdown all inbound and outbound windows. - */ - for (i = 0; i < 8; i++) { - iowrite32be(0, bridge->base + TSI148_LCSR_IT[i] + - TSI148_LCSR_OFFSET_ITAT); - iowrite32be(0, bridge->base + TSI148_LCSR_OT[i] + - TSI148_LCSR_OFFSET_OTAT); - } - - /* - * Shutdown Location monitor. - */ - iowrite32be(0, bridge->base + TSI148_LCSR_LMAT); - - /* - * Shutdown CRG map. - */ - iowrite32be(0, bridge->base + TSI148_LCSR_CSRAT); - - /* - * Clear error status. - */ - iowrite32be(0xFFFFFFFF, bridge->base + TSI148_LCSR_EDPAT); - iowrite32be(0xFFFFFFFF, bridge->base + TSI148_LCSR_VEAT); - iowrite32be(0x07000700, bridge->base + TSI148_LCSR_PSTAT); - - /* - * Remove VIRQ interrupt (if any) - */ - if (ioread32be(bridge->base + TSI148_LCSR_VICR) & 0x800) - iowrite32be(0x8000, bridge->base + TSI148_LCSR_VICR); - - /* - * Map all Interrupts to PCI INTA - */ - iowrite32be(0x0, bridge->base + TSI148_LCSR_INTM1); - iowrite32be(0x0, bridge->base + TSI148_LCSR_INTM2); - - tsi148_irq_exit(tsi148_bridge, pdev); - - vme_unregister_bridge(tsi148_bridge); - - tsi148_crcsr_exit(tsi148_bridge, pdev); - - /* resources are stored in link list */ - list_for_each_safe(pos, tmplist, &tsi148_bridge->dma_resources) { - dma_ctrlr = list_entry(pos, struct vme_dma_resource, list); - list_del(pos); - kfree(dma_ctrlr); - } - - /* resources are stored in link list */ - list_for_each_safe(pos, tmplist, &tsi148_bridge->slave_resources) { - slave_image = list_entry(pos, struct vme_slave_resource, list); - list_del(pos); - kfree(slave_image); - } - - /* resources are stored in link list */ - list_for_each_safe(pos, tmplist, &tsi148_bridge->master_resources) { - master_image = list_entry(pos, struct vme_master_resource, - list); - list_del(pos); - kfree(master_image); - } - - iounmap(bridge->base); - - pci_release_regions(pdev); - - pci_disable_device(pdev); - - kfree(tsi148_bridge->driver_priv); - - kfree(tsi148_bridge); -} - -module_pci_driver(tsi148_driver); - -MODULE_PARM_DESC(err_chk, "Check for VME errors on reads and writes"); -module_param(err_chk, bool, 0); - -MODULE_PARM_DESC(geoid, "Override geographical addressing"); -module_param(geoid, int, 0); - -MODULE_DESCRIPTION("VME driver for the Tundra Tempe VME bridge"); -MODULE_LICENSE("GPL"); diff --git a/drivers/vme/bridges/vme_tsi148.h b/drivers/vme/bridges/vme_tsi148.h deleted file mode 100644 index 226fedc6f167..000000000000 --- a/drivers/vme/bridges/vme_tsi148.h +++ /dev/null @@ -1,1407 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * tsi148.h - * - * Support for the Tundra TSI148 VME Bridge chip - * - * Author: Tom Armistead - * Updated and maintained by Ajit Prem - * Copyright 2004 Motorola Inc. - */ - -#ifndef TSI148_H -#define TSI148_H - -#ifndef PCI_VENDOR_ID_TUNDRA -#define PCI_VENDOR_ID_TUNDRA 0x10e3 -#endif - -#ifndef PCI_DEVICE_ID_TUNDRA_TSI148 -#define PCI_DEVICE_ID_TUNDRA_TSI148 0x148 -#endif - -/* - * Define the number of each that the Tsi148 supports. - */ -#define TSI148_MAX_MASTER 8 /* Max Master Windows */ -#define TSI148_MAX_SLAVE 8 /* Max Slave Windows */ -#define TSI148_MAX_DMA 2 /* Max DMA Controllers */ -#define TSI148_MAX_MAILBOX 4 /* Max Mail Box registers */ -#define TSI148_MAX_SEMAPHORE 8 /* Max Semaphores */ - -/* Structure used to hold driver specific information */ -struct tsi148_driver { - void __iomem *base; /* Base Address of device registers */ - wait_queue_head_t dma_queue[2]; - wait_queue_head_t iack_queue; - void (*lm_callback[4])(void *); /* Called in interrupt handler */ - void *lm_data[4]; - void *crcsr_kernel; - dma_addr_t crcsr_bus; - struct vme_master_resource *flush_image; - struct mutex vme_rmw; /* Only one RMW cycle at a time */ - struct mutex vme_int; /* - * Only one VME interrupt can be - * generated at a time, provide locking - */ -}; - -/* - * Layout of a DMAC Linked-List Descriptor - * - * Note: This structure is accessed via the chip and therefore must be - * correctly laid out - It must also be aligned on 64-bit boundaries. - */ -struct tsi148_dma_descriptor { - __be32 dsau; /* Source Address */ - __be32 dsal; - __be32 ddau; /* Destination Address */ - __be32 ddal; - __be32 dsat; /* Source attributes */ - __be32 ddat; /* Destination attributes */ - __be32 dnlau; /* Next link address */ - __be32 dnlal; - __be32 dcnt; /* Byte count */ - __be32 ddbs; /* 2eSST Broadcast select */ -}; - -struct tsi148_dma_entry { - /* - * The descriptor needs to be aligned on a 64-bit boundary, we increase - * the chance of this by putting it first in the structure. - */ - struct tsi148_dma_descriptor descriptor; - struct list_head list; - dma_addr_t dma_handle; -}; - -/* - * TSI148 ASIC register structure overlays and bit field definitions. - * - * Note: Tsi148 Register Group (CRG) consists of the following - * combination of registers: - * PCFS - PCI Configuration Space Registers - * LCSR - Local Control and Status Registers - * GCSR - Global Control and Status Registers - * CR/CSR - Subset of Configuration ROM / - * Control and Status Registers - */ - - -/* - * Command/Status Registers (CRG + $004) - */ -#define TSI148_PCFS_ID 0x0 -#define TSI148_PCFS_CSR 0x4 -#define TSI148_PCFS_CLASS 0x8 -#define TSI148_PCFS_MISC0 0xC -#define TSI148_PCFS_MBARL 0x10 -#define TSI148_PCFS_MBARU 0x14 - -#define TSI148_PCFS_SUBID 0x28 - -#define TSI148_PCFS_CAPP 0x34 - -#define TSI148_PCFS_MISC1 0x3C - -#define TSI148_PCFS_XCAPP 0x40 -#define TSI148_PCFS_XSTAT 0x44 - -/* - * LCSR definitions - */ - -/* - * Outbound Translations - */ -#define TSI148_LCSR_OT0_OTSAU 0x100 -#define TSI148_LCSR_OT0_OTSAL 0x104 -#define TSI148_LCSR_OT0_OTEAU 0x108 -#define TSI148_LCSR_OT0_OTEAL 0x10C -#define TSI148_LCSR_OT0_OTOFU 0x110 -#define TSI148_LCSR_OT0_OTOFL 0x114 -#define TSI148_LCSR_OT0_OTBS 0x118 -#define TSI148_LCSR_OT0_OTAT 0x11C - -#define TSI148_LCSR_OT1_OTSAU 0x120 -#define TSI148_LCSR_OT1_OTSAL 0x124 -#define TSI148_LCSR_OT1_OTEAU 0x128 -#define TSI148_LCSR_OT1_OTEAL 0x12C -#define TSI148_LCSR_OT1_OTOFU 0x130 -#define TSI148_LCSR_OT1_OTOFL 0x134 -#define TSI148_LCSR_OT1_OTBS 0x138 -#define TSI148_LCSR_OT1_OTAT 0x13C - -#define TSI148_LCSR_OT2_OTSAU 0x140 -#define TSI148_LCSR_OT2_OTSAL 0x144 -#define TSI148_LCSR_OT2_OTEAU 0x148 -#define TSI148_LCSR_OT2_OTEAL 0x14C -#define TSI148_LCSR_OT2_OTOFU 0x150 -#define TSI148_LCSR_OT2_OTOFL 0x154 -#define TSI148_LCSR_OT2_OTBS 0x158 -#define TSI148_LCSR_OT2_OTAT 0x15C - -#define TSI148_LCSR_OT3_OTSAU 0x160 -#define TSI148_LCSR_OT3_OTSAL 0x164 -#define TSI148_LCSR_OT3_OTEAU 0x168 -#define TSI148_LCSR_OT3_OTEAL 0x16C -#define TSI148_LCSR_OT3_OTOFU 0x170 -#define TSI148_LCSR_OT3_OTOFL 0x174 -#define TSI148_LCSR_OT3_OTBS 0x178 -#define TSI148_LCSR_OT3_OTAT 0x17C - -#define TSI148_LCSR_OT4_OTSAU 0x180 -#define TSI148_LCSR_OT4_OTSAL 0x184 -#define TSI148_LCSR_OT4_OTEAU 0x188 -#define TSI148_LCSR_OT4_OTEAL 0x18C -#define TSI148_LCSR_OT4_OTOFU 0x190 -#define TSI148_LCSR_OT4_OTOFL 0x194 -#define TSI148_LCSR_OT4_OTBS 0x198 -#define TSI148_LCSR_OT4_OTAT 0x19C - -#define TSI148_LCSR_OT5_OTSAU 0x1A0 -#define TSI148_LCSR_OT5_OTSAL 0x1A4 -#define TSI148_LCSR_OT5_OTEAU 0x1A8 -#define TSI148_LCSR_OT5_OTEAL 0x1AC -#define TSI148_LCSR_OT5_OTOFU 0x1B0 -#define TSI148_LCSR_OT5_OTOFL 0x1B4 -#define TSI148_LCSR_OT5_OTBS 0x1B8 -#define TSI148_LCSR_OT5_OTAT 0x1BC - -#define TSI148_LCSR_OT6_OTSAU 0x1C0 -#define TSI148_LCSR_OT6_OTSAL 0x1C4 -#define TSI148_LCSR_OT6_OTEAU 0x1C8 -#define TSI148_LCSR_OT6_OTEAL 0x1CC -#define TSI148_LCSR_OT6_OTOFU 0x1D0 -#define TSI148_LCSR_OT6_OTOFL 0x1D4 -#define TSI148_LCSR_OT6_OTBS 0x1D8 -#define TSI148_LCSR_OT6_OTAT 0x1DC - -#define TSI148_LCSR_OT7_OTSAU 0x1E0 -#define TSI148_LCSR_OT7_OTSAL 0x1E4 -#define TSI148_LCSR_OT7_OTEAU 0x1E8 -#define TSI148_LCSR_OT7_OTEAL 0x1EC -#define TSI148_LCSR_OT7_OTOFU 0x1F0 -#define TSI148_LCSR_OT7_OTOFL 0x1F4 -#define TSI148_LCSR_OT7_OTBS 0x1F8 -#define TSI148_LCSR_OT7_OTAT 0x1FC - -#define TSI148_LCSR_OT0 0x100 -#define TSI148_LCSR_OT1 0x120 -#define TSI148_LCSR_OT2 0x140 -#define TSI148_LCSR_OT3 0x160 -#define TSI148_LCSR_OT4 0x180 -#define TSI148_LCSR_OT5 0x1A0 -#define TSI148_LCSR_OT6 0x1C0 -#define TSI148_LCSR_OT7 0x1E0 - -static const int TSI148_LCSR_OT[8] = { TSI148_LCSR_OT0, TSI148_LCSR_OT1, - TSI148_LCSR_OT2, TSI148_LCSR_OT3, - TSI148_LCSR_OT4, TSI148_LCSR_OT5, - TSI148_LCSR_OT6, TSI148_LCSR_OT7 }; - -#define TSI148_LCSR_OFFSET_OTSAU 0x0 -#define TSI148_LCSR_OFFSET_OTSAL 0x4 -#define TSI148_LCSR_OFFSET_OTEAU 0x8 -#define TSI148_LCSR_OFFSET_OTEAL 0xC -#define TSI148_LCSR_OFFSET_OTOFU 0x10 -#define TSI148_LCSR_OFFSET_OTOFL 0x14 -#define TSI148_LCSR_OFFSET_OTBS 0x18 -#define TSI148_LCSR_OFFSET_OTAT 0x1C - -/* - * VMEbus interrupt ack - * offset 200 - */ -#define TSI148_LCSR_VIACK1 0x204 -#define TSI148_LCSR_VIACK2 0x208 -#define TSI148_LCSR_VIACK3 0x20C -#define TSI148_LCSR_VIACK4 0x210 -#define TSI148_LCSR_VIACK5 0x214 -#define TSI148_LCSR_VIACK6 0x218 -#define TSI148_LCSR_VIACK7 0x21C - -static const int TSI148_LCSR_VIACK[8] = { 0, TSI148_LCSR_VIACK1, - TSI148_LCSR_VIACK2, TSI148_LCSR_VIACK3, - TSI148_LCSR_VIACK4, TSI148_LCSR_VIACK5, - TSI148_LCSR_VIACK6, TSI148_LCSR_VIACK7 }; - -/* - * RMW - * offset 220 - */ -#define TSI148_LCSR_RMWAU 0x220 -#define TSI148_LCSR_RMWAL 0x224 -#define TSI148_LCSR_RMWEN 0x228 -#define TSI148_LCSR_RMWC 0x22C -#define TSI148_LCSR_RMWS 0x230 - -/* - * VMEbus control - * offset 234 - */ -#define TSI148_LCSR_VMCTRL 0x234 -#define TSI148_LCSR_VCTRL 0x238 -#define TSI148_LCSR_VSTAT 0x23C - -/* - * PCI status - * offset 240 - */ -#define TSI148_LCSR_PSTAT 0x240 - -/* - * VME filter. - * offset 250 - */ -#define TSI148_LCSR_VMEFL 0x250 - - /* - * VME exception. - * offset 260 - */ -#define TSI148_LCSR_VEAU 0x260 -#define TSI148_LCSR_VEAL 0x264 -#define TSI148_LCSR_VEAT 0x268 - - /* - * PCI error - * offset 270 - */ -#define TSI148_LCSR_EDPAU 0x270 -#define TSI148_LCSR_EDPAL 0x274 -#define TSI148_LCSR_EDPXA 0x278 -#define TSI148_LCSR_EDPXS 0x27C -#define TSI148_LCSR_EDPAT 0x280 - - /* - * Inbound Translations - * offset 300 - */ -#define TSI148_LCSR_IT0_ITSAU 0x300 -#define TSI148_LCSR_IT0_ITSAL 0x304 -#define TSI148_LCSR_IT0_ITEAU 0x308 -#define TSI148_LCSR_IT0_ITEAL 0x30C -#define TSI148_LCSR_IT0_ITOFU 0x310 -#define TSI148_LCSR_IT0_ITOFL 0x314 -#define TSI148_LCSR_IT0_ITAT 0x318 - -#define TSI148_LCSR_IT1_ITSAU 0x320 -#define TSI148_LCSR_IT1_ITSAL 0x324 -#define TSI148_LCSR_IT1_ITEAU 0x328 -#define TSI148_LCSR_IT1_ITEAL 0x32C -#define TSI148_LCSR_IT1_ITOFU 0x330 -#define TSI148_LCSR_IT1_ITOFL 0x334 -#define TSI148_LCSR_IT1_ITAT 0x338 - -#define TSI148_LCSR_IT2_ITSAU 0x340 -#define TSI148_LCSR_IT2_ITSAL 0x344 -#define TSI148_LCSR_IT2_ITEAU 0x348 -#define TSI148_LCSR_IT2_ITEAL 0x34C -#define TSI148_LCSR_IT2_ITOFU 0x350 -#define TSI148_LCSR_IT2_ITOFL 0x354 -#define TSI148_LCSR_IT2_ITAT 0x358 - -#define TSI148_LCSR_IT3_ITSAU 0x360 -#define TSI148_LCSR_IT3_ITSAL 0x364 -#define TSI148_LCSR_IT3_ITEAU 0x368 -#define TSI148_LCSR_IT3_ITEAL 0x36C -#define TSI148_LCSR_IT3_ITOFU 0x370 -#define TSI148_LCSR_IT3_ITOFL 0x374 -#define TSI148_LCSR_IT3_ITAT 0x378 - -#define TSI148_LCSR_IT4_ITSAU 0x380 -#define TSI148_LCSR_IT4_ITSAL 0x384 -#define TSI148_LCSR_IT4_ITEAU 0x388 -#define TSI148_LCSR_IT4_ITEAL 0x38C -#define TSI148_LCSR_IT4_ITOFU 0x390 -#define TSI148_LCSR_IT4_ITOFL 0x394 -#define TSI148_LCSR_IT4_ITAT 0x398 - -#define TSI148_LCSR_IT5_ITSAU 0x3A0 -#define TSI148_LCSR_IT5_ITSAL 0x3A4 -#define TSI148_LCSR_IT5_ITEAU 0x3A8 -#define TSI148_LCSR_IT5_ITEAL 0x3AC -#define TSI148_LCSR_IT5_ITOFU 0x3B0 -#define TSI148_LCSR_IT5_ITOFL 0x3B4 -#define TSI148_LCSR_IT5_ITAT 0x3B8 - -#define TSI148_LCSR_IT6_ITSAU 0x3C0 -#define TSI148_LCSR_IT6_ITSAL 0x3C4 -#define TSI148_LCSR_IT6_ITEAU 0x3C8 -#define TSI148_LCSR_IT6_ITEAL 0x3CC -#define TSI148_LCSR_IT6_ITOFU 0x3D0 -#define TSI148_LCSR_IT6_ITOFL 0x3D4 -#define TSI148_LCSR_IT6_ITAT 0x3D8 - -#define TSI148_LCSR_IT7_ITSAU 0x3E0 -#define TSI148_LCSR_IT7_ITSAL 0x3E4 -#define TSI148_LCSR_IT7_ITEAU 0x3E8 -#define TSI148_LCSR_IT7_ITEAL 0x3EC -#define TSI148_LCSR_IT7_ITOFU 0x3F0 -#define TSI148_LCSR_IT7_ITOFL 0x3F4 -#define TSI148_LCSR_IT7_ITAT 0x3F8 - - -#define TSI148_LCSR_IT0 0x300 -#define TSI148_LCSR_IT1 0x320 -#define TSI148_LCSR_IT2 0x340 -#define TSI148_LCSR_IT3 0x360 -#define TSI148_LCSR_IT4 0x380 -#define TSI148_LCSR_IT5 0x3A0 -#define TSI148_LCSR_IT6 0x3C0 -#define TSI148_LCSR_IT7 0x3E0 - -static const int TSI148_LCSR_IT[8] = { TSI148_LCSR_IT0, TSI148_LCSR_IT1, - TSI148_LCSR_IT2, TSI148_LCSR_IT3, - TSI148_LCSR_IT4, TSI148_LCSR_IT5, - TSI148_LCSR_IT6, TSI148_LCSR_IT7 }; - -#define TSI148_LCSR_OFFSET_ITSAU 0x0 -#define TSI148_LCSR_OFFSET_ITSAL 0x4 -#define TSI148_LCSR_OFFSET_ITEAU 0x8 -#define TSI148_LCSR_OFFSET_ITEAL 0xC -#define TSI148_LCSR_OFFSET_ITOFU 0x10 -#define TSI148_LCSR_OFFSET_ITOFL 0x14 -#define TSI148_LCSR_OFFSET_ITAT 0x18 - - /* - * Inbound Translation GCSR - * offset 400 - */ -#define TSI148_LCSR_GBAU 0x400 -#define TSI148_LCSR_GBAL 0x404 -#define TSI148_LCSR_GCSRAT 0x408 - - /* - * Inbound Translation CRG - * offset 40C - */ -#define TSI148_LCSR_CBAU 0x40C -#define TSI148_LCSR_CBAL 0x410 -#define TSI148_LCSR_CSRAT 0x414 - - /* - * Inbound Translation CR/CSR - * CRG - * offset 418 - */ -#define TSI148_LCSR_CROU 0x418 -#define TSI148_LCSR_CROL 0x41C -#define TSI148_LCSR_CRAT 0x420 - - /* - * Inbound Translation Location Monitor - * offset 424 - */ -#define TSI148_LCSR_LMBAU 0x424 -#define TSI148_LCSR_LMBAL 0x428 -#define TSI148_LCSR_LMAT 0x42C - - /* - * VMEbus Interrupt Control. - * offset 430 - */ -#define TSI148_LCSR_BCU 0x430 -#define TSI148_LCSR_BCL 0x434 -#define TSI148_LCSR_BPGTR 0x438 -#define TSI148_LCSR_BPCTR 0x43C -#define TSI148_LCSR_VICR 0x440 - - /* - * Local Bus Interrupt Control. - * offset 448 - */ -#define TSI148_LCSR_INTEN 0x448 -#define TSI148_LCSR_INTEO 0x44C -#define TSI148_LCSR_INTS 0x450 -#define TSI148_LCSR_INTC 0x454 -#define TSI148_LCSR_INTM1 0x458 -#define TSI148_LCSR_INTM2 0x45C - - /* - * DMA Controllers - * offset 500 - */ -#define TSI148_LCSR_DCTL0 0x500 -#define TSI148_LCSR_DSTA0 0x504 -#define TSI148_LCSR_DCSAU0 0x508 -#define TSI148_LCSR_DCSAL0 0x50C -#define TSI148_LCSR_DCDAU0 0x510 -#define TSI148_LCSR_DCDAL0 0x514 -#define TSI148_LCSR_DCLAU0 0x518 -#define TSI148_LCSR_DCLAL0 0x51C -#define TSI148_LCSR_DSAU0 0x520 -#define TSI148_LCSR_DSAL0 0x524 -#define TSI148_LCSR_DDAU0 0x528 -#define TSI148_LCSR_DDAL0 0x52C -#define TSI148_LCSR_DSAT0 0x530 -#define TSI148_LCSR_DDAT0 0x534 -#define TSI148_LCSR_DNLAU0 0x538 -#define TSI148_LCSR_DNLAL0 0x53C -#define TSI148_LCSR_DCNT0 0x540 -#define TSI148_LCSR_DDBS0 0x544 - -#define TSI148_LCSR_DCTL1 0x580 -#define TSI148_LCSR_DSTA1 0x584 -#define TSI148_LCSR_DCSAU1 0x588 -#define TSI148_LCSR_DCSAL1 0x58C -#define TSI148_LCSR_DCDAU1 0x590 -#define TSI148_LCSR_DCDAL1 0x594 -#define TSI148_LCSR_DCLAU1 0x598 -#define TSI148_LCSR_DCLAL1 0x59C -#define TSI148_LCSR_DSAU1 0x5A0 -#define TSI148_LCSR_DSAL1 0x5A4 -#define TSI148_LCSR_DDAU1 0x5A8 -#define TSI148_LCSR_DDAL1 0x5AC -#define TSI148_LCSR_DSAT1 0x5B0 -#define TSI148_LCSR_DDAT1 0x5B4 -#define TSI148_LCSR_DNLAU1 0x5B8 -#define TSI148_LCSR_DNLAL1 0x5BC -#define TSI148_LCSR_DCNT1 0x5C0 -#define TSI148_LCSR_DDBS1 0x5C4 - -#define TSI148_LCSR_DMA0 0x500 -#define TSI148_LCSR_DMA1 0x580 - - -static const int TSI148_LCSR_DMA[TSI148_MAX_DMA] = { TSI148_LCSR_DMA0, - TSI148_LCSR_DMA1 }; - -#define TSI148_LCSR_OFFSET_DCTL 0x0 -#define TSI148_LCSR_OFFSET_DSTA 0x4 -#define TSI148_LCSR_OFFSET_DCSAU 0x8 -#define TSI148_LCSR_OFFSET_DCSAL 0xC -#define TSI148_LCSR_OFFSET_DCDAU 0x10 -#define TSI148_LCSR_OFFSET_DCDAL 0x14 -#define TSI148_LCSR_OFFSET_DCLAU 0x18 -#define TSI148_LCSR_OFFSET_DCLAL 0x1C -#define TSI148_LCSR_OFFSET_DSAU 0x20 -#define TSI148_LCSR_OFFSET_DSAL 0x24 -#define TSI148_LCSR_OFFSET_DDAU 0x28 -#define TSI148_LCSR_OFFSET_DDAL 0x2C -#define TSI148_LCSR_OFFSET_DSAT 0x30 -#define TSI148_LCSR_OFFSET_DDAT 0x34 -#define TSI148_LCSR_OFFSET_DNLAU 0x38 -#define TSI148_LCSR_OFFSET_DNLAL 0x3C -#define TSI148_LCSR_OFFSET_DCNT 0x40 -#define TSI148_LCSR_OFFSET_DDBS 0x44 - - /* - * GCSR Register Group - */ - - /* - * GCSR CRG - * offset 00 600 - DEVI/VENI - * offset 04 604 - CTRL/GA/REVID - * offset 08 608 - Semaphore3/2/1/0 - * offset 0C 60C - Seamphore7/6/5/4 - */ -#define TSI148_GCSR_ID 0x600 -#define TSI148_GCSR_CSR 0x604 -#define TSI148_GCSR_SEMA0 0x608 -#define TSI148_GCSR_SEMA1 0x60C - - /* - * Mail Box - * GCSR CRG - * offset 10 610 - Mailbox0 - */ -#define TSI148_GCSR_MBOX0 0x610 -#define TSI148_GCSR_MBOX1 0x614 -#define TSI148_GCSR_MBOX2 0x618 -#define TSI148_GCSR_MBOX3 0x61C - -static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0, - TSI148_GCSR_MBOX1, - TSI148_GCSR_MBOX2, - TSI148_GCSR_MBOX3 }; - - /* - * CR/CSR - */ - - /* - * CR/CSR CRG - * offset 7FFF4 FF4 - CSRBCR - * offset 7FFF8 FF8 - CSRBSR - * offset 7FFFC FFC - CBAR - */ -#define TSI148_CSRBCR 0xFF4 -#define TSI148_CSRBSR 0xFF8 -#define TSI148_CBAR 0xFFC - - - - - /* - * TSI148 Register Bit Definitions - */ - - /* - * PFCS Register Set - */ -#define TSI148_PCFS_CMMD_SERR (1<<8) /* SERR_L out pin ssys err */ -#define TSI148_PCFS_CMMD_PERR (1<<6) /* PERR_L out pin parity */ -#define TSI148_PCFS_CMMD_MSTR (1<<2) /* PCI bus master */ -#define TSI148_PCFS_CMMD_MEMSP (1<<1) /* PCI mem space access */ -#define TSI148_PCFS_CMMD_IOSP (1<<0) /* PCI I/O space enable */ - -#define TSI148_PCFS_STAT_RCPVE (1<<15) /* Detected Parity Error */ -#define TSI148_PCFS_STAT_SIGSE (1<<14) /* Signalled System Error */ -#define TSI148_PCFS_STAT_RCVMA (1<<13) /* Received Master Abort */ -#define TSI148_PCFS_STAT_RCVTA (1<<12) /* Received Target Abort */ -#define TSI148_PCFS_STAT_SIGTA (1<<11) /* Signalled Target Abort */ -#define TSI148_PCFS_STAT_SELTIM (3<<9) /* DELSEL Timing */ -#define TSI148_PCFS_STAT_DPAR (1<<8) /* Data Parity Err Reported */ -#define TSI148_PCFS_STAT_FAST (1<<7) /* Fast back-to-back Cap */ -#define TSI148_PCFS_STAT_P66M (1<<5) /* 66 MHz Capable */ -#define TSI148_PCFS_STAT_CAPL (1<<4) /* Capab List - address $34 */ - -/* - * Revision ID/Class Code Registers (CRG +$008) - */ -#define TSI148_PCFS_CLAS_M (0xFF<<24) /* Class ID */ -#define TSI148_PCFS_SUBCLAS_M (0xFF<<16) /* Sub-Class ID */ -#define TSI148_PCFS_PROGIF_M (0xFF<<8) /* Sub-Class ID */ -#define TSI148_PCFS_REVID_M (0xFF<<0) /* Rev ID */ - -/* - * Cache Line Size/ Master Latency Timer/ Header Type Registers (CRG + $00C) - */ -#define TSI148_PCFS_HEAD_M (0xFF<<16) /* Master Lat Timer */ -#define TSI148_PCFS_MLAT_M (0xFF<<8) /* Master Lat Timer */ -#define TSI148_PCFS_CLSZ_M (0xFF<<0) /* Cache Line Size */ - -/* - * Memory Base Address Lower Reg (CRG + $010) - */ -#define TSI148_PCFS_MBARL_BASEL_M (0xFFFFF<<12) /* Base Addr Lower Mask */ -#define TSI148_PCFS_MBARL_PRE (1<<3) /* Prefetch */ -#define TSI148_PCFS_MBARL_MTYPE_M (3<<1) /* Memory Type Mask */ -#define TSI148_PCFS_MBARL_IOMEM (1<<0) /* I/O Space Indicator */ - -/* - * Message Signaled Interrupt Capabilities Register (CRG + $040) - */ -#define TSI148_PCFS_MSICAP_64BAC (1<<7) /* 64-bit Address Capable */ -#define TSI148_PCFS_MSICAP_MME_M (7<<4) /* Multiple Msg Enable Mask */ -#define TSI148_PCFS_MSICAP_MMC_M (7<<1) /* Multiple Msg Capable Mask */ -#define TSI148_PCFS_MSICAP_MSIEN (1<<0) /* Msg signaled INT Enable */ - -/* - * Message Address Lower Register (CRG +$044) - */ -#define TSI148_PCFS_MSIAL_M (0x3FFFFFFF<<2) /* Mask */ - -/* - * Message Data Register (CRG + 4C) - */ -#define TSI148_PCFS_MSIMD_M (0xFFFF<<0) /* Mask */ - -/* - * PCI-X Capabilities Register (CRG + $050) - */ -#define TSI148_PCFS_PCIXCAP_MOST_M (7<<4) /* Max outstanding Split Tran */ -#define TSI148_PCFS_PCIXCAP_MMRBC_M (3<<2) /* Max Mem Read byte cnt */ -#define TSI148_PCFS_PCIXCAP_ERO (1<<1) /* Enable Relaxed Ordering */ -#define TSI148_PCFS_PCIXCAP_DPERE (1<<0) /* Data Parity Recover Enable */ - -/* - * PCI-X Status Register (CRG +$054) - */ -#define TSI148_PCFS_PCIXSTAT_RSCEM (1<<29) /* Received Split Comp Error */ -#define TSI148_PCFS_PCIXSTAT_DMCRS_M (7<<26) /* max Cumulative Read Size */ -#define TSI148_PCFS_PCIXSTAT_DMOST_M (7<<23) /* max outstanding Split Trans - */ -#define TSI148_PCFS_PCIXSTAT_DMMRC_M (3<<21) /* max mem read byte count */ -#define TSI148_PCFS_PCIXSTAT_DC (1<<20) /* Device Complexity */ -#define TSI148_PCFS_PCIXSTAT_USC (1<<19) /* Unexpected Split comp */ -#define TSI148_PCFS_PCIXSTAT_SCD (1<<18) /* Split completion discard */ -#define TSI148_PCFS_PCIXSTAT_133C (1<<17) /* 133MHz capable */ -#define TSI148_PCFS_PCIXSTAT_64D (1<<16) /* 64 bit device */ -#define TSI148_PCFS_PCIXSTAT_BN_M (0xFF<<8) /* Bus number */ -#define TSI148_PCFS_PCIXSTAT_DN_M (0x1F<<3) /* Device number */ -#define TSI148_PCFS_PCIXSTAT_FN_M (7<<0) /* Function Number */ - -/* - * LCSR Registers - */ - -/* - * Outbound Translation Starting Address Lower - */ -#define TSI148_LCSR_OTSAL_M (0xFFFF<<16) /* Mask */ - -/* - * Outbound Translation Ending Address Lower - */ -#define TSI148_LCSR_OTEAL_M (0xFFFF<<16) /* Mask */ - -/* - * Outbound Translation Offset Lower - */ -#define TSI148_LCSR_OTOFFL_M (0xFFFF<<16) /* Mask */ - -/* - * Outbound Translation 2eSST Broadcast Select - */ -#define TSI148_LCSR_OTBS_M (0xFFFFF<<0) /* Mask */ - -/* - * Outbound Translation Attribute - */ -#define TSI148_LCSR_OTAT_EN (1<<31) /* Window Enable */ -#define TSI148_LCSR_OTAT_MRPFD (1<<18) /* Prefetch Disable */ - -#define TSI148_LCSR_OTAT_PFS_M (3<<16) /* Prefetch Size Mask */ -#define TSI148_LCSR_OTAT_PFS_2 (0<<16) /* 2 Cache Lines P Size */ -#define TSI148_LCSR_OTAT_PFS_4 (1<<16) /* 4 Cache Lines P Size */ -#define TSI148_LCSR_OTAT_PFS_8 (2<<16) /* 8 Cache Lines P Size */ -#define TSI148_LCSR_OTAT_PFS_16 (3<<16) /* 16 Cache Lines P Size */ - -#define TSI148_LCSR_OTAT_2eSSTM_M (7<<11) /* 2eSST Xfer Rate Mask */ -#define TSI148_LCSR_OTAT_2eSSTM_160 (0<<11) /* 160MB/s 2eSST Xfer Rate */ -#define TSI148_LCSR_OTAT_2eSSTM_267 (1<<11) /* 267MB/s 2eSST Xfer Rate */ -#define TSI148_LCSR_OTAT_2eSSTM_320 (2<<11) /* 320MB/s 2eSST Xfer Rate */ - -#define TSI148_LCSR_OTAT_TM_M (7<<8) /* Xfer Protocol Mask */ -#define TSI148_LCSR_OTAT_TM_SCT (0<<8) /* SCT Xfer Protocol */ -#define TSI148_LCSR_OTAT_TM_BLT (1<<8) /* BLT Xfer Protocol */ -#define TSI148_LCSR_OTAT_TM_MBLT (2<<8) /* MBLT Xfer Protocol */ -#define TSI148_LCSR_OTAT_TM_2eVME (3<<8) /* 2eVME Xfer Protocol */ -#define TSI148_LCSR_OTAT_TM_2eSST (4<<8) /* 2eSST Xfer Protocol */ -#define TSI148_LCSR_OTAT_TM_2eSSTB (5<<8) /* 2eSST Bcast Xfer Protocol */ - -#define TSI148_LCSR_OTAT_DBW_M (3<<6) /* Max Data Width */ -#define TSI148_LCSR_OTAT_DBW_16 (0<<6) /* 16-bit Data Width */ -#define TSI148_LCSR_OTAT_DBW_32 (1<<6) /* 32-bit Data Width */ - -#define TSI148_LCSR_OTAT_SUP (1<<5) /* Supervisory Access */ -#define TSI148_LCSR_OTAT_PGM (1<<4) /* Program Access */ - -#define TSI148_LCSR_OTAT_AMODE_M (0xf<<0) /* Address Mode Mask */ -#define TSI148_LCSR_OTAT_AMODE_A16 (0<<0) /* A16 Address Space */ -#define TSI148_LCSR_OTAT_AMODE_A24 (1<<0) /* A24 Address Space */ -#define TSI148_LCSR_OTAT_AMODE_A32 (2<<0) /* A32 Address Space */ -#define TSI148_LCSR_OTAT_AMODE_A64 (4<<0) /* A32 Address Space */ -#define TSI148_LCSR_OTAT_AMODE_CRCSR (5<<0) /* CR/CSR Address Space */ -#define TSI148_LCSR_OTAT_AMODE_USER1 (8<<0) /* User1 Address Space */ -#define TSI148_LCSR_OTAT_AMODE_USER2 (9<<0) /* User2 Address Space */ -#define TSI148_LCSR_OTAT_AMODE_USER3 (10<<0) /* User3 Address Space */ -#define TSI148_LCSR_OTAT_AMODE_USER4 (11<<0) /* User4 Address Space */ - -/* - * VME Master Control Register CRG+$234 - */ -#define TSI148_LCSR_VMCTRL_VSA (1<<27) /* VMEbus Stop Ack */ -#define TSI148_LCSR_VMCTRL_VS (1<<26) /* VMEbus Stop */ -#define TSI148_LCSR_VMCTRL_DHB (1<<25) /* Device Has Bus */ -#define TSI148_LCSR_VMCTRL_DWB (1<<24) /* Device Wants Bus */ - -#define TSI148_LCSR_VMCTRL_RMWEN (1<<20) /* RMW Enable */ - -#define TSI148_LCSR_VMCTRL_ATO_M (7<<16) /* Master Access Time-out Mask - */ -#define TSI148_LCSR_VMCTRL_ATO_32 (0<<16) /* 32 us */ -#define TSI148_LCSR_VMCTRL_ATO_128 (1<<16) /* 128 us */ -#define TSI148_LCSR_VMCTRL_ATO_512 (2<<16) /* 512 us */ -#define TSI148_LCSR_VMCTRL_ATO_2M (3<<16) /* 2 ms */ -#define TSI148_LCSR_VMCTRL_ATO_8M (4<<16) /* 8 ms */ -#define TSI148_LCSR_VMCTRL_ATO_32M (5<<16) /* 32 ms */ -#define TSI148_LCSR_VMCTRL_ATO_128M (6<<16) /* 128 ms */ -#define TSI148_LCSR_VMCTRL_ATO_DIS (7<<16) /* Disabled */ - -#define TSI148_LCSR_VMCTRL_VTOFF_M (7<<12) /* VMEbus Master Time off */ -#define TSI148_LCSR_VMCTRL_VTOFF_0 (0<<12) /* 0us */ -#define TSI148_LCSR_VMCTRL_VTOFF_1 (1<<12) /* 1us */ -#define TSI148_LCSR_VMCTRL_VTOFF_2 (2<<12) /* 2us */ -#define TSI148_LCSR_VMCTRL_VTOFF_4 (3<<12) /* 4us */ -#define TSI148_LCSR_VMCTRL_VTOFF_8 (4<<12) /* 8us */ -#define TSI148_LCSR_VMCTRL_VTOFF_16 (5<<12) /* 16us */ -#define TSI148_LCSR_VMCTRL_VTOFF_32 (6<<12) /* 32us */ -#define TSI148_LCSR_VMCTRL_VTOFF_64 (7<<12) /* 64us */ - -#define TSI148_LCSR_VMCTRL_VTON_M (7<<8) /* VMEbus Master Time On */ -#define TSI148_LCSR_VMCTRL_VTON_4 (0<<8) /* 8us */ -#define TSI148_LCSR_VMCTRL_VTON_8 (1<<8) /* 8us */ -#define TSI148_LCSR_VMCTRL_VTON_16 (2<<8) /* 16us */ -#define TSI148_LCSR_VMCTRL_VTON_32 (3<<8) /* 32us */ -#define TSI148_LCSR_VMCTRL_VTON_64 (4<<8) /* 64us */ -#define TSI148_LCSR_VMCTRL_VTON_128 (5<<8) /* 128us */ -#define TSI148_LCSR_VMCTRL_VTON_256 (6<<8) /* 256us */ -#define TSI148_LCSR_VMCTRL_VTON_512 (7<<8) /* 512us */ - -#define TSI148_LCSR_VMCTRL_VREL_M (3<<3) /* VMEbus Master Rel Mode Mask - */ -#define TSI148_LCSR_VMCTRL_VREL_T_D (0<<3) /* Time on or Done */ -#define TSI148_LCSR_VMCTRL_VREL_T_R_D (1<<3) /* Time on and REQ or Done */ -#define TSI148_LCSR_VMCTRL_VREL_T_B_D (2<<3) /* Time on and BCLR or Done */ -#define TSI148_LCSR_VMCTRL_VREL_T_D_R (3<<3) /* Time on or Done and REQ */ - -#define TSI148_LCSR_VMCTRL_VFAIR (1<<2) /* VMEbus Master Fair Mode */ -#define TSI148_LCSR_VMCTRL_VREQL_M (3<<0) /* VMEbus Master Req Level Mask - */ - -/* - * VMEbus Control Register CRG+$238 - */ -#define TSI148_LCSR_VCTRL_LRE (1<<31) /* Late Retry Enable */ - -#define TSI148_LCSR_VCTRL_DLT_M (0xF<<24) /* Deadlock Timer */ -#define TSI148_LCSR_VCTRL_DLT_OFF (0<<24) /* Deadlock Timer Off */ -#define TSI148_LCSR_VCTRL_DLT_16 (1<<24) /* 16 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_32 (2<<24) /* 32 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_64 (3<<24) /* 64 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_128 (4<<24) /* 128 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_256 (5<<24) /* 256 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_512 (6<<24) /* 512 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_1024 (7<<24) /* 1024 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_2048 (8<<24) /* 2048 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_4096 (9<<24) /* 4096 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_8192 (0xA<<24) /* 8192 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_16384 (0xB<<24) /* 16384 VCLKS */ -#define TSI148_LCSR_VCTRL_DLT_32768 (0xC<<24) /* 32768 VCLKS */ - -#define TSI148_LCSR_VCTRL_NERBB (1<<20) /* No Early Release of Bus Busy - */ - -#define TSI148_LCSR_VCTRL_SRESET (1<<17) /* System Reset */ -#define TSI148_LCSR_VCTRL_LRESET (1<<16) /* Local Reset */ - -#define TSI148_LCSR_VCTRL_SFAILAI (1<<15) /* SYSFAIL Auto Slot ID */ -#define TSI148_LCSR_VCTRL_BID_M (0x1F<<8) /* Broadcast ID Mask */ - -#define TSI148_LCSR_VCTRL_ATOEN (1<<7) /* Arbiter Time-out Enable */ -#define TSI148_LCSR_VCTRL_ROBIN (1<<6) /* VMEbus Round Robin */ - -#define TSI148_LCSR_VCTRL_GTO_M (7<<0) /* VMEbus Global Time-out Mask - */ -#define TSI148_LCSR_VCTRL_GTO_8 (0<<0) /* 8 us */ -#define TSI148_LCSR_VCTRL_GTO_16 (1<<0) /* 16 us */ -#define TSI148_LCSR_VCTRL_GTO_32 (2<<0) /* 32 us */ -#define TSI148_LCSR_VCTRL_GTO_64 (3<<0) /* 64 us */ -#define TSI148_LCSR_VCTRL_GTO_128 (4<<0) /* 128 us */ -#define TSI148_LCSR_VCTRL_GTO_256 (5<<0) /* 256 us */ -#define TSI148_LCSR_VCTRL_GTO_512 (6<<0) /* 512 us */ -#define TSI148_LCSR_VCTRL_GTO_DIS (7<<0) /* Disabled */ - -/* - * VMEbus Status Register CRG + $23C - */ -#define TSI148_LCSR_VSTAT_CPURST (1<<15) /* Clear power up reset */ -#define TSI148_LCSR_VSTAT_BRDFL (1<<14) /* Board fail */ -#define TSI148_LCSR_VSTAT_PURSTS (1<<12) /* Power up reset status */ -#define TSI148_LCSR_VSTAT_BDFAILS (1<<11) /* Board Fail Status */ -#define TSI148_LCSR_VSTAT_SYSFAILS (1<<10) /* System Fail Status */ -#define TSI148_LCSR_VSTAT_ACFAILS (1<<9) /* AC fail status */ -#define TSI148_LCSR_VSTAT_SCONS (1<<8) /* System Cont Status */ -#define TSI148_LCSR_VSTAT_GAP (1<<5) /* Geographic Addr Parity */ -#define TSI148_LCSR_VSTAT_GA_M (0x1F<<0) /* Geographic Addr Mask */ - -/* - * PCI Configuration Status Register CRG+$240 - */ -#define TSI148_LCSR_PSTAT_REQ64S (1<<6) /* Request 64 status set */ -#define TSI148_LCSR_PSTAT_M66ENS (1<<5) /* M66ENS 66Mhz enable */ -#define TSI148_LCSR_PSTAT_FRAMES (1<<4) /* Frame Status */ -#define TSI148_LCSR_PSTAT_IRDYS (1<<3) /* IRDY status */ -#define TSI148_LCSR_PSTAT_DEVSELS (1<<2) /* DEVL status */ -#define TSI148_LCSR_PSTAT_STOPS (1<<1) /* STOP status */ -#define TSI148_LCSR_PSTAT_TRDYS (1<<0) /* TRDY status */ - -/* - * VMEbus Exception Attributes Register CRG + $268 - */ -#define TSI148_LCSR_VEAT_VES (1<<31) /* Status */ -#define TSI148_LCSR_VEAT_VEOF (1<<30) /* Overflow */ -#define TSI148_LCSR_VEAT_VESCL (1<<29) /* Status Clear */ -#define TSI148_LCSR_VEAT_2EOT (1<<21) /* 2e Odd Termination */ -#define TSI148_LCSR_VEAT_2EST (1<<20) /* 2e Slave terminated */ -#define TSI148_LCSR_VEAT_BERR (1<<19) /* Bus Error */ -#define TSI148_LCSR_VEAT_LWORD (1<<18) /* LWORD_ signal state */ -#define TSI148_LCSR_VEAT_WRITE (1<<17) /* WRITE_ signal state */ -#define TSI148_LCSR_VEAT_IACK (1<<16) /* IACK_ signal state */ -#define TSI148_LCSR_VEAT_DS1 (1<<15) /* DS1_ signal state */ -#define TSI148_LCSR_VEAT_DS0 (1<<14) /* DS0_ signal state */ -#define TSI148_LCSR_VEAT_AM_M (0x3F<<8) /* Address Mode Mask */ -#define TSI148_LCSR_VEAT_XAM_M (0xFF<<0) /* Master AMode Mask */ - - -/* - * VMEbus PCI Error Diagnostics PCI/X Attributes Register CRG + $280 - */ -#define TSI148_LCSR_EDPAT_EDPCL (1<<29) - -/* - * Inbound Translation Starting Address Lower - */ -#define TSI148_LCSR_ITSAL6432_M (0xFFFF<<16) /* Mask */ -#define TSI148_LCSR_ITSAL24_M (0x00FFF<<12) /* Mask */ -#define TSI148_LCSR_ITSAL16_M (0x0000FFF<<4) /* Mask */ - -/* - * Inbound Translation Ending Address Lower - */ -#define TSI148_LCSR_ITEAL6432_M (0xFFFF<<16) /* Mask */ -#define TSI148_LCSR_ITEAL24_M (0x00FFF<<12) /* Mask */ -#define TSI148_LCSR_ITEAL16_M (0x0000FFF<<4) /* Mask */ - -/* - * Inbound Translation Offset Lower - */ -#define TSI148_LCSR_ITOFFL6432_M (0xFFFF<<16) /* Mask */ -#define TSI148_LCSR_ITOFFL24_M (0xFFFFF<<12) /* Mask */ -#define TSI148_LCSR_ITOFFL16_M (0xFFFFFFF<<4) /* Mask */ - -/* - * Inbound Translation Attribute - */ -#define TSI148_LCSR_ITAT_EN (1<<31) /* Window Enable */ -#define TSI148_LCSR_ITAT_TH (1<<18) /* Prefetch Threshold */ - -#define TSI148_LCSR_ITAT_VFS_M (3<<16) /* Virtual FIFO Size Mask */ -#define TSI148_LCSR_ITAT_VFS_64 (0<<16) /* 64 bytes Virtual FIFO Size */ -#define TSI148_LCSR_ITAT_VFS_128 (1<<16) /* 128 bytes Virtual FIFO Sz */ -#define TSI148_LCSR_ITAT_VFS_256 (2<<16) /* 256 bytes Virtual FIFO Sz */ -#define TSI148_LCSR_ITAT_VFS_512 (3<<16) /* 512 bytes Virtual FIFO Sz */ - -#define TSI148_LCSR_ITAT_2eSSTM_M (7<<12) /* 2eSST Xfer Rate Mask */ -#define TSI148_LCSR_ITAT_2eSSTM_160 (0<<12) /* 160MB/s 2eSST Xfer Rate */ -#define TSI148_LCSR_ITAT_2eSSTM_267 (1<<12) /* 267MB/s 2eSST Xfer Rate */ -#define TSI148_LCSR_ITAT_2eSSTM_320 (2<<12) /* 320MB/s 2eSST Xfer Rate */ - -#define TSI148_LCSR_ITAT_2eSSTB (1<<11) /* 2eSST Bcast Xfer Protocol */ -#define TSI148_LCSR_ITAT_2eSST (1<<10) /* 2eSST Xfer Protocol */ -#define TSI148_LCSR_ITAT_2eVME (1<<9) /* 2eVME Xfer Protocol */ -#define TSI148_LCSR_ITAT_MBLT (1<<8) /* MBLT Xfer Protocol */ -#define TSI148_LCSR_ITAT_BLT (1<<7) /* BLT Xfer Protocol */ - -#define TSI148_LCSR_ITAT_AS_M (7<<4) /* Address Space Mask */ -#define TSI148_LCSR_ITAT_AS_A16 (0<<4) /* A16 Address Space */ -#define TSI148_LCSR_ITAT_AS_A24 (1<<4) /* A24 Address Space */ -#define TSI148_LCSR_ITAT_AS_A32 (2<<4) /* A32 Address Space */ -#define TSI148_LCSR_ITAT_AS_A64 (4<<4) /* A64 Address Space */ - -#define TSI148_LCSR_ITAT_SUPR (1<<3) /* Supervisor Access */ -#define TSI148_LCSR_ITAT_NPRIV (1<<2) /* Non-Priv (User) Access */ -#define TSI148_LCSR_ITAT_PGM (1<<1) /* Program Access */ -#define TSI148_LCSR_ITAT_DATA (1<<0) /* Data Access */ - -/* - * GCSR Base Address Lower Address CRG +$404 - */ -#define TSI148_LCSR_GBAL_M (0x7FFFFFF<<5) /* Mask */ - -/* - * GCSR Attribute Register CRG + $408 - */ -#define TSI148_LCSR_GCSRAT_EN (1<<7) /* Enable access to GCSR */ - -#define TSI148_LCSR_GCSRAT_AS_M (7<<4) /* Address Space Mask */ -#define TSI148_LCSR_GCSRAT_AS_A16 (0<<4) /* Address Space 16 */ -#define TSI148_LCSR_GCSRAT_AS_A24 (1<<4) /* Address Space 24 */ -#define TSI148_LCSR_GCSRAT_AS_A32 (2<<4) /* Address Space 32 */ -#define TSI148_LCSR_GCSRAT_AS_A64 (4<<4) /* Address Space 64 */ - -#define TSI148_LCSR_GCSRAT_SUPR (1<<3) /* Sup set -GCSR decoder */ -#define TSI148_LCSR_GCSRAT_NPRIV (1<<2) /* Non-Privliged set - CGSR */ -#define TSI148_LCSR_GCSRAT_PGM (1<<1) /* Program set - GCSR decoder */ -#define TSI148_LCSR_GCSRAT_DATA (1<<0) /* DATA set GCSR decoder */ - -/* - * CRG Base Address Lower Address CRG + $410 - */ -#define TSI148_LCSR_CBAL_M (0xFFFFF<<12) - -/* - * CRG Attribute Register CRG + $414 - */ -#define TSI148_LCSR_CRGAT_EN (1<<7) /* Enable PRG Access */ - -#define TSI148_LCSR_CRGAT_AS_M (7<<4) /* Address Space */ -#define TSI148_LCSR_CRGAT_AS_A16 (0<<4) /* Address Space 16 */ -#define TSI148_LCSR_CRGAT_AS_A24 (1<<4) /* Address Space 24 */ -#define TSI148_LCSR_CRGAT_AS_A32 (2<<4) /* Address Space 32 */ -#define TSI148_LCSR_CRGAT_AS_A64 (4<<4) /* Address Space 64 */ - -#define TSI148_LCSR_CRGAT_SUPR (1<<3) /* Supervisor Access */ -#define TSI148_LCSR_CRGAT_NPRIV (1<<2) /* Non-Privliged(User) Access */ -#define TSI148_LCSR_CRGAT_PGM (1<<1) /* Program Access */ -#define TSI148_LCSR_CRGAT_DATA (1<<0) /* Data Access */ - -/* - * CR/CSR Offset Lower Register CRG + $41C - */ -#define TSI148_LCSR_CROL_M (0x1FFF<<19) /* Mask */ - -/* - * CR/CSR Attribute register CRG + $420 - */ -#define TSI148_LCSR_CRAT_EN (1<<7) /* Enable access to CR/CSR */ - -/* - * Location Monitor base address lower register CRG + $428 - */ -#define TSI148_LCSR_LMBAL_M (0x7FFFFFF<<5) /* Mask */ - -/* - * Location Monitor Attribute Register CRG + $42C - */ -#define TSI148_LCSR_LMAT_EN (1<<7) /* Enable Location Monitor */ - -#define TSI148_LCSR_LMAT_AS_M (7<<4) /* Address Space MASK */ -#define TSI148_LCSR_LMAT_AS_A16 (0<<4) /* A16 */ -#define TSI148_LCSR_LMAT_AS_A24 (1<<4) /* A24 */ -#define TSI148_LCSR_LMAT_AS_A32 (2<<4) /* A32 */ -#define TSI148_LCSR_LMAT_AS_A64 (4<<4) /* A64 */ - -#define TSI148_LCSR_LMAT_SUPR (1<<3) /* Supervisor Access */ -#define TSI148_LCSR_LMAT_NPRIV (1<<2) /* Non-Priv (User) Access */ -#define TSI148_LCSR_LMAT_PGM (1<<1) /* Program Access */ -#define TSI148_LCSR_LMAT_DATA (1<<0) /* Data Access */ - -/* - * Broadcast Pulse Generator Timer Register CRG + $438 - */ -#define TSI148_LCSR_BPGTR_BPGT_M (0xFFFF<<0) /* Mask */ - -/* - * Broadcast Programmable Clock Timer Register CRG + $43C - */ -#define TSI148_LCSR_BPCTR_BPCT_M (0xFFFFFF<<0) /* Mask */ - -/* - * VMEbus Interrupt Control Register CRG + $43C - */ -#define TSI148_LCSR_VICR_CNTS_M (3<<22) /* Cntr Source MASK */ -#define TSI148_LCSR_VICR_CNTS_DIS (1<<22) /* Cntr Disable */ -#define TSI148_LCSR_VICR_CNTS_IRQ1 (2<<22) /* IRQ1 to Cntr */ -#define TSI148_LCSR_VICR_CNTS_IRQ2 (3<<22) /* IRQ2 to Cntr */ - -#define TSI148_LCSR_VICR_EDGIS_M (3<<20) /* Edge interrupt MASK */ -#define TSI148_LCSR_VICR_EDGIS_DIS (1<<20) /* Edge interrupt Disable */ -#define TSI148_LCSR_VICR_EDGIS_IRQ1 (2<<20) /* IRQ1 to Edge */ -#define TSI148_LCSR_VICR_EDGIS_IRQ2 (3<<20) /* IRQ2 to Edge */ - -#define TSI148_LCSR_VICR_IRQIF_M (3<<18) /* IRQ1* Function MASK */ -#define TSI148_LCSR_VICR_IRQIF_NORM (1<<18) /* Normal */ -#define TSI148_LCSR_VICR_IRQIF_PULSE (2<<18) /* Pulse Generator */ -#define TSI148_LCSR_VICR_IRQIF_PROG (3<<18) /* Programmable Clock */ -#define TSI148_LCSR_VICR_IRQIF_1U (4<<18) /* 1us Clock */ - -#define TSI148_LCSR_VICR_IRQ2F_M (3<<16) /* IRQ2* Function MASK */ -#define TSI148_LCSR_VICR_IRQ2F_NORM (1<<16) /* Normal */ -#define TSI148_LCSR_VICR_IRQ2F_PULSE (2<<16) /* Pulse Generator */ -#define TSI148_LCSR_VICR_IRQ2F_PROG (3<<16) /* Programmable Clock */ -#define TSI148_LCSR_VICR_IRQ2F_1U (4<<16) /* 1us Clock */ - -#define TSI148_LCSR_VICR_BIP (1<<15) /* Broadcast Interrupt Pulse */ - -#define TSI148_LCSR_VICR_IRQC (1<<12) /* VMEbus IRQ Clear */ -#define TSI148_LCSR_VICR_IRQS (1<<11) /* VMEbus IRQ Status */ - -#define TSI148_LCSR_VICR_IRQL_M (7<<8) /* VMEbus SW IRQ Level Mask */ -#define TSI148_LCSR_VICR_IRQL_1 (1<<8) /* VMEbus SW IRQ Level 1 */ -#define TSI148_LCSR_VICR_IRQL_2 (2<<8) /* VMEbus SW IRQ Level 2 */ -#define TSI148_LCSR_VICR_IRQL_3 (3<<8) /* VMEbus SW IRQ Level 3 */ -#define TSI148_LCSR_VICR_IRQL_4 (4<<8) /* VMEbus SW IRQ Level 4 */ -#define TSI148_LCSR_VICR_IRQL_5 (5<<8) /* VMEbus SW IRQ Level 5 */ -#define TSI148_LCSR_VICR_IRQL_6 (6<<8) /* VMEbus SW IRQ Level 6 */ -#define TSI148_LCSR_VICR_IRQL_7 (7<<8) /* VMEbus SW IRQ Level 7 */ - -static const int TSI148_LCSR_VICR_IRQL[8] = { 0, TSI148_LCSR_VICR_IRQL_1, - TSI148_LCSR_VICR_IRQL_2, TSI148_LCSR_VICR_IRQL_3, - TSI148_LCSR_VICR_IRQL_4, TSI148_LCSR_VICR_IRQL_5, - TSI148_LCSR_VICR_IRQL_6, TSI148_LCSR_VICR_IRQL_7 }; - -#define TSI148_LCSR_VICR_STID_M (0xFF<<0) /* Status/ID Mask */ - -/* - * Interrupt Enable Register CRG + $440 - */ -#define TSI148_LCSR_INTEN_DMA1EN (1<<25) /* DMAC 1 */ -#define TSI148_LCSR_INTEN_DMA0EN (1<<24) /* DMAC 0 */ -#define TSI148_LCSR_INTEN_LM3EN (1<<23) /* Location Monitor 3 */ -#define TSI148_LCSR_INTEN_LM2EN (1<<22) /* Location Monitor 2 */ -#define TSI148_LCSR_INTEN_LM1EN (1<<21) /* Location Monitor 1 */ -#define TSI148_LCSR_INTEN_LM0EN (1<<20) /* Location Monitor 0 */ -#define TSI148_LCSR_INTEN_MB3EN (1<<19) /* Mail Box 3 */ -#define TSI148_LCSR_INTEN_MB2EN (1<<18) /* Mail Box 2 */ -#define TSI148_LCSR_INTEN_MB1EN (1<<17) /* Mail Box 1 */ -#define TSI148_LCSR_INTEN_MB0EN (1<<16) /* Mail Box 0 */ -#define TSI148_LCSR_INTEN_PERREN (1<<13) /* PCI/X Error */ -#define TSI148_LCSR_INTEN_VERREN (1<<12) /* VMEbus Error */ -#define TSI148_LCSR_INTEN_VIEEN (1<<11) /* VMEbus IRQ Edge */ -#define TSI148_LCSR_INTEN_IACKEN (1<<10) /* IACK */ -#define TSI148_LCSR_INTEN_SYSFLEN (1<<9) /* System Fail */ -#define TSI148_LCSR_INTEN_ACFLEN (1<<8) /* AC Fail */ -#define TSI148_LCSR_INTEN_IRQ7EN (1<<7) /* IRQ7 */ -#define TSI148_LCSR_INTEN_IRQ6EN (1<<6) /* IRQ6 */ -#define TSI148_LCSR_INTEN_IRQ5EN (1<<5) /* IRQ5 */ -#define TSI148_LCSR_INTEN_IRQ4EN (1<<4) /* IRQ4 */ -#define TSI148_LCSR_INTEN_IRQ3EN (1<<3) /* IRQ3 */ -#define TSI148_LCSR_INTEN_IRQ2EN (1<<2) /* IRQ2 */ -#define TSI148_LCSR_INTEN_IRQ1EN (1<<1) /* IRQ1 */ - -static const int TSI148_LCSR_INTEN_LMEN[4] = { TSI148_LCSR_INTEN_LM0EN, - TSI148_LCSR_INTEN_LM1EN, - TSI148_LCSR_INTEN_LM2EN, - TSI148_LCSR_INTEN_LM3EN }; - -static const int TSI148_LCSR_INTEN_IRQEN[7] = { TSI148_LCSR_INTEN_IRQ1EN, - TSI148_LCSR_INTEN_IRQ2EN, - TSI148_LCSR_INTEN_IRQ3EN, - TSI148_LCSR_INTEN_IRQ4EN, - TSI148_LCSR_INTEN_IRQ5EN, - TSI148_LCSR_INTEN_IRQ6EN, - TSI148_LCSR_INTEN_IRQ7EN }; - -/* - * Interrupt Enable Out Register CRG + $444 - */ -#define TSI148_LCSR_INTEO_DMA1EO (1<<25) /* DMAC 1 */ -#define TSI148_LCSR_INTEO_DMA0EO (1<<24) /* DMAC 0 */ -#define TSI148_LCSR_INTEO_LM3EO (1<<23) /* Loc Monitor 3 */ -#define TSI148_LCSR_INTEO_LM2EO (1<<22) /* Loc Monitor 2 */ -#define TSI148_LCSR_INTEO_LM1EO (1<<21) /* Loc Monitor 1 */ -#define TSI148_LCSR_INTEO_LM0EO (1<<20) /* Location Monitor 0 */ -#define TSI148_LCSR_INTEO_MB3EO (1<<19) /* Mail Box 3 */ -#define TSI148_LCSR_INTEO_MB2EO (1<<18) /* Mail Box 2 */ -#define TSI148_LCSR_INTEO_MB1EO (1<<17) /* Mail Box 1 */ -#define TSI148_LCSR_INTEO_MB0EO (1<<16) /* Mail Box 0 */ -#define TSI148_LCSR_INTEO_PERREO (1<<13) /* PCI/X Error */ -#define TSI148_LCSR_INTEO_VERREO (1<<12) /* VMEbus Error */ -#define TSI148_LCSR_INTEO_VIEEO (1<<11) /* VMEbus IRQ Edge */ -#define TSI148_LCSR_INTEO_IACKEO (1<<10) /* IACK */ -#define TSI148_LCSR_INTEO_SYSFLEO (1<<9) /* System Fail */ -#define TSI148_LCSR_INTEO_ACFLEO (1<<8) /* AC Fail */ -#define TSI148_LCSR_INTEO_IRQ7EO (1<<7) /* IRQ7 */ -#define TSI148_LCSR_INTEO_IRQ6EO (1<<6) /* IRQ6 */ -#define TSI148_LCSR_INTEO_IRQ5EO (1<<5) /* IRQ5 */ -#define TSI148_LCSR_INTEO_IRQ4EO (1<<4) /* IRQ4 */ -#define TSI148_LCSR_INTEO_IRQ3EO (1<<3) /* IRQ3 */ -#define TSI148_LCSR_INTEO_IRQ2EO (1<<2) /* IRQ2 */ -#define TSI148_LCSR_INTEO_IRQ1EO (1<<1) /* IRQ1 */ - -static const int TSI148_LCSR_INTEO_LMEO[4] = { TSI148_LCSR_INTEO_LM0EO, - TSI148_LCSR_INTEO_LM1EO, - TSI148_LCSR_INTEO_LM2EO, - TSI148_LCSR_INTEO_LM3EO }; - -static const int TSI148_LCSR_INTEO_IRQEO[7] = { TSI148_LCSR_INTEO_IRQ1EO, - TSI148_LCSR_INTEO_IRQ2EO, - TSI148_LCSR_INTEO_IRQ3EO, - TSI148_LCSR_INTEO_IRQ4EO, - TSI148_LCSR_INTEO_IRQ5EO, - TSI148_LCSR_INTEO_IRQ6EO, - TSI148_LCSR_INTEO_IRQ7EO }; - -/* - * Interrupt Status Register CRG + $448 - */ -#define TSI148_LCSR_INTS_DMA1S (1<<25) /* DMA 1 */ -#define TSI148_LCSR_INTS_DMA0S (1<<24) /* DMA 0 */ -#define TSI148_LCSR_INTS_LM3S (1<<23) /* Location Monitor 3 */ -#define TSI148_LCSR_INTS_LM2S (1<<22) /* Location Monitor 2 */ -#define TSI148_LCSR_INTS_LM1S (1<<21) /* Location Monitor 1 */ -#define TSI148_LCSR_INTS_LM0S (1<<20) /* Location Monitor 0 */ -#define TSI148_LCSR_INTS_MB3S (1<<19) /* Mail Box 3 */ -#define TSI148_LCSR_INTS_MB2S (1<<18) /* Mail Box 2 */ -#define TSI148_LCSR_INTS_MB1S (1<<17) /* Mail Box 1 */ -#define TSI148_LCSR_INTS_MB0S (1<<16) /* Mail Box 0 */ -#define TSI148_LCSR_INTS_PERRS (1<<13) /* PCI/X Error */ -#define TSI148_LCSR_INTS_VERRS (1<<12) /* VMEbus Error */ -#define TSI148_LCSR_INTS_VIES (1<<11) /* VMEbus IRQ Edge */ -#define TSI148_LCSR_INTS_IACKS (1<<10) /* IACK */ -#define TSI148_LCSR_INTS_SYSFLS (1<<9) /* System Fail */ -#define TSI148_LCSR_INTS_ACFLS (1<<8) /* AC Fail */ -#define TSI148_LCSR_INTS_IRQ7S (1<<7) /* IRQ7 */ -#define TSI148_LCSR_INTS_IRQ6S (1<<6) /* IRQ6 */ -#define TSI148_LCSR_INTS_IRQ5S (1<<5) /* IRQ5 */ -#define TSI148_LCSR_INTS_IRQ4S (1<<4) /* IRQ4 */ -#define TSI148_LCSR_INTS_IRQ3S (1<<3) /* IRQ3 */ -#define TSI148_LCSR_INTS_IRQ2S (1<<2) /* IRQ2 */ -#define TSI148_LCSR_INTS_IRQ1S (1<<1) /* IRQ1 */ - -static const int TSI148_LCSR_INTS_LMS[4] = { TSI148_LCSR_INTS_LM0S, - TSI148_LCSR_INTS_LM1S, - TSI148_LCSR_INTS_LM2S, - TSI148_LCSR_INTS_LM3S }; - -static const int TSI148_LCSR_INTS_MBS[4] = { TSI148_LCSR_INTS_MB0S, - TSI148_LCSR_INTS_MB1S, - TSI148_LCSR_INTS_MB2S, - TSI148_LCSR_INTS_MB3S }; - -/* - * Interrupt Clear Register CRG + $44C - */ -#define TSI148_LCSR_INTC_DMA1C (1<<25) /* DMA 1 */ -#define TSI148_LCSR_INTC_DMA0C (1<<24) /* DMA 0 */ -#define TSI148_LCSR_INTC_LM3C (1<<23) /* Location Monitor 3 */ -#define TSI148_LCSR_INTC_LM2C (1<<22) /* Location Monitor 2 */ -#define TSI148_LCSR_INTC_LM1C (1<<21) /* Location Monitor 1 */ -#define TSI148_LCSR_INTC_LM0C (1<<20) /* Location Monitor 0 */ -#define TSI148_LCSR_INTC_MB3C (1<<19) /* Mail Box 3 */ -#define TSI148_LCSR_INTC_MB2C (1<<18) /* Mail Box 2 */ -#define TSI148_LCSR_INTC_MB1C (1<<17) /* Mail Box 1 */ -#define TSI148_LCSR_INTC_MB0C (1<<16) /* Mail Box 0 */ -#define TSI148_LCSR_INTC_PERRC (1<<13) /* VMEbus Error */ -#define TSI148_LCSR_INTC_VERRC (1<<12) /* VMEbus Access Time-out */ -#define TSI148_LCSR_INTC_VIEC (1<<11) /* VMEbus IRQ Edge */ -#define TSI148_LCSR_INTC_IACKC (1<<10) /* IACK */ -#define TSI148_LCSR_INTC_SYSFLC (1<<9) /* System Fail */ -#define TSI148_LCSR_INTC_ACFLC (1<<8) /* AC Fail */ - -static const int TSI148_LCSR_INTC_LMC[4] = { TSI148_LCSR_INTC_LM0C, - TSI148_LCSR_INTC_LM1C, - TSI148_LCSR_INTC_LM2C, - TSI148_LCSR_INTC_LM3C }; - -static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C, - TSI148_LCSR_INTC_MB1C, - TSI148_LCSR_INTC_MB2C, - TSI148_LCSR_INTC_MB3C }; - -/* - * Interrupt Map Register 1 CRG + $458 - */ -#define TSI148_LCSR_INTM1_DMA1M_M (3<<18) /* DMA 1 */ -#define TSI148_LCSR_INTM1_DMA0M_M (3<<16) /* DMA 0 */ -#define TSI148_LCSR_INTM1_LM3M_M (3<<14) /* Location Monitor 3 */ -#define TSI148_LCSR_INTM1_LM2M_M (3<<12) /* Location Monitor 2 */ -#define TSI148_LCSR_INTM1_LM1M_M (3<<10) /* Location Monitor 1 */ -#define TSI148_LCSR_INTM1_LM0M_M (3<<8) /* Location Monitor 0 */ -#define TSI148_LCSR_INTM1_MB3M_M (3<<6) /* Mail Box 3 */ -#define TSI148_LCSR_INTM1_MB2M_M (3<<4) /* Mail Box 2 */ -#define TSI148_LCSR_INTM1_MB1M_M (3<<2) /* Mail Box 1 */ -#define TSI148_LCSR_INTM1_MB0M_M (3<<0) /* Mail Box 0 */ - -/* - * Interrupt Map Register 2 CRG + $45C - */ -#define TSI148_LCSR_INTM2_PERRM_M (3<<26) /* PCI Bus Error */ -#define TSI148_LCSR_INTM2_VERRM_M (3<<24) /* VMEbus Error */ -#define TSI148_LCSR_INTM2_VIEM_M (3<<22) /* VMEbus IRQ Edge */ -#define TSI148_LCSR_INTM2_IACKM_M (3<<20) /* IACK */ -#define TSI148_LCSR_INTM2_SYSFLM_M (3<<18) /* System Fail */ -#define TSI148_LCSR_INTM2_ACFLM_M (3<<16) /* AC Fail */ -#define TSI148_LCSR_INTM2_IRQ7M_M (3<<14) /* IRQ7 */ -#define TSI148_LCSR_INTM2_IRQ6M_M (3<<12) /* IRQ6 */ -#define TSI148_LCSR_INTM2_IRQ5M_M (3<<10) /* IRQ5 */ -#define TSI148_LCSR_INTM2_IRQ4M_M (3<<8) /* IRQ4 */ -#define TSI148_LCSR_INTM2_IRQ3M_M (3<<6) /* IRQ3 */ -#define TSI148_LCSR_INTM2_IRQ2M_M (3<<4) /* IRQ2 */ -#define TSI148_LCSR_INTM2_IRQ1M_M (3<<2) /* IRQ1 */ - -/* - * DMA Control (0-1) Registers CRG + $500 - */ -#define TSI148_LCSR_DCTL_ABT (1<<27) /* Abort */ -#define TSI148_LCSR_DCTL_PAU (1<<26) /* Pause */ -#define TSI148_LCSR_DCTL_DGO (1<<25) /* DMA Go */ - -#define TSI148_LCSR_DCTL_MOD (1<<23) /* Mode */ - -#define TSI148_LCSR_DCTL_VBKS_M (7<<12) /* VMEbus block Size MASK */ -#define TSI148_LCSR_DCTL_VBKS_32 (0<<12) /* VMEbus block Size 32 */ -#define TSI148_LCSR_DCTL_VBKS_64 (1<<12) /* VMEbus block Size 64 */ -#define TSI148_LCSR_DCTL_VBKS_128 (2<<12) /* VMEbus block Size 128 */ -#define TSI148_LCSR_DCTL_VBKS_256 (3<<12) /* VMEbus block Size 256 */ -#define TSI148_LCSR_DCTL_VBKS_512 (4<<12) /* VMEbus block Size 512 */ -#define TSI148_LCSR_DCTL_VBKS_1024 (5<<12) /* VMEbus block Size 1024 */ -#define TSI148_LCSR_DCTL_VBKS_2048 (6<<12) /* VMEbus block Size 2048 */ -#define TSI148_LCSR_DCTL_VBKS_4096 (7<<12) /* VMEbus block Size 4096 */ - -#define TSI148_LCSR_DCTL_VBOT_M (7<<8) /* VMEbus back-off MASK */ -#define TSI148_LCSR_DCTL_VBOT_0 (0<<8) /* VMEbus back-off 0us */ -#define TSI148_LCSR_DCTL_VBOT_1 (1<<8) /* VMEbus back-off 1us */ -#define TSI148_LCSR_DCTL_VBOT_2 (2<<8) /* VMEbus back-off 2us */ -#define TSI148_LCSR_DCTL_VBOT_4 (3<<8) /* VMEbus back-off 4us */ -#define TSI148_LCSR_DCTL_VBOT_8 (4<<8) /* VMEbus back-off 8us */ -#define TSI148_LCSR_DCTL_VBOT_16 (5<<8) /* VMEbus back-off 16us */ -#define TSI148_LCSR_DCTL_VBOT_32 (6<<8) /* VMEbus back-off 32us */ -#define TSI148_LCSR_DCTL_VBOT_64 (7<<8) /* VMEbus back-off 64us */ - -#define TSI148_LCSR_DCTL_PBKS_M (7<<4) /* PCI block size MASK */ -#define TSI148_LCSR_DCTL_PBKS_32 (0<<4) /* PCI block size 32 bytes */ -#define TSI148_LCSR_DCTL_PBKS_64 (1<<4) /* PCI block size 64 bytes */ -#define TSI148_LCSR_DCTL_PBKS_128 (2<<4) /* PCI block size 128 bytes */ -#define TSI148_LCSR_DCTL_PBKS_256 (3<<4) /* PCI block size 256 bytes */ -#define TSI148_LCSR_DCTL_PBKS_512 (4<<4) /* PCI block size 512 bytes */ -#define TSI148_LCSR_DCTL_PBKS_1024 (5<<4) /* PCI block size 1024 bytes */ -#define TSI148_LCSR_DCTL_PBKS_2048 (6<<4) /* PCI block size 2048 bytes */ -#define TSI148_LCSR_DCTL_PBKS_4096 (7<<4) /* PCI block size 4096 bytes */ - -#define TSI148_LCSR_DCTL_PBOT_M (7<<0) /* PCI back off MASK */ -#define TSI148_LCSR_DCTL_PBOT_0 (0<<0) /* PCI back off 0us */ -#define TSI148_LCSR_DCTL_PBOT_1 (1<<0) /* PCI back off 1us */ -#define TSI148_LCSR_DCTL_PBOT_2 (2<<0) /* PCI back off 2us */ -#define TSI148_LCSR_DCTL_PBOT_4 (3<<0) /* PCI back off 3us */ -#define TSI148_LCSR_DCTL_PBOT_8 (4<<0) /* PCI back off 4us */ -#define TSI148_LCSR_DCTL_PBOT_16 (5<<0) /* PCI back off 8us */ -#define TSI148_LCSR_DCTL_PBOT_32 (6<<0) /* PCI back off 16us */ -#define TSI148_LCSR_DCTL_PBOT_64 (7<<0) /* PCI back off 32us */ - -/* - * DMA Status Registers (0-1) CRG + $504 - */ -#define TSI148_LCSR_DSTA_SMA (1<<31) /* PCI Signalled Master Abt */ -#define TSI148_LCSR_DSTA_RTA (1<<30) /* PCI Received Target Abt */ -#define TSI148_LCSR_DSTA_MRC (1<<29) /* PCI Max Retry Count */ -#define TSI148_LCSR_DSTA_VBE (1<<28) /* VMEbus error */ -#define TSI148_LCSR_DSTA_ABT (1<<27) /* Abort */ -#define TSI148_LCSR_DSTA_PAU (1<<26) /* Pause */ -#define TSI148_LCSR_DSTA_DON (1<<25) /* Done */ -#define TSI148_LCSR_DSTA_BSY (1<<24) /* Busy */ - -/* - * DMA Current Link Address Lower (0-1) - */ -#define TSI148_LCSR_DCLAL_M (0x3FFFFFF<<6) /* Mask */ - -/* - * DMA Source Attribute (0-1) Reg - */ -#define TSI148_LCSR_DSAT_TYP_M (3<<28) /* Source Bus Type */ -#define TSI148_LCSR_DSAT_TYP_PCI (0<<28) /* PCI Bus */ -#define TSI148_LCSR_DSAT_TYP_VME (1<<28) /* VMEbus */ -#define TSI148_LCSR_DSAT_TYP_PAT (2<<28) /* Data Pattern */ - -#define TSI148_LCSR_DSAT_PSZ (1<<25) /* Pattern Size */ -#define TSI148_LCSR_DSAT_NIN (1<<24) /* No Increment */ - -#define TSI148_LCSR_DSAT_2eSSTM_M (3<<11) /* 2eSST Trans Rate Mask */ -#define TSI148_LCSR_DSAT_2eSSTM_160 (0<<11) /* 160 MB/s */ -#define TSI148_LCSR_DSAT_2eSSTM_267 (1<<11) /* 267 MB/s */ -#define TSI148_LCSR_DSAT_2eSSTM_320 (2<<11) /* 320 MB/s */ - -#define TSI148_LCSR_DSAT_TM_M (7<<8) /* Bus Transfer Protocol Mask */ -#define TSI148_LCSR_DSAT_TM_SCT (0<<8) /* SCT */ -#define TSI148_LCSR_DSAT_TM_BLT (1<<8) /* BLT */ -#define TSI148_LCSR_DSAT_TM_MBLT (2<<8) /* MBLT */ -#define TSI148_LCSR_DSAT_TM_2eVME (3<<8) /* 2eVME */ -#define TSI148_LCSR_DSAT_TM_2eSST (4<<8) /* 2eSST */ -#define TSI148_LCSR_DSAT_TM_2eSSTB (5<<8) /* 2eSST Broadcast */ - -#define TSI148_LCSR_DSAT_DBW_M (3<<6) /* Max Data Width MASK */ -#define TSI148_LCSR_DSAT_DBW_16 (0<<6) /* 16 Bits */ -#define TSI148_LCSR_DSAT_DBW_32 (1<<6) /* 32 Bits */ - -#define TSI148_LCSR_DSAT_SUP (1<<5) /* Supervisory Mode */ -#define TSI148_LCSR_DSAT_PGM (1<<4) /* Program Mode */ - -#define TSI148_LCSR_DSAT_AMODE_M (0xf<<0) /* Address Space Mask */ -#define TSI148_LCSR_DSAT_AMODE_A16 (0<<0) /* A16 */ -#define TSI148_LCSR_DSAT_AMODE_A24 (1<<0) /* A24 */ -#define TSI148_LCSR_DSAT_AMODE_A32 (2<<0) /* A32 */ -#define TSI148_LCSR_DSAT_AMODE_A64 (4<<0) /* A64 */ -#define TSI148_LCSR_DSAT_AMODE_CRCSR (5<<0) /* CR/CSR */ -#define TSI148_LCSR_DSAT_AMODE_USER1 (8<<0) /* User1 */ -#define TSI148_LCSR_DSAT_AMODE_USER2 (9<<0) /* User2 */ -#define TSI148_LCSR_DSAT_AMODE_USER3 (0xa<<0) /* User3 */ -#define TSI148_LCSR_DSAT_AMODE_USER4 (0xb<<0) /* User4 */ - -/* - * DMA Destination Attribute Registers (0-1) - */ -#define TSI148_LCSR_DDAT_TYP_PCI (0<<28) /* Destination PCI Bus */ -#define TSI148_LCSR_DDAT_TYP_VME (1<<28) /* Destination VMEbus */ - -#define TSI148_LCSR_DDAT_2eSSTM_M (3<<11) /* 2eSST Transfer Rate Mask */ -#define TSI148_LCSR_DDAT_2eSSTM_160 (0<<11) /* 160 MB/s */ -#define TSI148_LCSR_DDAT_2eSSTM_267 (1<<11) /* 267 MB/s */ -#define TSI148_LCSR_DDAT_2eSSTM_320 (2<<11) /* 320 MB/s */ - -#define TSI148_LCSR_DDAT_TM_M (7<<8) /* Bus Transfer Protocol Mask */ -#define TSI148_LCSR_DDAT_TM_SCT (0<<8) /* SCT */ -#define TSI148_LCSR_DDAT_TM_BLT (1<<8) /* BLT */ -#define TSI148_LCSR_DDAT_TM_MBLT (2<<8) /* MBLT */ -#define TSI148_LCSR_DDAT_TM_2eVME (3<<8) /* 2eVME */ -#define TSI148_LCSR_DDAT_TM_2eSST (4<<8) /* 2eSST */ -#define TSI148_LCSR_DDAT_TM_2eSSTB (5<<8) /* 2eSST Broadcast */ - -#define TSI148_LCSR_DDAT_DBW_M (3<<6) /* Max Data Width MASK */ -#define TSI148_LCSR_DDAT_DBW_16 (0<<6) /* 16 Bits */ -#define TSI148_LCSR_DDAT_DBW_32 (1<<6) /* 32 Bits */ - -#define TSI148_LCSR_DDAT_SUP (1<<5) /* Supervisory/User Access */ -#define TSI148_LCSR_DDAT_PGM (1<<4) /* Program/Data Access */ - -#define TSI148_LCSR_DDAT_AMODE_M (0xf<<0) /* Address Space Mask */ -#define TSI148_LCSR_DDAT_AMODE_A16 (0<<0) /* A16 */ -#define TSI148_LCSR_DDAT_AMODE_A24 (1<<0) /* A24 */ -#define TSI148_LCSR_DDAT_AMODE_A32 (2<<0) /* A32 */ -#define TSI148_LCSR_DDAT_AMODE_A64 (4<<0) /* A64 */ -#define TSI148_LCSR_DDAT_AMODE_CRCSR (5<<0) /* CRC/SR */ -#define TSI148_LCSR_DDAT_AMODE_USER1 (8<<0) /* User1 */ -#define TSI148_LCSR_DDAT_AMODE_USER2 (9<<0) /* User2 */ -#define TSI148_LCSR_DDAT_AMODE_USER3 (0xa<<0) /* User3 */ -#define TSI148_LCSR_DDAT_AMODE_USER4 (0xb<<0) /* User4 */ - -/* - * DMA Next Link Address Lower - */ -#define TSI148_LCSR_DNLAL_DNLAL_M (0x3FFFFFF<<6) /* Address Mask */ -#define TSI148_LCSR_DNLAL_LLA (1<<0) /* Last Link Address Indicator */ - -/* - * DMA 2eSST Broadcast Select - */ -#define TSI148_LCSR_DBS_M (0x1FFFFF<<0) /* Mask */ - -/* - * GCSR Register Group - */ - -/* - * GCSR Control and Status Register CRG + $604 - */ -#define TSI148_GCSR_GCTRL_LRST (1<<15) /* Local Reset */ -#define TSI148_GCSR_GCTRL_SFAILEN (1<<14) /* System Fail enable */ -#define TSI148_GCSR_GCTRL_BDFAILS (1<<13) /* Board Fail Status */ -#define TSI148_GCSR_GCTRL_SCON (1<<12) /* System Copntroller */ -#define TSI148_GCSR_GCTRL_MEN (1<<11) /* Module Enable (READY) */ - -#define TSI148_GCSR_GCTRL_LMI3S (1<<7) /* Loc Monitor 3 Int Status */ -#define TSI148_GCSR_GCTRL_LMI2S (1<<6) /* Loc Monitor 2 Int Status */ -#define TSI148_GCSR_GCTRL_LMI1S (1<<5) /* Loc Monitor 1 Int Status */ -#define TSI148_GCSR_GCTRL_LMI0S (1<<4) /* Loc Monitor 0 Int Status */ -#define TSI148_GCSR_GCTRL_MBI3S (1<<3) /* Mail box 3 Int Status */ -#define TSI148_GCSR_GCTRL_MBI2S (1<<2) /* Mail box 2 Int Status */ -#define TSI148_GCSR_GCTRL_MBI1S (1<<1) /* Mail box 1 Int Status */ -#define TSI148_GCSR_GCTRL_MBI0S (1<<0) /* Mail box 0 Int Status */ - -#define TSI148_GCSR_GAP (1<<5) /* Geographic Addr Parity */ -#define TSI148_GCSR_GA_M (0x1F<<0) /* Geographic Address Mask */ - -/* - * CR/CSR Register Group - */ - -/* - * CR/CSR Bit Clear Register CRG + $FF4 - */ -#define TSI148_CRCSR_CSRBCR_LRSTC (1<<7) /* Local Reset Clear */ -#define TSI148_CRCSR_CSRBCR_SFAILC (1<<6) /* System Fail Enable Clear */ -#define TSI148_CRCSR_CSRBCR_BDFAILS (1<<5) /* Board Fail Status */ -#define TSI148_CRCSR_CSRBCR_MENC (1<<4) /* Module Enable Clear */ -#define TSI148_CRCSR_CSRBCR_BERRSC (1<<3) /* Bus Error Status Clear */ - -/* - * CR/CSR Bit Set Register CRG+$FF8 - */ -#define TSI148_CRCSR_CSRBSR_LISTS (1<<7) /* Local Reset Clear */ -#define TSI148_CRCSR_CSRBSR_SFAILS (1<<6) /* System Fail Enable Clear */ -#define TSI148_CRCSR_CSRBSR_BDFAILS (1<<5) /* Board Fail Status */ -#define TSI148_CRCSR_CSRBSR_MENS (1<<4) /* Module Enable Clear */ -#define TSI148_CRCSR_CSRBSR_BERRS (1<<3) /* Bus Error Status Clear */ - -/* - * CR/CSR Base Address Register CRG + FFC - */ -#define TSI148_CRCSR_CBAR_M (0x1F<<3) /* Mask */ - -#endif /* TSI148_H */ diff --git a/drivers/vme/vme.c b/drivers/vme/vme.c deleted file mode 100644 index 8dba20186be3..000000000000 --- a/drivers/vme/vme.c +++ /dev/null @@ -1,2015 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * VME Bridge Framework - * - * Author: Martyn Welch - * Copyright 2008 GE Intelligent Platforms Embedded Systems, Inc. - * - * Based on work by Tom Armistead and Ajit Prem - * Copyright 2004 Motorola Inc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "vme_bridge.h" - -/* Bitmask and list of registered buses both protected by common mutex */ -static unsigned int vme_bus_numbers; -static LIST_HEAD(vme_bus_list); -static DEFINE_MUTEX(vme_buses_lock); - -static int __init vme_init(void); - -static struct vme_dev *dev_to_vme_dev(struct device *dev) -{ - return container_of(dev, struct vme_dev, dev); -} - -/* - * Find the bridge that the resource is associated with. - */ -static struct vme_bridge *find_bridge(struct vme_resource *resource) -{ - /* Get list to search */ - switch (resource->type) { - case VME_MASTER: - return list_entry(resource->entry, struct vme_master_resource, - list)->parent; - case VME_SLAVE: - return list_entry(resource->entry, struct vme_slave_resource, - list)->parent; - case VME_DMA: - return list_entry(resource->entry, struct vme_dma_resource, - list)->parent; - case VME_LM: - return list_entry(resource->entry, struct vme_lm_resource, - list)->parent; - default: - printk(KERN_ERR "Unknown resource type\n"); - return NULL; - } -} - -/** - * vme_alloc_consistent - Allocate contiguous memory. - * @resource: Pointer to VME resource. - * @size: Size of allocation required. - * @dma: Pointer to variable to store physical address of allocation. - * - * Allocate a contiguous block of memory for use by the driver. This is used to - * create the buffers for the slave windows. - * - * Return: Virtual address of allocation on success, NULL on failure. - */ -void *vme_alloc_consistent(struct vme_resource *resource, size_t size, - dma_addr_t *dma) -{ - struct vme_bridge *bridge; - - if (!resource) { - printk(KERN_ERR "No resource\n"); - return NULL; - } - - bridge = find_bridge(resource); - if (!bridge) { - printk(KERN_ERR "Can't find bridge\n"); - return NULL; - } - - if (!bridge->parent) { - printk(KERN_ERR "Dev entry NULL for bridge %s\n", bridge->name); - return NULL; - } - - if (!bridge->alloc_consistent) { - printk(KERN_ERR "alloc_consistent not supported by bridge %s\n", - bridge->name); - return NULL; - } - - return bridge->alloc_consistent(bridge->parent, size, dma); -} -EXPORT_SYMBOL(vme_alloc_consistent); - -/** - * vme_free_consistent - Free previously allocated memory. - * @resource: Pointer to VME resource. - * @size: Size of allocation to free. - * @vaddr: Virtual address of allocation. - * @dma: Physical address of allocation. - * - * Free previously allocated block of contiguous memory. - */ -void vme_free_consistent(struct vme_resource *resource, size_t size, - void *vaddr, dma_addr_t dma) -{ - struct vme_bridge *bridge; - - if (!resource) { - printk(KERN_ERR "No resource\n"); - return; - } - - bridge = find_bridge(resource); - if (!bridge) { - printk(KERN_ERR "Can't find bridge\n"); - return; - } - - if (!bridge->parent) { - printk(KERN_ERR "Dev entry NULL for bridge %s\n", bridge->name); - return; - } - - if (!bridge->free_consistent) { - printk(KERN_ERR "free_consistent not supported by bridge %s\n", - bridge->name); - return; - } - - bridge->free_consistent(bridge->parent, size, vaddr, dma); -} -EXPORT_SYMBOL(vme_free_consistent); - -/** - * vme_get_size - Helper function returning size of a VME window - * @resource: Pointer to VME slave or master resource. - * - * Determine the size of the VME window provided. This is a helper - * function, wrappering the call to vme_master_get or vme_slave_get - * depending on the type of window resource handed to it. - * - * Return: Size of the window on success, zero on failure. - */ -size_t vme_get_size(struct vme_resource *resource) -{ - int enabled, retval; - unsigned long long base, size; - dma_addr_t buf_base; - u32 aspace, cycle, dwidth; - - switch (resource->type) { - case VME_MASTER: - retval = vme_master_get(resource, &enabled, &base, &size, - &aspace, &cycle, &dwidth); - if (retval) - return 0; - - return size; - case VME_SLAVE: - retval = vme_slave_get(resource, &enabled, &base, &size, - &buf_base, &aspace, &cycle); - if (retval) - return 0; - - return size; - case VME_DMA: - return 0; - default: - printk(KERN_ERR "Unknown resource type\n"); - return 0; - } -} -EXPORT_SYMBOL(vme_get_size); - -int vme_check_window(u32 aspace, unsigned long long vme_base, - unsigned long long size) -{ - int retval = 0; - - if (vme_base + size < size) - return -EINVAL; - - switch (aspace) { - case VME_A16: - if (vme_base + size > VME_A16_MAX) - retval = -EFAULT; - break; - case VME_A24: - if (vme_base + size > VME_A24_MAX) - retval = -EFAULT; - break; - case VME_A32: - if (vme_base + size > VME_A32_MAX) - retval = -EFAULT; - break; - case VME_A64: - /* The VME_A64_MAX limit is actually U64_MAX + 1 */ - break; - case VME_CRCSR: - if (vme_base + size > VME_CRCSR_MAX) - retval = -EFAULT; - break; - case VME_USER1: - case VME_USER2: - case VME_USER3: - case VME_USER4: - /* User Defined */ - break; - default: - printk(KERN_ERR "Invalid address space\n"); - retval = -EINVAL; - break; - } - - return retval; -} -EXPORT_SYMBOL(vme_check_window); - -static u32 vme_get_aspace(int am) -{ - switch (am) { - case 0x29: - case 0x2D: - return VME_A16; - case 0x38: - case 0x39: - case 0x3A: - case 0x3B: - case 0x3C: - case 0x3D: - case 0x3E: - case 0x3F: - return VME_A24; - case 0x8: - case 0x9: - case 0xA: - case 0xB: - case 0xC: - case 0xD: - case 0xE: - case 0xF: - return VME_A32; - case 0x0: - case 0x1: - case 0x3: - return VME_A64; - } - - return 0; -} - -/** - * vme_slave_request - Request a VME slave window resource. - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * @address: Required VME address space. - * @cycle: Required VME data transfer cycle type. - * - * Request use of a VME window resource capable of being set for the requested - * address space and data transfer cycle. - * - * Return: Pointer to VME resource on success, NULL on failure. - */ -struct vme_resource *vme_slave_request(struct vme_dev *vdev, u32 address, - u32 cycle) -{ - struct vme_bridge *bridge; - struct list_head *slave_pos = NULL; - struct vme_slave_resource *allocated_image = NULL; - struct vme_slave_resource *slave_image = NULL; - struct vme_resource *resource = NULL; - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - goto err_bus; - } - - /* Loop through slave resources */ - list_for_each(slave_pos, &bridge->slave_resources) { - slave_image = list_entry(slave_pos, - struct vme_slave_resource, list); - - if (!slave_image) { - printk(KERN_ERR "Registered NULL Slave resource\n"); - continue; - } - - /* Find an unlocked and compatible image */ - mutex_lock(&slave_image->mtx); - if (((slave_image->address_attr & address) == address) && - ((slave_image->cycle_attr & cycle) == cycle) && - (slave_image->locked == 0)) { - - slave_image->locked = 1; - mutex_unlock(&slave_image->mtx); - allocated_image = slave_image; - break; - } - mutex_unlock(&slave_image->mtx); - } - - /* No free image */ - if (!allocated_image) - goto err_image; - - resource = kmalloc(sizeof(*resource), GFP_KERNEL); - if (!resource) - goto err_alloc; - - resource->type = VME_SLAVE; - resource->entry = &allocated_image->list; - - return resource; - -err_alloc: - /* Unlock image */ - mutex_lock(&slave_image->mtx); - slave_image->locked = 0; - mutex_unlock(&slave_image->mtx); -err_image: -err_bus: - return NULL; -} -EXPORT_SYMBOL(vme_slave_request); - -/** - * vme_slave_set - Set VME slave window configuration. - * @resource: Pointer to VME slave resource. - * @enabled: State to which the window should be configured. - * @vme_base: Base address for the window. - * @size: Size of the VME window. - * @buf_base: Based address of buffer used to provide VME slave window storage. - * @aspace: VME address space for the VME window. - * @cycle: VME data transfer cycle type for the VME window. - * - * Set configuration for provided VME slave window. - * - * Return: Zero on success, -EINVAL if operation is not supported on this - * device, if an invalid resource has been provided or invalid - * attributes are provided. Hardware specific errors may also be - * returned. - */ -int vme_slave_set(struct vme_resource *resource, int enabled, - unsigned long long vme_base, unsigned long long size, - dma_addr_t buf_base, u32 aspace, u32 cycle) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_slave_resource *image; - int retval; - - if (resource->type != VME_SLAVE) { - printk(KERN_ERR "Not a slave resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_slave_resource, list); - - if (!bridge->slave_set) { - printk(KERN_ERR "Function not supported\n"); - return -ENOSYS; - } - - if (!(((image->address_attr & aspace) == aspace) && - ((image->cycle_attr & cycle) == cycle))) { - printk(KERN_ERR "Invalid attributes\n"); - return -EINVAL; - } - - retval = vme_check_window(aspace, vme_base, size); - if (retval) - return retval; - - return bridge->slave_set(image, enabled, vme_base, size, buf_base, - aspace, cycle); -} -EXPORT_SYMBOL(vme_slave_set); - -/** - * vme_slave_get - Retrieve VME slave window configuration. - * @resource: Pointer to VME slave resource. - * @enabled: Pointer to variable for storing state. - * @vme_base: Pointer to variable for storing window base address. - * @size: Pointer to variable for storing window size. - * @buf_base: Pointer to variable for storing slave buffer base address. - * @aspace: Pointer to variable for storing VME address space. - * @cycle: Pointer to variable for storing VME data transfer cycle type. - * - * Return configuration for provided VME slave window. - * - * Return: Zero on success, -EINVAL if operation is not supported on this - * device or if an invalid resource has been provided. - */ -int vme_slave_get(struct vme_resource *resource, int *enabled, - unsigned long long *vme_base, unsigned long long *size, - dma_addr_t *buf_base, u32 *aspace, u32 *cycle) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_slave_resource *image; - - if (resource->type != VME_SLAVE) { - printk(KERN_ERR "Not a slave resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_slave_resource, list); - - if (!bridge->slave_get) { - printk(KERN_ERR "vme_slave_get not supported\n"); - return -EINVAL; - } - - return bridge->slave_get(image, enabled, vme_base, size, buf_base, - aspace, cycle); -} -EXPORT_SYMBOL(vme_slave_get); - -/** - * vme_slave_free - Free VME slave window - * @resource: Pointer to VME slave resource. - * - * Free the provided slave resource so that it may be reallocated. - */ -void vme_slave_free(struct vme_resource *resource) -{ - struct vme_slave_resource *slave_image; - - if (resource->type != VME_SLAVE) { - printk(KERN_ERR "Not a slave resource\n"); - return; - } - - slave_image = list_entry(resource->entry, struct vme_slave_resource, - list); - if (!slave_image) { - printk(KERN_ERR "Can't find slave resource\n"); - return; - } - - /* Unlock image */ - mutex_lock(&slave_image->mtx); - if (slave_image->locked == 0) - printk(KERN_ERR "Image is already free\n"); - - slave_image->locked = 0; - mutex_unlock(&slave_image->mtx); - - /* Free up resource memory */ - kfree(resource); -} -EXPORT_SYMBOL(vme_slave_free); - -/** - * vme_master_request - Request a VME master window resource. - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * @address: Required VME address space. - * @cycle: Required VME data transfer cycle type. - * @dwidth: Required VME data transfer width. - * - * Request use of a VME window resource capable of being set for the requested - * address space, data transfer cycle and width. - * - * Return: Pointer to VME resource on success, NULL on failure. - */ -struct vme_resource *vme_master_request(struct vme_dev *vdev, u32 address, - u32 cycle, u32 dwidth) -{ - struct vme_bridge *bridge; - struct list_head *master_pos = NULL; - struct vme_master_resource *allocated_image = NULL; - struct vme_master_resource *master_image = NULL; - struct vme_resource *resource = NULL; - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - goto err_bus; - } - - /* Loop through master resources */ - list_for_each(master_pos, &bridge->master_resources) { - master_image = list_entry(master_pos, - struct vme_master_resource, list); - - if (!master_image) { - printk(KERN_WARNING "Registered NULL master resource\n"); - continue; - } - - /* Find an unlocked and compatible image */ - spin_lock(&master_image->lock); - if (((master_image->address_attr & address) == address) && - ((master_image->cycle_attr & cycle) == cycle) && - ((master_image->width_attr & dwidth) == dwidth) && - (master_image->locked == 0)) { - - master_image->locked = 1; - spin_unlock(&master_image->lock); - allocated_image = master_image; - break; - } - spin_unlock(&master_image->lock); - } - - /* Check to see if we found a resource */ - if (!allocated_image) { - printk(KERN_ERR "Can't find a suitable resource\n"); - goto err_image; - } - - resource = kmalloc(sizeof(*resource), GFP_KERNEL); - if (!resource) - goto err_alloc; - - resource->type = VME_MASTER; - resource->entry = &allocated_image->list; - - return resource; - -err_alloc: - /* Unlock image */ - spin_lock(&master_image->lock); - master_image->locked = 0; - spin_unlock(&master_image->lock); -err_image: -err_bus: - return NULL; -} -EXPORT_SYMBOL(vme_master_request); - -/** - * vme_master_set - Set VME master window configuration. - * @resource: Pointer to VME master resource. - * @enabled: State to which the window should be configured. - * @vme_base: Base address for the window. - * @size: Size of the VME window. - * @aspace: VME address space for the VME window. - * @cycle: VME data transfer cycle type for the VME window. - * @dwidth: VME data transfer width for the VME window. - * - * Set configuration for provided VME master window. - * - * Return: Zero on success, -EINVAL if operation is not supported on this - * device, if an invalid resource has been provided or invalid - * attributes are provided. Hardware specific errors may also be - * returned. - */ -int vme_master_set(struct vme_resource *resource, int enabled, - unsigned long long vme_base, unsigned long long size, u32 aspace, - u32 cycle, u32 dwidth) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_master_resource *image; - int retval; - - if (resource->type != VME_MASTER) { - printk(KERN_ERR "Not a master resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_master_resource, list); - - if (!bridge->master_set) { - printk(KERN_WARNING "vme_master_set not supported\n"); - return -EINVAL; - } - - if (!(((image->address_attr & aspace) == aspace) && - ((image->cycle_attr & cycle) == cycle) && - ((image->width_attr & dwidth) == dwidth))) { - printk(KERN_WARNING "Invalid attributes\n"); - return -EINVAL; - } - - retval = vme_check_window(aspace, vme_base, size); - if (retval) - return retval; - - return bridge->master_set(image, enabled, vme_base, size, aspace, - cycle, dwidth); -} -EXPORT_SYMBOL(vme_master_set); - -/** - * vme_master_get - Retrieve VME master window configuration. - * @resource: Pointer to VME master resource. - * @enabled: Pointer to variable for storing state. - * @vme_base: Pointer to variable for storing window base address. - * @size: Pointer to variable for storing window size. - * @aspace: Pointer to variable for storing VME address space. - * @cycle: Pointer to variable for storing VME data transfer cycle type. - * @dwidth: Pointer to variable for storing VME data transfer width. - * - * Return configuration for provided VME master window. - * - * Return: Zero on success, -EINVAL if operation is not supported on this - * device or if an invalid resource has been provided. - */ -int vme_master_get(struct vme_resource *resource, int *enabled, - unsigned long long *vme_base, unsigned long long *size, u32 *aspace, - u32 *cycle, u32 *dwidth) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_master_resource *image; - - if (resource->type != VME_MASTER) { - printk(KERN_ERR "Not a master resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_master_resource, list); - - if (!bridge->master_get) { - printk(KERN_WARNING "%s not supported\n", __func__); - return -EINVAL; - } - - return bridge->master_get(image, enabled, vme_base, size, aspace, - cycle, dwidth); -} -EXPORT_SYMBOL(vme_master_get); - -/** - * vme_master_read - Read data from VME space into a buffer. - * @resource: Pointer to VME master resource. - * @buf: Pointer to buffer where data should be transferred. - * @count: Number of bytes to transfer. - * @offset: Offset into VME master window at which to start transfer. - * - * Perform read of count bytes of data from location on VME bus which maps into - * the VME master window at offset to buf. - * - * Return: Number of bytes read, -EINVAL if resource is not a VME master - * resource or read operation is not supported. -EFAULT returned if - * invalid offset is provided. Hardware specific errors may also be - * returned. - */ -ssize_t vme_master_read(struct vme_resource *resource, void *buf, size_t count, - loff_t offset) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_master_resource *image; - size_t length; - - if (!bridge->master_read) { - printk(KERN_WARNING "Reading from resource not supported\n"); - return -EINVAL; - } - - if (resource->type != VME_MASTER) { - printk(KERN_ERR "Not a master resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_master_resource, list); - - length = vme_get_size(resource); - - if (offset > length) { - printk(KERN_WARNING "Invalid Offset\n"); - return -EFAULT; - } - - if ((offset + count) > length) - count = length - offset; - - return bridge->master_read(image, buf, count, offset); - -} -EXPORT_SYMBOL(vme_master_read); - -/** - * vme_master_write - Write data out to VME space from a buffer. - * @resource: Pointer to VME master resource. - * @buf: Pointer to buffer holding data to transfer. - * @count: Number of bytes to transfer. - * @offset: Offset into VME master window at which to start transfer. - * - * Perform write of count bytes of data from buf to location on VME bus which - * maps into the VME master window at offset. - * - * Return: Number of bytes written, -EINVAL if resource is not a VME master - * resource or write operation is not supported. -EFAULT returned if - * invalid offset is provided. Hardware specific errors may also be - * returned. - */ -ssize_t vme_master_write(struct vme_resource *resource, void *buf, - size_t count, loff_t offset) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_master_resource *image; - size_t length; - - if (!bridge->master_write) { - printk(KERN_WARNING "Writing to resource not supported\n"); - return -EINVAL; - } - - if (resource->type != VME_MASTER) { - printk(KERN_ERR "Not a master resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_master_resource, list); - - length = vme_get_size(resource); - - if (offset > length) { - printk(KERN_WARNING "Invalid Offset\n"); - return -EFAULT; - } - - if ((offset + count) > length) - count = length - offset; - - return bridge->master_write(image, buf, count, offset); -} -EXPORT_SYMBOL(vme_master_write); - -/** - * vme_master_rmw - Perform read-modify-write cycle. - * @resource: Pointer to VME master resource. - * @mask: Bits to be compared and swapped in operation. - * @compare: Bits to be compared with data read from offset. - * @swap: Bits to be swapped in data read from offset. - * @offset: Offset into VME master window at which to perform operation. - * - * Perform read-modify-write cycle on provided location: - * - Location on VME bus is read. - * - Bits selected by mask are compared with compare. - * - Where a selected bit matches that in compare and are selected in swap, - * the bit is swapped. - * - Result written back to location on VME bus. - * - * Return: Bytes written on success, -EINVAL if resource is not a VME master - * resource or RMW operation is not supported. Hardware specific - * errors may also be returned. - */ -unsigned int vme_master_rmw(struct vme_resource *resource, unsigned int mask, - unsigned int compare, unsigned int swap, loff_t offset) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_master_resource *image; - - if (!bridge->master_rmw) { - printk(KERN_WARNING "Writing to resource not supported\n"); - return -EINVAL; - } - - if (resource->type != VME_MASTER) { - printk(KERN_ERR "Not a master resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_master_resource, list); - - return bridge->master_rmw(image, mask, compare, swap, offset); -} -EXPORT_SYMBOL(vme_master_rmw); - -/** - * vme_master_mmap - Mmap region of VME master window. - * @resource: Pointer to VME master resource. - * @vma: Pointer to definition of user mapping. - * - * Memory map a region of the VME master window into user space. - * - * Return: Zero on success, -EINVAL if resource is not a VME master - * resource or -EFAULT if map exceeds window size. Other generic mmap - * errors may also be returned. - */ -int vme_master_mmap(struct vme_resource *resource, struct vm_area_struct *vma) -{ - struct vme_master_resource *image; - phys_addr_t phys_addr; - unsigned long vma_size; - - if (resource->type != VME_MASTER) { - pr_err("Not a master resource\n"); - return -EINVAL; - } - - image = list_entry(resource->entry, struct vme_master_resource, list); - phys_addr = image->bus_resource.start + (vma->vm_pgoff << PAGE_SHIFT); - vma_size = vma->vm_end - vma->vm_start; - - if (phys_addr + vma_size > image->bus_resource.end + 1) { - pr_err("Map size cannot exceed the window size\n"); - return -EFAULT; - } - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - return vm_iomap_memory(vma, phys_addr, vma->vm_end - vma->vm_start); -} -EXPORT_SYMBOL(vme_master_mmap); - -/** - * vme_master_free - Free VME master window - * @resource: Pointer to VME master resource. - * - * Free the provided master resource so that it may be reallocated. - */ -void vme_master_free(struct vme_resource *resource) -{ - struct vme_master_resource *master_image; - - if (resource->type != VME_MASTER) { - printk(KERN_ERR "Not a master resource\n"); - return; - } - - master_image = list_entry(resource->entry, struct vme_master_resource, - list); - if (!master_image) { - printk(KERN_ERR "Can't find master resource\n"); - return; - } - - /* Unlock image */ - spin_lock(&master_image->lock); - if (master_image->locked == 0) - printk(KERN_ERR "Image is already free\n"); - - master_image->locked = 0; - spin_unlock(&master_image->lock); - - /* Free up resource memory */ - kfree(resource); -} -EXPORT_SYMBOL(vme_master_free); - -/** - * vme_dma_request - Request a DMA controller. - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * @route: Required src/destination combination. - * - * Request a VME DMA controller with capability to perform transfers bewteen - * requested source/destination combination. - * - * Return: Pointer to VME DMA resource on success, NULL on failure. - */ -struct vme_resource *vme_dma_request(struct vme_dev *vdev, u32 route) -{ - struct vme_bridge *bridge; - struct list_head *dma_pos = NULL; - struct vme_dma_resource *allocated_ctrlr = NULL; - struct vme_dma_resource *dma_ctrlr = NULL; - struct vme_resource *resource = NULL; - - /* XXX Not checking resource attributes */ - printk(KERN_ERR "No VME resource Attribute tests done\n"); - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - goto err_bus; - } - - /* Loop through DMA resources */ - list_for_each(dma_pos, &bridge->dma_resources) { - dma_ctrlr = list_entry(dma_pos, - struct vme_dma_resource, list); - if (!dma_ctrlr) { - printk(KERN_ERR "Registered NULL DMA resource\n"); - continue; - } - - /* Find an unlocked and compatible controller */ - mutex_lock(&dma_ctrlr->mtx); - if (((dma_ctrlr->route_attr & route) == route) && - (dma_ctrlr->locked == 0)) { - - dma_ctrlr->locked = 1; - mutex_unlock(&dma_ctrlr->mtx); - allocated_ctrlr = dma_ctrlr; - break; - } - mutex_unlock(&dma_ctrlr->mtx); - } - - /* Check to see if we found a resource */ - if (!allocated_ctrlr) - goto err_ctrlr; - - resource = kmalloc(sizeof(*resource), GFP_KERNEL); - if (!resource) - goto err_alloc; - - resource->type = VME_DMA; - resource->entry = &allocated_ctrlr->list; - - return resource; - -err_alloc: - /* Unlock image */ - mutex_lock(&dma_ctrlr->mtx); - dma_ctrlr->locked = 0; - mutex_unlock(&dma_ctrlr->mtx); -err_ctrlr: -err_bus: - return NULL; -} -EXPORT_SYMBOL(vme_dma_request); - -/** - * vme_new_dma_list - Create new VME DMA list. - * @resource: Pointer to VME DMA resource. - * - * Create a new VME DMA list. It is the responsibility of the user to free - * the list once it is no longer required with vme_dma_list_free(). - * - * Return: Pointer to new VME DMA list, NULL on allocation failure or invalid - * VME DMA resource. - */ -struct vme_dma_list *vme_new_dma_list(struct vme_resource *resource) -{ - struct vme_dma_list *dma_list; - - if (resource->type != VME_DMA) { - printk(KERN_ERR "Not a DMA resource\n"); - return NULL; - } - - dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL); - if (!dma_list) - return NULL; - - INIT_LIST_HEAD(&dma_list->entries); - dma_list->parent = list_entry(resource->entry, - struct vme_dma_resource, - list); - mutex_init(&dma_list->mtx); - - return dma_list; -} -EXPORT_SYMBOL(vme_new_dma_list); - -/** - * vme_dma_pattern_attribute - Create "Pattern" type VME DMA list attribute. - * @pattern: Value to use used as pattern - * @type: Type of pattern to be written. - * - * Create VME DMA list attribute for pattern generation. It is the - * responsibility of the user to free used attributes using - * vme_dma_free_attribute(). - * - * Return: Pointer to VME DMA attribute, NULL on failure. - */ -struct vme_dma_attr *vme_dma_pattern_attribute(u32 pattern, u32 type) -{ - struct vme_dma_attr *attributes; - struct vme_dma_pattern *pattern_attr; - - attributes = kmalloc(sizeof(*attributes), GFP_KERNEL); - if (!attributes) - goto err_attr; - - pattern_attr = kmalloc(sizeof(*pattern_attr), GFP_KERNEL); - if (!pattern_attr) - goto err_pat; - - attributes->type = VME_DMA_PATTERN; - attributes->private = (void *)pattern_attr; - - pattern_attr->pattern = pattern; - pattern_attr->type = type; - - return attributes; - -err_pat: - kfree(attributes); -err_attr: - return NULL; -} -EXPORT_SYMBOL(vme_dma_pattern_attribute); - -/** - * vme_dma_pci_attribute - Create "PCI" type VME DMA list attribute. - * @address: PCI base address for DMA transfer. - * - * Create VME DMA list attribute pointing to a location on PCI for DMA - * transfers. It is the responsibility of the user to free used attributes - * using vme_dma_free_attribute(). - * - * Return: Pointer to VME DMA attribute, NULL on failure. - */ -struct vme_dma_attr *vme_dma_pci_attribute(dma_addr_t address) -{ - struct vme_dma_attr *attributes; - struct vme_dma_pci *pci_attr; - - /* XXX Run some sanity checks here */ - - attributes = kmalloc(sizeof(*attributes), GFP_KERNEL); - if (!attributes) - goto err_attr; - - pci_attr = kmalloc(sizeof(*pci_attr), GFP_KERNEL); - if (!pci_attr) - goto err_pci; - - attributes->type = VME_DMA_PCI; - attributes->private = (void *)pci_attr; - - pci_attr->address = address; - - return attributes; - -err_pci: - kfree(attributes); -err_attr: - return NULL; -} -EXPORT_SYMBOL(vme_dma_pci_attribute); - -/** - * vme_dma_vme_attribute - Create "VME" type VME DMA list attribute. - * @address: VME base address for DMA transfer. - * @aspace: VME address space to use for DMA transfer. - * @cycle: VME bus cycle to use for DMA transfer. - * @dwidth: VME data width to use for DMA transfer. - * - * Create VME DMA list attribute pointing to a location on the VME bus for DMA - * transfers. It is the responsibility of the user to free used attributes - * using vme_dma_free_attribute(). - * - * Return: Pointer to VME DMA attribute, NULL on failure. - */ -struct vme_dma_attr *vme_dma_vme_attribute(unsigned long long address, - u32 aspace, u32 cycle, u32 dwidth) -{ - struct vme_dma_attr *attributes; - struct vme_dma_vme *vme_attr; - - attributes = kmalloc(sizeof(*attributes), GFP_KERNEL); - if (!attributes) - goto err_attr; - - vme_attr = kmalloc(sizeof(*vme_attr), GFP_KERNEL); - if (!vme_attr) - goto err_vme; - - attributes->type = VME_DMA_VME; - attributes->private = (void *)vme_attr; - - vme_attr->address = address; - vme_attr->aspace = aspace; - vme_attr->cycle = cycle; - vme_attr->dwidth = dwidth; - - return attributes; - -err_vme: - kfree(attributes); -err_attr: - return NULL; -} -EXPORT_SYMBOL(vme_dma_vme_attribute); - -/** - * vme_dma_free_attribute - Free DMA list attribute. - * @attributes: Pointer to DMA list attribute. - * - * Free VME DMA list attribute. VME DMA list attributes can be safely freed - * once vme_dma_list_add() has returned. - */ -void vme_dma_free_attribute(struct vme_dma_attr *attributes) -{ - kfree(attributes->private); - kfree(attributes); -} -EXPORT_SYMBOL(vme_dma_free_attribute); - -/** - * vme_dma_list_add - Add enty to a VME DMA list. - * @list: Pointer to VME list. - * @src: Pointer to DMA list attribute to use as source. - * @dest: Pointer to DMA list attribute to use as destination. - * @count: Number of bytes to transfer. - * - * Add an entry to the provided VME DMA list. Entry requires pointers to source - * and destination DMA attributes and a count. - * - * Please note, the attributes supported as source and destinations for - * transfers are hardware dependent. - * - * Return: Zero on success, -EINVAL if operation is not supported on this - * device or if the link list has already been submitted for execution. - * Hardware specific errors also possible. - */ -int vme_dma_list_add(struct vme_dma_list *list, struct vme_dma_attr *src, - struct vme_dma_attr *dest, size_t count) -{ - struct vme_bridge *bridge = list->parent->parent; - int retval; - - if (!bridge->dma_list_add) { - printk(KERN_WARNING "Link List DMA generation not supported\n"); - return -EINVAL; - } - - if (!mutex_trylock(&list->mtx)) { - printk(KERN_ERR "Link List already submitted\n"); - return -EINVAL; - } - - retval = bridge->dma_list_add(list, src, dest, count); - - mutex_unlock(&list->mtx); - - return retval; -} -EXPORT_SYMBOL(vme_dma_list_add); - -/** - * vme_dma_list_exec - Queue a VME DMA list for execution. - * @list: Pointer to VME list. - * - * Queue the provided VME DMA list for execution. The call will return once the - * list has been executed. - * - * Return: Zero on success, -EINVAL if operation is not supported on this - * device. Hardware specific errors also possible. - */ -int vme_dma_list_exec(struct vme_dma_list *list) -{ - struct vme_bridge *bridge = list->parent->parent; - int retval; - - if (!bridge->dma_list_exec) { - printk(KERN_ERR "Link List DMA execution not supported\n"); - return -EINVAL; - } - - mutex_lock(&list->mtx); - - retval = bridge->dma_list_exec(list); - - mutex_unlock(&list->mtx); - - return retval; -} -EXPORT_SYMBOL(vme_dma_list_exec); - -/** - * vme_dma_list_free - Free a VME DMA list. - * @list: Pointer to VME list. - * - * Free the provided DMA list and all its entries. - * - * Return: Zero on success, -EINVAL on invalid VME resource, -EBUSY if resource - * is still in use. Hardware specific errors also possible. - */ -int vme_dma_list_free(struct vme_dma_list *list) -{ - struct vme_bridge *bridge = list->parent->parent; - int retval; - - if (!bridge->dma_list_empty) { - printk(KERN_WARNING "Emptying of Link Lists not supported\n"); - return -EINVAL; - } - - if (!mutex_trylock(&list->mtx)) { - printk(KERN_ERR "Link List in use\n"); - return -EBUSY; - } - - /* - * Empty out all of the entries from the DMA list. We need to go to the - * low level driver as DMA entries are driver specific. - */ - retval = bridge->dma_list_empty(list); - if (retval) { - printk(KERN_ERR "Unable to empty link-list entries\n"); - mutex_unlock(&list->mtx); - return retval; - } - mutex_unlock(&list->mtx); - kfree(list); - - return retval; -} -EXPORT_SYMBOL(vme_dma_list_free); - -/** - * vme_dma_free - Free a VME DMA resource. - * @resource: Pointer to VME DMA resource. - * - * Free the provided DMA resource so that it may be reallocated. - * - * Return: Zero on success, -EINVAL on invalid VME resource, -EBUSY if resource - * is still active. - */ -int vme_dma_free(struct vme_resource *resource) -{ - struct vme_dma_resource *ctrlr; - - if (resource->type != VME_DMA) { - printk(KERN_ERR "Not a DMA resource\n"); - return -EINVAL; - } - - ctrlr = list_entry(resource->entry, struct vme_dma_resource, list); - - if (!mutex_trylock(&ctrlr->mtx)) { - printk(KERN_ERR "Resource busy, can't free\n"); - return -EBUSY; - } - - if (!(list_empty(&ctrlr->pending) && list_empty(&ctrlr->running))) { - printk(KERN_WARNING "Resource still processing transfers\n"); - mutex_unlock(&ctrlr->mtx); - return -EBUSY; - } - - ctrlr->locked = 0; - - mutex_unlock(&ctrlr->mtx); - - kfree(resource); - - return 0; -} -EXPORT_SYMBOL(vme_dma_free); - -void vme_bus_error_handler(struct vme_bridge *bridge, - unsigned long long address, int am) -{ - struct list_head *handler_pos = NULL; - struct vme_error_handler *handler; - int handler_triggered = 0; - u32 aspace = vme_get_aspace(am); - - list_for_each(handler_pos, &bridge->vme_error_handlers) { - handler = list_entry(handler_pos, struct vme_error_handler, - list); - if ((aspace == handler->aspace) && - (address >= handler->start) && - (address < handler->end)) { - if (!handler->num_errors) - handler->first_error = address; - if (handler->num_errors != UINT_MAX) - handler->num_errors++; - handler_triggered = 1; - } - } - - if (!handler_triggered) - dev_err(bridge->parent, - "Unhandled VME access error at address 0x%llx\n", - address); -} -EXPORT_SYMBOL(vme_bus_error_handler); - -struct vme_error_handler *vme_register_error_handler( - struct vme_bridge *bridge, u32 aspace, - unsigned long long address, size_t len) -{ - struct vme_error_handler *handler; - - handler = kmalloc(sizeof(*handler), GFP_ATOMIC); - if (!handler) - return NULL; - - handler->aspace = aspace; - handler->start = address; - handler->end = address + len; - handler->num_errors = 0; - handler->first_error = 0; - list_add_tail(&handler->list, &bridge->vme_error_handlers); - - return handler; -} -EXPORT_SYMBOL(vme_register_error_handler); - -void vme_unregister_error_handler(struct vme_error_handler *handler) -{ - list_del(&handler->list); - kfree(handler); -} -EXPORT_SYMBOL(vme_unregister_error_handler); - -void vme_irq_handler(struct vme_bridge *bridge, int level, int statid) -{ - void (*call)(int, int, void *); - void *priv_data; - - call = bridge->irq[level - 1].callback[statid].func; - priv_data = bridge->irq[level - 1].callback[statid].priv_data; - if (call) - call(level, statid, priv_data); - else - printk(KERN_WARNING "Spurious VME interrupt, level:%x, vector:%x\n", - level, statid); -} -EXPORT_SYMBOL(vme_irq_handler); - -/** - * vme_irq_request - Request a specific VME interrupt. - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * @level: Interrupt priority being requested. - * @statid: Interrupt vector being requested. - * @callback: Pointer to callback function called when VME interrupt/vector - * received. - * @priv_data: Generic pointer that will be passed to the callback function. - * - * Request callback to be attached as a handler for VME interrupts with provided - * level and statid. - * - * Return: Zero on success, -EINVAL on invalid vme device, level or if the - * function is not supported, -EBUSY if the level/statid combination is - * already in use. Hardware specific errors also possible. - */ -int vme_irq_request(struct vme_dev *vdev, int level, int statid, - void (*callback)(int, int, void *), - void *priv_data) -{ - struct vme_bridge *bridge; - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - return -EINVAL; - } - - if ((level < 1) || (level > 7)) { - printk(KERN_ERR "Invalid interrupt level\n"); - return -EINVAL; - } - - if (!bridge->irq_set) { - printk(KERN_ERR "Configuring interrupts not supported\n"); - return -EINVAL; - } - - mutex_lock(&bridge->irq_mtx); - - if (bridge->irq[level - 1].callback[statid].func) { - mutex_unlock(&bridge->irq_mtx); - printk(KERN_WARNING "VME Interrupt already taken\n"); - return -EBUSY; - } - - bridge->irq[level - 1].count++; - bridge->irq[level - 1].callback[statid].priv_data = priv_data; - bridge->irq[level - 1].callback[statid].func = callback; - - /* Enable IRQ level */ - bridge->irq_set(bridge, level, 1, 1); - - mutex_unlock(&bridge->irq_mtx); - - return 0; -} -EXPORT_SYMBOL(vme_irq_request); - -/** - * vme_irq_free - Free a VME interrupt. - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * @level: Interrupt priority of interrupt being freed. - * @statid: Interrupt vector of interrupt being freed. - * - * Remove previously attached callback from VME interrupt priority/vector. - */ -void vme_irq_free(struct vme_dev *vdev, int level, int statid) -{ - struct vme_bridge *bridge; - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - return; - } - - if ((level < 1) || (level > 7)) { - printk(KERN_ERR "Invalid interrupt level\n"); - return; - } - - if (!bridge->irq_set) { - printk(KERN_ERR "Configuring interrupts not supported\n"); - return; - } - - mutex_lock(&bridge->irq_mtx); - - bridge->irq[level - 1].count--; - - /* Disable IRQ level if no more interrupts attached at this level*/ - if (bridge->irq[level - 1].count == 0) - bridge->irq_set(bridge, level, 0, 1); - - bridge->irq[level - 1].callback[statid].func = NULL; - bridge->irq[level - 1].callback[statid].priv_data = NULL; - - mutex_unlock(&bridge->irq_mtx); -} -EXPORT_SYMBOL(vme_irq_free); - -/** - * vme_irq_generate - Generate VME interrupt. - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * @level: Interrupt priority at which to assert the interrupt. - * @statid: Interrupt vector to associate with the interrupt. - * - * Generate a VME interrupt of the provided level and with the provided - * statid. - * - * Return: Zero on success, -EINVAL on invalid vme device, level or if the - * function is not supported. Hardware specific errors also possible. - */ -int vme_irq_generate(struct vme_dev *vdev, int level, int statid) -{ - struct vme_bridge *bridge; - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - return -EINVAL; - } - - if ((level < 1) || (level > 7)) { - printk(KERN_WARNING "Invalid interrupt level\n"); - return -EINVAL; - } - - if (!bridge->irq_generate) { - printk(KERN_WARNING "Interrupt generation not supported\n"); - return -EINVAL; - } - - return bridge->irq_generate(bridge, level, statid); -} -EXPORT_SYMBOL(vme_irq_generate); - -/** - * vme_lm_request - Request a VME location monitor - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * - * Allocate a location monitor resource to the driver. A location monitor - * allows the driver to monitor accesses to a contiguous number of - * addresses on the VME bus. - * - * Return: Pointer to a VME resource on success or NULL on failure. - */ -struct vme_resource *vme_lm_request(struct vme_dev *vdev) -{ - struct vme_bridge *bridge; - struct list_head *lm_pos = NULL; - struct vme_lm_resource *allocated_lm = NULL; - struct vme_lm_resource *lm = NULL; - struct vme_resource *resource = NULL; - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - goto err_bus; - } - - /* Loop through LM resources */ - list_for_each(lm_pos, &bridge->lm_resources) { - lm = list_entry(lm_pos, - struct vme_lm_resource, list); - if (!lm) { - printk(KERN_ERR "Registered NULL Location Monitor resource\n"); - continue; - } - - /* Find an unlocked controller */ - mutex_lock(&lm->mtx); - if (lm->locked == 0) { - lm->locked = 1; - mutex_unlock(&lm->mtx); - allocated_lm = lm; - break; - } - mutex_unlock(&lm->mtx); - } - - /* Check to see if we found a resource */ - if (!allocated_lm) - goto err_lm; - - resource = kmalloc(sizeof(*resource), GFP_KERNEL); - if (!resource) - goto err_alloc; - - resource->type = VME_LM; - resource->entry = &allocated_lm->list; - - return resource; - -err_alloc: - /* Unlock image */ - mutex_lock(&lm->mtx); - lm->locked = 0; - mutex_unlock(&lm->mtx); -err_lm: -err_bus: - return NULL; -} -EXPORT_SYMBOL(vme_lm_request); - -/** - * vme_lm_count - Determine number of VME Addresses monitored - * @resource: Pointer to VME location monitor resource. - * - * The number of contiguous addresses monitored is hardware dependent. - * Return the number of contiguous addresses monitored by the - * location monitor. - * - * Return: Count of addresses monitored or -EINVAL when provided with an - * invalid location monitor resource. - */ -int vme_lm_count(struct vme_resource *resource) -{ - struct vme_lm_resource *lm; - - if (resource->type != VME_LM) { - printk(KERN_ERR "Not a Location Monitor resource\n"); - return -EINVAL; - } - - lm = list_entry(resource->entry, struct vme_lm_resource, list); - - return lm->monitors; -} -EXPORT_SYMBOL(vme_lm_count); - -/** - * vme_lm_set - Configure location monitor - * @resource: Pointer to VME location monitor resource. - * @lm_base: Base address to monitor. - * @aspace: VME address space to monitor. - * @cycle: VME bus cycle type to monitor. - * - * Set the base address, address space and cycle type of accesses to be - * monitored by the location monitor. - * - * Return: Zero on success, -EINVAL when provided with an invalid location - * monitor resource or function is not supported. Hardware specific - * errors may also be returned. - */ -int vme_lm_set(struct vme_resource *resource, unsigned long long lm_base, - u32 aspace, u32 cycle) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_lm_resource *lm; - - if (resource->type != VME_LM) { - printk(KERN_ERR "Not a Location Monitor resource\n"); - return -EINVAL; - } - - lm = list_entry(resource->entry, struct vme_lm_resource, list); - - if (!bridge->lm_set) { - printk(KERN_ERR "vme_lm_set not supported\n"); - return -EINVAL; - } - - return bridge->lm_set(lm, lm_base, aspace, cycle); -} -EXPORT_SYMBOL(vme_lm_set); - -/** - * vme_lm_get - Retrieve location monitor settings - * @resource: Pointer to VME location monitor resource. - * @lm_base: Pointer used to output the base address monitored. - * @aspace: Pointer used to output the address space monitored. - * @cycle: Pointer used to output the VME bus cycle type monitored. - * - * Retrieve the base address, address space and cycle type of accesses to - * be monitored by the location monitor. - * - * Return: Zero on success, -EINVAL when provided with an invalid location - * monitor resource or function is not supported. Hardware specific - * errors may also be returned. - */ -int vme_lm_get(struct vme_resource *resource, unsigned long long *lm_base, - u32 *aspace, u32 *cycle) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_lm_resource *lm; - - if (resource->type != VME_LM) { - printk(KERN_ERR "Not a Location Monitor resource\n"); - return -EINVAL; - } - - lm = list_entry(resource->entry, struct vme_lm_resource, list); - - if (!bridge->lm_get) { - printk(KERN_ERR "vme_lm_get not supported\n"); - return -EINVAL; - } - - return bridge->lm_get(lm, lm_base, aspace, cycle); -} -EXPORT_SYMBOL(vme_lm_get); - -/** - * vme_lm_attach - Provide callback for location monitor address - * @resource: Pointer to VME location monitor resource. - * @monitor: Offset to which callback should be attached. - * @callback: Pointer to callback function called when triggered. - * @data: Generic pointer that will be passed to the callback function. - * - * Attach a callback to the specificed offset into the location monitors - * monitored addresses. A generic pointer is provided to allow data to be - * passed to the callback when called. - * - * Return: Zero on success, -EINVAL when provided with an invalid location - * monitor resource or function is not supported. Hardware specific - * errors may also be returned. - */ -int vme_lm_attach(struct vme_resource *resource, int monitor, - void (*callback)(void *), void *data) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_lm_resource *lm; - - if (resource->type != VME_LM) { - printk(KERN_ERR "Not a Location Monitor resource\n"); - return -EINVAL; - } - - lm = list_entry(resource->entry, struct vme_lm_resource, list); - - if (!bridge->lm_attach) { - printk(KERN_ERR "vme_lm_attach not supported\n"); - return -EINVAL; - } - - return bridge->lm_attach(lm, monitor, callback, data); -} -EXPORT_SYMBOL(vme_lm_attach); - -/** - * vme_lm_detach - Remove callback for location monitor address - * @resource: Pointer to VME location monitor resource. - * @monitor: Offset to which callback should be removed. - * - * Remove the callback associated with the specificed offset into the - * location monitors monitored addresses. - * - * Return: Zero on success, -EINVAL when provided with an invalid location - * monitor resource or function is not supported. Hardware specific - * errors may also be returned. - */ -int vme_lm_detach(struct vme_resource *resource, int monitor) -{ - struct vme_bridge *bridge = find_bridge(resource); - struct vme_lm_resource *lm; - - if (resource->type != VME_LM) { - printk(KERN_ERR "Not a Location Monitor resource\n"); - return -EINVAL; - } - - lm = list_entry(resource->entry, struct vme_lm_resource, list); - - if (!bridge->lm_detach) { - printk(KERN_ERR "vme_lm_detach not supported\n"); - return -EINVAL; - } - - return bridge->lm_detach(lm, monitor); -} -EXPORT_SYMBOL(vme_lm_detach); - -/** - * vme_lm_free - Free allocated VME location monitor - * @resource: Pointer to VME location monitor resource. - * - * Free allocation of a VME location monitor. - * - * WARNING: This function currently expects that any callbacks that have - * been attached to the location monitor have been removed. - * - * Return: Zero on success, -EINVAL when provided with an invalid location - * monitor resource. - */ -void vme_lm_free(struct vme_resource *resource) -{ - struct vme_lm_resource *lm; - - if (resource->type != VME_LM) { - printk(KERN_ERR "Not a Location Monitor resource\n"); - return; - } - - lm = list_entry(resource->entry, struct vme_lm_resource, list); - - mutex_lock(&lm->mtx); - - /* XXX - * Check to see that there aren't any callbacks still attached, if - * there are we should probably be detaching them! - */ - - lm->locked = 0; - - mutex_unlock(&lm->mtx); - - kfree(resource); -} -EXPORT_SYMBOL(vme_lm_free); - -/** - * vme_slot_num - Retrieve slot ID - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * - * Retrieve the slot ID associated with the provided VME device. - * - * Return: The slot ID on success, -EINVAL if VME bridge cannot be determined - * or the function is not supported. Hardware specific errors may also - * be returned. - */ -int vme_slot_num(struct vme_dev *vdev) -{ - struct vme_bridge *bridge; - - bridge = vdev->bridge; - if (!bridge) { - printk(KERN_ERR "Can't find VME bus\n"); - return -EINVAL; - } - - if (!bridge->slot_get) { - printk(KERN_WARNING "vme_slot_num not supported\n"); - return -EINVAL; - } - - return bridge->slot_get(bridge); -} -EXPORT_SYMBOL(vme_slot_num); - -/** - * vme_bus_num - Retrieve bus number - * @vdev: Pointer to VME device struct vme_dev assigned to driver instance. - * - * Retrieve the bus enumeration associated with the provided VME device. - * - * Return: The bus number on success, -EINVAL if VME bridge cannot be - * determined. - */ -int vme_bus_num(struct vme_dev *vdev) -{ - struct vme_bridge *bridge; - - bridge = vdev->bridge; - if (!bridge) { - pr_err("Can't find VME bus\n"); - return -EINVAL; - } - - return bridge->num; -} -EXPORT_SYMBOL(vme_bus_num); - -/* - Bridge Registration --------------------------------------------------- */ - -static void vme_dev_release(struct device *dev) -{ - kfree(dev_to_vme_dev(dev)); -} - -/* Common bridge initialization */ -struct vme_bridge *vme_init_bridge(struct vme_bridge *bridge) -{ - INIT_LIST_HEAD(&bridge->vme_error_handlers); - INIT_LIST_HEAD(&bridge->master_resources); - INIT_LIST_HEAD(&bridge->slave_resources); - INIT_LIST_HEAD(&bridge->dma_resources); - INIT_LIST_HEAD(&bridge->lm_resources); - mutex_init(&bridge->irq_mtx); - - return bridge; -} -EXPORT_SYMBOL(vme_init_bridge); - -int vme_register_bridge(struct vme_bridge *bridge) -{ - int i; - int ret = -1; - - mutex_lock(&vme_buses_lock); - for (i = 0; i < sizeof(vme_bus_numbers) * 8; i++) { - if ((vme_bus_numbers & (1 << i)) == 0) { - vme_bus_numbers |= (1 << i); - bridge->num = i; - INIT_LIST_HEAD(&bridge->devices); - list_add_tail(&bridge->bus_list, &vme_bus_list); - ret = 0; - break; - } - } - mutex_unlock(&vme_buses_lock); - - return ret; -} -EXPORT_SYMBOL(vme_register_bridge); - -void vme_unregister_bridge(struct vme_bridge *bridge) -{ - struct vme_dev *vdev; - struct vme_dev *tmp; - - mutex_lock(&vme_buses_lock); - vme_bus_numbers &= ~(1 << bridge->num); - list_for_each_entry_safe(vdev, tmp, &bridge->devices, bridge_list) { - list_del(&vdev->drv_list); - list_del(&vdev->bridge_list); - device_unregister(&vdev->dev); - } - list_del(&bridge->bus_list); - mutex_unlock(&vme_buses_lock); -} -EXPORT_SYMBOL(vme_unregister_bridge); - -/* - Driver Registration --------------------------------------------------- */ - -static int __vme_register_driver_bus(struct vme_driver *drv, - struct vme_bridge *bridge, unsigned int ndevs) -{ - int err; - unsigned int i; - struct vme_dev *vdev; - struct vme_dev *tmp; - - for (i = 0; i < ndevs; i++) { - vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); - if (!vdev) { - err = -ENOMEM; - goto err_devalloc; - } - vdev->num = i; - vdev->bridge = bridge; - vdev->dev.platform_data = drv; - vdev->dev.release = vme_dev_release; - vdev->dev.parent = bridge->parent; - vdev->dev.bus = &vme_bus_type; - dev_set_name(&vdev->dev, "%s.%u-%u", drv->name, bridge->num, - vdev->num); - - err = device_register(&vdev->dev); - if (err) - goto err_reg; - - if (vdev->dev.platform_data) { - list_add_tail(&vdev->drv_list, &drv->devices); - list_add_tail(&vdev->bridge_list, &bridge->devices); - } else - device_unregister(&vdev->dev); - } - return 0; - -err_reg: - put_device(&vdev->dev); -err_devalloc: - list_for_each_entry_safe(vdev, tmp, &drv->devices, drv_list) { - list_del(&vdev->drv_list); - list_del(&vdev->bridge_list); - device_unregister(&vdev->dev); - } - return err; -} - -static int __vme_register_driver(struct vme_driver *drv, unsigned int ndevs) -{ - struct vme_bridge *bridge; - int err = 0; - - mutex_lock(&vme_buses_lock); - list_for_each_entry(bridge, &vme_bus_list, bus_list) { - /* - * This cannot cause trouble as we already have vme_buses_lock - * and if the bridge is removed, it will have to go through - * vme_unregister_bridge() to do it (which calls remove() on - * the bridge which in turn tries to acquire vme_buses_lock and - * will have to wait). - */ - err = __vme_register_driver_bus(drv, bridge, ndevs); - if (err) - break; - } - mutex_unlock(&vme_buses_lock); - return err; -} - -/** - * vme_register_driver - Register a VME driver - * @drv: Pointer to VME driver structure to register. - * @ndevs: Maximum number of devices to allow to be enumerated. - * - * Register a VME device driver with the VME subsystem. - * - * Return: Zero on success, error value on registration failure. - */ -int vme_register_driver(struct vme_driver *drv, unsigned int ndevs) -{ - int err; - - drv->driver.name = drv->name; - drv->driver.bus = &vme_bus_type; - INIT_LIST_HEAD(&drv->devices); - - err = driver_register(&drv->driver); - if (err) - return err; - - err = __vme_register_driver(drv, ndevs); - if (err) - driver_unregister(&drv->driver); - - return err; -} -EXPORT_SYMBOL(vme_register_driver); - -/** - * vme_unregister_driver - Unregister a VME driver - * @drv: Pointer to VME driver structure to unregister. - * - * Unregister a VME device driver from the VME subsystem. - */ -void vme_unregister_driver(struct vme_driver *drv) -{ - struct vme_dev *dev, *dev_tmp; - - mutex_lock(&vme_buses_lock); - list_for_each_entry_safe(dev, dev_tmp, &drv->devices, drv_list) { - list_del(&dev->drv_list); - list_del(&dev->bridge_list); - device_unregister(&dev->dev); - } - mutex_unlock(&vme_buses_lock); - - driver_unregister(&drv->driver); -} -EXPORT_SYMBOL(vme_unregister_driver); - -/* - Bus Registration ------------------------------------------------------ */ - -static int vme_bus_match(struct device *dev, struct device_driver *drv) -{ - struct vme_driver *vme_drv; - - vme_drv = container_of(drv, struct vme_driver, driver); - - if (dev->platform_data == vme_drv) { - struct vme_dev *vdev = dev_to_vme_dev(dev); - - if (vme_drv->match && vme_drv->match(vdev)) - return 1; - - dev->platform_data = NULL; - } - return 0; -} - -static int vme_bus_probe(struct device *dev) -{ - struct vme_driver *driver; - struct vme_dev *vdev = dev_to_vme_dev(dev); - - driver = dev->platform_data; - if (driver->probe) - return driver->probe(vdev); - - return -ENODEV; -} - -static void vme_bus_remove(struct device *dev) -{ - struct vme_driver *driver; - struct vme_dev *vdev = dev_to_vme_dev(dev); - - driver = dev->platform_data; - if (driver->remove) - driver->remove(vdev); -} - -struct bus_type vme_bus_type = { - .name = "vme", - .match = vme_bus_match, - .probe = vme_bus_probe, - .remove = vme_bus_remove, -}; -EXPORT_SYMBOL(vme_bus_type); - -static int __init vme_init(void) -{ - return bus_register(&vme_bus_type); -} -subsys_initcall(vme_init); diff --git a/drivers/vme/vme_bridge.h b/drivers/vme/vme_bridge.h deleted file mode 100644 index 42ecf961004e..000000000000 --- a/drivers/vme/vme_bridge.h +++ /dev/null @@ -1,190 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _VME_BRIDGE_H_ -#define _VME_BRIDGE_H_ - -#include - -#define VME_CRCSR_BUF_SIZE (508*1024) -/* - * Resource structures - */ -struct vme_master_resource { - struct list_head list; - struct vme_bridge *parent; - /* - * We are likely to need to access the VME bus in interrupt context, so - * protect master routines with a spinlock rather than a mutex. - */ - spinlock_t lock; - int locked; - int number; - u32 address_attr; - u32 cycle_attr; - u32 width_attr; - struct resource bus_resource; - void __iomem *kern_base; -}; - -struct vme_slave_resource { - struct list_head list; - struct vme_bridge *parent; - struct mutex mtx; - int locked; - int number; - u32 address_attr; - u32 cycle_attr; -}; - -struct vme_dma_pattern { - u32 pattern; - u32 type; -}; - -struct vme_dma_pci { - dma_addr_t address; -}; - -struct vme_dma_vme { - unsigned long long address; - u32 aspace; - u32 cycle; - u32 dwidth; -}; - -struct vme_dma_list { - struct list_head list; - struct vme_dma_resource *parent; - struct list_head entries; - struct mutex mtx; -}; - -struct vme_dma_resource { - struct list_head list; - struct vme_bridge *parent; - struct mutex mtx; - int locked; - int number; - struct list_head pending; - struct list_head running; - u32 route_attr; -}; - -struct vme_lm_resource { - struct list_head list; - struct vme_bridge *parent; - struct mutex mtx; - int locked; - int number; - int monitors; -}; - -struct vme_error_handler { - struct list_head list; - unsigned long long start; /* Beginning of error window */ - unsigned long long end; /* End of error window */ - unsigned long long first_error; /* Address of the first error */ - u32 aspace; /* Address space of error window*/ - unsigned num_errors; /* Number of errors */ -}; - -struct vme_callback { - void (*func)(int, int, void*); - void *priv_data; -}; - -struct vme_irq { - int count; - struct vme_callback callback[VME_NUM_STATUSID]; -}; - -/* Allow 16 characters for name (including null character) */ -#define VMENAMSIZ 16 - -/* This structure stores all the information about one bridge - * The structure should be dynamically allocated by the driver and one instance - * of the structure should be present for each VME chip present in the system. - */ -struct vme_bridge { - char name[VMENAMSIZ]; - int num; - struct list_head master_resources; - struct list_head slave_resources; - struct list_head dma_resources; - struct list_head lm_resources; - - /* List for registered errors handlers */ - struct list_head vme_error_handlers; - /* List of devices on this bridge */ - struct list_head devices; - - /* Bridge Info - XXX Move to private structure? */ - struct device *parent; /* Parent device (eg. pdev->dev for PCI) */ - void *driver_priv; /* Private pointer for the bridge driver */ - struct list_head bus_list; /* list of VME buses */ - - /* Interrupt callbacks */ - struct vme_irq irq[7]; - /* Locking for VME irq callback configuration */ - struct mutex irq_mtx; - - /* Slave Functions */ - int (*slave_get) (struct vme_slave_resource *, int *, - unsigned long long *, unsigned long long *, dma_addr_t *, - u32 *, u32 *); - int (*slave_set) (struct vme_slave_resource *, int, unsigned long long, - unsigned long long, dma_addr_t, u32, u32); - - /* Master Functions */ - int (*master_get) (struct vme_master_resource *, int *, - unsigned long long *, unsigned long long *, u32 *, u32 *, - u32 *); - int (*master_set) (struct vme_master_resource *, int, - unsigned long long, unsigned long long, u32, u32, u32); - ssize_t (*master_read) (struct vme_master_resource *, void *, size_t, - loff_t); - ssize_t (*master_write) (struct vme_master_resource *, void *, size_t, - loff_t); - unsigned int (*master_rmw) (struct vme_master_resource *, unsigned int, - unsigned int, unsigned int, loff_t); - - /* DMA Functions */ - int (*dma_list_add) (struct vme_dma_list *, struct vme_dma_attr *, - struct vme_dma_attr *, size_t); - int (*dma_list_exec) (struct vme_dma_list *); - int (*dma_list_empty) (struct vme_dma_list *); - - /* Interrupt Functions */ - void (*irq_set) (struct vme_bridge *, int, int, int); - int (*irq_generate) (struct vme_bridge *, int, int); - - /* Location monitor functions */ - int (*lm_set) (struct vme_lm_resource *, unsigned long long, u32, u32); - int (*lm_get) (struct vme_lm_resource *, unsigned long long *, u32 *, - u32 *); - int (*lm_attach)(struct vme_lm_resource *, int, - void (*callback)(void *), void *); - int (*lm_detach) (struct vme_lm_resource *, int); - - /* CR/CSR space functions */ - int (*slot_get) (struct vme_bridge *); - - /* Bridge parent interface */ - void *(*alloc_consistent)(struct device *dev, size_t size, - dma_addr_t *dma); - void (*free_consistent)(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma); -}; - -void vme_bus_error_handler(struct vme_bridge *bridge, - unsigned long long address, int am); -void vme_irq_handler(struct vme_bridge *, int, int); - -struct vme_bridge *vme_init_bridge(struct vme_bridge *); -int vme_register_bridge(struct vme_bridge *); -void vme_unregister_bridge(struct vme_bridge *); -struct vme_error_handler *vme_register_error_handler( - struct vme_bridge *bridge, u32 aspace, - unsigned long long address, size_t len); -void vme_unregister_error_handler(struct vme_error_handler *handler); - -#endif /* _VME_BRIDGE_H_ */ diff --git a/include/linux/vme.h b/include/linux/vme.h deleted file mode 100644 index b204a9b4be1b..000000000000 --- a/include/linux/vme.h +++ /dev/null @@ -1,190 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _VME_H_ -#define _VME_H_ - -/* Resource Type */ -enum vme_resource_type { - VME_MASTER, - VME_SLAVE, - VME_DMA, - VME_LM -}; - -/* VME Address Spaces */ -#define VME_A16 0x1 -#define VME_A24 0x2 -#define VME_A32 0x4 -#define VME_A64 0x8 -#define VME_CRCSR 0x10 -#define VME_USER1 0x20 -#define VME_USER2 0x40 -#define VME_USER3 0x80 -#define VME_USER4 0x100 - -#define VME_A16_MAX 0x10000ULL -#define VME_A24_MAX 0x1000000ULL -#define VME_A32_MAX 0x100000000ULL -#define VME_A64_MAX 0x10000000000000000ULL -#define VME_CRCSR_MAX 0x1000000ULL - - -/* VME Cycle Types */ -#define VME_SCT 0x1 -#define VME_BLT 0x2 -#define VME_MBLT 0x4 -#define VME_2eVME 0x8 -#define VME_2eSST 0x10 -#define VME_2eSSTB 0x20 - -#define VME_2eSST160 0x100 -#define VME_2eSST267 0x200 -#define VME_2eSST320 0x400 - -#define VME_SUPER 0x1000 -#define VME_USER 0x2000 -#define VME_PROG 0x4000 -#define VME_DATA 0x8000 - -/* VME Data Widths */ -#define VME_D8 0x1 -#define VME_D16 0x2 -#define VME_D32 0x4 -#define VME_D64 0x8 - -/* Arbitration Scheduling Modes */ -#define VME_R_ROBIN_MODE 0x1 -#define VME_PRIORITY_MODE 0x2 - -#define VME_DMA_PATTERN (1<<0) -#define VME_DMA_PCI (1<<1) -#define VME_DMA_VME (1<<2) - -#define VME_DMA_PATTERN_BYTE (1<<0) -#define VME_DMA_PATTERN_WORD (1<<1) -#define VME_DMA_PATTERN_INCREMENT (1<<2) - -#define VME_DMA_VME_TO_MEM (1<<0) -#define VME_DMA_MEM_TO_VME (1<<1) -#define VME_DMA_VME_TO_VME (1<<2) -#define VME_DMA_MEM_TO_MEM (1<<3) -#define VME_DMA_PATTERN_TO_VME (1<<4) -#define VME_DMA_PATTERN_TO_MEM (1<<5) - -struct vme_dma_attr { - u32 type; - void *private; -}; - -struct vme_resource { - enum vme_resource_type type; - struct list_head *entry; -}; - -extern struct bus_type vme_bus_type; - -/* Number of VME interrupt vectors */ -#define VME_NUM_STATUSID 256 - -/* VME_MAX_BRIDGES comes from the type of vme_bus_numbers */ -#define VME_MAX_BRIDGES (sizeof(unsigned int)*8) -#define VME_MAX_SLOTS 32 - -#define VME_SLOT_CURRENT -1 -#define VME_SLOT_ALL -2 - -/** - * struct vme_dev - Structure representing a VME device - * @num: The device number - * @bridge: Pointer to the bridge device this device is on - * @dev: Internal device structure - * @drv_list: List of devices (per driver) - * @bridge_list: List of devices (per bridge) - */ -struct vme_dev { - int num; - struct vme_bridge *bridge; - struct device dev; - struct list_head drv_list; - struct list_head bridge_list; -}; - -/** - * struct vme_driver - Structure representing a VME driver - * @name: Driver name, should be unique among VME drivers and usually the same - * as the module name. - * @match: Callback used to determine whether probe should be run. - * @probe: Callback for device binding, called when new device is detected. - * @remove: Callback, called on device removal. - * @driver: Underlying generic device driver structure. - * @devices: List of VME devices (struct vme_dev) associated with this driver. - */ -struct vme_driver { - const char *name; - int (*match)(struct vme_dev *); - int (*probe)(struct vme_dev *); - void (*remove)(struct vme_dev *); - struct device_driver driver; - struct list_head devices; -}; - -void *vme_alloc_consistent(struct vme_resource *, size_t, dma_addr_t *); -void vme_free_consistent(struct vme_resource *, size_t, void *, - dma_addr_t); - -size_t vme_get_size(struct vme_resource *); -int vme_check_window(u32 aspace, unsigned long long vme_base, - unsigned long long size); - -struct vme_resource *vme_slave_request(struct vme_dev *, u32, u32); -int vme_slave_set(struct vme_resource *, int, unsigned long long, - unsigned long long, dma_addr_t, u32, u32); -int vme_slave_get(struct vme_resource *, int *, unsigned long long *, - unsigned long long *, dma_addr_t *, u32 *, u32 *); -void vme_slave_free(struct vme_resource *); - -struct vme_resource *vme_master_request(struct vme_dev *, u32, u32, u32); -int vme_master_set(struct vme_resource *, int, unsigned long long, - unsigned long long, u32, u32, u32); -int vme_master_get(struct vme_resource *, int *, unsigned long long *, - unsigned long long *, u32 *, u32 *, u32 *); -ssize_t vme_master_read(struct vme_resource *, void *, size_t, loff_t); -ssize_t vme_master_write(struct vme_resource *, void *, size_t, loff_t); -unsigned int vme_master_rmw(struct vme_resource *, unsigned int, unsigned int, - unsigned int, loff_t); -int vme_master_mmap(struct vme_resource *resource, struct vm_area_struct *vma); -void vme_master_free(struct vme_resource *); - -struct vme_resource *vme_dma_request(struct vme_dev *, u32); -struct vme_dma_list *vme_new_dma_list(struct vme_resource *); -struct vme_dma_attr *vme_dma_pattern_attribute(u32, u32); -struct vme_dma_attr *vme_dma_pci_attribute(dma_addr_t); -struct vme_dma_attr *vme_dma_vme_attribute(unsigned long long, u32, u32, u32); -void vme_dma_free_attribute(struct vme_dma_attr *); -int vme_dma_list_add(struct vme_dma_list *, struct vme_dma_attr *, - struct vme_dma_attr *, size_t); -int vme_dma_list_exec(struct vme_dma_list *); -int vme_dma_list_free(struct vme_dma_list *); -int vme_dma_free(struct vme_resource *); - -int vme_irq_request(struct vme_dev *, int, int, - void (*callback)(int, int, void *), void *); -void vme_irq_free(struct vme_dev *, int, int); -int vme_irq_generate(struct vme_dev *, int, int); - -struct vme_resource *vme_lm_request(struct vme_dev *); -int vme_lm_count(struct vme_resource *); -int vme_lm_set(struct vme_resource *, unsigned long long, u32, u32); -int vme_lm_get(struct vme_resource *, unsigned long long *, u32 *, u32 *); -int vme_lm_attach(struct vme_resource *, int, void (*callback)(void *), void *); -int vme_lm_detach(struct vme_resource *, int); -void vme_lm_free(struct vme_resource *); - -int vme_slot_num(struct vme_dev *); -int vme_bus_num(struct vme_dev *); - -int vme_register_driver(struct vme_driver *, unsigned int); -void vme_unregister_driver(struct vme_driver *); - - -#endif /* _VME_H_ */ - -- cgit v1.2.3 From 2f8c3ae8288e4a4018330ed5c4e758b878d9c555 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Wed, 1 Jun 2022 00:07:00 -0700 Subject: driver core: Add wait_for_init_devices_probe helper function Some devices might need to be probed and bound successfully before the kernel boot sequence can finish and move on to init/userspace. For example, a network interface might need to be bound to be able to mount a NFS rootfs. With fw_devlink=on by default, some of these devices might be blocked from probing because they are waiting on a optional supplier that doesn't have a driver. While fw_devlink will eventually identify such devices and unblock the probing automatically, it might be too late by the time it unblocks the probing of devices. For example, the IP4 autoconfig might timeout before fw_devlink unblocks probing of the network interface. This function is available to temporarily try and probe all devices that have a driver even if some of their suppliers haven't been added or don't have drivers. The drivers can then decide which of the suppliers are optional vs mandatory and probe the device if possible. By the time this function returns, all such "best effort" probes are guaranteed to be completed. If a device successfully probes in this mode, we delete all fw_devlink discovered dependencies of that device where the supplier hasn't yet probed successfully because they have to be optional dependencies. This also means that some devices that aren't needed for init and could have waited for their optional supplier to probe (when the supplier's module is loaded later on) would end up probing prematurely with limited functionality. So call this function only when boot would fail without it. Tested-by: Geert Uytterhoeven Signed-off-by: Saravana Kannan Link: https://lore.kernel.org/r/20220601070707.3946847-5-saravanak@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/base.h | 1 + drivers/base/core.c | 100 +++++++++++++++++++++++++++++++++++++++--- drivers/base/dd.c | 19 +++++--- include/linux/device/driver.h | 1 + 4 files changed, 110 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/base.h b/drivers/base/base.h index ab71403d102f..b3a43a164dcd 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -160,6 +160,7 @@ extern int devres_release_all(struct device *dev); extern void device_block_probing(void); extern void device_unblock_probing(void); extern void deferred_probe_extend_timeout(void); +extern void driver_deferred_probe_trigger(void); /* /sys/devices directory */ extern struct kset *devices_kset; diff --git a/drivers/base/core.c b/drivers/base/core.c index 7cd789c4985d..61fdfe99b348 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -54,6 +54,7 @@ static unsigned int defer_sync_state_count = 1; static DEFINE_MUTEX(fwnode_link_lock); static bool fw_devlink_is_permissive(void); static bool fw_devlink_drv_reg_done; +static bool fw_devlink_best_effort; /** * fwnode_link_add - Create a link between two fwnode_handles. @@ -965,6 +966,11 @@ static void device_links_missing_supplier(struct device *dev) } } +static bool dev_is_best_effort(struct device *dev) +{ + return fw_devlink_best_effort && dev->can_match; +} + /** * device_links_check_suppliers - Check presence of supplier drivers. * @dev: Consumer device. @@ -984,7 +990,7 @@ static void device_links_missing_supplier(struct device *dev) int device_links_check_suppliers(struct device *dev) { struct device_link *link; - int ret = 0; + int ret = 0, fwnode_ret = 0; struct fwnode_handle *sup_fw; /* @@ -997,12 +1003,17 @@ int device_links_check_suppliers(struct device *dev) sup_fw = list_first_entry(&dev->fwnode->suppliers, struct fwnode_link, c_hook)->supplier; - dev_err_probe(dev, -EPROBE_DEFER, "wait for supplier %pfwP\n", - sup_fw); - mutex_unlock(&fwnode_link_lock); - return -EPROBE_DEFER; + if (!dev_is_best_effort(dev)) { + fwnode_ret = -EPROBE_DEFER; + dev_err_probe(dev, -EPROBE_DEFER, + "wait for supplier %pfwP\n", sup_fw); + } else { + fwnode_ret = -EAGAIN; + } } mutex_unlock(&fwnode_link_lock); + if (fwnode_ret == -EPROBE_DEFER) + return fwnode_ret; device_links_write_lock(); @@ -1012,6 +1023,14 @@ int device_links_check_suppliers(struct device *dev) if (link->status != DL_STATE_AVAILABLE && !(link->flags & DL_FLAG_SYNC_STATE_ONLY)) { + + if (dev_is_best_effort(dev) && + link->flags & DL_FLAG_INFERRED && + !link->supplier->can_match) { + ret = -EAGAIN; + continue; + } + device_links_missing_supplier(dev); dev_err_probe(dev, -EPROBE_DEFER, "supplier %s not ready\n", @@ -1024,7 +1043,8 @@ int device_links_check_suppliers(struct device *dev) dev->links.status = DL_DEV_PROBING; device_links_write_unlock(); - return ret; + + return ret ? ret : fwnode_ret; } /** @@ -1289,6 +1309,18 @@ void device_links_driver_bound(struct device *dev) * save to drop the managed link completely. */ device_link_drop_managed(link); + } else if (dev_is_best_effort(dev) && + link->flags & DL_FLAG_INFERRED && + link->status != DL_STATE_CONSUMER_PROBE && + !link->supplier->can_match) { + /* + * When dev_is_best_effort() is true, we ignore device + * links to suppliers that don't have a driver. If the + * consumer device still managed to probe, there's no + * point in maintaining a device link in a weird state + * (consumer probed before supplier). So delete it. + */ + device_link_drop_managed(link); } else { WARN_ON(link->status != DL_STATE_CONSUMER_PROBE); WRITE_ONCE(link->status, DL_STATE_ACTIVE); @@ -1655,6 +1687,62 @@ void fw_devlink_drivers_done(void) device_links_write_unlock(); } +/** + * wait_for_init_devices_probe - Try to probe any device needed for init + * + * Some devices might need to be probed and bound successfully before the kernel + * boot sequence can finish and move on to init/userspace. For example, a + * network interface might need to be bound to be able to mount a NFS rootfs. + * + * With fw_devlink=on by default, some of these devices might be blocked from + * probing because they are waiting on a optional supplier that doesn't have a + * driver. While fw_devlink will eventually identify such devices and unblock + * the probing automatically, it might be too late by the time it unblocks the + * probing of devices. For example, the IP4 autoconfig might timeout before + * fw_devlink unblocks probing of the network interface. + * + * This function is available to temporarily try and probe all devices that have + * a driver even if some of their suppliers haven't been added or don't have + * drivers. + * + * The drivers can then decide which of the suppliers are optional vs mandatory + * and probe the device if possible. By the time this function returns, all such + * "best effort" probes are guaranteed to be completed. If a device successfully + * probes in this mode, we delete all fw_devlink discovered dependencies of that + * device where the supplier hasn't yet probed successfully because they have to + * be optional dependencies. + * + * Any devices that didn't successfully probe go back to being treated as if + * this function was never called. + * + * This also means that some devices that aren't needed for init and could have + * waited for their optional supplier to probe (when the supplier's module is + * loaded later on) would end up probing prematurely with limited functionality. + * So call this function only when boot would fail without it. + */ +void __init wait_for_init_devices_probe(void) +{ + if (!fw_devlink_flags || fw_devlink_is_permissive()) + return; + + /* + * Wait for all ongoing probes to finish so that the "best effort" is + * only applied to devices that can't probe otherwise. + */ + wait_for_device_probe(); + + pr_info("Trying to probe devices needed for running init ...\n"); + fw_devlink_best_effort = true; + driver_deferred_probe_trigger(); + + /* + * Wait for all "best effort" probes to finish before going back to + * normal enforcement. + */ + wait_for_device_probe(); + fw_devlink_best_effort = false; +} + static void fw_devlink_unblock_consumers(struct device *dev) { struct device_link *link; diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 11b0fb6414d3..4a55fbb7e0da 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -172,7 +172,7 @@ static bool driver_deferred_probe_enable; * changes in the midst of a probe, then deferred processing should be triggered * again. */ -static void driver_deferred_probe_trigger(void) +void driver_deferred_probe_trigger(void) { if (!driver_deferred_probe_enable) return; @@ -580,7 +580,7 @@ static int really_probe(struct device *dev, struct device_driver *drv) { bool test_remove = IS_ENABLED(CONFIG_DEBUG_TEST_DRIVER_REMOVE) && !drv->suppress_bind_attrs; - int ret; + int ret, link_ret; if (defer_all_probes) { /* @@ -592,9 +592,9 @@ static int really_probe(struct device *dev, struct device_driver *drv) return -EPROBE_DEFER; } - ret = device_links_check_suppliers(dev); - if (ret) - return ret; + link_ret = device_links_check_suppliers(dev); + if (link_ret == -EPROBE_DEFER) + return link_ret; pr_debug("bus: '%s': %s: probing driver %s with device %s\n", drv->bus->name, __func__, drv->name, dev_name(dev)); @@ -633,6 +633,15 @@ re_probe: ret = call_driver_probe(dev, drv); if (ret) { + /* + * If fw_devlink_best_effort is active (denoted by -EAGAIN), the + * device might actually probe properly once some of its missing + * suppliers have probed. So, treat this as if the driver + * returned -EPROBE_DEFER. + */ + if (link_ret == -EAGAIN) + ret = -EPROBE_DEFER; + /* * Return probe errors as positive values so that the callers * can distinguish them from other errors. diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index 700453017e1c..2114d65b862f 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -129,6 +129,7 @@ extern struct device_driver *driver_find(const char *name, struct bus_type *bus); extern int driver_probe_done(void); extern void wait_for_device_probe(void); +void __init wait_for_init_devices_probe(void); /* sysfs interface for exporting driver attributes */ -- cgit v1.2.3 From 9cbffc7a59561be950ecc675d19a3d2b45202b2b Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Wed, 1 Jun 2022 00:07:05 -0700 Subject: driver core: Delete driver_deferred_probe_check_state() The function is no longer used. So delete it. Tested-by: Geert Uytterhoeven Signed-off-by: Saravana Kannan Link: https://lore.kernel.org/r/20220601070707.3946847-10-saravanak@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/dd.c | 30 ------------------------------ include/linux/device/driver.h | 1 - 2 files changed, 31 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 335e71d3a618..e600dd2afc35 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -274,42 +274,12 @@ static int __init deferred_probe_timeout_setup(char *str) } __setup("deferred_probe_timeout=", deferred_probe_timeout_setup); -/** - * driver_deferred_probe_check_state() - Check deferred probe state - * @dev: device to check - * - * Return: - * * -ENODEV if initcalls have completed and modules are disabled. - * * -ETIMEDOUT if the deferred probe timeout was set and has expired - * and modules are enabled. - * * -EPROBE_DEFER in other cases. - * - * Drivers or subsystems can opt-in to calling this function instead of directly - * returning -EPROBE_DEFER. - */ -int driver_deferred_probe_check_state(struct device *dev) -{ - if (!IS_ENABLED(CONFIG_MODULES) && initcalls_done) { - dev_warn(dev, "ignoring dependency for device, assuming no driver\n"); - return -ENODEV; - } - - if (!driver_deferred_probe_timeout && initcalls_done) { - dev_warn(dev, "deferred probe timeout, ignoring dependency\n"); - return -ETIMEDOUT; - } - - return -EPROBE_DEFER; -} -EXPORT_SYMBOL_GPL(driver_deferred_probe_check_state); - static void deferred_probe_timeout_work_func(struct work_struct *work) { struct device_private *p; fw_devlink_drivers_done(); - driver_deferred_probe_timeout = 0; driver_deferred_probe_trigger(); flush_work(&deferred_probe_work); diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index 2114d65b862f..7acaabde5396 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -242,7 +242,6 @@ driver_find_device_by_acpi_dev(struct device_driver *drv, const void *adev) extern int driver_deferred_probe_timeout; void driver_deferred_probe_add(struct device *dev); -int driver_deferred_probe_check_state(struct device *dev); void driver_init(void); /** -- cgit v1.2.3 From 82b070beae1ef55b0049768c8dc91d87565bb191 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 10 Jun 2022 15:02:18 +0300 Subject: driver core: Introduce device_find_any_child() helper There are several places in the kernel where this kind of functionality is being used. Provide a generic helper for such cases. Reviewed-by: Rafael J. Wysocki Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20220610120219.18988-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 20 ++++++++++++++++++++ include/linux/device.h | 2 ++ 2 files changed, 22 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 977b379a495b..839f64485a55 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -3920,6 +3920,26 @@ struct device *device_find_child_by_name(struct device *parent, } EXPORT_SYMBOL_GPL(device_find_child_by_name); +static int match_any(struct device *dev, void *unused) +{ + return 1; +} + +/** + * device_find_any_child - device iterator for locating a child device, if any. + * @parent: parent struct device + * + * This is similar to the device_find_child() function above, but it + * returns a reference to a child device, if any. + * + * NOTE: you will need to drop the reference with put_device() after use. + */ +struct device *device_find_any_child(struct device *parent) +{ + return device_find_child(parent, NULL, match_any); +} +EXPORT_SYMBOL_GPL(device_find_any_child); + int __init devices_init(void) { devices_kset = kset_create_and_add("devices", &device_uevent_ops, NULL); diff --git a/include/linux/device.h b/include/linux/device.h index dc941997795c..424b55df0272 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -905,6 +905,8 @@ struct device *device_find_child(struct device *dev, void *data, int (*match)(struct device *dev, void *data)); struct device *device_find_child_by_name(struct device *parent, const char *name); +struct device *device_find_any_child(struct device *parent); + int device_rename(struct device *dev, const char *new_name); int device_move(struct device *dev, struct device *new_parent, enum dpm_order dpm_order); -- cgit v1.2.3 From 73b4b53276a1d6290cd4f47dbbc885b6e6e59ac6 Mon Sep 17 00:00:00 2001 From: Andrey Grodzovsky Date: Thu, 19 May 2022 09:47:28 -0400 Subject: Revert "workqueue: remove unused cancel_work()" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 6417250d3f894e66a68ba1cd93676143f2376a6f. amdpgu need this function in order to prematurly stop pending reset works when another reset work already in progress. Acked-by: Tejun Heo Signed-off-by: Andrey Grodzovsky Reviewed-by: Lai Jiangshan Reviewed-by: Christian König Signed-off-by: Alex Deucher --- include/linux/workqueue.h | 1 + kernel/workqueue.c | 9 +++++++++ 2 files changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index 7fee9b6cfede..9e41e1226193 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -453,6 +453,7 @@ extern int schedule_on_each_cpu(work_func_t func); int execute_in_process_context(work_func_t fn, struct execute_work *); extern bool flush_work(struct work_struct *work); +extern bool cancel_work(struct work_struct *work); extern bool cancel_work_sync(struct work_struct *work); extern bool flush_delayed_work(struct delayed_work *dwork); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 0d2514b4ff0d..20d226d5bbc2 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3258,6 +3258,15 @@ static bool __cancel_work(struct work_struct *work, bool is_dwork) return ret; } +/* + * See cancel_delayed_work() + */ +bool cancel_work(struct work_struct *work) +{ + return __cancel_work(work, false); +} +EXPORT_SYMBOL(cancel_work); + /** * cancel_delayed_work - cancel a delayed work * @dwork: delayed_work to cancel -- cgit v1.2.3 From 36518b6b4da7e8d4387bc19ad21e772f1060e9d7 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 7 Jun 2022 16:04:03 -0400 Subject: teach iomap_dio_rw() to suppress dsync New flag, equivalent to removal of IOCB_DSYNC from iocb flags. This mimics what btrfs is doing (and that's what btrfs will switch to). However, I'm not at all sure that we want to suppress REQ_FUA for those - all btrfs hack really cares about is suppression of generic_write_sync(). For now let's keep the existing behaviour, but I really want to hear more detailed arguments pro or contra. [folded brain fix from willy] Suggested-by: Christoph Hellwig Reviewed-by: Christian Brauner (Microsoft) Signed-off-by: Al Viro --- fs/iomap/direct-io.c | 20 +++++++++++--------- include/linux/iomap.h | 6 ++++++ 2 files changed, 17 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 370c3241618a..c10c69e2de24 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -548,17 +548,19 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, } /* for data sync or sync, we need sync completion processing */ - if (iocb->ki_flags & IOCB_DSYNC) + if (iocb->ki_flags & IOCB_DSYNC && + !(dio_flags & IOMAP_DIO_NOSYNC)) { dio->flags |= IOMAP_DIO_NEED_SYNC; - /* - * For datasync only writes, we optimistically try using FUA for - * this IO. Any non-FUA write that occurs will clear this flag, - * hence we know before completion whether a cache flush is - * necessary. - */ - if ((iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC)) == IOCB_DSYNC) - dio->flags |= IOMAP_DIO_WRITE_FUA; + /* + * For datasync only writes, we optimistically try + * using FUA for this IO. Any non-FUA write that + * occurs will clear this flag, hence we know before + * completion whether a cache flush is necessary. + */ + if (!(iocb->ki_flags & IOCB_SYNC)) + dio->flags |= IOMAP_DIO_WRITE_FUA; + } } if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) { diff --git a/include/linux/iomap.h b/include/linux/iomap.h index e552097c67e0..c8622d8f064e 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -353,6 +353,12 @@ struct iomap_dio_ops { */ #define IOMAP_DIO_PARTIAL (1 << 2) +/* + * The caller will sync the write if needed; do not sync it within + * iomap_dio_rw. Overrides IOMAP_DIO_FORCE_WAIT. + */ +#define IOMAP_DIO_NOSYNC (1 << 3) + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before); -- cgit v1.2.3 From e87f2c26c8085dac59978dee1beeb05cef31a9dd Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 22 May 2022 09:28:12 -0400 Subject: struct file: use anonymous union member for rcuhead and llist Once upon a time we couldn't afford anon unions; these days minimal gcc version had been raised enough to take care of that. Reviewed-by: Jan Kara Reviewed-by: Christian Brauner (Microsoft) Signed-off-by: Al Viro --- fs/file_table.c | 16 ++++++++-------- include/linux/fs.h | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/fs/file_table.c b/fs/file_table.c index 5424e3a8df5f..b989e33aacda 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -45,7 +45,7 @@ static struct percpu_counter nr_files __cacheline_aligned_in_smp; static void file_free_rcu(struct rcu_head *head) { - struct file *f = container_of(head, struct file, f_u.fu_rcuhead); + struct file *f = container_of(head, struct file, f_rcuhead); put_cred(f->f_cred); kmem_cache_free(filp_cachep, f); @@ -56,7 +56,7 @@ static inline void file_free(struct file *f) security_file_free(f); if (!(f->f_mode & FMODE_NOACCOUNT)) percpu_counter_dec(&nr_files); - call_rcu(&f->f_u.fu_rcuhead, file_free_rcu); + call_rcu(&f->f_rcuhead, file_free_rcu); } /* @@ -142,7 +142,7 @@ static struct file *__alloc_file(int flags, const struct cred *cred) f->f_cred = get_cred(cred); error = security_file_alloc(f); if (unlikely(error)) { - file_free_rcu(&f->f_u.fu_rcuhead); + file_free_rcu(&f->f_rcuhead); return ERR_PTR(error); } @@ -341,13 +341,13 @@ static void delayed_fput(struct work_struct *unused) struct llist_node *node = llist_del_all(&delayed_fput_list); struct file *f, *t; - llist_for_each_entry_safe(f, t, node, f_u.fu_llist) + llist_for_each_entry_safe(f, t, node, f_llist) __fput(f); } static void ____fput(struct callback_head *work) { - __fput(container_of(work, struct file, f_u.fu_rcuhead)); + __fput(container_of(work, struct file, f_rcuhead)); } /* @@ -374,8 +374,8 @@ void fput(struct file *file) struct task_struct *task = current; if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) { - init_task_work(&file->f_u.fu_rcuhead, ____fput); - if (!task_work_add(task, &file->f_u.fu_rcuhead, TWA_RESUME)) + init_task_work(&file->f_rcuhead, ____fput); + if (!task_work_add(task, &file->f_rcuhead, TWA_RESUME)) return; /* * After this task has run exit_task_work(), @@ -384,7 +384,7 @@ void fput(struct file *file) */ } - if (llist_add(&file->f_u.fu_llist, &delayed_fput_list)) + if (llist_add(&file->f_llist, &delayed_fput_list)) schedule_delayed_work(&delayed_fput_work, 1); } } diff --git a/include/linux/fs.h b/include/linux/fs.h index 9ad5e3520fae..6a2a4906041f 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -924,9 +924,9 @@ static inline int ra_has_index(struct file_ra_state *ra, pgoff_t index) struct file { union { - struct llist_node fu_llist; - struct rcu_head fu_rcuhead; - } f_u; + struct llist_node f_llist; + struct rcu_head f_rcuhead; + }; struct path f_path; struct inode *f_inode; /* cached value */ const struct file_operations *f_op; -- cgit v1.2.3 From 91b94c5d6ae55d1161633047ffeea644b110b35f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 22 May 2022 09:39:27 -0400 Subject: iocb: delay evaluation of IS_SYNC(...) until we want to check IOCB_DSYNC New helper to be used instead of direct checks for IOCB_DSYNC: iocb_is_dsync(iocb). Checks converted, which allows to avoid the IS_SYNC(iocb->ki_filp->f_mapping->host) part (4 cache lines) from iocb_flags() - it's checked in iocb_is_dsync() instead Reviewed-by: Christian Brauner (Microsoft) Signed-off-by: Al Viro --- block/fops.c | 2 +- fs/btrfs/file.c | 2 +- fs/direct-io.c | 2 +- fs/fuse/file.c | 2 +- fs/iomap/direct-io.c | 3 +-- fs/zonefs/super.c | 2 +- include/linux/fs.h | 10 ++++++++-- 7 files changed, 14 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/block/fops.c b/block/fops.c index d6b3276a6c68..6e86931ab847 100644 --- a/block/fops.c +++ b/block/fops.c @@ -37,7 +37,7 @@ static unsigned int dio_bio_write_op(struct kiocb *iocb) unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; /* avoid the need for a I/O completion work item */ - if (iocb->ki_flags & IOCB_DSYNC) + if (iocb_is_dsync(iocb)) op |= REQ_FUA; return op; } diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 98f81e304eb1..54358a5c9d56 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2021,7 +2021,7 @@ ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from, struct file *file = iocb->ki_filp; struct btrfs_inode *inode = BTRFS_I(file_inode(file)); ssize_t num_written, num_sync; - const bool sync = iocb->ki_flags & IOCB_DSYNC; + const bool sync = iocb_is_dsync(iocb); /* * If the fs flips readonly due to some impossible error, although we diff --git a/fs/direct-io.c b/fs/direct-io.c index 840752006f60..39647eb56904 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -1210,7 +1210,7 @@ ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, */ if (dio->is_async && iov_iter_rw(iter) == WRITE) { retval = 0; - if (iocb->ki_flags & IOCB_DSYNC) + if (iocb_is_dsync(iocb)) retval = dio_set_defer_completion(dio); else if (!dio->inode->i_sb->s_dio_done_wq) { /* diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 05caa2b9272e..00fa861aeead 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1042,7 +1042,7 @@ static unsigned int fuse_write_flags(struct kiocb *iocb) { unsigned int flags = iocb->ki_filp->f_flags; - if (iocb->ki_flags & IOCB_DSYNC) + if (iocb_is_dsync(iocb)) flags |= O_DSYNC; if (iocb->ki_flags & IOCB_SYNC) flags |= O_SYNC; diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index c10c69e2de24..31c7f1035b20 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -548,8 +548,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, } /* for data sync or sync, we need sync completion processing */ - if (iocb->ki_flags & IOCB_DSYNC && - !(dio_flags & IOMAP_DIO_NOSYNC)) { + if (iocb_is_dsync(iocb) && !(dio_flags & IOMAP_DIO_NOSYNC)) { dio->flags |= IOMAP_DIO_NEED_SYNC; /* diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index bcb21aea990a..04a98b4cd7ee 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -746,7 +746,7 @@ static ssize_t zonefs_file_dio_append(struct kiocb *iocb, struct iov_iter *from) REQ_OP_ZONE_APPEND | REQ_SYNC | REQ_IDLE, GFP_NOFS); bio->bi_iter.bi_sector = zi->i_zsector; bio->bi_ioprio = iocb->ki_ioprio; - if (iocb->ki_flags & IOCB_DSYNC) + if (iocb_is_dsync(iocb)) bio->bi_opf |= REQ_FUA; ret = bio_iov_iter_get_pages(bio, from); diff --git a/include/linux/fs.h b/include/linux/fs.h index 6a2a4906041f..380a1292f4f9 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2720,6 +2720,12 @@ extern int vfs_fsync(struct file *file, int datasync); extern int sync_file_range(struct file *file, loff_t offset, loff_t nbytes, unsigned int flags); +static inline bool iocb_is_dsync(const struct kiocb *iocb) +{ + return (iocb->ki_flags & IOCB_DSYNC) || + IS_SYNC(iocb->ki_filp->f_mapping->host); +} + /* * Sync the bytes written if this was a synchronous write. Expect ki_pos * to already be updated for the write, and will return either the amount @@ -2727,7 +2733,7 @@ extern int sync_file_range(struct file *file, loff_t offset, loff_t nbytes, */ static inline ssize_t generic_write_sync(struct kiocb *iocb, ssize_t count) { - if (iocb->ki_flags & IOCB_DSYNC) { + if (iocb_is_dsync(iocb)) { int ret = vfs_fsync_range(iocb->ki_filp, iocb->ki_pos - count, iocb->ki_pos - 1, (iocb->ki_flags & IOCB_SYNC) ? 0 : 1); @@ -3262,7 +3268,7 @@ static inline int iocb_flags(struct file *file) res |= IOCB_APPEND; if (file->f_flags & O_DIRECT) res |= IOCB_DIRECT; - if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host)) + if (file->f_flags & O_DSYNC) res |= IOCB_DSYNC; if (file->f_flags & __O_SYNC) res |= IOCB_SYNC; -- cgit v1.2.3 From 164f4064ca81eefcea29f7f5dcf394f92be1d0c0 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 22 May 2022 11:38:11 -0400 Subject: keep iocb_flags() result cached in struct file * calculate at the time we set FMODE_OPENED (do_dentry_open() for normal opens, alloc_file() for pipe()/socket()/etc.) * update when handling F_SETFL * keep in a new field - file->f_iocb_flags; since that thing is needed only before the refcount reaches zero, we can put it into the same anon union where ->f_rcuhead and ->f_llist live - those are used only after refcount reaches zero. Reviewed-by: Christian Brauner (Microsoft) Signed-off-by: Al Viro --- drivers/nvme/target/io-cmd-file.c | 2 +- fs/aio.c | 2 +- fs/fcntl.c | 1 + fs/file_table.c | 1 + fs/io_uring.c | 2 +- fs/open.c | 1 + include/linux/fs.h | 5 ++--- 7 files changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index f3d58abf11e0..64b47e2a4633 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -112,7 +112,7 @@ static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos, iocb->ki_pos = pos; iocb->ki_filp = req->ns->file; - iocb->ki_flags = ki_flags | iocb_flags(req->ns->file); + iocb->ki_flags = ki_flags | iocb->ki_filp->f_iocb_flags; return call_iter(iocb, &iter); } diff --git a/fs/aio.c b/fs/aio.c index 3c249b938632..2bdd444d408b 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1475,7 +1475,7 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) req->ki_complete = aio_complete_rw; req->private = NULL; req->ki_pos = iocb->aio_offset; - req->ki_flags = iocb_flags(req->ki_filp); + req->ki_flags = req->ki_filp->f_iocb_flags; if (iocb->aio_flags & IOCB_FLAG_RESFD) req->ki_flags |= IOCB_EVENTFD; if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { diff --git a/fs/fcntl.c b/fs/fcntl.c index 34a3faa4886d..146c9ab0cd4b 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -78,6 +78,7 @@ static int setfl(int fd, struct file * filp, unsigned long arg) } spin_lock(&filp->f_lock); filp->f_flags = (arg & SETFL_MASK) | (filp->f_flags & ~SETFL_MASK); + filp->f_iocb_flags = iocb_flags(filp); spin_unlock(&filp->f_lock); out: diff --git a/fs/file_table.c b/fs/file_table.c index b989e33aacda..905792b0521c 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -241,6 +241,7 @@ static struct file *alloc_file(const struct path *path, int flags, if ((file->f_mode & FMODE_WRITE) && likely(fop->write || fop->write_iter)) file->f_mode |= FMODE_CAN_WRITE; + file->f_iocb_flags = iocb_flags(file); file->f_mode |= FMODE_OPENED; file->f_op = fop; if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3aab4182fd89..53424b1f019f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4330,7 +4330,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) if (!io_req_ffs_set(req)) req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT; - kiocb->ki_flags = iocb_flags(file); + kiocb->ki_flags = file->f_iocb_flags; ret = kiocb_set_rw_flags(kiocb, req->rw.flags); if (unlikely(ret)) return ret; diff --git a/fs/open.c b/fs/open.c index 1d57fbde2feb..d80441a0bf17 100644 --- a/fs/open.c +++ b/fs/open.c @@ -862,6 +862,7 @@ static int do_dentry_open(struct file *f, f->f_mode |= FMODE_CAN_ODIRECT; f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC); + f->f_iocb_flags = iocb_flags(f); file_ra_state_init(&f->f_ra, f->f_mapping->host->i_mapping); diff --git a/include/linux/fs.h b/include/linux/fs.h index 380a1292f4f9..c82b9d442f56 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -926,6 +926,7 @@ struct file { union { struct llist_node f_llist; struct rcu_head f_rcuhead; + unsigned int f_iocb_flags; }; struct path f_path; struct inode *f_inode; /* cached value */ @@ -2199,13 +2200,11 @@ static inline bool HAS_UNMAPPED_ID(struct user_namespace *mnt_userns, !gid_valid(i_gid_into_mnt(mnt_userns, inode)); } -static inline int iocb_flags(struct file *file); - static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp) { *kiocb = (struct kiocb) { .ki_filp = filp, - .ki_flags = iocb_flags(filp), + .ki_flags = filp->f_iocb_flags, .ki_ioprio = get_current_ioprio(), }; } -- cgit v1.2.3 From 7cbb6681d7e5b88688234ad370e027a9346ff7a9 Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Wed, 27 Apr 2022 12:08:04 -0700 Subject: iio: common: cros_ec_sensors: Add label attribute When sensor location is known, populate iio sysfs "label" attribute: * "accel-base" : the sensor is in the base of the convertible (2-1) device. * "accel-display" : the sensor is in the lid/display plane of the device. * "accel-camera" : the sensor is in the swivel camera subassembly. The non-standard |location| attribute is removed, the field |loc| in cros_ec_sensors_core_state is removed. It apply to standalone accelerometer as well as IMU (accelerometer + gyroscope) and sensors where the location is known (light). Signed-off-by: Gwendal Grignou Link: https://lore.kernel.org/r/20220427190804.961697-3-gwendal@chromium.org Signed-off-by: Jonathan Cameron --- drivers/iio/accel/cros_ec_accel_legacy.c | 2 +- .../common/cros_ec_sensors/cros_ec_sensors_core.c | 30 +++++++--------------- drivers/iio/light/cros_ec_light_prox.c | 2 -- drivers/iio/pressure/cros_ec_baro.c | 2 -- include/linux/iio/common/cros_ec_sensors_core.h | 2 -- 5 files changed, 10 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/accel/cros_ec_accel_legacy.c b/drivers/iio/accel/cros_ec_accel_legacy.c index b6f3471b62dc..1c0171f26e99 100644 --- a/drivers/iio/accel/cros_ec_accel_legacy.c +++ b/drivers/iio/accel/cros_ec_accel_legacy.c @@ -230,7 +230,7 @@ static int cros_ec_accel_legacy_probe(struct platform_device *pdev) indio_dev->channels = cros_ec_accel_legacy_channels; indio_dev->num_channels = ARRAY_SIZE(cros_ec_accel_legacy_channels); /* The lid sensor needs to be presented inverted. */ - if (state->loc == MOTIONSENSE_LOC_LID) { + if (!strcmp(indio_dev->label, "accel-display")) { state->sign[CROS_EC_SENSOR_X] = -1; state->sign[CROS_EC_SENSOR_Z] = -1; } diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index 5976aca48e3b..e5ccedef13a8 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -29,12 +29,6 @@ */ #define CROS_EC_FIFO_SIZE (2048 * 2 / 3) -static char *cros_ec_loc[] = { - [MOTIONSENSE_LOC_BASE] = "base", - [MOTIONSENSE_LOC_LID] = "lid", - [MOTIONSENSE_LOC_MAX] = "unknown", -}; - static int cros_ec_get_host_cmd_version_mask(struct cros_ec_device *ec_dev, u16 cmd_offset, u16 cmd, u32 *mask) { @@ -287,6 +281,8 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, indio_dev->name = pdev->name; if (physical_device) { + enum motionsensor_location loc; + state->param.cmd = MOTIONSENSE_CMD_INFO; state->param.info.sensor_num = sensor_platform->sensor_num; ret = cros_ec_motion_send_host_cmd(state, 0); @@ -295,7 +291,13 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, return ret; } state->type = state->resp->info.type; - state->loc = state->resp->info.location; + loc = state->resp->info.location; + if (loc == MOTIONSENSE_LOC_BASE) + indio_dev->label = "accel-base"; + else if (loc == MOTIONSENSE_LOC_LID) + indio_dev->label = "accel-display"; + else if (loc == MOTIONSENSE_LOC_CAMERA) + indio_dev->label = "accel-camera"; /* Set sign vector, only used for backward compatibility. */ memset(state->sign, 1, CROS_EC_SENSOR_MAX_AXIS); @@ -442,15 +444,6 @@ static ssize_t cros_ec_sensors_id(struct iio_dev *indio_dev, return snprintf(buf, PAGE_SIZE, "%d\n", st->param.info.sensor_num); } -static ssize_t cros_ec_sensors_loc(struct iio_dev *indio_dev, - uintptr_t private, const struct iio_chan_spec *chan, - char *buf) -{ - struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); - - return snprintf(buf, PAGE_SIZE, "%s\n", cros_ec_loc[st->loc]); -} - const struct iio_chan_spec_ext_info cros_ec_sensors_ext_info[] = { { .name = "calibrate", @@ -462,11 +455,6 @@ const struct iio_chan_spec_ext_info cros_ec_sensors_ext_info[] = { .shared = IIO_SHARED_BY_ALL, .read = cros_ec_sensors_id }, - { - .name = "location", - .shared = IIO_SHARED_BY_ALL, - .read = cros_ec_sensors_loc - }, { }, }; EXPORT_SYMBOL_GPL(cros_ec_sensors_ext_info); diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c index de472f23d1cb..a00a8b3b86cf 100644 --- a/drivers/iio/light/cros_ec_light_prox.c +++ b/drivers/iio/light/cros_ec_light_prox.c @@ -188,8 +188,6 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev) indio_dev->info = &cros_ec_light_prox_info; state = iio_priv(indio_dev); - state->core.type = state->core.resp->info.type; - state->core.loc = state->core.resp->info.location; channel = state->channels; /* Common part */ diff --git a/drivers/iio/pressure/cros_ec_baro.c b/drivers/iio/pressure/cros_ec_baro.c index 2f882e109423..1d9d34ae3c0a 100644 --- a/drivers/iio/pressure/cros_ec_baro.c +++ b/drivers/iio/pressure/cros_ec_baro.c @@ -145,8 +145,6 @@ static int cros_ec_baro_probe(struct platform_device *pdev) indio_dev->info = &cros_ec_baro_info; state = iio_priv(indio_dev); - state->core.type = state->core.resp->info.type; - state->core.loc = state->core.resp->info.location; channel = state->channels; /* Common part */ channel->info_mask_separate = BIT(IIO_CHAN_INFO_RAW); diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index c582e1a14232..a8259c8822f5 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -41,7 +41,6 @@ typedef irqreturn_t (*cros_ec_sensors_capture_t)(int irq, void *p); * @param: motion sensor parameters structure * @resp: motion sensor response structure * @type: type of motion sensor - * @loc: location where the motion sensor is placed * @range_updated: True if the range of the sensor has been * updated. * @curr_range: If updated, the current range value. @@ -67,7 +66,6 @@ struct cros_ec_sensors_core_state { struct ec_response_motion_sense *resp; enum motionsensor_type type; - enum motionsensor_location loc; bool range_updated; int curr_range; -- cgit v1.2.3 From ccd8a9351f7b44bc1c0c8e4d39c0d6593b106fc4 Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Fri, 10 Jun 2022 23:30:08 +0900 Subject: can: skb: move can_dropped_invalid_skb() and can_skb_headroom_valid() to skb.c The functions can_dropped_invalid_skb() and can_skb_headroom_valid() grew a lot over the years to a point which it does not make much sense to have them defined as static inline in header files. Move those two functions to the .c counterpart of skb.h. can_skb_headroom_valid()'s only caller being can_dropped_invalid_skb(), the declaration is removed from the header. Only can_dropped_invalid_skb() gets its symbol exported. While doing so, do a small cleanup: add brackets around the else block in can_dropped_invalid_skb(). Link: https://lore.kernel.org/all/20220610143009.323579-7-mailhol.vincent@wanadoo.fr Signed-off-by: Vincent Mailhol Reported-by: kernel test robot Acked-by: Max Staudt Tested-by: Oliver Hartkopp Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/skb.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++ include/linux/can/skb.h | 59 +---------------------------------------------- 2 files changed, 59 insertions(+), 58 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index a4208f125b76..dc9da76c0470 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -259,3 +259,61 @@ struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf) return skb; } EXPORT_SYMBOL_GPL(alloc_can_err_skb); + +/* Check for outgoing skbs that have not been created by the CAN subsystem */ +static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb) +{ + /* af_packet creates a headroom of HH_DATA_MOD bytes which is fine */ + if (WARN_ON_ONCE(skb_headroom(skb) < sizeof(struct can_skb_priv))) + return false; + + /* af_packet does not apply CAN skb specific settings */ + if (skb->ip_summed == CHECKSUM_NONE) { + /* init headroom */ + can_skb_prv(skb)->ifindex = dev->ifindex; + can_skb_prv(skb)->skbcnt = 0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* perform proper loopback on capable devices */ + if (dev->flags & IFF_ECHO) + skb->pkt_type = PACKET_LOOPBACK; + else + skb->pkt_type = PACKET_HOST; + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + } + + return true; +} + +/* Drop a given socketbuffer if it does not contain a valid CAN frame. */ +bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb) +{ + const struct canfd_frame *cfd = (struct canfd_frame *)skb->data; + + if (skb->protocol == htons(ETH_P_CAN)) { + if (unlikely(skb->len != CAN_MTU || + cfd->len > CAN_MAX_DLEN)) + goto inval_skb; + } else if (skb->protocol == htons(ETH_P_CANFD)) { + if (unlikely(skb->len != CANFD_MTU || + cfd->len > CANFD_MAX_DLEN)) + goto inval_skb; + } else { + goto inval_skb; + } + + if (!can_skb_headroom_valid(dev, skb)) + goto inval_skb; + + return false; + +inval_skb: + kfree_skb(skb); + dev->stats.tx_dropped++; + return true; +} +EXPORT_SYMBOL_GPL(can_dropped_invalid_skb); diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index fdb22b00674a..182749e858b3 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -31,6 +31,7 @@ struct sk_buff *alloc_canfd_skb(struct net_device *dev, struct canfd_frame **cfd); struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf); +bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb); /* * The struct can_skb_priv is used to transport additional information along @@ -96,64 +97,6 @@ static inline struct sk_buff *can_create_echo_skb(struct sk_buff *skb) return nskb; } -/* Check for outgoing skbs that have not been created by the CAN subsystem */ -static inline bool can_skb_headroom_valid(struct net_device *dev, - struct sk_buff *skb) -{ - /* af_packet creates a headroom of HH_DATA_MOD bytes which is fine */ - if (WARN_ON_ONCE(skb_headroom(skb) < sizeof(struct can_skb_priv))) - return false; - - /* af_packet does not apply CAN skb specific settings */ - if (skb->ip_summed == CHECKSUM_NONE) { - /* init headroom */ - can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; - - skb->ip_summed = CHECKSUM_UNNECESSARY; - - /* perform proper loopback on capable devices */ - if (dev->flags & IFF_ECHO) - skb->pkt_type = PACKET_LOOPBACK; - else - skb->pkt_type = PACKET_HOST; - - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - } - - return true; -} - -/* Drop a given socketbuffer if it does not contain a valid CAN frame. */ -static inline bool can_dropped_invalid_skb(struct net_device *dev, - struct sk_buff *skb) -{ - const struct canfd_frame *cfd = (struct canfd_frame *)skb->data; - - if (skb->protocol == htons(ETH_P_CAN)) { - if (unlikely(skb->len != CAN_MTU || - cfd->len > CAN_MAX_DLEN)) - goto inval_skb; - } else if (skb->protocol == htons(ETH_P_CANFD)) { - if (unlikely(skb->len != CANFD_MTU || - cfd->len > CANFD_MAX_DLEN)) - goto inval_skb; - } else - goto inval_skb; - - if (!can_skb_headroom_valid(dev, skb)) - goto inval_skb; - - return false; - -inval_skb: - kfree_skb(skb); - dev->stats.tx_dropped++; - return true; -} - static inline bool can_is_canfd_skb(const struct sk_buff *skb) { /* the CAN specific type of skb is identified by its data length */ -- cgit v1.2.3 From 662a60102c122e44fdaf5c826f7f415eb57d48ad Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Mon, 2 May 2022 16:20:56 +0300 Subject: usb: typec: Separate USB Power Delivery from USB Type-C Introducing a small device class for USB Power Delivery. The idea with it is that we do not mix any more USB Power Delivery information into the USB Type-C connectors only. This separation will make it possible to register USB Power Delivery devices also from other places, for example from USB Type-C Bridges (see USB Type-C Bridge Specification). The device class will not always deal with only the messages and objects that were negotiated with the partner, but instead messages and objects that can be used in the negotiation. That allows the USB PD devices to be shared and reconfigured. The ports can decide which objects are to be advertised to the partner before the contract is negotiated. It is also possible to allow the user space to make that decision if needed. Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20220502132058.86236-2-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- .../ABI/testing/sysfs-class-usb_power_delivery | 240 +++++++ drivers/usb/typec/Makefile | 2 +- drivers/usb/typec/pd.c | 708 +++++++++++++++++++++ drivers/usb/typec/pd.h | 30 + include/linux/usb/pd.h | 38 ++ include/linux/usb/typec.h | 10 + 6 files changed, 1027 insertions(+), 1 deletion(-) create mode 100644 Documentation/ABI/testing/sysfs-class-usb_power_delivery create mode 100644 drivers/usb/typec/pd.c create mode 100644 drivers/usb/typec/pd.h (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-usb_power_delivery b/Documentation/ABI/testing/sysfs-class-usb_power_delivery new file mode 100644 index 000000000000..ce2b1b563cb3 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-usb_power_delivery @@ -0,0 +1,240 @@ +What: /sys/class/usb_power_delivery +Date: May 2022 +Contact: Heikki Krogerus +Description: + Directory for USB Power Delivery devices. + +What: /sys/class/usb_power_delivery/.../revision +Date: May 2022 +Contact: Heikki Krogerus +Description: + File showing the USB Power Delivery Specification Revision used + in communication. + +What: /sys/class/usb_power_delivery/.../version +Date: May 2022 +Contact: Heikki Krogerus +Description: + This is an optional attribute file showing the version of the + specific revision of the USB Power Delivery Specification. In + most cases the specification version is not known and the file + is not available. + +What: /sys/class/usb_power_delivery/.../source-capabilities +Date: May 2022 +Contact: Heikki Krogerus +Description: + The source capabilities message "Source_Capabilities" contains a + set of Power Data Objects (PDO), each representing a type of + power supply. The order of the PDO objects is defined in the USB + Power Delivery Specification. Each PDO - power supply - will + have its own device, and the PDO device name will start with the + object position number as the first character followed by the + power supply type name (":" as delimiter). + + /sys/class/usb_power_delivery/.../source_capabilities/: + +What: /sys/class/usb_power_delivery/.../sink-capabilities +Date: May 2022 +Contact: Heikki Krogerus +Description: + The sink capability message "Sink_Capabilities" contains a set + of Power Data Objects (PDO) just like with source capabilities, + but instead of describing the power capabilities, these objects + describe the power requirements. + + The order of the objects in the sink capability message is the + same as with the source capabilities message. + +Fixed Supplies + +What: /sys/class/usb_power_delivery/...//:fixed_supply +Date: May 2022 +Contact: Heikki Krogerus +Description: + Devices containing the attributes (the bit fields) defined for + Fixed Supplies. + + The device "1:fixed_supply" is special. USB Power Delivery + Specification dictates that the first PDO (at object position + 1), and the only mandatory PDO, is always the vSafe5V Fixed + Supply Object. vSafe5V Object has additional fields defined for + it that the other Fixed Supply Objects do not have and that are + related to the USB capabilities rather than power capabilities. + +What: /sys/class/usb_power_delivery/...//1:fixed_supply/dual_role_power +Date: May 2022 +Contact: Heikki Krogerus +Description: + This file contains boolean value that tells does the device + support both source and sink power roles. + +What: /sys/class/usb_power_delivery/...//1:fixed_supply/usb_suspend_supported +Date: May 2022 +Contact: Heikki Krogerus +Description: + This file shows the value of the USB Suspend Supported bit in + vSafe5V Fixed Supply Object. If the bit is set then the device + will follow the USB 2.0 and USB 3.2 rules for suspend and + resume. + +What: /sys/class/usb_power_delivery/...//1:fixed_supply/unconstrained_power +Date: May 2022 +Contact: Heikki Krogerus +Description: + This file shows the value of the Unconstrained Power bit in + vSafe5V Fixed Supply Object. The bit is set when an external + source of power, powerful enough to power the entire system on + its own, is available for the device. + +What: /sys/class/usb_power_delivery/...//1:fixed_supply/usb_communication_capable +Date: May 2022 +Contact: Heikki Krogerus +Description: + This file shows the value of the USB Communication Capable bit in + vSafe5V Fixed Supply Object. + +What: /sys/class/usb_power_delivery/...//1:fixed_supply/dual_role_data +Date: May 2022 +Contact: Heikki Krogerus +Description: + This file shows the value of the Dual-Role Data bit in vSafe5V + Fixed Supply Object. Dual role data means ability act as both + USB host and USB device. + +What: /sys/class/usb_power_delivery/...//1:fixed_supply/unchunked_extended_messages_supported +Date: May 2022 +Contact: Heikki Krogerus +Description: + This file shows the value of the Unchunked Extended Messages + Supported bit in vSafe5V Fixed Supply Object. + +What: /sys/class/usb_power_delivery/...//:fixed_supply/voltage +Date: May 2022 +Contact: Heikki Krogerus +Description: + The voltage the supply supports in millivolts. + +What: /sys/class/usb_power_delivery/.../source-capabilities/:fixed_supply/maximum_current +Date: May 2022 +Contact: Heikki Krogerus +Description: + Maximum current of the fixed source supply in milliamperes. + +What: /sys/class/usb_power_delivery/.../sink-capabilities/:fixed_supply/operational_current +Date: May 2022 +Contact: Heikki Krogerus +Description: + Operational current of the sink in milliamperes. + +What: /sys/class/usb_power_delivery/.../sink-capabilities/:fixed_supply/fast_role_swap_current +Date: May 2022 +Contact: Heikki Krogerus +Description: + This file contains the value of the "Fast Role Swap USB Type-C + Current" field that tells the current level the sink requires + after a Fast Role Swap. + 0 - Fast Swap not supported" + 1 - Default USB Power" + 2 - 1.5A@5V" + 3 - 3.0A@5V" + +Variable Supplies + +What: /sys/class/usb_power_delivery/...//:variable_supply +Date: May 2022 +Contact: Heikki Krogerus +Description: + Variable Power Supply PDO. + +What: /sys/class/usb_power_delivery/...//:variable_supply/maximum_voltage +Date: May 2022 +Contact: Heikki Krogerus +Description: + Maximum Voltage in millivolts. + +What: /sys/class/usb_power_delivery/...//:variable_supply/minimum_voltage +Date: May 2022 +Contact: Heikki Krogerus +Description: + Minimum Voltage in millivolts. + +What: /sys/class/usb_power_delivery/.../source-capabilities/:variable_supply/maximum_current +Date: May 2022 +Contact: Heikki Krogerus +Description: + The maximum current in milliamperes that the source can supply + at the given Voltage range. + +What: /sys/class/usb_power_delivery/.../sink-capabilities/:variable_supply/operational_current +Date: May 2022 +Contact: Heikki Krogerus +Description: + The operational current in milliamperes that the sink requires + at the given Voltage range. + +Battery Supplies + +What: /sys/class/usb_power_delivery/...//:battery +Date: May 2022 +Contact: Heikki Krogerus +Description: + Battery PDO. + +What: /sys/class/usb_power_delivery/...//:battery/maximum_voltage +Date: May 2022 +Contact: Heikki Krogerus +Description: + Maximum Voltage in millivolts. + +What: /sys/class/usb_power_delivery/...//:battery/minimum_voltage +Date: May 2022 +Contact: Heikki Krogerus +Description: + Minimum Voltage in millivolts. + +What: /sys/class/usb_power_delivery/.../source-capabilities/:battery/maximum_power +Date: May 2022 +Contact: Heikki Krogerus +Description: + Maximum allowable Power in milliwatts. + +What: /sys/class/usb_power_delivery/.../sink-capabilities/:battery/operational_power +Date: May 2022 +Contact: Heikki Krogerus +Description: + The operational power that the sink requires at the given + voltage range. + +Standard Power Range (SPR) Programmable Power Supplies + +What: /sys/class/usb_power_delivery/...//:programmable_supply +Date: May 2022 +Contact: Heikki Krogerus +Description: + Programmable Power Supply (PPS) Augmented PDO (APDO). + +What: /sys/class/usb_power_delivery/...//:programmable_supply/maximum_voltage +Date: May 2022 +Contact: Heikki Krogerus +Description: + Maximum Voltage in millivolts. + +What: /sys/class/usb_power_delivery/...//:programmable_supply/minimum_voltage +Date: May 2022 +Contact: Heikki Krogerus +Description: + Minimum Voltage in millivolts. + +What: /sys/class/usb_power_delivery/...//:programmable_supply/maximum_current +Date: May 2022 +Contact: Heikki Krogerus +Description: + Maximum Current in milliamperes. + +What: /sys/class/usb_power_delivery/.../source-capabilities/:programmable_supply/pps_power_limited +Date: May 2022 +Contact: Heikki Krogerus +Description: + The PPS Power Limited bit indicates whether or not the source + supply will exceed the rated output power if requested. diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index 43626acc0aaf..2f174cd3e5df 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_TYPEC) += typec.o -typec-y := class.o mux.o bus.o +typec-y := class.o mux.o bus.o pd.o typec-$(CONFIG_ACPI) += port-mapper.o obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ diff --git a/drivers/usb/typec/pd.c b/drivers/usb/typec/pd.c new file mode 100644 index 000000000000..dc72005d68db --- /dev/null +++ b/drivers/usb/typec/pd.c @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * USB Power Delivery sysfs entries + * + * Copyright (C) 2022, Intel Corporation + * Author: Heikki Krogerus + */ + +#include +#include + +#include "pd.h" + +static DEFINE_IDA(pd_ida); + +static struct class pd_class = { + .name = "usb_power_delivery", + .owner = THIS_MODULE, +}; + +#define to_pdo(o) container_of(o, struct pdo, dev) + +struct pdo { + struct device dev; + int object_position; + u32 pdo; +}; + +static void pdo_release(struct device *dev) +{ + kfree(to_pdo(dev)); +} + +/* -------------------------------------------------------------------------- */ +/* Fixed Supply */ + +static ssize_t +dual_role_power_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", !!(to_pdo(dev)->pdo & PDO_FIXED_DUAL_ROLE)); +} +static DEVICE_ATTR_RO(dual_role_power); + +static ssize_t +usb_suspend_supported_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", !!(to_pdo(dev)->pdo & PDO_FIXED_SUSPEND)); +} +static DEVICE_ATTR_RO(usb_suspend_supported); + +static ssize_t +unconstrained_power_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", !!(to_pdo(dev)->pdo & PDO_FIXED_EXTPOWER)); +} +static DEVICE_ATTR_RO(unconstrained_power); + +static ssize_t +usb_communication_capable_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", !!(to_pdo(dev)->pdo & PDO_FIXED_USB_COMM)); +} +static DEVICE_ATTR_RO(usb_communication_capable); + +static ssize_t +dual_role_data_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", !!(to_pdo(dev)->pdo & PDO_FIXED_DATA_SWAP)); +} +static DEVICE_ATTR_RO(dual_role_data); + +static ssize_t +unchunked_extended_messages_supported_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", !!(to_pdo(dev)->pdo & PDO_FIXED_UNCHUNK_EXT)); +} +static DEVICE_ATTR_RO(unchunked_extended_messages_supported); + +/* + * REVISIT: Peak Current requires access also to the RDO. +static ssize_t +peak_current_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + ... +} +*/ + +static ssize_t +fast_role_swap_current_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", to_pdo(dev)->pdo >> PDO_FIXED_FRS_CURR_SHIFT) & 3; +} +static DEVICE_ATTR_RO(fast_role_swap_current); + +static ssize_t voltage_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umV\n", pdo_fixed_voltage(to_pdo(dev)->pdo)); +} +static DEVICE_ATTR_RO(voltage); + +/* Shared with Variable supplies, both source and sink */ +static ssize_t current_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umA\n", pdo_max_current(to_pdo(dev)->pdo)); +} + +/* Shared with Variable type supplies */ +static struct device_attribute maximum_current_attr = { + .attr = { + .name = "maximum_current", + .mode = 0444, + }, + .show = current_show, +}; + +static struct device_attribute operational_current_attr = { + .attr = { + .name = "operational_current", + .mode = 0444, + }, + .show = current_show, +}; + +static struct attribute *source_fixed_supply_attrs[] = { + &dev_attr_dual_role_power.attr, + &dev_attr_usb_suspend_supported.attr, + &dev_attr_unconstrained_power.attr, + &dev_attr_usb_communication_capable.attr, + &dev_attr_dual_role_data.attr, + &dev_attr_unchunked_extended_messages_supported.attr, + /*&dev_attr_peak_current.attr,*/ + &dev_attr_voltage.attr, + &maximum_current_attr.attr, + NULL +}; + +static umode_t fixed_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + if (to_pdo(kobj_to_dev(kobj))->object_position && + /*attr != &dev_attr_peak_current.attr &&*/ + attr != &dev_attr_voltage.attr && + attr != &maximum_current_attr.attr && + attr != &operational_current_attr.attr) + return 0; + + return attr->mode; +} + +static const struct attribute_group source_fixed_supply_group = { + .is_visible = fixed_attr_is_visible, + .attrs = source_fixed_supply_attrs, +}; +__ATTRIBUTE_GROUPS(source_fixed_supply); + +static struct device_type source_fixed_supply_type = { + .name = "pdo", + .release = pdo_release, + .groups = source_fixed_supply_groups, +}; + +static struct attribute *sink_fixed_supply_attrs[] = { + &dev_attr_dual_role_power.attr, + &dev_attr_usb_suspend_supported.attr, + &dev_attr_unconstrained_power.attr, + &dev_attr_usb_communication_capable.attr, + &dev_attr_dual_role_data.attr, + &dev_attr_unchunked_extended_messages_supported.attr, + &dev_attr_fast_role_swap_current.attr, + &dev_attr_voltage.attr, + &operational_current_attr.attr, + NULL +}; + +static const struct attribute_group sink_fixed_supply_group = { + .is_visible = fixed_attr_is_visible, + .attrs = sink_fixed_supply_attrs, +}; +__ATTRIBUTE_GROUPS(sink_fixed_supply); + +static struct device_type sink_fixed_supply_type = { + .name = "pdo", + .release = pdo_release, + .groups = sink_fixed_supply_groups, +}; + +/* -------------------------------------------------------------------------- */ +/* Variable Supply */ + +static ssize_t +maximum_voltage_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umV\n", pdo_max_voltage(to_pdo(dev)->pdo)); +} +static DEVICE_ATTR_RO(maximum_voltage); + +static ssize_t +minimum_voltage_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umV\n", pdo_min_voltage(to_pdo(dev)->pdo)); +} +static DEVICE_ATTR_RO(minimum_voltage); + +static struct attribute *source_variable_supply_attrs[] = { + &dev_attr_maximum_voltage.attr, + &dev_attr_minimum_voltage.attr, + &maximum_current_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(source_variable_supply); + +static struct device_type source_variable_supply_type = { + .name = "pdo", + .release = pdo_release, + .groups = source_variable_supply_groups, +}; + +static struct attribute *sink_variable_supply_attrs[] = { + &dev_attr_maximum_voltage.attr, + &dev_attr_minimum_voltage.attr, + &operational_current_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(sink_variable_supply); + +static struct device_type sink_variable_supply_type = { + .name = "pdo", + .release = pdo_release, + .groups = sink_variable_supply_groups, +}; + +/* -------------------------------------------------------------------------- */ +/* Battery */ + +static ssize_t +maximum_power_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umW\n", pdo_max_power(to_pdo(dev)->pdo)); +} +static DEVICE_ATTR_RO(maximum_power); + +static ssize_t +operational_power_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umW\n", pdo_max_power(to_pdo(dev)->pdo)); +} +static DEVICE_ATTR_RO(operational_power); + +static struct attribute *source_battery_attrs[] = { + &dev_attr_maximum_voltage.attr, + &dev_attr_minimum_voltage.attr, + &dev_attr_maximum_power.attr, + NULL +}; +ATTRIBUTE_GROUPS(source_battery); + +static struct device_type source_battery_type = { + .name = "pdo", + .release = pdo_release, + .groups = source_battery_groups, +}; + +static struct attribute *sink_battery_attrs[] = { + &dev_attr_maximum_voltage.attr, + &dev_attr_minimum_voltage.attr, + &dev_attr_operational_power.attr, + NULL +}; +ATTRIBUTE_GROUPS(sink_battery); + +static struct device_type sink_battery_type = { + .name = "pdo", + .release = pdo_release, + .groups = sink_battery_groups, +}; + +/* -------------------------------------------------------------------------- */ +/* Standard Power Range (SPR) Programmable Power Supply (PPS) */ + +static ssize_t +pps_power_limited_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", !!(to_pdo(dev)->pdo & BIT(27))); +} +static DEVICE_ATTR_RO(pps_power_limited); + +static ssize_t +pps_max_voltage_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umV\n", pdo_pps_apdo_max_voltage(to_pdo(dev)->pdo)); +} + +static ssize_t +pps_min_voltage_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umV\n", pdo_pps_apdo_min_voltage(to_pdo(dev)->pdo)); +} + +static ssize_t +pps_max_current_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%umA\n", pdo_pps_apdo_max_current(to_pdo(dev)->pdo)); +} + +static struct device_attribute pps_max_voltage_attr = { + .attr = { + .name = "maximum_voltage", + .mode = 0444, + }, + .show = pps_max_voltage_show, +}; + +static struct device_attribute pps_min_voltage_attr = { + .attr = { + .name = "minimum_voltage", + .mode = 0444, + }, + .show = pps_min_voltage_show, +}; + +static struct device_attribute pps_max_current_attr = { + .attr = { + .name = "maximum_current", + .mode = 0444, + }, + .show = pps_max_current_show, +}; + +static struct attribute *source_pps_attrs[] = { + &dev_attr_pps_power_limited.attr, + &pps_max_voltage_attr.attr, + &pps_min_voltage_attr.attr, + &pps_max_current_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(source_pps); + +static struct device_type source_pps_type = { + .name = "pdo", + .release = pdo_release, + .groups = source_pps_groups, +}; + +static struct attribute *sink_pps_attrs[] = { + &pps_max_voltage_attr.attr, + &pps_min_voltage_attr.attr, + &pps_max_current_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(sink_pps); + +static struct device_type sink_pps_type = { + .name = "pdo", + .release = pdo_release, + .groups = sink_pps_groups, +}; + +/* -------------------------------------------------------------------------- */ + +static const char * const supply_name[] = { + [PDO_TYPE_FIXED] = "fixed_supply", + [PDO_TYPE_BATT] = "battery", + [PDO_TYPE_VAR] = "variable_supply", +}; + +static const char * const apdo_supply_name[] = { + [APDO_TYPE_PPS] = "programmable_supply", +}; + +static struct device_type *source_type[] = { + [PDO_TYPE_FIXED] = &source_fixed_supply_type, + [PDO_TYPE_BATT] = &source_battery_type, + [PDO_TYPE_VAR] = &source_variable_supply_type, +}; + +static struct device_type *source_apdo_type[] = { + [APDO_TYPE_PPS] = &source_pps_type, +}; + +static struct device_type *sink_type[] = { + [PDO_TYPE_FIXED] = &sink_fixed_supply_type, + [PDO_TYPE_BATT] = &sink_battery_type, + [PDO_TYPE_VAR] = &sink_variable_supply_type, +}; + +static struct device_type *sink_apdo_type[] = { + [APDO_TYPE_PPS] = &sink_pps_type, +}; + +/* REVISIT: Export when EPR_*_Capabilities need to be supported. */ +static int add_pdo(struct usb_power_delivery_capabilities *cap, u32 pdo, int position) +{ + struct device_type *type; + const char *name; + struct pdo *p; + int ret; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return -ENOMEM; + + p->pdo = pdo; + p->object_position = position; + + if (pdo_type(pdo) == PDO_TYPE_APDO) { + /* FIXME: Only PPS supported for now! Skipping others. */ + if (pdo_apdo_type(pdo) > APDO_TYPE_PPS) { + dev_warn(&cap->dev, "Unknown APDO type. PDO 0x%08x\n", pdo); + kfree(p); + return 0; + } + + if (is_source(cap->role)) + type = source_apdo_type[pdo_apdo_type(pdo)]; + else + type = sink_apdo_type[pdo_apdo_type(pdo)]; + + name = apdo_supply_name[pdo_apdo_type(pdo)]; + } else { + if (is_source(cap->role)) + type = source_type[pdo_type(pdo)]; + else + type = sink_type[pdo_type(pdo)]; + + name = supply_name[pdo_type(pdo)]; + } + + p->dev.parent = &cap->dev; + p->dev.type = type; + dev_set_name(&p->dev, "%u:%s", position + 1, name); + + ret = device_register(&p->dev); + if (ret) { + put_device(&p->dev); + return ret; + } + + return 0; +} + +static int remove_pdo(struct device *dev, void *data) +{ + device_unregister(dev); + return 0; +} + +/* -------------------------------------------------------------------------- */ + +static const char * const cap_name[] = { + [TYPEC_SINK] = "sink-capabilities", + [TYPEC_SOURCE] = "source-capabilities", +}; + +static void pd_capabilities_release(struct device *dev) +{ + kfree(to_usb_power_delivery_capabilities(dev)); +} + +static struct device_type pd_capabilities_type = { + .name = "capabilities", + .release = pd_capabilities_release, +}; + +/** + * usb_power_delivery_register_capabilities - Register a set of capabilities. + * @pd: The USB PD instance that the capabilities belong to. + * @desc: Description of the Capablities Message. + * + * This function registers a Capabilities Message described in @desc. The + * capabilities will have their own sub-directory under @pd in sysfs. + * + * The function returns pointer to struct usb_power_delivery_capabilities, or + * ERR_PRT(errno). + */ +struct usb_power_delivery_capabilities * +usb_power_delivery_register_capabilities(struct usb_power_delivery *pd, + struct usb_power_delivery_capabilities_desc *desc) +{ + struct usb_power_delivery_capabilities *cap; + int ret; + int i; + + cap = kzalloc(sizeof(*cap), GFP_KERNEL); + if (!cap) + return ERR_PTR(-ENOMEM); + + cap->pd = pd; + cap->role = desc->role; + + cap->dev.parent = &pd->dev; + cap->dev.type = &pd_capabilities_type; + dev_set_name(&cap->dev, "%s", cap_name[cap->role]); + + ret = device_register(&cap->dev); + if (ret) { + put_device(&cap->dev); + return ERR_PTR(ret); + } + + for (i = 0; i < PDO_MAX_OBJECTS && desc->pdo[i]; i++) { + ret = add_pdo(cap, desc->pdo[i], i); + if (ret) { + usb_power_delivery_unregister_capabilities(cap); + return ERR_PTR(ret); + } + } + + return cap; +} +EXPORT_SYMBOL_GPL(usb_power_delivery_register_capabilities); + +/** + * usb_power_delivery_unregister_capabilities - Unregister a set of capabilities + * @cap: The capabilities + */ +void usb_power_delivery_unregister_capabilities(struct usb_power_delivery_capabilities *cap) +{ + if (!cap) + return; + + device_for_each_child(&cap->dev, NULL, remove_pdo); + device_unregister(&cap->dev); +} +EXPORT_SYMBOL_GPL(usb_power_delivery_unregister_capabilities); + +/* -------------------------------------------------------------------------- */ + +static ssize_t revision_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct usb_power_delivery *pd = to_usb_power_delivery(dev); + + return sysfs_emit(buf, "%u.%u\n", (pd->revision >> 8) & 0xff, (pd->revision >> 4) & 0xf); +} +static DEVICE_ATTR_RO(revision); + +static ssize_t version_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct usb_power_delivery *pd = to_usb_power_delivery(dev); + + return sysfs_emit(buf, "%u.%u\n", (pd->version >> 8) & 0xff, (pd->version >> 4) & 0xf); +} +static DEVICE_ATTR_RO(version); + +static struct attribute *pd_attrs[] = { + &dev_attr_revision.attr, + &dev_attr_version.attr, + NULL +}; + +static umode_t pd_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + struct usb_power_delivery *pd = to_usb_power_delivery(kobj_to_dev(kobj)); + + if (attr == &dev_attr_version.attr && !pd->version) + return 0; + + return attr->mode; +} + +static const struct attribute_group pd_group = { + .is_visible = pd_attr_is_visible, + .attrs = pd_attrs, +}; +__ATTRIBUTE_GROUPS(pd); + +static void pd_release(struct device *dev) +{ + struct usb_power_delivery *pd = to_usb_power_delivery(dev); + + ida_simple_remove(&pd_ida, pd->id); + kfree(pd); +} + +static struct device_type pd_type = { + .name = "usb_power_delivery", + .release = pd_release, + .groups = pd_groups, +}; + +struct usb_power_delivery *usb_power_delivery_find(const char *name) +{ + struct device *dev; + + dev = class_find_device_by_name(&pd_class, name); + + return dev ? to_usb_power_delivery(dev) : NULL; +} + +/** + * usb_power_delivery_register - Register USB Power Delivery Support. + * @parent: Parent device. + * @desc: Description of the USB PD contract. + * + * This routine can be used to register USB Power Delivery capabilities that a + * device or devices can support. These capabilities represent all the + * capabilities that can be negotiated with a partner, so not only the Power + * Capabilities that are negotiated using the USB PD Capabilities Message. + * + * The USB Power Delivery Support object that this routine generates can be used + * as the parent object for all the actual USB Power Delivery Messages and + * objects that can be negotiated with the partner. + * + * Returns handle to struct usb_power_delivery or ERR_PTR. + */ +struct usb_power_delivery * +usb_power_delivery_register(struct device *parent, struct usb_power_delivery_desc *desc) +{ + struct usb_power_delivery *pd; + int ret; + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) + return ERR_PTR(-ENOMEM); + + ret = ida_simple_get(&pd_ida, 0, 0, GFP_KERNEL); + if (ret < 0) { + kfree(pd); + return ERR_PTR(ret); + } + + pd->id = ret; + pd->revision = desc->revision; + pd->version = desc->version; + + pd->dev.parent = parent; + pd->dev.type = &pd_type; + pd->dev.class = &pd_class; + dev_set_name(&pd->dev, "pd%d", pd->id); + + ret = device_register(&pd->dev); + if (ret) { + put_device(&pd->dev); + return ERR_PTR(ret); + } + + return pd; +} +EXPORT_SYMBOL_GPL(usb_power_delivery_register); + +/** + * usb_power_delivery_unregister - Unregister USB Power Delivery Support. + * @pd: The USB PD contract. + */ +void usb_power_delivery_unregister(struct usb_power_delivery *pd) +{ + if (IS_ERR_OR_NULL(pd)) + return; + + device_unregister(&pd->dev); +} +EXPORT_SYMBOL_GPL(usb_power_delivery_unregister); + +/** + * usb_power_delivery_link_device - Link device to its USB PD object. + * @pd: The USB PD instance. + * @dev: The device. + * + * This function can be used to create a symlink named "usb_power_delivery" for + * @dev that points to @pd. + */ +int usb_power_delivery_link_device(struct usb_power_delivery *pd, struct device *dev) +{ + int ret; + + if (IS_ERR_OR_NULL(pd) || !dev) + return 0; + + ret = sysfs_create_link(&dev->kobj, &pd->dev.kobj, "usb_power_delivery"); + if (ret) + return ret; + + get_device(&pd->dev); + get_device(dev); + + return 0; +} +EXPORT_SYMBOL_GPL(usb_power_delivery_link_device); + +/** + * usb_power_delivery_unlink_device - Unlink device from its USB PD object. + * @pd: The USB PD instance. + * @dev: The device. + * + * Remove the symlink that was previously created with pd_link_device(). + */ +void usb_power_delivery_unlink_device(struct usb_power_delivery *pd, struct device *dev) +{ + if (IS_ERR_OR_NULL(pd) || !dev) + return; + + sysfs_remove_link(&dev->kobj, "usb_power_delivery"); + put_device(&pd->dev); + put_device(dev); +} +EXPORT_SYMBOL_GPL(usb_power_delivery_unlink_device); + +/* -------------------------------------------------------------------------- */ + +int __init usb_power_delivery_init(void) +{ + return class_register(&pd_class); +} + +void __exit usb_power_delivery_exit(void) +{ + ida_destroy(&pd_ida); + class_unregister(&pd_class); +} diff --git a/drivers/usb/typec/pd.h b/drivers/usb/typec/pd.h new file mode 100644 index 000000000000..049a1aad440a --- /dev/null +++ b/drivers/usb/typec/pd.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __USB_POWER_DELIVERY__ +#define __USB_POWER_DELIVERY__ + +#include +#include + +struct usb_power_delivery { + struct device dev; + int id; + u16 revision; + u16 version; +}; + +struct usb_power_delivery_capabilities { + struct device dev; + struct usb_power_delivery *pd; + enum typec_role role; +}; + +#define to_usb_power_delivery_capabilities(o) container_of(o, struct usb_power_delivery_capabilities, dev) +#define to_usb_power_delivery(o) container_of(o, struct usb_power_delivery, dev) + +struct usb_power_delivery *usb_power_delivery_find(const char *name); + +int usb_power_delivery_init(void); +void usb_power_delivery_exit(void); + +#endif /* __USB_POWER_DELIVERY__ */ diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 96b7ff66f074..c59fb79a42e8 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -495,4 +495,42 @@ static inline unsigned int rdo_max_power(u32 rdo) #define PD_P_SNK_STDBY_MW 2500 /* 2500 mW */ +#if IS_ENABLED(CONFIG_TYPEC) + +struct usb_power_delivery; + +/** + * usb_power_delivery_desc - USB Power Delivery Descriptor + * @revision: USB Power Delivery Specification Revision + * @version: USB Power Delivery Specicication Version - optional + */ +struct usb_power_delivery_desc { + u16 revision; + u16 version; +}; + +/** + * usb_power_delivery_capabilities_desc - Description of USB Power Delivery Capabilities Message + * @pdo: The Power Data Objects in the Capability Message + * @role: Power role of the capabilities + */ +struct usb_power_delivery_capabilities_desc { + u32 pdo[PDO_MAX_OBJECTS]; + enum typec_role role; +}; + +struct usb_power_delivery_capabilities * +usb_power_delivery_register_capabilities(struct usb_power_delivery *pd, + struct usb_power_delivery_capabilities_desc *desc); +void usb_power_delivery_unregister_capabilities(struct usb_power_delivery_capabilities *cap); + +struct usb_power_delivery *usb_power_delivery_register(struct device *parent, + struct usb_power_delivery_desc *desc); +void usb_power_delivery_unregister(struct usb_power_delivery *pd); + +int usb_power_delivery_link_device(struct usb_power_delivery *pd, struct device *dev); +void usb_power_delivery_unlink_device(struct usb_power_delivery *pd, struct device *dev); + +#endif /* CONFIG_TYPEC */ + #endif /* __LINUX_USB_PD_H */ diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index fdf737d48b3b..45e28d14ae56 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -52,6 +52,16 @@ enum typec_role { TYPEC_SOURCE, }; +static inline int is_sink(enum typec_role role) +{ + return role == TYPEC_SINK; +} + +static inline int is_source(enum typec_role role) +{ + return role == TYPEC_SOURCE; +} + enum typec_pwr_opmode { TYPEC_PWR_MODE_USB, TYPEC_PWR_MODE_1_5A, -- cgit v1.2.3 From a7cff92f0635c794e2198a69a7ff4ecfe0decab9 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Mon, 2 May 2022 16:20:57 +0300 Subject: usb: typec: USB Power Delivery helpers for ports and partners All the USB Type-C Connector Class devices are protected, so the drivers can not directly access them. This will adds a few helpers that can be used to link the ports and partners to the correct USB Power Delivery objects. For ports a new optional sysfs attribute file is also added that can be used to select the USB Power Delivery capabilities that the port will advertise to the partner. Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20220502132058.86236-3-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/testing/sysfs-class-typec | 8 ++ drivers/usb/typec/class.c | 149 ++++++++++++++++++++++++++++ drivers/usb/typec/class.h | 4 + include/linux/usb/typec.h | 13 +++ 4 files changed, 174 insertions(+) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-typec b/Documentation/ABI/testing/sysfs-class-typec index 75088ecad202..281b995beb05 100644 --- a/Documentation/ABI/testing/sysfs-class-typec +++ b/Documentation/ABI/testing/sysfs-class-typec @@ -141,6 +141,14 @@ Description: - "reverse": CC2 orientation - "unknown": Orientation cannot be determined. +What: /sys/class/typec//select_usb_power_delivery +Date: May 2022 +Contact: Heikki Krogerus +Description: + Lists the USB Power Delivery Capabilities that the port can + advertise to the partner. The currently used capabilities are in + brackets. Selection happens by writing to the file. + USB Type-C partner devices (eg. /sys/class/typec/port0-partner/) What: /sys/class/typec/-partner/accessory_mode diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index ee0e520707dd..bbc46b14f99a 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -15,6 +15,7 @@ #include "bus.h" #include "class.h" +#include "pd.h" static DEFINE_IDA(typec_index_ida); @@ -720,6 +721,39 @@ void typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revisio } EXPORT_SYMBOL_GPL(typec_partner_set_pd_revision); +/** + * typec_partner_set_usb_power_delivery - Declare USB Power Delivery Contract. + * @partner: The partner device. + * @pd: The USB PD instance. + * + * This routine can be used to declare USB Power Delivery Contract with @partner + * by linking @partner to @pd which contains the objects that were used during the + * negotiation of the contract. + * + * If @pd is NULL, the link is removed and the contract with @partner has ended. + */ +int typec_partner_set_usb_power_delivery(struct typec_partner *partner, + struct usb_power_delivery *pd) +{ + int ret; + + if (IS_ERR_OR_NULL(partner) || partner->pd == pd) + return 0; + + if (pd) { + ret = usb_power_delivery_link_device(pd, &partner->dev); + if (ret) + return ret; + } else { + usb_power_delivery_unlink_device(partner->pd, &partner->dev); + } + + partner->pd = pd; + + return 0; +} +EXPORT_SYMBOL_GPL(typec_partner_set_usb_power_delivery); + /** * typec_partner_set_num_altmodes - Set the number of available partner altmodes * @partner: The partner to be updated. @@ -1170,6 +1204,104 @@ EXPORT_SYMBOL_GPL(typec_unregister_cable); /* ------------------------------------------------------------------------- */ /* USB Type-C ports */ +/** + * typec_port_set_usb_power_delivery - Assign USB PD for port. + * @port: USB Type-C port. + * @pd: USB PD instance. + * + * This routine can be used to set the USB Power Delivery Capabilities for @port + * that it will advertise to the partner. + * + * If @pd is NULL, the assignment is removed. + */ +int typec_port_set_usb_power_delivery(struct typec_port *port, struct usb_power_delivery *pd) +{ + int ret; + + if (IS_ERR_OR_NULL(port) || port->pd == pd) + return 0; + + if (pd) { + ret = usb_power_delivery_link_device(pd, &port->dev); + if (ret) + return ret; + } else { + usb_power_delivery_unlink_device(port->pd, &port->dev); + } + + port->pd = pd; + + return 0; +} +EXPORT_SYMBOL_GPL(typec_port_set_usb_power_delivery); + +static ssize_t select_usb_power_delivery_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct typec_port *port = to_typec_port(dev); + struct usb_power_delivery *pd; + + if (!port->ops || !port->ops->pd_set) + return -EOPNOTSUPP; + + pd = usb_power_delivery_find(buf); + if (!pd) + return -EINVAL; + + return port->ops->pd_set(port, pd); +} + +static ssize_t select_usb_power_delivery_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct typec_port *port = to_typec_port(dev); + struct usb_power_delivery **pds; + struct usb_power_delivery *pd; + int ret = 0; + + if (!port->ops || !port->ops->pd_get) + return -EOPNOTSUPP; + + pds = port->ops->pd_get(port); + if (!pds) + return 0; + + for (pd = pds[0]; pd; pd++) { + if (pd == port->pd) + ret += sysfs_emit(buf + ret, "[%s] ", dev_name(&pd->dev)); + else + ret += sysfs_emit(buf + ret, "%s ", dev_name(&pd->dev)); + } + + buf[ret - 1] = '\n'; + + return ret; +} +static DEVICE_ATTR_RW(select_usb_power_delivery); + +static struct attribute *port_attrs[] = { + &dev_attr_select_usb_power_delivery.attr, + NULL +}; + +static umode_t port_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + struct typec_port *port = to_typec_port(kobj_to_dev(kobj)); + + if (!port->pd || !port->ops || !port->ops->pd_get) + return 0; + if (!port->ops->pd_set) + return 0444; + + return attr->mode; +} + +static const struct attribute_group pd_group = { + .is_visible = port_attr_is_visible, + .attrs = port_attrs, +}; + static const char * const typec_orientations[] = { [TYPEC_ORIENTATION_NONE] = "unknown", [TYPEC_ORIENTATION_NORMAL] = "normal", @@ -1581,6 +1713,7 @@ static const struct attribute_group typec_group = { static const struct attribute_group *typec_groups[] = { &typec_group, + &pd_group, NULL }; @@ -2123,6 +2256,13 @@ struct typec_port *typec_register_port(struct device *parent, return ERR_PTR(ret); } + ret = typec_port_set_usb_power_delivery(port, cap->pd); + if (ret) { + dev_err(&port->dev, "failed to link pd\n"); + device_unregister(&port->dev); + return ERR_PTR(ret); + } + ret = typec_link_ports(port); if (ret) dev_warn(&port->dev, "failed to create symlinks (%d)\n", ret); @@ -2141,6 +2281,7 @@ void typec_unregister_port(struct typec_port *port) { if (!IS_ERR_OR_NULL(port)) { typec_unlink_ports(port); + typec_port_set_usb_power_delivery(port, NULL); device_unregister(&port->dev); } } @@ -2162,8 +2303,15 @@ static int __init typec_init(void) if (ret) goto err_unregister_mux_class; + ret = usb_power_delivery_init(); + if (ret) + goto err_unregister_class; + return 0; +err_unregister_class: + class_unregister(&typec_class); + err_unregister_mux_class: class_unregister(&typec_mux_class); @@ -2176,6 +2324,7 @@ subsys_initcall(typec_init); static void __exit typec_exit(void) { + usb_power_delivery_exit(); class_unregister(&typec_class); ida_destroy(&typec_index_ida); bus_unregister(&typec_bus); diff --git a/drivers/usb/typec/class.h b/drivers/usb/typec/class.h index 0f1bd6d19d67..b531f9853bc0 100644 --- a/drivers/usb/typec/class.h +++ b/drivers/usb/typec/class.h @@ -33,6 +33,8 @@ struct typec_partner { int num_altmodes; u16 pd_revision; /* 0300H = "3.0" */ enum usb_pd_svdm_ver svdm_version; + + struct usb_power_delivery *pd; }; struct typec_port { @@ -40,6 +42,8 @@ struct typec_port { struct device dev; struct ida mode_ids; + struct usb_power_delivery *pd; + int prefer_role; enum typec_data_role data_role; enum typec_role pwr_role; diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index 45e28d14ae56..7751bedcae5d 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -22,6 +22,8 @@ struct typec_altmode_ops; struct fwnode_handle; struct device; +struct usb_power_delivery; + enum typec_port_type { TYPEC_PORT_SRC, TYPEC_PORT_SNK, @@ -223,6 +225,8 @@ struct typec_partner_desc { * @pr_set: Set Power Role * @vconn_set: Source VCONN * @port_type_set: Set port type + * @pd_get: Get available USB Power Delivery Capabilities. + * @pd_set: Set USB Power Delivery Capabilities. */ struct typec_operations { int (*try_role)(struct typec_port *port, int role); @@ -231,6 +235,8 @@ struct typec_operations { int (*vconn_set)(struct typec_port *port, enum typec_role role); int (*port_type_set)(struct typec_port *port, enum typec_port_type type); + struct usb_power_delivery **(*pd_get)(struct typec_port *port); + int (*pd_set)(struct typec_port *port, struct usb_power_delivery *pd); }; enum usb_pd_svdm_ver { @@ -250,6 +256,7 @@ enum usb_pd_svdm_ver { * @accessory: Supported Accessory Modes * @fwnode: Optional fwnode of the port * @driver_data: Private pointer for driver specific info + * @pd: Optional USB Power Delivery Support * @ops: Port operations vector * * Static capabilities of a single USB Type-C port. @@ -267,6 +274,8 @@ struct typec_capability { struct fwnode_handle *fwnode; void *driver_data; + struct usb_power_delivery *pd; + const struct typec_operations *ops; }; @@ -318,4 +327,8 @@ void typec_partner_set_svdm_version(struct typec_partner *partner, enum usb_pd_svdm_ver svdm_version); int typec_get_negotiated_svdm_version(struct typec_port *port); +int typec_port_set_usb_power_delivery(struct typec_port *port, struct usb_power_delivery *pd); +int typec_partner_set_usb_power_delivery(struct typec_partner *partner, + struct usb_power_delivery *pd); + #endif /* __LINUX_USB_TYPEC_H */ -- cgit v1.2.3 From e146caf303493c4f2458173d7f1598b76a9b1396 Mon Sep 17 00:00:00 2001 From: Mathias Nyman Date: Fri, 6 May 2022 19:18:07 +0300 Subject: usb: Avoid extra usb SET_SEL requests when enabling link power management The host needs to tell the device the exit latencies using the SET_SEL request before device initiated link powermanagement can be enabled. The exit latency values do not change after enumeration, it's enough to set them once. So do like Windows 10 and issue the SET_SEL request once just before setting the configuration. This is also the sequence described in USB 3.2 specs "9.1.2 Bus enumeration". SET_SEL is issued once before the Set Configuration request, and won't be cleared by the Set Configuration, Set Interface or ClearFeature (STALL) requests. Only warm reset, hot reset, set Address 0 clears the exit latencies. See USB 3.2 section 9.4.14 Table 9-10 Device parameters and events Add udev->lpm_devinit_allow, and set it if SET_SEL was successful. If not set, then don't try to enable device initiated LPM We used to issue a SET_SEL request every time lpm is enabled for either U1 or U2 link states, meaning a SET_SEL was issued twice after every Set Configuration and Set Interface requests, easily accumulating to over 15 SET_SEL requets during a USB3 webcam enumeration. Signed-off-by: Mathias Nyman Link: https://lore.kernel.org/r/20220506161807.3369439-1-mathias.nyman@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/core/hub.c | 60 ++++++++++++++++++-------------------------------- include/linux/usb.h | 2 ++ 2 files changed, 23 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index ba406b8d688d..b7f66dcd1fe0 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3947,7 +3947,7 @@ static const char * const usb3_lpm_names[] = { * This function will fail if the SEL or PEL values for udev are greater than * the maximum allowed values for the link state to be enabled. */ -static int usb_req_set_sel(struct usb_device *udev, enum usb3_link_state state) +static int usb_req_set_sel(struct usb_device *udev) { struct usb_set_sel_req *sel_values; unsigned long long u1_sel; @@ -3956,7 +3956,7 @@ static int usb_req_set_sel(struct usb_device *udev, enum usb3_link_state state) unsigned long long u2_pel; int ret; - if (udev->state != USB_STATE_CONFIGURED) + if (!udev->parent || udev->speed < USB_SPEED_SUPER || !udev->lpm_capable) return 0; /* Convert SEL and PEL stored in ns to us */ @@ -3973,34 +3973,14 @@ static int usb_req_set_sel(struct usb_device *udev, enum usb3_link_state state) * latency for the link state, and could start a device-initiated * U1/U2 when the exit latencies are too high. */ - if ((state == USB3_LPM_U1 && - (u1_sel > USB3_LPM_MAX_U1_SEL_PEL || - u1_pel > USB3_LPM_MAX_U1_SEL_PEL)) || - (state == USB3_LPM_U2 && - (u2_sel > USB3_LPM_MAX_U2_SEL_PEL || - u2_pel > USB3_LPM_MAX_U2_SEL_PEL))) { - dev_dbg(&udev->dev, "Device-initiated %s disabled due to long SEL %llu us or PEL %llu us\n", - usb3_lpm_names[state], u1_sel, u1_pel); + if (u1_sel > USB3_LPM_MAX_U1_SEL_PEL || + u1_pel > USB3_LPM_MAX_U1_SEL_PEL || + u2_sel > USB3_LPM_MAX_U2_SEL_PEL || + u2_pel > USB3_LPM_MAX_U2_SEL_PEL) { + dev_dbg(&udev->dev, "Device-initiated U1/U2 disabled due to long SEL or PEL\n"); return -EINVAL; } - /* - * If we're enabling device-initiated LPM for one link state, - * but the other link state has a too high SEL or PEL value, - * just set those values to the max in the Set SEL request. - */ - if (u1_sel > USB3_LPM_MAX_U1_SEL_PEL) - u1_sel = USB3_LPM_MAX_U1_SEL_PEL; - - if (u1_pel > USB3_LPM_MAX_U1_SEL_PEL) - u1_pel = USB3_LPM_MAX_U1_SEL_PEL; - - if (u2_sel > USB3_LPM_MAX_U2_SEL_PEL) - u2_sel = USB3_LPM_MAX_U2_SEL_PEL; - - if (u2_pel > USB3_LPM_MAX_U2_SEL_PEL) - u2_pel = USB3_LPM_MAX_U2_SEL_PEL; - /* * usb_enable_lpm() can be called as part of a failed device reset, * which may be initiated by an error path of a mass storage driver. @@ -4022,6 +4002,10 @@ static int usb_req_set_sel(struct usb_device *udev, enum usb3_link_state state) sel_values, sizeof *(sel_values), USB_CTRL_SET_TIMEOUT); kfree(sel_values); + + if (ret > 0) + udev->lpm_devinit_allow = 1; + return ret; } @@ -4137,6 +4121,9 @@ static bool usb_device_may_initiate_lpm(struct usb_device *udev, unsigned int sel; /* us */ int i, j; + if (!udev->lpm_devinit_allow) + return false; + if (state == USB3_LPM_U1) sel = DIV_ROUND_UP(udev->u1_params.sel, 1000); else if (state == USB3_LPM_U2) @@ -4185,7 +4172,7 @@ static bool usb_device_may_initiate_lpm(struct usb_device *udev, static void usb_enable_link_state(struct usb_hcd *hcd, struct usb_device *udev, enum usb3_link_state state) { - int timeout, ret; + int timeout; __u8 u1_mel = udev->bos->ss_cap->bU1devExitLat; __le16 u2_mel = udev->bos->ss_cap->bU2DevExitLat; @@ -4197,17 +4184,6 @@ static void usb_enable_link_state(struct usb_hcd *hcd, struct usb_device *udev, (state == USB3_LPM_U2 && u2_mel == 0)) return; - /* - * First, let the device know about the exit latencies - * associated with the link state we're about to enable. - */ - ret = usb_req_set_sel(udev, state); - if (ret < 0) { - dev_warn(&udev->dev, "Set SEL for device-initiated %s failed.\n", - usb3_lpm_names[state]); - return; - } - /* We allow the host controller to set the U1/U2 timeout internally * first, so that it can change its schedule to account for the * additional latency to send data to a device in a lower power @@ -4487,6 +4463,11 @@ static int hub_handle_remote_wakeup(struct usb_hub *hub, unsigned int port, return 0; } +static int usb_req_set_sel(struct usb_device *udev) +{ + return 0; +} + #endif /* CONFIG_PM */ /* @@ -5012,6 +4993,7 @@ hub_port_init(struct usb_hub *hub, struct usb_device *udev, int port1, udev->lpm_capable = usb_device_supports_lpm(udev); udev->lpm_disable_count = 1; usb_set_lpm_parameters(udev); + usb_req_set_sel(udev); } } diff --git a/include/linux/usb.h b/include/linux/usb.h index 60bee864d897..f7a9914fc97f 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -584,6 +584,7 @@ struct usb3_lpm_parameters { * @authenticated: Crypto authentication passed * @wusb: device is Wireless USB * @lpm_capable: device supports LPM + * @lpm_devinit_allow: Allow USB3 device initiated LPM, exit latency is in range * @usb2_hw_lpm_capable: device can perform USB2 hardware LPM * @usb2_hw_lpm_besl_capable: device can perform USB2 hardware BESL LPM * @usb2_hw_lpm_enabled: USB2 hardware LPM is enabled @@ -666,6 +667,7 @@ struct usb_device { unsigned authenticated:1; unsigned wusb:1; unsigned lpm_capable:1; + unsigned lpm_devinit_allow:1; unsigned usb2_hw_lpm_capable:1; unsigned usb2_hw_lpm_besl_capable:1; unsigned usb2_hw_lpm_enabled:1; -- cgit v1.2.3 From c04245328dd7e915e21ac6395ffd218616e22754 Mon Sep 17 00:00:00 2001 From: Yajun Deng Date: Fri, 10 Jun 2022 17:10:17 +0800 Subject: net: make __sys_accept4_file() static __sys_accept4_file() isn't used outside of the file, make it static. As the same time, move file_flags and nofile parameters into __sys_accept4_file(). Signed-off-by: Yajun Deng Signed-off-by: David S. Miller --- include/linux/socket.h | 4 ---- net/socket.c | 15 ++++++--------- 2 files changed, 6 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index 17311ad9f9af..414b8c7bb8f7 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -428,10 +428,6 @@ extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size, extern int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags, struct sockaddr __user *addr, int addr_len); -extern int __sys_accept4_file(struct file *file, unsigned file_flags, - struct sockaddr __user *upeer_sockaddr, - int __user *upeer_addrlen, int flags, - unsigned long nofile); extern struct file *do_accept(struct file *file, unsigned file_flags, struct sockaddr __user *upeer_sockaddr, int __user *upeer_addrlen, int flags); diff --git a/net/socket.c b/net/socket.c index 2bc8773d9dc5..1b6f5e2ebef5 100644 --- a/net/socket.c +++ b/net/socket.c @@ -1878,10 +1878,8 @@ out_fd: return ERR_PTR(err); } -int __sys_accept4_file(struct file *file, unsigned file_flags, - struct sockaddr __user *upeer_sockaddr, - int __user *upeer_addrlen, int flags, - unsigned long nofile) +static int __sys_accept4_file(struct file *file, struct sockaddr __user *upeer_sockaddr, + int __user *upeer_addrlen, int flags) { struct file *newfile; int newfd; @@ -1892,11 +1890,11 @@ int __sys_accept4_file(struct file *file, unsigned file_flags, if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK)) flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK; - newfd = __get_unused_fd_flags(flags, nofile); + newfd = get_unused_fd_flags(flags); if (unlikely(newfd < 0)) return newfd; - newfile = do_accept(file, file_flags, upeer_sockaddr, upeer_addrlen, + newfile = do_accept(file, 0, upeer_sockaddr, upeer_addrlen, flags); if (IS_ERR(newfile)) { put_unused_fd(newfd); @@ -1926,9 +1924,8 @@ int __sys_accept4(int fd, struct sockaddr __user *upeer_sockaddr, f = fdget(fd); if (f.file) { - ret = __sys_accept4_file(f.file, 0, upeer_sockaddr, - upeer_addrlen, flags, - rlimit(RLIMIT_NOFILE)); + ret = __sys_accept4_file(f.file, upeer_sockaddr, + upeer_addrlen, flags); fdput(f); } -- cgit v1.2.3 From 0eb6584068642767baab17c2e4385a6b9c029caa Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 27 May 2022 04:34:36 +0200 Subject: platform/surface: aggregator: Allow is_ssam_device() to be used when CONFIG_SURFACE_AGGREGATOR_BUS is disabled In SSAM subsystem drivers that handle both ACPI and SSAM-native client devices, we may want to check whether we have a SSAM (native) client device. Further, we may want to do this even when instantiation thereof cannot happen due to CONFIG_SURFACE_AGGREGATOR_BUS=n. Currently, doing so causes an error due to an undefined reference error due to ssam_device_type being placed in the bus source unit. Therefore, if CONFIG_SURFACE_AGGREGATOR_BUS is not defined, simply let is_ssam_device() return false to prevent this error. Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20220527023447.2460025-2-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- include/linux/surface_aggregator/device.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include/linux') diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index cc257097eb05..62b38b4487eb 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -177,6 +177,8 @@ struct ssam_device_driver { void (*remove)(struct ssam_device *sdev); }; +#ifdef CONFIG_SURFACE_AGGREGATOR_BUS + extern struct bus_type ssam_bus_type; extern const struct device_type ssam_device_type; @@ -193,6 +195,15 @@ static inline bool is_ssam_device(struct device *d) return d->type == &ssam_device_type; } +#else /* CONFIG_SURFACE_AGGREGATOR_BUS */ + +static inline bool is_ssam_device(struct device *d) +{ + return false; +} + +#endif /* CONFIG_SURFACE_AGGREGATOR_BUS */ + /** * to_ssam_device() - Casts the given device to a SSAM client device. * @d: The device to cast. -- cgit v1.2.3 From dc0393c76f378f68961587fd4f32de29fb8f0c79 Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 27 May 2022 04:34:37 +0200 Subject: platform/surface: aggregator: Allow devices to be marked as hot-removed Some SSAM devices, notably the keyboard cover (keyboard and touchpad) on the Surface Pro 8, can be hot-removed. When this occurs, communication with the device may fail and time out. This timeout can unnecessarily block and slow down device removal and even cause issues when the devices are detached and re-attached quickly. Thus, communication should generally be avoided once hot-removal is detected. While we already remove a device as soon as we detect its (hot-)removal, the corresponding device driver may still attempt to communicate with the device during teardown. This is especially critical as communication failure may also extend to disabling of events, which is typically done at that stage. Add a flag to allow marking devices as hot-removed. This can then be used during client driver teardown to check if any communication attempts should be avoided. Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20220527023447.2460025-3-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- include/linux/surface_aggregator/device.h | 48 +++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index 62b38b4487eb..6df7c8d4e50e 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -148,17 +148,30 @@ struct ssam_device_uid { #define SSAM_SDEV(cat, tid, iid, fun) \ SSAM_DEVICE(SSAM_DOMAIN_SERIALHUB, SSAM_SSH_TC_##cat, tid, iid, fun) +/* + * enum ssam_device_flags - Flags for SSAM client devices. + * @SSAM_DEVICE_HOT_REMOVED_BIT: + * The device has been hot-removed. Further communication with it may time + * out and should be avoided. + */ +enum ssam_device_flags { + SSAM_DEVICE_HOT_REMOVED_BIT = 0, +}; + /** * struct ssam_device - SSAM client device. - * @dev: Driver model representation of the device. - * @ctrl: SSAM controller managing this device. - * @uid: UID identifying the device. + * @dev: Driver model representation of the device. + * @ctrl: SSAM controller managing this device. + * @uid: UID identifying the device. + * @flags: Device state flags, see &enum ssam_device_flags. */ struct ssam_device { struct device dev; struct ssam_controller *ctrl; struct ssam_device_uid uid; + + unsigned long flags; }; /** @@ -251,6 +264,35 @@ struct ssam_device *ssam_device_alloc(struct ssam_controller *ctrl, int ssam_device_add(struct ssam_device *sdev); void ssam_device_remove(struct ssam_device *sdev); +/** + * ssam_device_mark_hot_removed() - Mark the given device as hot-removed. + * @sdev: The device to mark as hot-removed. + * + * Mark the device as having been hot-removed. This signals drivers using the + * device that communication with the device should be avoided and may lead to + * timeouts. + */ +static inline void ssam_device_mark_hot_removed(struct ssam_device *sdev) +{ + dev_dbg(&sdev->dev, "marking device as hot-removed\n"); + set_bit(SSAM_DEVICE_HOT_REMOVED_BIT, &sdev->flags); +} + +/** + * ssam_device_is_hot_removed() - Check if the given device has been + * hot-removed. + * @sdev: The device to check. + * + * Checks if the given device has been marked as hot-removed. See + * ssam_device_mark_hot_removed() for more details. + * + * Return: Returns ``true`` if the device has been marked as hot-removed. + */ +static inline bool ssam_device_is_hot_removed(struct ssam_device *sdev) +{ + return test_bit(SSAM_DEVICE_HOT_REMOVED_BIT, &sdev->flags); +} + /** * ssam_device_get() - Increment reference count of SSAM client device. * @sdev: The device to increment the reference count of. -- cgit v1.2.3 From 5c1e88b98c60e4074796e9a05d3c674479ab1919 Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 27 May 2022 04:34:38 +0200 Subject: platform/surface: aggregator: Allow notifiers to avoid communication on unregistering When SSAM client devices have been (physically) hot-removed, communication attempts with those devices may fail and time out. This can even extend to event notifiers, due to which timeouts may occur during device removal, slowing down that process. Add a parameter to the notifier unregister function that allows skipping communication with the EC to prevent this. Furthermore, add wrappers for registering and unregistering notifiers belonging to SSAM client devices that automatically check if the device has been marked as hot-removed and communication should be avoided. Note that non-SSAM client devices can generally not be hot-removed, so also add a convenience wrapper for those, defaulting to allow communication. Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20220527023447.2460025-4-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- .../driver-api/surface_aggregator/client.rst | 6 +- drivers/platform/surface/aggregator/controller.c | 53 +++++++++++------ include/linux/surface_aggregator/controller.h | 24 +++++++- include/linux/surface_aggregator/device.h | 66 ++++++++++++++++++++++ 4 files changed, 128 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/surface_aggregator/client.rst b/Documentation/driver-api/surface_aggregator/client.rst index e519d374c378..27f95abdbe99 100644 --- a/Documentation/driver-api/surface_aggregator/client.rst +++ b/Documentation/driver-api/surface_aggregator/client.rst @@ -17,6 +17,8 @@ .. |SSAM_DEVICE| replace:: :c:func:`SSAM_DEVICE` .. |ssam_notifier_register| replace:: :c:func:`ssam_notifier_register` .. |ssam_notifier_unregister| replace:: :c:func:`ssam_notifier_unregister` +.. |ssam_device_notifier_register| replace:: :c:func:`ssam_device_notifier_register` +.. |ssam_device_notifier_unregister| replace:: :c:func:`ssam_device_notifier_unregister` .. |ssam_request_sync| replace:: :c:func:`ssam_request_sync` .. |ssam_event_mask| replace:: :c:type:`enum ssam_event_mask ` @@ -312,7 +314,9 @@ Handling Events To receive events from the SAM EC, an event notifier must be registered for the desired event via |ssam_notifier_register|. The notifier must be unregistered via |ssam_notifier_unregister| once it is not required any -more. +more. For |ssam_device| type clients, the |ssam_device_notifier_register| and +|ssam_device_notifier_unregister| wrappers should be preferred as they properly +handle hot-removal of client devices. Event notifiers are registered by providing (at minimum) a callback to call in case an event has been received, the registry specifying how the event diff --git a/drivers/platform/surface/aggregator/controller.c b/drivers/platform/surface/aggregator/controller.c index b8c377b3f932..6de834b52b63 100644 --- a/drivers/platform/surface/aggregator/controller.c +++ b/drivers/platform/surface/aggregator/controller.c @@ -2199,16 +2199,26 @@ static int ssam_nf_refcount_enable(struct ssam_controller *ctrl, } /** - * ssam_nf_refcount_disable_free() - Disable event for reference count entry if it is - * no longer in use and free the corresponding entry. + * ssam_nf_refcount_disable_free() - Disable event for reference count entry if + * it is no longer in use and free the corresponding entry. * @ctrl: The controller to disable the event on. * @entry: The reference count entry for the event to be disabled. * @flags: The flags used for enabling the event on the EC. + * @ec: Flag specifying if the event should actually be disabled on the EC. * - * If the reference count equals zero, i.e. the event is no longer requested by - * any client, the event will be disabled and the corresponding reference count - * entry freed. The reference count entry must not be used any more after a - * call to this function. + * If ``ec`` equals ``true`` and the reference count equals zero (i.e. the + * event is no longer requested by any client), the specified event will be + * disabled on the EC via the corresponding request. + * + * If ``ec`` equals ``false``, no request will be sent to the EC and the event + * can be considered in a detached state (i.e. no longer used but still + * enabled). Disabling an event via this method may be required for + * hot-removable devices, where event disable requests may time out after the + * device has been physically removed. + * + * In both cases, if the reference count equals zero, the corresponding + * reference count entry will be freed. The reference count entry must not be + * used any more after a call to this function. * * Also checks if the flags used for disabling the event match the flags used * for enabling the event and warns if they do not (regardless of reference @@ -2223,7 +2233,7 @@ static int ssam_nf_refcount_enable(struct ssam_controller *ctrl, * returns the status of the event-enable EC command. */ static int ssam_nf_refcount_disable_free(struct ssam_controller *ctrl, - struct ssam_nf_refcount_entry *entry, u8 flags) + struct ssam_nf_refcount_entry *entry, u8 flags, bool ec) { const struct ssam_event_registry reg = entry->key.reg; const struct ssam_event_id id = entry->key.id; @@ -2232,8 +2242,9 @@ static int ssam_nf_refcount_disable_free(struct ssam_controller *ctrl, lockdep_assert_held(&nf->lock); - ssam_dbg(ctrl, "disabling event (reg: %#04x, tc: %#04x, iid: %#04x, rc: %d)\n", - reg.target_category, id.target_category, id.instance, entry->refcount); + ssam_dbg(ctrl, "%s event (reg: %#04x, tc: %#04x, iid: %#04x, rc: %d)\n", + ec ? "disabling" : "detaching", reg.target_category, id.target_category, + id.instance, entry->refcount); if (entry->flags != flags) { ssam_warn(ctrl, @@ -2242,7 +2253,7 @@ static int ssam_nf_refcount_disable_free(struct ssam_controller *ctrl, id.instance); } - if (entry->refcount == 0) { + if (ec && entry->refcount == 0) { status = ssam_ssh_event_disable(ctrl, reg, id, flags); kfree(entry); } @@ -2322,20 +2333,26 @@ int ssam_notifier_register(struct ssam_controller *ctrl, struct ssam_event_notif EXPORT_SYMBOL_GPL(ssam_notifier_register); /** - * ssam_notifier_unregister() - Unregister an event notifier. - * @ctrl: The controller the notifier has been registered on. - * @n: The event notifier to unregister. + * __ssam_notifier_unregister() - Unregister an event notifier. + * @ctrl: The controller the notifier has been registered on. + * @n: The event notifier to unregister. + * @disable: Whether to disable the corresponding event on the EC. * * Unregister an event notifier. Decrement the usage counter of the associated * SAM event if the notifier is not marked as an observer. If the usage counter - * reaches zero, the event will be disabled. + * reaches zero and ``disable`` equals ``true``, the event will be disabled. + * + * Useful for hot-removable devices, where communication may fail once the + * device has been physically removed. In that case, specifying ``disable`` as + * ``false`` avoids communication with the EC. * * Return: Returns zero on success, %-ENOENT if the given notifier block has * not been registered on the controller. If the given notifier block was the * last one associated with its specific event, returns the status of the * event-disable EC-command. */ -int ssam_notifier_unregister(struct ssam_controller *ctrl, struct ssam_event_notifier *n) +int __ssam_notifier_unregister(struct ssam_controller *ctrl, struct ssam_event_notifier *n, + bool disable) { u16 rqid = ssh_tc_to_rqid(n->event.id.target_category); struct ssam_nf_refcount_entry *entry; @@ -2373,7 +2390,7 @@ int ssam_notifier_unregister(struct ssam_controller *ctrl, struct ssam_event_not goto remove; } - status = ssam_nf_refcount_disable_free(ctrl, entry, n->event.flags); + status = ssam_nf_refcount_disable_free(ctrl, entry, n->event.flags, disable); } remove: @@ -2383,7 +2400,7 @@ remove: return status; } -EXPORT_SYMBOL_GPL(ssam_notifier_unregister); +EXPORT_SYMBOL_GPL(__ssam_notifier_unregister); /** * ssam_controller_event_enable() - Enable the specified event. @@ -2477,7 +2494,7 @@ int ssam_controller_event_disable(struct ssam_controller *ctrl, return -ENOENT; } - status = ssam_nf_refcount_disable_free(ctrl, entry, flags); + status = ssam_nf_refcount_disable_free(ctrl, entry, flags, true); mutex_unlock(&nf->lock); return status; diff --git a/include/linux/surface_aggregator/controller.h b/include/linux/surface_aggregator/controller.h index 74bfdffaf7b0..50a2b4926c06 100644 --- a/include/linux/surface_aggregator/controller.h +++ b/include/linux/surface_aggregator/controller.h @@ -835,8 +835,28 @@ struct ssam_event_notifier { int ssam_notifier_register(struct ssam_controller *ctrl, struct ssam_event_notifier *n); -int ssam_notifier_unregister(struct ssam_controller *ctrl, - struct ssam_event_notifier *n); +int __ssam_notifier_unregister(struct ssam_controller *ctrl, + struct ssam_event_notifier *n, bool disable); + +/** + * ssam_notifier_unregister() - Unregister an event notifier. + * @ctrl: The controller the notifier has been registered on. + * @n: The event notifier to unregister. + * + * Unregister an event notifier. Decrement the usage counter of the associated + * SAM event if the notifier is not marked as an observer. If the usage counter + * reaches zero, the event will be disabled. + * + * Return: Returns zero on success, %-ENOENT if the given notifier block has + * not been registered on the controller. If the given notifier block was the + * last one associated with its specific event, returns the status of the + * event-disable EC-command. + */ +static inline int ssam_notifier_unregister(struct ssam_controller *ctrl, + struct ssam_event_notifier *n) +{ + return __ssam_notifier_unregister(ctrl, n, true); +} int ssam_controller_event_enable(struct ssam_controller *ctrl, struct ssam_event_registry reg, diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index 6df7c8d4e50e..c418f7f2732d 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -483,4 +483,70 @@ static inline void ssam_remove_clients(struct device *dev) {} sdev->uid.instance, ret); \ } + +/* -- Helpers for client-device notifiers. ---------------------------------- */ + +/** + * ssam_device_notifier_register() - Register an event notifier for the + * specified client device. + * @sdev: The device the notifier should be registered on. + * @n: The event notifier to register. + * + * Register an event notifier. Increment the usage counter of the associated + * SAM event if the notifier is not marked as an observer. If the event is not + * marked as an observer and is currently not enabled, it will be enabled + * during this call. If the notifier is marked as an observer, no attempt will + * be made at enabling any event and no reference count will be modified. + * + * Notifiers marked as observers do not need to be associated with one specific + * event, i.e. as long as no event matching is performed, only the event target + * category needs to be set. + * + * Return: Returns zero on success, %-ENOSPC if there have already been + * %INT_MAX notifiers for the event ID/type associated with the notifier block + * registered, %-ENOMEM if the corresponding event entry could not be + * allocated, %-ENODEV if the device is marked as hot-removed. If this is the + * first time that a notifier block is registered for the specific associated + * event, returns the status of the event-enable EC-command. + */ +static inline int ssam_device_notifier_register(struct ssam_device *sdev, + struct ssam_event_notifier *n) +{ + /* + * Note that this check does not provide any guarantees whatsoever as + * hot-removal could happen at any point and we can't protect against + * it. Nevertheless, if we can detect hot-removal, bail early to avoid + * communication timeouts. + */ + if (ssam_device_is_hot_removed(sdev)) + return -ENODEV; + + return ssam_notifier_register(sdev->ctrl, n); +} + +/** + * ssam_device_notifier_unregister() - Unregister an event notifier for the + * specified client device. + * @sdev: The device the notifier has been registered on. + * @n: The event notifier to unregister. + * + * Unregister an event notifier. Decrement the usage counter of the associated + * SAM event if the notifier is not marked as an observer. If the usage counter + * reaches zero, the event will be disabled. + * + * In case the device has been marked as hot-removed, the event will not be + * disabled on the EC, as in those cases any attempt at doing so may time out. + * + * Return: Returns zero on success, %-ENOENT if the given notifier block has + * not been registered on the controller. If the given notifier block was the + * last one associated with its specific event, returns the status of the + * event-disable EC-command. + */ +static inline int ssam_device_notifier_unregister(struct ssam_device *sdev, + struct ssam_event_notifier *n) +{ + return __ssam_notifier_unregister(sdev->ctrl, n, + !ssam_device_is_hot_removed(sdev)); +} + #endif /* _LINUX_SURFACE_AGGREGATOR_DEVICE_H */ -- cgit v1.2.3 From 25e2ca7301bd3ca5a63a6be41d729eb42202bc21 Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 27 May 2022 04:34:43 +0200 Subject: platform/surface: aggregator: Add comment for KIP subsystem category The KIP subsystem (full name unknown, abbreviation has been obtained through reverse engineering) handles detachable peripherals such as the keyboard cover on the Surface Pro X and Surface Pro 8. It is currently not entirely clear what this subsystem entails, but at the very least it provides event notifications for when the keyboard cover on the Surface Pro X and Surface Pro 8 have been detached or re-attached, as well as the state that the keyboard cover is currently in (e.g. folded-back, folded laptop-like, closed, etc.). Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20220527023447.2460025-9-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- include/linux/surface_aggregator/serial_hub.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/surface_aggregator/serial_hub.h b/include/linux/surface_aggregator/serial_hub.h index c3de43edcffa..26b95ec12733 100644 --- a/include/linux/surface_aggregator/serial_hub.h +++ b/include/linux/surface_aggregator/serial_hub.h @@ -306,7 +306,7 @@ enum ssam_ssh_tc { SSAM_SSH_TC_LPC = 0x0b, SSAM_SSH_TC_TCL = 0x0c, SSAM_SSH_TC_SFL = 0x0d, - SSAM_SSH_TC_KIP = 0x0e, + SSAM_SSH_TC_KIP = 0x0e, /* Manages detachable peripherals (Pro X/8 keyboard cover) */ SSAM_SSH_TC_EXT = 0x0f, SSAM_SSH_TC_BLD = 0x10, SSAM_SSH_TC_BAS = 0x11, /* Detachment system (Surface Book 2/3). */ -- cgit v1.2.3 From 546093206ba16623c18e344630dbfdd71a4327e0 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Sat, 11 Jun 2022 17:23:04 +0800 Subject: audit: make is_audit_feature_set() static Currently nobody use is_audit_feature_set() outside this file, so make it static. Signed-off-by: Xiu Jianfeng Signed-off-by: Paul Moore --- include/linux/audit.h | 2 -- kernel/audit.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/audit.h b/include/linux/audit.h index cece70231138..00f7a80f1a3e 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -119,8 +119,6 @@ enum audit_nfcfgop { AUDIT_NFT_OP_INVALID, }; -extern int is_audit_feature_set(int which); - extern int __init audit_register_class(int class, unsigned *list); extern int audit_classify_syscall(int abi, unsigned syscall); extern int audit_classify_arch(int arch); diff --git a/kernel/audit.c b/kernel/audit.c index 0749211d5552..a75978ae38ad 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1100,7 +1100,7 @@ static inline void audit_log_user_recv_msg(struct audit_buffer **ab, audit_log_common_recv_msg(NULL, ab, msg_type); } -int is_audit_feature_set(int i) +static int is_audit_feature_set(int i) { return af.features & AUDIT_FEATURE_TO_MASK(i); } -- cgit v1.2.3 From 795e10b450a88b2943a241a5eaa6e86ae4f47694 Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Tue, 7 Jun 2022 15:47:43 +0300 Subject: net/mlx5: Introduce header-modify-pattern ICM properties Added new fields for device memory capabilities, in order to support creation of ICM memory for modify header patterns. Signed-off-by: Erez Shitrit Signed-off-by: Yevgeny Kliteynik Signed-off-by: Leon Romanovsky Acked-by: Saeed Mahameed Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index fd7d083a34d3..789d64400744 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1086,11 +1086,14 @@ struct mlx5_ifc_device_mem_cap_bits { u8 log_sw_icm_alloc_granularity[0x6]; u8 log_steering_sw_icm_size[0x8]; - u8 reserved_at_120[0x20]; + u8 reserved_at_120[0x18]; + u8 log_header_modify_pattern_sw_icm_size[0x8]; u8 header_modify_sw_icm_start_address[0x40]; - u8 reserved_at_180[0x80]; + u8 reserved_at_180[0x40]; + + u8 header_modify_pattern_sw_icm_start_address[0x40]; u8 memic_operations[0x20]; -- cgit v1.2.3 From 667658364b2056f344a2769280b939a5e45610be Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Tue, 7 Jun 2022 15:47:44 +0300 Subject: net/mlx5: Manage ICM of type modify-header pattern Added support for managing new type of ICM for devices that support sw_owner_v2. Signed-off-by: Erez Shitrit Signed-off-by: Yevgeny Kliteynik Signed-off-by: Leon Romanovsky Acked-by: Saeed Mahameed Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c | 42 ++++++++++++++++++++++++ include/linux/mlx5/driver.h | 1 + 2 files changed, 43 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c index 3d5e57ff558c..7e02cbe8c3b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c @@ -12,13 +12,16 @@ struct mlx5_dm { spinlock_t lock; unsigned long *steering_sw_icm_alloc_blocks; unsigned long *header_modify_sw_icm_alloc_blocks; + unsigned long *header_modify_pattern_sw_icm_alloc_blocks; }; struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) { + u64 header_modify_pattern_icm_blocks = 0; u64 header_modify_icm_blocks = 0; u64 steering_icm_blocks = 0; struct mlx5_dm *dm; + bool support_v2; if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)) return NULL; @@ -53,8 +56,27 @@ struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) goto err_modify_hdr; } + support_v2 = MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) && + MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2) && + MLX5_CAP64_DEV_MEM(dev, header_modify_pattern_sw_icm_start_address); + + if (support_v2) { + header_modify_pattern_icm_blocks = + BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_pattern_sw_icm_size) - + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); + + dm->header_modify_pattern_sw_icm_alloc_blocks = + kcalloc(BITS_TO_LONGS(header_modify_pattern_icm_blocks), + sizeof(unsigned long), GFP_KERNEL); + if (!dm->header_modify_pattern_sw_icm_alloc_blocks) + goto err_pattern; + } + return dm; +err_pattern: + kfree(dm->header_modify_sw_icm_alloc_blocks); + err_modify_hdr: kfree(dm->steering_sw_icm_alloc_blocks); @@ -86,6 +108,14 @@ void mlx5_dm_cleanup(struct mlx5_core_dev *dev) kfree(dm->header_modify_sw_icm_alloc_blocks); } + if (dm->header_modify_pattern_sw_icm_alloc_blocks) { + WARN_ON(!bitmap_empty(dm->header_modify_pattern_sw_icm_alloc_blocks, + BIT(MLX5_CAP_DEV_MEM(dev, + log_header_modify_pattern_sw_icm_size) - + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); + kfree(dm->header_modify_pattern_sw_icm_alloc_blocks); + } + kfree(dm); } @@ -130,6 +160,13 @@ int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, log_header_modify_sw_icm_size); block_map = dm->header_modify_sw_icm_alloc_blocks; break; + case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, + header_modify_pattern_sw_icm_start_address); + log_icm_size = MLX5_CAP_DEV_MEM(dev, + log_header_modify_pattern_sw_icm_size); + block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; + break; default: return -EINVAL; } @@ -203,6 +240,11 @@ int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); block_map = dm->header_modify_sw_icm_alloc_blocks; break; + case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, + header_modify_pattern_sw_icm_start_address); + block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; + break; default: return -EINVAL; } diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 5040cd774c5a..76d7661e3e63 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -676,6 +676,7 @@ struct mlx5e_resources { enum mlx5_sw_icm_type { MLX5_SW_ICM_TYPE_STEERING, MLX5_SW_ICM_TYPE_HEADER_MODIFY, + MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN, }; #define MLX5_MAX_RESERVED_GIDS 8 -- cgit v1.2.3 From f5d23ee137e51b4e5cd5d263b144d5e6719f6e52 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 8 Jun 2022 13:04:47 -0700 Subject: net/mlx5: Add IFC bits and enums for flow meter Add/extend structure layouts and defines for flow meter. Signed-off-by: Jianbo Liu Reviewed-by: Ariel Levkovich Signed-off-by: Saeed Mahameed --- include/linux/mlx5/device.h | 1 + include/linux/mlx5/mlx5_ifc.h | 114 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 604b85dd770a..15ac02eeed4f 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -455,6 +455,7 @@ enum { MLX5_OPCODE_UMR = 0x25, + MLX5_OPCODE_ACCESS_ASO = 0x2d, }; enum { diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 789d64400744..91872afb2bfe 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -442,7 +442,9 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 max_modify_header_actions[0x8]; u8 max_ft_level[0x8]; - u8 reserved_at_40[0x20]; + u8 reserved_at_40[0x6]; + u8 execute_aso[0x1]; + u8 reserved_at_47[0x19]; u8 reserved_at_60[0x2]; u8 reformat_insert[0x1]; @@ -940,7 +942,17 @@ struct mlx5_ifc_qos_cap_bits { u8 max_tsar_bw_share[0x20]; - u8 reserved_at_100[0x700]; + u8 reserved_at_100[0x20]; + + u8 reserved_at_120[0x3]; + u8 log_meter_aso_granularity[0x5]; + u8 reserved_at_128[0x3]; + u8 log_meter_aso_max_alloc[0x5]; + u8 reserved_at_130[0x3]; + u8 log_max_num_meter_aso[0x5]; + u8 reserved_at_138[0x8]; + + u8 reserved_at_140[0x6c0]; }; struct mlx5_ifc_debug_cap_bits { @@ -3280,6 +3292,7 @@ enum { MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 = 0x800, MLX5_FLOW_CONTEXT_ACTION_IPSEC_DECRYPT = 0x1000, MLX5_FLOW_CONTEXT_ACTION_IPSEC_ENCRYPT = 0x2000, + MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO = 0x4000, }; enum { @@ -3295,6 +3308,38 @@ struct mlx5_ifc_vlan_bits { u8 vid[0xc]; }; +enum { + MLX5_FLOW_METER_COLOR_RED = 0x0, + MLX5_FLOW_METER_COLOR_YELLOW = 0x1, + MLX5_FLOW_METER_COLOR_GREEN = 0x2, + MLX5_FLOW_METER_COLOR_UNDEFINED = 0x3, +}; + +enum { + MLX5_EXE_ASO_FLOW_METER = 0x2, +}; + +struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits { + u8 return_reg_id[0x4]; + u8 aso_type[0x4]; + u8 reserved_at_8[0x14]; + u8 action[0x1]; + u8 init_color[0x2]; + u8 meter_id[0x1]; +}; + +union mlx5_ifc_exe_aso_ctrl { + struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits exe_aso_ctrl_flow_meter; +}; + +struct mlx5_ifc_execute_aso_bits { + u8 valid[0x1]; + u8 reserved_at_1[0x7]; + u8 aso_object_id[0x18]; + + union mlx5_ifc_exe_aso_ctrl exe_aso_ctrl; +}; + struct mlx5_ifc_flow_context_bits { struct mlx5_ifc_vlan_bits push_vlan; @@ -3326,7 +3371,9 @@ struct mlx5_ifc_flow_context_bits { struct mlx5_ifc_fte_match_param_bits match_value; - u8 reserved_at_1200[0x600]; + struct mlx5_ifc_execute_aso_bits execute_aso[4]; + + u8 reserved_at_1300[0x500]; union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[]; }; @@ -5973,7 +6020,9 @@ struct mlx5_ifc_general_obj_in_cmd_hdr_bits { u8 obj_id[0x20]; - u8 reserved_at_60[0x20]; + u8 reserved_at_60[0x3]; + u8 log_obj_range[0x5]; + u8 reserved_at_68[0x18]; }; struct mlx5_ifc_general_obj_out_cmd_hdr_bits { @@ -11373,12 +11422,14 @@ enum { MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY = BIT_ULL(0xc), MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC = BIT_ULL(0x13), MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_SAMPLER = BIT_ULL(0x20), + MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_FLOW_METER_ASO = BIT_ULL(0x24), }; enum { MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY = 0xc, MLX5_GENERAL_OBJECT_TYPES_IPSEC = 0x13, MLX5_GENERAL_OBJECT_TYPES_SAMPLER = 0x20, + MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO = 0x24, }; enum { @@ -11451,6 +11502,61 @@ struct mlx5_ifc_create_encryption_key_in_bits { struct mlx5_ifc_encryption_key_obj_bits encryption_key_object; }; +enum { + MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH = 0x0, + MLX5_FLOW_METER_MODE_BYTES_CALC_WITH_L2 = 0x1, + MLX5_FLOW_METER_MODE_BYTES_CALC_WITH_L2_IPG = 0x2, + MLX5_FLOW_METER_MODE_NUM_PACKETS = 0x3, +}; + +struct mlx5_ifc_flow_meter_parameters_bits { + u8 valid[0x1]; + u8 bucket_overflow[0x1]; + u8 start_color[0x2]; + u8 both_buckets_on_green[0x1]; + u8 reserved_at_5[0x1]; + u8 meter_mode[0x2]; + u8 reserved_at_8[0x18]; + + u8 reserved_at_20[0x20]; + + u8 reserved_at_40[0x3]; + u8 cbs_exponent[0x5]; + u8 cbs_mantissa[0x8]; + u8 reserved_at_50[0x3]; + u8 cir_exponent[0x5]; + u8 cir_mantissa[0x8]; + + u8 reserved_at_60[0x20]; + + u8 reserved_at_80[0x3]; + u8 ebs_exponent[0x5]; + u8 ebs_mantissa[0x8]; + u8 reserved_at_90[0x3]; + u8 eir_exponent[0x5]; + u8 eir_mantissa[0x8]; + + u8 reserved_at_a0[0x60]; +}; + +struct mlx5_ifc_flow_meter_aso_obj_bits { + u8 modify_field_select[0x40]; + + u8 reserved_at_40[0x40]; + + u8 reserved_at_80[0x8]; + u8 meter_aso_access_pd[0x18]; + + u8 reserved_at_a0[0x160]; + + struct mlx5_ifc_flow_meter_parameters_bits flow_meter_parameters[2]; +}; + +struct mlx5_ifc_create_flow_meter_aso_obj_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr; + struct mlx5_ifc_flow_meter_aso_obj_bits flow_meter_aso_obj; +}; + struct mlx5_ifc_sampler_obj_bits { u8 modify_field_select[0x40]; -- cgit v1.2.3 From 3e94e61bd44d90070dcda53b647fdc826097ef26 Mon Sep 17 00:00:00 2001 From: Saeed Mahameed Date: Wed, 8 Jun 2022 13:04:48 -0700 Subject: net/mlx5: Add HW definitions of vport debug counters total_q_under_processor_handle - number of queues in error state due to an async error or errored command. send_queue_priority_update_flow - number of QP/SQ priority/SL update events. cq_overrun - number of times CQ entered an error state due to an overflow. async_eq_overrun -number of time an EQ mapped to async events was overrun. comp_eq_overrun - number of time an EQ mapped to completion events was overrun. quota_exceeded_command - number of commands issued and failed due to quota exceeded. invalid_command - number of commands issued and failed dues to any reason other than quota exceeded. Signed-off-by: Saeed Mahameed Signed-off-by: Michael Guralnik Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 91872afb2bfe..f678ad88a7d5 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1441,7 +1441,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_120[0xa]; u8 log_max_ra_req_dc[0x6]; - u8 reserved_at_130[0xa]; + u8 reserved_at_130[0x9]; + u8 vnic_env_cq_overrun[0x1]; u8 log_max_ra_res_dc[0x6]; u8 reserved_at_140[0x5]; @@ -1636,7 +1637,11 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 nic_receive_steering_discard[0x1]; u8 receive_discard_vport_down[0x1]; u8 transmit_discard_vport_down[0x1]; - u8 reserved_at_343[0x5]; + u8 eq_overrun_count[0x1]; + u8 reserved_at_344[0x1]; + u8 invalid_command_count[0x1]; + u8 quota_exceeded_count[0x1]; + u8 reserved_at_347[0x1]; u8 log_max_flow_counter_bulk[0x8]; u8 max_flow_counter_15_0[0x10]; @@ -3441,11 +3446,21 @@ struct mlx5_ifc_vnic_diagnostic_statistics_bits { u8 transmit_discard_vport_down[0x40]; - u8 reserved_at_140[0xa0]; + u8 async_eq_overrun[0x20]; + + u8 comp_eq_overrun[0x20]; + + u8 reserved_at_180[0x20]; + + u8 invalid_command[0x20]; + + u8 quota_exceeded_command[0x20]; u8 internal_rq_out_of_buffer[0x20]; - u8 reserved_at_200[0xe00]; + u8 cq_overrun[0x20]; + + u8 reserved_at_220[0xde0]; }; struct mlx5_ifc_traffic_counter_bits { -- cgit v1.2.3 From 91707779a481aab9c7f1d7a5ea3033ce87dc4fd6 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 8 Jun 2022 13:04:49 -0700 Subject: net/mlx5: Add support EXECUTE_ASO action for flow entry Attach flow meter to FTE with object id and index. Use metadata register C5 to store the packet color meter result. Signed-off-by: Jianbo Liu Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c | 33 ++++++++++++++++++++++++ include/linux/mlx5/fs.h | 14 ++++++++++ 2 files changed, 47 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 2ccf7bef9b05..735dc805dad7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -479,6 +479,30 @@ static int mlx5_set_extended_dest(struct mlx5_core_dev *dev, return 0; } + +static void +mlx5_cmd_set_fte_flow_meter(struct fs_fte *fte, void *in_flow_context) +{ + void *exe_aso_ctrl; + void *execute_aso; + + execute_aso = MLX5_ADDR_OF(flow_context, in_flow_context, + execute_aso[0]); + MLX5_SET(execute_aso, execute_aso, valid, 1); + MLX5_SET(execute_aso, execute_aso, aso_object_id, + fte->action.exe_aso.object_id); + + exe_aso_ctrl = MLX5_ADDR_OF(execute_aso, execute_aso, exe_aso_ctrl); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, return_reg_id, + fte->action.exe_aso.return_reg_id); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, aso_type, + fte->action.exe_aso.type); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, init_color, + fte->action.exe_aso.flow_meter.init_color); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, meter_id, + fte->action.exe_aso.flow_meter.meter_idx); +} + static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, int opmod, int modify_mask, struct mlx5_flow_table *ft, @@ -663,6 +687,15 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, list_size); } + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) { + if (fte->action.exe_aso.type == MLX5_EXE_ASO_FLOW_METER) { + mlx5_cmd_set_fte_flow_meter(fte, in_flow_context); + } else { + err = -EOPNOTSUPP; + goto err_out; + } + } + err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); err_out: kvfree(in); diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index 8135713b0d2d..ece3e35622d7 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -212,6 +212,19 @@ struct mlx5_flow_group * mlx5_create_flow_group(struct mlx5_flow_table *ft, u32 *in); void mlx5_destroy_flow_group(struct mlx5_flow_group *fg); +struct mlx5_exe_aso { + u32 object_id; + u8 type; + u8 return_reg_id; + union { + u32 ctrl_data; + struct { + u8 meter_idx; + u8 init_color; + } flow_meter; + }; +}; + struct mlx5_fs_vlan { u16 ethtype; u16 vid; @@ -237,6 +250,7 @@ struct mlx5_flow_act { struct mlx5_fs_vlan vlan[MLX5_FS_VLAN_DEPTH]; struct ib_counters *counters; struct mlx5_flow_group *fg; + struct mlx5_exe_aso exe_aso; }; #define MLX5_DECLARE_FLOW_ACT(name) \ -- cgit v1.2.3 From d107ba1f7c067b08eb4b8ca7c51187fb7c4d97f2 Mon Sep 17 00:00:00 2001 From: Shay Drory Date: Wed, 8 Jun 2022 13:04:51 -0700 Subject: net/mlx5: Remove not used MLX5_CAP_BITS_RW_MASK Remove not used MLX5_CAP_BITS_RW_MASK. While at it, remove CAP_MASK, MLX5_CAP_OFF_CMDIF_CSUM and MLX5_DEV_CAP_FLAG_*, since MLX5_CAP_BITS_RW_MASK was their only user. Signed-off-by: Shay Drory Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/main.c | 7 ------- include/linux/mlx5/device.h | 19 ------------------- 2 files changed, 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index c9b4e50a593e..2078d9f03a5f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -314,13 +314,6 @@ struct mlx5_reg_host_endianness { u8 rsvd[15]; }; -#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) - -enum { - MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | - MLX5_DEV_CAP_FLAG_DCT, -}; - static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) { switch (size) { diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 15ac02eeed4f..95a4fa0fd40a 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -386,21 +386,6 @@ enum { MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG = 9, }; -enum { - MLX5_DEV_CAP_FLAG_XRC = 1LL << 3, - MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR = 1LL << 8, - MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1LL << 9, - MLX5_DEV_CAP_FLAG_APM = 1LL << 17, - MLX5_DEV_CAP_FLAG_ATOMIC = 1LL << 18, - MLX5_DEV_CAP_FLAG_BLOCK_MCAST = 1LL << 23, - MLX5_DEV_CAP_FLAG_ON_DMND_PG = 1LL << 24, - MLX5_DEV_CAP_FLAG_CQ_MODER = 1LL << 29, - MLX5_DEV_CAP_FLAG_RESIZE_CQ = 1LL << 30, - MLX5_DEV_CAP_FLAG_DCT = 1LL << 37, - MLX5_DEV_CAP_FLAG_SIG_HAND_OVER = 1LL << 40, - MLX5_DEV_CAP_FLAG_CMDIF_CSUM = 3LL << 46, -}; - enum { MLX5_ROCE_VERSION_1 = 0, MLX5_ROCE_VERSION_2 = 2, @@ -496,10 +481,6 @@ enum { MLX5_MAX_PAGE_SHIFT = 31 }; -enum { - MLX5_CAP_OFF_CMDIF_CSUM = 46, -}; - enum { /* * Max wqe size for rdma read is 512 bytes, so this -- cgit v1.2.3 From cdcdce948d64139aea1c6dfea4b04f5c8ad2784e Mon Sep 17 00:00:00 2001 From: Ofer Levi Date: Wed, 8 Jun 2022 13:04:52 -0700 Subject: net/mlx5: Add bits and fields to support enhanced CQE compression Expose ifc bits and add needed structure fields and methods to support enhanced CQE compression feature. The enhanced CQE compression feature improves cpu utiliziation with better packet latency from nic to host. Signed-off-by: Ofer Levi Signed-off-by: Saeed Mahameed --- include/linux/mlx5/device.h | 16 +++++++++++++++- include/linux/mlx5/mlx5_ifc.h | 7 +++++-- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 95a4fa0fd40a..b5f58fd37a0f 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -822,7 +822,10 @@ struct mlx5_cqe64 { __be32 timestamp_l; __be32 sop_drop_qpn; __be16 wqe_counter; - u8 signature; + union { + u8 signature; + u8 validity_iteration_count; + }; u8 op_own; }; @@ -854,6 +857,11 @@ enum { MLX5_CQE_FORMAT_CSUM_STRIDX = 0x3, }; +enum { + MLX5_CQE_COMPRESS_LAYOUT_BASIC = 0, + MLX5_CQE_COMPRESS_LAYOUT_ENHANCED = 1, +}; + #define MLX5_MINI_CQE_ARRAY_SIZE 8 static inline u8 mlx5_get_cqe_format(struct mlx5_cqe64 *cqe) @@ -866,6 +874,12 @@ static inline u8 get_cqe_opcode(struct mlx5_cqe64 *cqe) return cqe->op_own >> 4; } +static inline u8 get_cqe_enhanced_num_mini_cqes(struct mlx5_cqe64 *cqe) +{ + /* num_of_mini_cqes is zero based */ + return get_cqe_opcode(cqe) + 1; +} + static inline u8 get_cqe_lro_tcppsh(struct mlx5_cqe64 *cqe) { return (cqe->lro.tcppsh_abort_dupack >> 6) & 1; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index f678ad88a7d5..8e87eb47f9dc 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1739,7 +1739,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 log_max_dci_errored_streams[0x5]; u8 reserved_at_598[0x8]; - u8 reserved_at_5a0[0x13]; + u8 reserved_at_5a0[0x10]; + u8 enhanced_cqe_compression[0x1]; + u8 reserved_at_5b1[0x2]; u8 log_max_dek[0x5]; u8 reserved_at_5b8[0x4]; u8 mini_cqe_resp_stride_index[0x1]; @@ -4139,7 +4141,8 @@ struct mlx5_ifc_cqc_bits { u8 cqe_comp_en[0x1]; u8 mini_cqe_res_format[0x2]; u8 st[0x4]; - u8 reserved_at_18[0x8]; + u8 reserved_at_18[0x6]; + u8 cqe_compression_layout[0x2]; u8 reserved_at_20[0x20]; -- cgit v1.2.3 From 9822bb87cee12a9d1e3321b652ba537af1f174aa Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 20 Feb 2022 16:33:27 +0000 Subject: iio: core: drop iio_get_time_res() This function was introduced with the ability to pick a clock. There are no upstream users so presumably it isn't as obviously useful as it seemed at the time. Hence drop it. Signed-off-by: Jonathan Cameron Link: https://lore.kernel.org/r/20220220163327.424696-1-jic23@kernel.org --- drivers/iio/industrialio-core.c | 23 ----------------------- include/linux/iio/iio.h | 1 - 2 files changed, 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 7517deec501e..dd42df854a57 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -334,29 +334,6 @@ s64 iio_get_time_ns(const struct iio_dev *indio_dev) } EXPORT_SYMBOL(iio_get_time_ns); -/** - * iio_get_time_res() - utility function to get time stamp clock resolution in - * nano seconds. - * @indio_dev: device - */ -unsigned int iio_get_time_res(const struct iio_dev *indio_dev) -{ - switch (iio_device_get_clock(indio_dev)) { - case CLOCK_REALTIME: - case CLOCK_MONOTONIC: - case CLOCK_MONOTONIC_RAW: - case CLOCK_BOOTTIME: - case CLOCK_TAI: - return hrtimer_resolution; - case CLOCK_REALTIME_COARSE: - case CLOCK_MONOTONIC_COARSE: - return LOW_RES_NSEC; - default: - BUG(); - } -} -EXPORT_SYMBOL(iio_get_time_res); - static int __init iio_init(void) { int ret; diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 233d2e6b7721..f6ea2ed99457 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -313,7 +313,6 @@ static inline bool iio_channel_has_available(const struct iio_chan_spec *chan, } s64 iio_get_time_ns(const struct iio_dev *indio_dev); -unsigned int iio_get_time_res(const struct iio_dev *indio_dev); /* * Device operating modes -- cgit v1.2.3 From 12c4efe3509b8018e76ea3ebda8227cb53bf5887 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 8 May 2022 18:55:41 +0100 Subject: iio: core: Fix IIO_ALIGN and rename as it was not sufficiently large MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discussion of the series: https://lore.kernel.org/all/20220405135758.774016-1-catalin.marinas@arm.com/ mm, arm64: Reduce ARCH_KMALLOC_MINALIGN brought to my attention that our current IIO usage of L1CACHE_ALIGN is insufficient as their are Arm platforms out their with non coherent DMA and larger cache lines at at higher levels of their cache hierarchy. Rename the define to make it's purpose more explicit. It will be used much more widely going forwards (to replace incorrect ____cacheline_aligned markings. Note this patch will greatly reduce the padding on some architectures that have smaller requirements for DMA safe buffers. The history of changing values of ARCH_KMALLOC_MINALIGN via ARCH_DMA_MINALIGN on arm64 is rather complex. I'm not tagging this as fixing a particular patch from that route as it's not clear what to tag. Most recently a change to bring them back inline was reverted because of some Qualcomm Kryo cores with an L2 cache with 128-byte lines sitting above the point of coherency. c1132702c71f Revert "arm64: cache: Lower ARCH_DMA_MINALIGN to 64 (L1_CACHE_BYTES)" That reverts: 65688d2a05de arm64: cache: Lower ARCH_DMA_MINALIGN to 64 (L1_CACHE_BYTES) which refers to the change originally being motivated by Thunder x1 performance rather than correctness. Fixes: 6f7c8ee585e9d ("staging:iio: Add ability to allocate private data space to iio_allocate_device") Signed-off-by: Jonathan Cameron Acked-by: Nuno Sá Link: https://lore.kernel.org/r/20220508175712.647246-2-jic23@kernel.org --- drivers/iio/accel/bma400_core.c | 2 +- drivers/iio/adc/adi-axi-adc.c | 7 ++++--- drivers/iio/industrialio-core.c | 4 ++-- include/linux/iio/iio.h | 10 ++++++++-- 4 files changed, 15 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/accel/bma400_core.c b/drivers/iio/accel/bma400_core.c index 517920400df1..c31bdd9b168e 100644 --- a/drivers/iio/accel/bma400_core.c +++ b/drivers/iio/accel/bma400_core.c @@ -93,7 +93,7 @@ struct bma400_data { __le16 buff[3]; u8 temperature; s64 ts __aligned(8); - } buffer __aligned(IIO_ALIGN); + } buffer __aligned(IIO_DMA_MINALIGN); __le16 status; __be16 duration; }; diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c index cf07d522925e..bd6fd0dd04b9 100644 --- a/drivers/iio/adc/adi-axi-adc.c +++ b/drivers/iio/adc/adi-axi-adc.c @@ -84,7 +84,8 @@ void *adi_axi_adc_conv_priv(struct adi_axi_adc_conv *conv) { struct adi_axi_adc_client *cl = conv_to_client(conv); - return (char *)cl + ALIGN(sizeof(struct adi_axi_adc_client), IIO_ALIGN); + return (char *)cl + ALIGN(sizeof(struct adi_axi_adc_client), + IIO_DMA_MINALIGN); } EXPORT_SYMBOL_NS_GPL(adi_axi_adc_conv_priv, IIO_ADI_AXI); @@ -169,9 +170,9 @@ static struct adi_axi_adc_conv *adi_axi_adc_conv_register(struct device *dev, struct adi_axi_adc_client *cl; size_t alloc_size; - alloc_size = ALIGN(sizeof(struct adi_axi_adc_client), IIO_ALIGN); + alloc_size = ALIGN(sizeof(struct adi_axi_adc_client), IIO_DMA_MINALIGN); if (sizeof_priv) - alloc_size += ALIGN(sizeof_priv, IIO_ALIGN); + alloc_size += ALIGN(sizeof_priv, IIO_DMA_MINALIGN); cl = kzalloc(alloc_size, GFP_KERNEL); if (!cl) diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index dd42df854a57..0703c8c7c180 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -1630,7 +1630,7 @@ struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv) alloc_size = sizeof(struct iio_dev_opaque); if (sizeof_priv) { - alloc_size = ALIGN(alloc_size, IIO_ALIGN); + alloc_size = ALIGN(alloc_size, IIO_DMA_MINALIGN); alloc_size += sizeof_priv; } @@ -1640,7 +1640,7 @@ struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv) indio_dev = &iio_dev_opaque->indio_dev; indio_dev->priv = (char *)iio_dev_opaque + - ALIGN(sizeof(struct iio_dev_opaque), IIO_ALIGN); + ALIGN(sizeof(struct iio_dev_opaque), IIO_DMA_MINALIGN); indio_dev->dev.parent = parent; indio_dev->dev.type = &iio_device_type; diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index f6ea2ed99457..4e21a82b3756 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -9,6 +9,7 @@ #include #include +#include #include #include /* IIO TODO LIST */ @@ -708,8 +709,13 @@ static inline void *iio_device_get_drvdata(const struct iio_dev *indio_dev) return dev_get_drvdata(&indio_dev->dev); } -/* Can we make this smaller? */ -#define IIO_ALIGN L1_CACHE_BYTES +/* + * Used to ensure the iio_priv() structure is aligned to allow that structure + * to in turn include IIO_DMA_MINALIGN'd elements such as buffers which + * must not share cachelines with the rest of the structure, thus making + * them safe for use with non-coherent DMA. + */ +#define IIO_DMA_MINALIGN ARCH_KMALLOC_MINALIGN struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv); /* The information at the returned address is guaranteed to be cacheline aligned */ -- cgit v1.2.3 From 6dbdc9f35360d4ef4704462c265f63b32fcb5354 Mon Sep 17 00:00:00 2001 From: Hongyi Lu Date: Mon, 13 Jun 2022 21:16:33 +0000 Subject: bpf: Fix spelling in bpf_verifier.h Minor spelling fix spotted in bpf_verifier.h. Spelling is no big deal, but it is still an improvement when reading through the code. Signed-off-by: Hongyi Lu Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220613211633.58647-1-jwnhy0@gmail.com --- include/linux/bpf_verifier.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e8439f6cbe57..3930c963fa67 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -299,7 +299,7 @@ struct bpf_verifier_state { * If is_state_visited() sees a state with branches > 0 it means * there is a loop. If such state is exactly equal to the current state * it's an infinite loop. Note states_equal() checks for states - * equvalency, so two states being 'states_equal' does not mean + * equivalency, so two states being 'states_equal' does not mean * infinite loop. The exact comparison is provided by * states_maybe_looping() function. It's a stronger pre-check and * much faster than states_equal(). -- cgit v1.2.3 From 064520e8aeaa2569f6504a50a37ac801b73656bc Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Wed, 15 Jun 2022 16:43:48 +0800 Subject: ASoC: SOF: Intel: Add support for MeteorLake (MTL) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add platform abstraction for the Meteor Lake platform. This platform has significant differences compared to the TGL/ADL generation: it relies on new hardware using the code name 'ACE' and only supports the INTEL_IPC4 protocol and firmware architecture based on the Zephyr RTOS Co-developed-by: Ranjani Sridharan Signed-off-by: Ranjani Sridharan Signed-off-by: Bard Liao Reviewed-by: Péter Ujfalusi Reviewed-by: Rander Wang Link: https://lore.kernel.org/r/20220615084348.3489-3-yung-chuan.liao@linux.intel.com Signed-off-by: Mark Brown --- include/linux/soundwire/sdw_intel.h | 2 + sound/soc/sof/intel/Kconfig | 16 + sound/soc/sof/intel/Makefile | 4 +- sound/soc/sof/intel/hda.h | 3 + sound/soc/sof/intel/mtl.c | 800 ++++++++++++++++++++++++++++++++++++ sound/soc/sof/intel/mtl.h | 76 ++++ sound/soc/sof/intel/pci-mtl.c | 71 ++++ sound/soc/sof/intel/shim.h | 1 + 8 files changed, 972 insertions(+), 1 deletion(-) create mode 100644 sound/soc/sof/intel/mtl.c create mode 100644 sound/soc/sof/intel/mtl.h create mode 100644 sound/soc/sof/intel/pci-mtl.c (limited to 'include/linux') diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h index 67e0d3e750b5..b5b489ea1aef 100644 --- a/include/linux/soundwire/sdw_intel.h +++ b/include/linux/soundwire/sdw_intel.h @@ -9,6 +9,8 @@ #define SDW_SHIM_BASE 0x2C000 #define SDW_ALH_BASE 0x2C800 +#define SDW_SHIM_BASE_ACE 0x38000 +#define SDW_ALH_BASE_ACE 0x24000 #define SDW_LINK_BASE 0x30000 #define SDW_LINK_SIZE 0x10000 diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig index 80cdc3788bbe..3f54678e810b 100644 --- a/sound/soc/sof/intel/Kconfig +++ b/sound/soc/sof/intel/Kconfig @@ -221,6 +221,22 @@ config SND_SOC_SOF_ALDERLAKE Say Y if you have such a device. If unsure select "N". +config SND_SOC_SOF_INTEL_MTL + tristate + select SND_SOC_SOF_HDA_COMMON + select SND_SOC_SOF_INTEL_SOUNDWIRE_LINK_BASELINE + select SND_SOC_SOF_INTEL_IPC4 + +config SND_SOC_SOF_METEORLAKE + tristate "SOF support for Meteorlake" + default SND_SOC_SOF_PCI + select SND_SOC_SOF_INTEL_MTL + help + This adds support for Sound Open Firmware for Intel(R) platforms + using the Meteorlake processors. + Say Y if you have such a device. + If unsure select "N". + config SND_SOC_SOF_HDA_COMMON tristate select SND_SOC_SOF_INTEL_COMMON diff --git a/sound/soc/sof/intel/Makefile b/sound/soc/sof/intel/Makefile index b9d51dc39ffa..a079159bb2f0 100644 --- a/sound/soc/sof/intel/Makefile +++ b/sound/soc/sof/intel/Makefile @@ -6,7 +6,7 @@ snd-sof-acpi-intel-bdw-objs := bdw.o snd-sof-intel-hda-common-objs := hda.o hda-loader.o hda-stream.o hda-trace.o \ hda-dsp.o hda-ipc.o hda-ctrl.o hda-pcm.o \ hda-dai.o hda-bus.o \ - apl.o cnl.o tgl.o icl.o hda-common-ops.o + apl.o cnl.o tgl.o icl.o mtl.o hda-common-ops.o snd-sof-intel-hda-common-$(CONFIG_SND_SOC_SOF_HDA_PROBES) += hda-probes.o snd-sof-intel-hda-objs := hda-codec.o @@ -24,9 +24,11 @@ snd-sof-pci-intel-apl-objs := pci-apl.o snd-sof-pci-intel-cnl-objs := pci-cnl.o snd-sof-pci-intel-icl-objs := pci-icl.o snd-sof-pci-intel-tgl-objs := pci-tgl.o +snd-sof-pci-intel-mtl-objs := pci-mtl.o obj-$(CONFIG_SND_SOC_SOF_MERRIFIELD) += snd-sof-pci-intel-tng.o obj-$(CONFIG_SND_SOC_SOF_INTEL_APL) += snd-sof-pci-intel-apl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_CNL) += snd-sof-pci-intel-cnl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_ICL) += snd-sof-pci-intel-icl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_TGL) += snd-sof-pci-intel-tgl.o +obj-$(CONFIG_SND_SOC_SOF_INTEL_MTL) += snd-sof-pci-intel-mtl.o diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h index 8b7f3c07d478..a3118499e34f 100644 --- a/sound/soc/sof/intel/hda.h +++ b/sound/soc/sof/intel/hda.h @@ -714,6 +714,8 @@ extern struct snd_sof_dsp_ops sof_tgl_ops; int sof_tgl_ops_init(struct snd_sof_dev *sdev); extern struct snd_sof_dsp_ops sof_icl_ops; int sof_icl_ops_init(struct snd_sof_dev *sdev); +extern struct snd_sof_dsp_ops sof_mtl_ops; +int sof_mtl_ops_init(struct snd_sof_dev *sdev); extern const struct sof_intel_dsp_desc apl_chip_info; extern const struct sof_intel_dsp_desc cnl_chip_info; @@ -723,6 +725,7 @@ extern const struct sof_intel_dsp_desc tglh_chip_info; extern const struct sof_intel_dsp_desc ehl_chip_info; extern const struct sof_intel_dsp_desc jsl_chip_info; extern const struct sof_intel_dsp_desc adls_chip_info; +extern const struct sof_intel_dsp_desc mtl_chip_info; /* Probes support */ #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) diff --git a/sound/soc/sof/intel/mtl.c b/sound/soc/sof/intel/mtl.c new file mode 100644 index 000000000000..37be77beb415 --- /dev/null +++ b/sound/soc/sof/intel/mtl.c @@ -0,0 +1,800 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// Copyright(c) 2022 Intel Corporation. All rights reserved. +// +// Authors: Ranjani Sridharan +// + +/* + * Hardware interface for audio DSP on Meteorlake. + */ + +#include +#include +#include "../ipc4-priv.h" +#include "../ops.h" +#include "hda.h" +#include "hda-ipc.h" +#include "../sof-audio.h" +#include "mtl.h" + +static const struct snd_sof_debugfs_map mtl_dsp_debugfs[] = { + {"hda", HDA_DSP_HDA_BAR, 0, 0x4000, SOF_DEBUGFS_ACCESS_ALWAYS}, + {"pp", HDA_DSP_PP_BAR, 0, 0x1000, SOF_DEBUGFS_ACCESS_ALWAYS}, + {"dsp", HDA_DSP_BAR, 0, 0x10000, SOF_DEBUGFS_ACCESS_ALWAYS}, +}; + +static void mtl_ipc_host_done(struct snd_sof_dev *sdev) +{ + /* + * clear busy interrupt to tell dsp controller this interrupt has been accepted, + * not trigger it again + */ + snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDR, + MTL_DSP_REG_HFIPCXTDR_BUSY, MTL_DSP_REG_HFIPCXTDR_BUSY); + /* + * clear busy bit to ack dsp the msg has been processed and send reply msg to dsp + */ + snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDA, + MTL_DSP_REG_HFIPCXTDA_BUSY, 0); +} + +static void mtl_ipc_dsp_done(struct snd_sof_dev *sdev) +{ + /* + * set DONE bit - tell DSP we have received the reply msg from DSP, and processed it, + * don't send more reply to host + */ + snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDA, + MTL_DSP_REG_HFIPCXIDA_DONE, MTL_DSP_REG_HFIPCXIDA_DONE); + + /* unmask Done interrupt */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXCTL, + MTL_DSP_REG_HFIPCXCTL_DONE, MTL_DSP_REG_HFIPCXCTL_DONE); +} + +/* Check if an IPC IRQ occurred */ +static bool mtl_dsp_check_ipc_irq(struct snd_sof_dev *sdev) +{ + u32 irq_status; + u32 hfintipptr; + + /* read Interrupt IP Pointer */ + hfintipptr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFINTIPPTR) & MTL_HFINTIPPTR_PTR_MASK; + irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr + MTL_DSP_IRQSTS); + + dev_vdbg(sdev->dev, "irq handler: irq_status:0x%x\n", irq_status); + + if (irq_status != U32_MAX && (irq_status & MTL_DSP_IRQSTS_IPC)) + return true; + + return false; +} + +/* Check if an SDW IRQ occurred */ +static bool mtl_dsp_check_sdw_irq(struct snd_sof_dev *sdev) +{ + u32 irq_status; + u32 hfintipptr; + + /* read Interrupt IP Pointer */ + hfintipptr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFINTIPPTR) & MTL_HFINTIPPTR_PTR_MASK; + irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr + MTL_DSP_IRQSTS); + + if (irq_status != U32_MAX && (irq_status & MTL_DSP_IRQSTS_SDW)) + return true; + + return false; +} + +static int mtl_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) +{ + struct sof_ipc4_msg *msg_data = msg->msg_data; + + /* send the message via mailbox */ + if (msg_data->data_size) + sof_mailbox_write(sdev, sdev->host_box.offset, msg_data->data_ptr, + msg_data->data_size); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDDY, + msg_data->extension); + snd_sof_dsp_write(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDR, + msg_data->primary | MTL_DSP_REG_HFIPCXIDR_BUSY); + + return 0; +} + +static void mtl_enable_ipc_interrupts(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; + + /* enable IPC DONE and BUSY interrupts */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl, + MTL_DSP_REG_HFIPCXCTL_BUSY | MTL_DSP_REG_HFIPCXCTL_DONE, + MTL_DSP_REG_HFIPCXCTL_BUSY | MTL_DSP_REG_HFIPCXCTL_DONE); +} + +static void mtl_disable_ipc_interrupts(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; + + /* disable IPC DONE and BUSY interrupts */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl, + MTL_DSP_REG_HFIPCXCTL_BUSY | MTL_DSP_REG_HFIPCXCTL_DONE, 0); +} + +static int mtl_enable_interrupts(struct snd_sof_dev *sdev) +{ + u32 hfintipptr; + u32 irqinten; + u32 host_ipc; + u32 hipcie; + int ret; + + /* read Interrupt IP Pointer */ + hfintipptr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFINTIPPTR) & MTL_HFINTIPPTR_PTR_MASK; + + /* Enable Host IPC and SOUNDWIRE */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, hfintipptr, + MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK, + MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK); + + /* check if operation was successful */ + host_ipc = MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK; + irqinten = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr); + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, hfintipptr, irqinten, + (irqinten & host_ipc) == host_ipc, + HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); + if (ret < 0) { + dev_err(sdev->dev, "failed to enable Host IPC and/or SOUNDWIRE\n"); + return ret; + } + + /* Set Host IPC interrupt enable */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, + MTL_DSP_REG_HfHIPCIE_IE_MASK, MTL_DSP_REG_HfHIPCIE_IE_MASK); + + /* check if operation was successful */ + host_ipc = MTL_DSP_REG_HfHIPCIE_IE_MASK; + hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE); + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, hipcie, + (hipcie & host_ipc) == host_ipc, + HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); + if (ret < 0) { + dev_err(sdev->dev, "failed to set Host IPC interrupt enable\n"); + return ret; + } + + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, + MTL_DSP_REG_HfSNDWIE_IE_MASK, MTL_DSP_REG_HfSNDWIE_IE_MASK); + host_ipc = MTL_DSP_REG_HfSNDWIE_IE_MASK; + hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE); + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, hipcie, + (hipcie & host_ipc) == host_ipc, + HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); + if (ret < 0) + dev_err(sdev->dev, "failed to set SoundWire IPC interrupt enable\n"); + + return ret; +} + +static int mtl_disable_interrupts(struct snd_sof_dev *sdev) +{ + u32 hfintipptr; + u32 irqinten; + u32 host_ipc; + u32 hipcie; + int ret1; + int ret; + + /* read Interrupt IP Pointer */ + hfintipptr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFINTIPPTR) & MTL_HFINTIPPTR_PTR_MASK; + + /* Disable Host IPC and SOUNDWIRE */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, hfintipptr, + MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK, 0); + + /* check if operation was successful */ + host_ipc = MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK; + irqinten = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr); + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, hfintipptr, irqinten, + (irqinten & host_ipc) == 0, + HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); + /* Continue to disable other interrupts when error happens */ + if (ret < 0) + dev_err(sdev->dev, "failed to disable Host IPC and SoundWire\n"); + + /* Set Host IPC interrupt disable */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, + MTL_DSP_REG_HfHIPCIE_IE_MASK, 0); + + /* check if operation was successful */ + host_ipc = MTL_DSP_REG_HfHIPCIE_IE_MASK; + hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE); + ret1 = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, hipcie, + (hipcie & host_ipc) == 0, + HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_RESET_TIMEOUT_US); + if (ret1 < 0) { + dev_err(sdev->dev, "failed to set Host IPC interrupt disable\n"); + if (!ret) + ret = ret1; + } + + /* Set SoundWire IPC interrupt disable */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, + MTL_DSP_REG_HfSNDWIE_IE_MASK, 0); + host_ipc = MTL_DSP_REG_HfSNDWIE_IE_MASK; + hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE); + ret1 = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, hipcie, + (hipcie & host_ipc) == 0, + HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_RESET_TIMEOUT_US); + if (ret1 < 0) { + dev_err(sdev->dev, "failed to set SoundWire IPC interrupt disable\n"); + if (!ret) + ret = ret1; + } + + return ret; +} + +/* pre fw run operations */ +static int mtl_dsp_pre_fw_run(struct snd_sof_dev *sdev) +{ + u32 dsphfpwrsts; + u32 dsphfdsscs; + u32 cpa; + u32 pgs; + int ret; + + /* Set the DSP subsystem power on */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFDSSCS, + MTL_HFDSSCS_SPA_MASK, MTL_HFDSSCS_SPA_MASK); + + /* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */ + usleep_range(1000, 1010); + + /* poll with timeout to check if operation successful */ + cpa = MTL_HFDSSCS_CPA_MASK; + dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS); + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs, + (dsphfdsscs & cpa) == cpa, HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_RESET_TIMEOUT_US); + if (ret < 0) { + dev_err(sdev->dev, "failed to enable DSP subsystem\n"); + return ret; + } + + /* Power up gated-DSP-0 domain in order to access the DSP shim register block. */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFPWRCTL, + MTL_HFPWRCTL_WPDSPHPXPG, MTL_HFPWRCTL_WPDSPHPXPG); + + usleep_range(1000, 1010); + + /* poll with timeout to check if operation successful */ + pgs = MTL_HFPWRSTS_DSPHPXPGS_MASK; + dsphfpwrsts = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFPWRSTS); + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFPWRSTS, dsphfpwrsts, + (dsphfpwrsts & pgs) == pgs, + HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_RESET_TIMEOUT_US); + if (ret < 0) + dev_err(sdev->dev, "failed to power up gated DSP domain\n"); + + /* make sure SoundWire is not power-gated */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, MTL_HFPWRCTL, + MTL_HfPWRCTL_WPIOXPG(1), MTL_HfPWRCTL_WPIOXPG(1)); + return ret; +} + +static int mtl_dsp_post_fw_run(struct snd_sof_dev *sdev) +{ + int ret; + + if (sdev->first_boot) { + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; + + ret = hda_sdw_startup(sdev); + if (ret < 0) { + dev_err(sdev->dev, "could not startup SoundWire links\n"); + return ret; + } + + /* Check if IMR boot is usable */ + if (!sof_debug_check_flag(SOF_DBG_IGNORE_D3_PERSISTENT)) + hdev->imrboot_supported = true; + } + + hda_sdw_int_enable(sdev, true); + return 0; +} + +static void mtl_dsp_dump(struct snd_sof_dev *sdev, u32 flags) +{ + char *level = (flags & SOF_DBG_DUMP_OPTIONAL) ? KERN_DEBUG : KERN_ERR; + u32 romdbgsts; + u32 romdbgerr; + u32 fwsts; + u32 fwlec; + + fwsts = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_ROM_STS); + fwlec = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_ROM_ERROR); + romdbgsts = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFFLGPXQWY); + romdbgerr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFFLGPXQWY_ERROR); + + dev_err(sdev->dev, "ROM status: %#x, ROM error: %#x\n", fwsts, fwlec); + dev_err(sdev->dev, "ROM debug status: %#x, ROM debug error: %#x\n", romdbgsts, + romdbgerr); + romdbgsts = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFFLGPXQWY + 0x8 * 3); + dev_printk(level, sdev->dev, "ROM feature bit%s enabled\n", + romdbgsts & BIT(24) ? "" : " not"); +} + +static bool mtl_dsp_primary_core_is_enabled(struct snd_sof_dev *sdev) +{ + int val; + + val = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP2CXCTL_PRIMARY_CORE); + if (val != U32_MAX && val & MTL_DSP2CXCTL_PRIMARY_CORE_CPA_MASK) + return true; + + return false; +} + +static int mtl_dsp_core_power_up(struct snd_sof_dev *sdev, int core) +{ + unsigned int cpa; + u32 dspcxctl; + int ret; + + /* Only the primary core can be powered up by the host */ + if (core != SOF_DSP_PRIMARY_CORE || mtl_dsp_primary_core_is_enabled(sdev)) + return 0; + + /* Program the owner of the IP & shim registers (10: Host CPU) */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP2CXCTL_PRIMARY_CORE, + MTL_DSP2CXCTL_PRIMARY_CORE_OSEL, + 0x2 << MTL_DSP2CXCTL_PRIMARY_CORE_OSEL_SHIFT); + + /* enable SPA bit */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP2CXCTL_PRIMARY_CORE, + MTL_DSP2CXCTL_PRIMARY_CORE_SPA_MASK, + MTL_DSP2CXCTL_PRIMARY_CORE_SPA_MASK); + + /* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */ + usleep_range(1000, 1010); + + /* poll with timeout to check if operation successful */ + cpa = MTL_DSP2CXCTL_PRIMARY_CORE_CPA_MASK; + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP2CXCTL_PRIMARY_CORE, dspcxctl, + (dspcxctl & cpa) == cpa, HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_RESET_TIMEOUT_US); + if (ret < 0) { + dev_err(sdev->dev, "%s: timeout on MTL_DSP2CXCTL_PRIMARY_CORE read\n", + __func__); + return ret; + } + + /* did core power up ? */ + dspcxctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP2CXCTL_PRIMARY_CORE); + if ((dspcxctl & MTL_DSP2CXCTL_PRIMARY_CORE_CPA_MASK) + != MTL_DSP2CXCTL_PRIMARY_CORE_CPA_MASK) { + dev_err(sdev->dev, "power up core failed core %d adspcs %#x\n", + core, dspcxctl); + ret = -EIO; + } + + return ret; +} + +static int mtl_dsp_core_power_down(struct snd_sof_dev *sdev, int core) +{ + u32 dspcxctl; + int ret; + + /* Only the primary core can be powered down by the host */ + if (core != SOF_DSP_PRIMARY_CORE || !mtl_dsp_primary_core_is_enabled(sdev)) + return 0; + + /* disable SPA bit */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP2CXCTL_PRIMARY_CORE, + MTL_DSP2CXCTL_PRIMARY_CORE_SPA_MASK, 0); + + /* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */ + usleep_range(1000, 1010); + + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP2CXCTL_PRIMARY_CORE, dspcxctl, + !(dspcxctl & MTL_DSP2CXCTL_PRIMARY_CORE_CPA_MASK), + HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_PD_TIMEOUT * USEC_PER_MSEC); + if (ret < 0) + dev_err(sdev->dev, "failed to power down primary core\n"); + + return ret; +} + +static int mtl_dsp_cl_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; + unsigned int status; + u32 ipc_hdr; + int ret; + + /* step 1: purge FW request */ + ipc_hdr = chip->ipc_req_mask | HDA_DSP_ROM_IPC_CONTROL; + if (!imr_boot) + ipc_hdr |= HDA_DSP_ROM_IPC_PURGE_FW | ((stream_tag - 1) << 9); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, chip->ipc_req, ipc_hdr); + + /* step 2: power up primary core */ + ret = mtl_dsp_core_power_up(sdev, SOF_DSP_PRIMARY_CORE); + if (ret < 0) { + if (hda->boot_iteration == HDA_FW_BOOT_ATTEMPTS) + dev_err(sdev->dev, "dsp core 0/1 power up failed\n"); + goto err; + } + + dev_dbg(sdev->dev, "Primary core power up successful\n"); + + /* step 3: wait for IPC DONE bit from ROM */ + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, chip->ipc_ack, status, + ((status & chip->ipc_ack_mask) == chip->ipc_ack_mask), + HDA_DSP_REG_POLL_INTERVAL_US, MTL_DSP_PURGE_TIMEOUT_US); + if (ret < 0) { + if (hda->boot_iteration == HDA_FW_BOOT_ATTEMPTS) + dev_err(sdev->dev, "timeout waiting for purge IPC done\n"); + goto err; + } + + /* set DONE bit to clear the reply IPC message */ + snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR, chip->ipc_ack, chip->ipc_ack_mask, + chip->ipc_ack_mask); + + /* step 4: enable interrupts */ + ret = mtl_enable_interrupts(sdev); + if (ret < 0) { + if (hda->boot_iteration == HDA_FW_BOOT_ATTEMPTS) + dev_err(sdev->dev, "%s: failed to enable interrupts\n", __func__); + goto err; + } + + mtl_enable_ipc_interrupts(sdev); + + /* + * ACE workaround: don't wait for ROM INIT. + * The platform cannot catch ROM_INIT_DONE because of a very short + * timing window. Follow the recommendations and skip this part. + */ + + return 0; + +err: + snd_sof_dsp_dbg_dump(sdev, "MTL DSP init fail", 0); + mtl_dsp_core_power_down(sdev, SOF_DSP_PRIMARY_CORE); + return ret; +} + +static irqreturn_t mtl_ipc_irq_thread(int irq, void *context) +{ + struct sof_ipc4_msg notification_data = {{ 0 }}; + struct snd_sof_dev *sdev = context; + bool ipc_irq = false; + u32 hipcida; + u32 hipctdr; + + hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDA); + + /* reply message from DSP */ + if (hipcida & MTL_DSP_REG_HFIPCXIDA_DONE) { + /* DSP received the message */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXCTL, + MTL_DSP_REG_HFIPCXCTL_DONE, 0); + + mtl_ipc_dsp_done(sdev); + + ipc_irq = true; + } + + hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDR); + if (hipctdr & MTL_DSP_REG_HFIPCXTDR_BUSY) { + /* Message from DSP (reply or notification) */ + u32 extension = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDDY); + u32 primary = hipctdr & MTL_DSP_REG_HFIPCXTDR_MSG_MASK; + + /* + * ACE fw sends a new fw ipc message to host to + * notify the status of the last host ipc message + */ + if (primary & SOF_IPC4_MSG_DIR_MASK) { + /* Reply received */ + struct sof_ipc4_msg *data = sdev->ipc->msg.reply_data; + + data->primary = primary; + data->extension = extension; + + spin_lock_irq(&sdev->ipc_lock); + + snd_sof_ipc_get_reply(sdev); + snd_sof_ipc_reply(sdev, data->primary); + + spin_unlock_irq(&sdev->ipc_lock); + } else { + /* Notification received */ + notification_data.primary = primary; + notification_data.extension = extension; + + sdev->ipc->msg.rx_data = ¬ification_data; + snd_sof_ipc_msgs_rx(sdev); + sdev->ipc->msg.rx_data = NULL; + } + + mtl_ipc_host_done(sdev); + + ipc_irq = true; + } + + if (!ipc_irq) { + /* This interrupt is not shared so no need to return IRQ_NONE. */ + dev_dbg_ratelimited(sdev->dev, "%s nothing to do in IPC IRQ thread\n", + __func__); + } + + return IRQ_HANDLED; +} + +static int mtl_dsp_ipc_get_mailbox_offset(struct snd_sof_dev *sdev) +{ + return MTL_DSP_MBOX_UPLINK_OFFSET; +} + +static int mtl_dsp_ipc_get_window_offset(struct snd_sof_dev *sdev, u32 id) +{ + return MTL_SRAM_WINDOW_OFFSET(id); +} + +static int mtl_suspend(struct snd_sof_dev *sdev, bool runtime_suspend) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + struct hdac_bus *bus = sof_to_bus(sdev); +#endif + u32 dsphfdsscs; + u32 cpa; + int ret; + int i; + + mtl_disable_ipc_interrupts(sdev); + ret = mtl_disable_interrupts(sdev); + if (ret) + return ret; + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + hda_codec_jack_wake_enable(sdev, runtime_suspend); + /* power down all hda link */ + snd_hdac_ext_bus_link_power_down_all(bus); +#endif + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFPWRCTL, + MTL_HFPWRCTL_WPDSPHPXPG, 0); + + /* Set the DSP subsystem power down */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFDSSCS, + MTL_HFDSSCS_SPA_MASK, 0); + + /* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */ + usleep_range(1000, 1010); + + /* poll with timeout to check if operation successful */ + cpa = MTL_HFDSSCS_CPA_MASK; + dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS); + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs, + (dsphfdsscs & cpa) == 0, HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_RESET_TIMEOUT_US); + if (ret < 0) + dev_err(sdev->dev, "failed to disable DSP subsystem\n"); + + /* reset ref counts for all cores */ + for (i = 0; i < chip->cores_num; i++) + sdev->dsp_core_ref_count[i] = 0; + + /* TODO: need to reset controller? */ + + /* display codec can be powered off after link reset */ + hda_codec_i915_display_power(sdev, false); + + return 0; +} + +static int mtl_dsp_suspend(struct snd_sof_dev *sdev, u32 target_state) +{ + const struct sof_dsp_power_state target_dsp_state = { + .state = target_state, + .substate = target_state == SOF_DSP_PM_D0 ? + SOF_HDA_DSP_PM_D0I3 : 0, + }; + int ret; + + ret = mtl_suspend(sdev, false); + if (ret < 0) + return ret; + + return snd_sof_dsp_set_power_state(sdev, &target_dsp_state); +} + +static int mtl_dsp_runtime_suspend(struct snd_sof_dev *sdev) +{ + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D3, + }; + int ret; + + ret = mtl_suspend(sdev, true); + if (ret < 0) + return ret; + + return snd_sof_dsp_set_power_state(sdev, &target_state); +} + +static int mtl_resume(struct snd_sof_dev *sdev, bool runtime_resume) +{ +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + struct hdac_bus *bus = sof_to_bus(sdev); + struct hdac_ext_link *hlink = NULL; +#endif + + /* display codec must be powered before link reset */ + hda_codec_i915_display_power(sdev, true); + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + /* check jack status */ + if (runtime_resume) { + hda_codec_jack_wake_enable(sdev, false); + if (sdev->system_suspend_target == SOF_SUSPEND_NONE) + hda_codec_jack_check(sdev); + } + + /* turn off the links that were off before suspend */ + list_for_each_entry(hlink, &bus->hlink_list, list) { + if (!hlink->ref_count) + snd_hdac_ext_bus_link_power_down(hlink); + } + + /* check dma status and clean up CORB/RIRB buffers */ + if (!bus->cmd_dma_state) + snd_hdac_bus_stop_cmd_io(bus); +#endif + + return 0; +} + +static int mtl_dsp_resume(struct snd_sof_dev *sdev) +{ + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D0, + .substate = SOF_HDA_DSP_PM_D0I0, + }; + int ret; + + ret = mtl_resume(sdev, false); + if (ret < 0) + return ret; + + return snd_sof_dsp_set_power_state(sdev, &target_state); +} + +static int mtl_dsp_runtime_resume(struct snd_sof_dev *sdev) +{ + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D0, + }; + int ret; + + ret = mtl_resume(sdev, true); + if (ret < 0) + return ret; + + return snd_sof_dsp_set_power_state(sdev, &target_state); +} + +static void mtl_ipc_dump(struct snd_sof_dev *sdev) +{ + u32 hipcctl; + u32 hipcida; + u32 hipctdr; + + /* read IPC status */ + hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDA); + hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXCTL); + hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDR); + + /* dump the IPC regs */ + /* TODO: parse the raw msg */ + dev_err(sdev->dev, + "error: host status 0x%8.8x dsp status 0x%8.8x mask 0x%8.8x\n", + hipcida, hipctdr, hipcctl); +} + +/* Meteorlake ops */ +struct snd_sof_dsp_ops sof_mtl_ops; +EXPORT_SYMBOL_NS(sof_mtl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); + +int sof_mtl_ops_init(struct snd_sof_dev *sdev) +{ + struct sof_ipc4_fw_data *ipc4_data; + + /* common defaults */ + memcpy(&sof_mtl_ops, &sof_hda_common_ops, sizeof(struct snd_sof_dsp_ops)); + + /* shutdown */ + sof_mtl_ops.shutdown = hda_dsp_shutdown; + + /* doorbell */ + sof_mtl_ops.irq_thread = mtl_ipc_irq_thread; + + /* ipc */ + sof_mtl_ops.send_msg = mtl_ipc_send_msg; + sof_mtl_ops.get_mailbox_offset = mtl_dsp_ipc_get_mailbox_offset; + sof_mtl_ops.get_window_offset = mtl_dsp_ipc_get_window_offset; + + /* debug */ + sof_mtl_ops.debug_map = mtl_dsp_debugfs; + sof_mtl_ops.debug_map_count = ARRAY_SIZE(mtl_dsp_debugfs); + sof_mtl_ops.dbg_dump = mtl_dsp_dump; + sof_mtl_ops.ipc_dump = mtl_ipc_dump; + + /* pre/post fw run */ + sof_mtl_ops.pre_fw_run = mtl_dsp_pre_fw_run; + sof_mtl_ops.post_fw_run = mtl_dsp_post_fw_run; + + /* parse platform specific extended manifest */ + sof_mtl_ops.parse_platform_ext_manifest = NULL; + + /* dsp core get/put */ + /* TODO: add core_get and core_put */ + + /* PM */ + sof_mtl_ops.suspend = mtl_dsp_suspend; + sof_mtl_ops.resume = mtl_dsp_resume; + sof_mtl_ops.runtime_suspend = mtl_dsp_runtime_suspend; + sof_mtl_ops.runtime_resume = mtl_dsp_runtime_resume; + + sdev->private = devm_kzalloc(sdev->dev, sizeof(struct sof_ipc4_fw_data), GFP_KERNEL); + if (!sdev->private) + return -ENOMEM; + + ipc4_data = sdev->private; + ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; + + /* set DAI ops */ + hda_set_dai_drv_ops(sdev, &sof_mtl_ops); + + return 0; +}; +EXPORT_SYMBOL_NS(sof_mtl_ops_init, SND_SOC_SOF_INTEL_HDA_COMMON); + +const struct sof_intel_dsp_desc mtl_chip_info = { + .cores_num = 3, + .init_core_mask = BIT(0), + .host_managed_cores_mask = BIT(0), + .ipc_req = MTL_DSP_REG_HFIPCXIDR, + .ipc_req_mask = MTL_DSP_REG_HFIPCXIDR_BUSY, + .ipc_ack = MTL_DSP_REG_HFIPCXIDA, + .ipc_ack_mask = MTL_DSP_REG_HFIPCXIDA_DONE, + .ipc_ctl = MTL_DSP_REG_HFIPCXCTL, + .rom_status_reg = MTL_DSP_ROM_STS, + .rom_init_timeout = 300, + .ssp_count = ICL_SSP_COUNT, + .ssp_base_offset = CNL_SSP_BASE_OFFSET, + .sdw_shim_base = SDW_SHIM_BASE_ACE, + .sdw_alh_base = SDW_ALH_BASE_ACE, + .check_sdw_irq = mtl_dsp_check_sdw_irq, + .check_ipc_irq = mtl_dsp_check_ipc_irq, + .cl_init = mtl_dsp_cl_init, + .hw_ip_version = SOF_INTEL_ACE_1_0, +}; +EXPORT_SYMBOL_NS(mtl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/mtl.h b/sound/soc/sof/intel/mtl.h new file mode 100644 index 000000000000..788bf0e3ea87 --- /dev/null +++ b/sound/soc/sof/intel/mtl.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * Copyright(c) 2020-2022 Intel Corporation. All rights reserved. + */ + +/* DSP Registers */ +#define MTL_HFDSSCS 0x1000 +#define MTL_HFDSSCS_SPA_MASK BIT(16) +#define MTL_HFDSSCS_CPA_MASK BIT(24) +#define MTL_HFSNDWIE 0x114C +#define MTL_HFPWRCTL 0x1D18 +#define MTL_HfPWRCTL_WPIOXPG(x) BIT((x) + 8) +#define MTL_HFPWRCTL_WPDSPHPXPG BIT(0) +#define MTL_HFPWRSTS 0x1D1C +#define MTL_HFPWRSTS_DSPHPXPGS_MASK BIT(0) +#define MTL_HFINTIPPTR 0x1108 +#define MTL_IRQ_INTEN_L_HOST_IPC_MASK BIT(0) +#define MTL_IRQ_INTEN_L_SOUNDWIRE_MASK BIT(6) +#define MTL_HFINTIPPTR_PTR_MASK GENMASK(20, 0) + +#define MTL_DSP2CXCAP_PRIMARY_CORE 0x178D00 +#define MTL_DSP2CXCTL_PRIMARY_CORE 0x178D04 +#define MTL_DSP2CXCTL_PRIMARY_CORE_SPA_MASK BIT(0) +#define MTL_DSP2CXCTL_PRIMARY_CORE_CPA_MASK BIT(8) +#define MTL_DSP2CXCTL_PRIMARY_CORE_OSEL GENMASK(25, 24) +#define MTL_DSP2CXCTL_PRIMARY_CORE_OSEL_SHIFT 24 + +/* IPC Registers */ +#define MTL_DSP_REG_HFIPCXTDR 0x73200 +#define MTL_DSP_REG_HFIPCXTDR_BUSY BIT(31) +#define MTL_DSP_REG_HFIPCXTDR_MSG_MASK GENMASK(30, 0) +#define MTL_DSP_REG_HFIPCXTDA 0x73204 +#define MTL_DSP_REG_HFIPCXTDA_BUSY BIT(31) +#define MTL_DSP_REG_HFIPCXIDR 0x73210 +#define MTL_DSP_REG_HFIPCXIDR_BUSY BIT(31) +#define MTL_DSP_REG_HFIPCXIDR_MSG_MASK GENMASK(30, 0) +#define MTL_DSP_REG_HFIPCXIDA 0x73214 +#define MTL_DSP_REG_HFIPCXIDA_DONE BIT(31) +#define MTL_DSP_REG_HFIPCXIDA_MSG_MASK GENMASK(30, 0) +#define MTL_DSP_REG_HFIPCXCTL 0x73228 +#define MTL_DSP_REG_HFIPCXCTL_BUSY BIT(0) +#define MTL_DSP_REG_HFIPCXCTL_DONE BIT(1) +#define MTL_DSP_REG_HFIPCXTDDY 0x73300 +#define MTL_DSP_REG_HFIPCXIDDY 0x73380 +#define MTL_DSP_REG_HfHIPCIE 0x1140 +#define MTL_DSP_REG_HfHIPCIE_IE_MASK BIT(0) +#define MTL_DSP_REG_HfSNDWIE 0x114C +#define MTL_DSP_REG_HfSNDWIE_IE_MASK GENMASK(3, 0) + +#define MTL_DSP_IRQSTS 0x20 +#define MTL_DSP_IRQSTS_IPC BIT(0) +#define MTL_DSP_IRQSTS_SDW BIT(6) + +#define MTL_DSP_PURGE_TIMEOUT_US 20000000 /* 20s */ +#define MTL_DSP_REG_POLL_INTERVAL_US 10 /* 10 us */ + +/* Memory windows */ +#define MTL_SRAM_WINDOW_OFFSET(x) (0x180000 + 0x8000 * (x)) + +#define MTL_DSP_MBOX_UPLINK_OFFSET (MTL_SRAM_WINDOW_OFFSET(0) + 0x1000) +#define MTL_DSP_MBOX_UPLINK_SIZE 0x1000 +#define MTL_DSP_MBOX_DOWNLINK_OFFSET MTL_SRAM_WINDOW_OFFSET(1) +#define MTL_DSP_MBOX_DOWNLINK_SIZE 0x1000 + +/* FW registers */ +#define MTL_DSP_ROM_STS MTL_SRAM_WINDOW_OFFSET(0) /* ROM status */ +#define MTL_DSP_ROM_ERROR (MTL_SRAM_WINDOW_OFFSET(0) + 0x4) /* ROM error code */ + +#define MTL_DSP_REG_HFFLGPXQWY 0x163200 /* ROM debug status */ +#define MTL_DSP_REG_HFFLGPXQWY_ERROR 0x163204 /* ROM debug error code */ +#define MTL_DSP_REG_HfIMRIS1 0x162088 +#define MTL_DSP_REG_HfIMRIS1_IU_MASK BIT(0) + diff --git a/sound/soc/sof/intel/pci-mtl.c b/sound/soc/sof/intel/pci-mtl.c new file mode 100644 index 000000000000..899b00d53d64 --- /dev/null +++ b/sound/soc/sof/intel/pci-mtl.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2018-2022 Intel Corporation. All rights reserved. +// +// Author: Ranjani Sridharan +// + +#include +#include +#include +#include +#include +#include "../ops.h" +#include "../sof-pci-dev.h" + +/* platform specific devices */ +#include "hda.h" +#include "mtl.h" + +static const struct sof_dev_desc mtl_desc = { + .use_acpi_target_states = true, + .machines = snd_soc_acpi_intel_mtl_machines, + .alt_machines = snd_soc_acpi_intel_mtl_sdw_machines, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .irqindex_host_ipc = -1, + .chip_info = &mtl_chip_info, + .ipc_supported_mask = BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_INTEL_IPC4, + .default_fw_path = { + [SOF_INTEL_IPC4] = "intel/sof-ipc4/mtl", + }, + .default_tplg_path = { + [SOF_INTEL_IPC4] = "intel/sof-ace-tplg", + }, + .default_fw_filename = { + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, + .nocodec_tplg_filename = "sof-mtl-nocodec.tplg", + .ops = &sof_mtl_ops, + .ops_init = sof_mtl_ops_init, +}; + +/* PCI IDs */ +static const struct pci_device_id sof_pci_ids[] = { + { PCI_DEVICE(0x8086, 0x7E28), /* MTL */ + .driver_data = (unsigned long)&mtl_desc}, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, sof_pci_ids); + +/* pci_driver definition */ +static struct pci_driver snd_sof_pci_intel_mtl_driver = { + .name = "sof-audio-pci-intel-mtl", + .id_table = sof_pci_ids, + .probe = hda_pci_intel_probe, + .remove = sof_pci_remove, + .shutdown = sof_pci_shutdown, + .driver = { + .pm = &sof_pci_pm, + }, +}; +module_pci_driver(snd_sof_pci_intel_mtl_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_IMPORT_NS(SND_SOC_SOF_INTEL_HDA_COMMON); +MODULE_IMPORT_NS(SND_SOC_SOF_PCI_DEV); diff --git a/sound/soc/sof/intel/shim.h b/sound/soc/sof/intel/shim.h index 371991fa474f..638159bee864 100644 --- a/sound/soc/sof/intel/shim.h +++ b/sound/soc/sof/intel/shim.h @@ -20,6 +20,7 @@ enum sof_intel_hw_ip_version { SOF_INTEL_CAVS_1_8, /* CannonLake, CometLake, CoffeeLake */ SOF_INTEL_CAVS_2_0, /* IceLake, JasperLake */ SOF_INTEL_CAVS_2_5, /* TigerLake, AlderLake */ + SOF_INTEL_ACE_1_0, /* MeteorLake */ }; /* -- cgit v1.2.3 From 6365a1935c5151455812e96d8de434c551dc0d98 Mon Sep 17 00:00:00 2001 From: Ma Wupeng Date: Tue, 14 Jun 2022 17:21:52 +0800 Subject: efi: Make code to find mirrored memory ranges generic Commit b05b9f5f9dcf ("x86, mirror: x86 enabling - find mirrored memory ranges") introduce the efi_find_mirror() function on x86. In order to reuse the API we make it public. Arm64 can support mirrored memory too, so function efi_find_mirror() is added to efi_init() to this support for arm64. Since efi_init() is shared by ARM, arm64 and riscv, this patch will bring mirror memory support for these architectures, but this support is only tested in arm64. Signed-off-by: Ma Wupeng Link: https://lore.kernel.org/r/20220614092156.1972846-2-mawupeng1@huawei.com [ardb: fix subject to better reflect the payload] Acked-by: Mike Rapoport Signed-off-by: Ard Biesheuvel --- arch/x86/include/asm/efi.h | 4 ---- arch/x86/platform/efi/efi.c | 23 ----------------------- drivers/firmware/efi/efi-init.c | 1 + drivers/firmware/efi/efi.c | 23 +++++++++++++++++++++++ include/linux/efi.h | 3 +++ 5 files changed, 27 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index 71943dce691e..eb90206eae80 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -383,7 +383,6 @@ static inline bool efi_is_64bit(void) extern bool efi_reboot_required(void); extern bool efi_is_table_address(unsigned long phys_addr); -extern void efi_find_mirror(void); extern void efi_reserve_boot_services(void); #else static inline void parse_efi_setup(u64 phys_addr, u32 data_len) {} @@ -395,9 +394,6 @@ static inline bool efi_is_table_address(unsigned long phys_addr) { return false; } -static inline void efi_find_mirror(void) -{ -} static inline void efi_reserve_boot_services(void) { } diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 1591d67e0bcd..6e598bd78eef 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -108,29 +108,6 @@ static int __init setup_add_efi_memmap(char *arg) } early_param("add_efi_memmap", setup_add_efi_memmap); -void __init efi_find_mirror(void) -{ - efi_memory_desc_t *md; - u64 mirror_size = 0, total_size = 0; - - if (!efi_enabled(EFI_MEMMAP)) - return; - - for_each_efi_memory_desc(md) { - unsigned long long start = md->phys_addr; - unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; - - total_size += size; - if (md->attribute & EFI_MEMORY_MORE_RELIABLE) { - memblock_mark_mirror(start, size); - mirror_size += size; - } - } - if (mirror_size) - pr_info("Memory: %lldM/%lldM mirrored memory\n", - mirror_size>>20, total_size>>20); -} - /* * Tell the kernel about the EFI memory map. This might include * more than the max 128 entries that can fit in the passed in e820 diff --git a/drivers/firmware/efi/efi-init.c b/drivers/firmware/efi/efi-init.c index b2c829e95bd1..3928dbff76d0 100644 --- a/drivers/firmware/efi/efi-init.c +++ b/drivers/firmware/efi/efi-init.c @@ -240,6 +240,7 @@ void __init efi_init(void) * And now, memblock is fully populated, it is time to do capping. */ early_init_dt_check_for_usable_mem_range(); + efi_find_mirror(); efi_esrt_init(); efi_mokvar_table_init(); diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 860534bcfdac..79c232e07de7 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -446,6 +446,29 @@ err_put: subsys_initcall(efisubsys_init); +void __init efi_find_mirror(void) +{ + efi_memory_desc_t *md; + u64 mirror_size = 0, total_size = 0; + + if (!efi_enabled(EFI_MEMMAP)) + return; + + for_each_efi_memory_desc(md) { + unsigned long long start = md->phys_addr; + unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; + + total_size += size; + if (md->attribute & EFI_MEMORY_MORE_RELIABLE) { + memblock_mark_mirror(start, size); + mirror_size += size; + } + } + if (mirror_size) + pr_info("Memory: %lldM/%lldM mirrored memory\n", + mirror_size>>20, total_size>>20); +} + /* * Find the efi memory descriptor for a given physical address. Given a * physical address, determine if it exists within an EFI Memory Map entry, diff --git a/include/linux/efi.h b/include/linux/efi.h index 7d9b0bb47eb3..53f64c14a525 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -872,6 +872,7 @@ static inline bool efi_rt_services_supported(unsigned int mask) { return (efi.runtime_supported_mask & mask) == mask; } +extern void efi_find_mirror(void); #else static inline bool efi_enabled(int feature) { @@ -889,6 +890,8 @@ static inline bool efi_rt_services_supported(unsigned int mask) { return false; } + +static inline void efi_find_mirror(void) {} #endif extern int efi_status_to_err(efi_status_t status); -- cgit v1.2.3 From f67be8b7ee90c292948c3ec6395673963cccaee6 Mon Sep 17 00:00:00 2001 From: Li Chen Date: Sun, 22 May 2022 20:26:58 -0700 Subject: regmap: provide regmap_field helpers for simple bit operations We have set/clear/test operations for regmap, but not for regmap_field yet. So let's introduce regmap_field helpers too. In many instances regmap_field_update_bits() is used for simple bit setting and clearing. In these cases the last argument is redundant and we can hide it with a static inline function. This adds three new helpers for simple bit operations: set_bits, clear_bits and test_bits (the last one defined as a regular function). Signed-off-by: Li Chen Link: https://lore.kernel.org/r/180eef422c3.deae9cd960729.8518395646822099769@zohomail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap.c | 22 ++++++++++++++++++++++ include/linux/regmap.h | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 2221d9863831..cb0be5e7b100 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -2220,6 +2220,28 @@ int regmap_field_update_bits_base(struct regmap_field *field, } EXPORT_SYMBOL_GPL(regmap_field_update_bits_base); +/** + * regmap_field_test_bits() - Check if all specified bits are set in a + * register field. + * + * @field: Register field to operate on + * @bits: Bits to test + * + * Returns -1 if the underlying regmap_field_read() fails, 0 if at least one of the + * tested bits is not set and 1 if all tested bits are set. + */ +int regmap_field_test_bits(struct regmap_field *field, unsigned int bits) +{ + unsigned int val, ret; + + ret = regmap_field_read(field, &val); + if (ret) + return ret; + + return (val & bits) == bits; +} +EXPORT_SYMBOL_GPL(regmap_field_test_bits); + /** * regmap_fields_update_bits_base() - Perform a read/modify/write cycle a * register field with port ID diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 8952fa3d0d59..d5b08f4f0dc0 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1336,6 +1336,22 @@ static inline int regmap_field_update_bits(struct regmap_field *field, NULL, false, false); } +static inline int regmap_field_set_bits(struct regmap_field *field, + unsigned int bits) +{ + return regmap_field_update_bits_base(field, bits, bits, NULL, false, + false); +} + +static inline int regmap_field_clear_bits(struct regmap_field *field, + unsigned int bits) +{ + return regmap_field_update_bits_base(field, bits, 0, NULL, false, + false); +} + +int regmap_field_test_bits(struct regmap_field *field, unsigned int bits); + static inline int regmap_field_force_update_bits(struct regmap_field *field, unsigned int mask, unsigned int val) @@ -1769,6 +1785,27 @@ regmap_field_force_update_bits(struct regmap_field *field, return -EINVAL; } +static inline int regmap_field_set_bits(struct regmap_field *field, + unsigned int bits) +{ + WARN_ONCE(1, "regmap API is disabled"); + return -EINVAL; +} + +static inline int regmap_field_clear_bits(struct regmap_field *field, + unsigned int bits) +{ + WARN_ONCE(1, "regmap API is disabled"); + return -EINVAL; +} + +static inline int regmap_field_test_bits(struct regmap_field *field, + unsigned int bits) +{ + WARN_ONCE(1, "regmap API is disabled"); + return -EINVAL; +} + static inline int regmap_fields_write(struct regmap_field *field, unsigned int id, unsigned int val) { -- cgit v1.2.3 From 003cbe046171596809c2f37dc07e69df1b4d9f95 Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Wed, 1 Jun 2022 20:58:55 +0530 Subject: pinctrl: Add pingroup and define PINCTRL_PINGROUP Add 'struct pingroup' to represent pingroup and 'PINCTRL_PINGROUP' macro for inline use. Both are used to manage and represent larger number of pingroups. Signed-off-by: Basavaraj Natikar Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20220601152900.1012813-2-Basavaraj.Natikar@amd.com Signed-off-by: Linus Walleij --- include/linux/pinctrl/pinctrl.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pinctrl/pinctrl.h b/include/linux/pinctrl/pinctrl.h index 70b45d28e7a9..487117ccb1bc 100644 --- a/include/linux/pinctrl/pinctrl.h +++ b/include/linux/pinctrl/pinctrl.h @@ -26,6 +26,26 @@ struct pin_config_item; struct gpio_chip; struct device_node; +/** + * struct pingroup - provides information on pingroup + * @name: a name for pingroup + * @pins: an array of pins in the pingroup + * @npins: number of pins in the pingroup + */ +struct pingroup { + const char *name; + const unsigned int *pins; + size_t npins; +}; + +/* Convenience macro to define a single named or anonymous pingroup */ +#define PINCTRL_PINGROUP(_name, _pins, _npins) \ +(struct pingroup){ \ + .name = _name, \ + .pins = _pins, \ + .npins = _npins, \ +} + /** * struct pinctrl_pin_desc - boards/machines provide information on their * pins, pads or other muxable units in this struct -- cgit v1.2.3 From 10f09307199da274584b8170a41228ca6dfed6d3 Mon Sep 17 00:00:00 2001 From: Nuno Sá Date: Fri, 10 Jun 2022 10:45:30 +0200 Subject: iio: core: drop of.h from iio.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no reason to include OF as we only need to forward declare 'of_phandle_args'. Previously, some drivers were actually relying on this for some headers (those were already fixed). Signed-off-by: Nuno Sá Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20220610084545.547700-20-nuno.sa@analog.com Signed-off-by: Jonathan Cameron --- include/linux/iio/iio.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 4e21a82b3756..d9b4a9ca9a0f 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -11,13 +11,14 @@ #include #include #include -#include /* IIO TODO LIST */ /* * Provide means of adjusting timer accuracy. * Currently assumes nano seconds. */ +struct of_phandle_args; + enum iio_shared_by { IIO_SEPARATE, IIO_SHARED_BY_TYPE, -- cgit v1.2.3 From af89cd45603483135bdd238fcb3fa871155a0ae1 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Fri, 20 May 2022 09:57:34 +0200 Subject: clk: Improve documentation for devm_clk_get() and its optional variant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make use of "Context:" and "Return:". Mention that the clk is not to be expected to be prepared, previously only not being enabled was mentioned which probably dates from the times when the concept of clk preparation wasn't invented yet. Also describe devm_clk_get_optional() fully instead of just referencing devm_clk_get(). Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20220520075737.758761-2-u.kleine-koenig@pengutronix.de Reviewed-by: Russell King (Oracle) Signed-off-by: Stephen Boyd --- include/linux/clk.h | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/clk.h b/include/linux/clk.h index 39faa54efe88..c8fc398d2ad7 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -443,15 +443,16 @@ int __must_check devm_clk_bulk_get_all(struct device *dev, * @dev: device for clock "consumer" * @id: clock consumer ID * - * Returns a struct clk corresponding to the clock producer, or + * Context: May sleep. + * + * Return: a struct clk corresponding to the clock producer, or * valid IS_ERR() condition containing errno. The implementation * uses @dev and @id to determine the clock consumer, and thereby * the clock producer. (IOW, @id may be identical strings, but * clk_get may return different clock producers depending on @dev.) * - * Drivers must assume that the clock source is not enabled. - * - * devm_clk_get should not be called from within interrupt context. + * Drivers must assume that the clock source is neither prepared nor + * enabled. * * The clock will automatically be freed when the device is unbound * from the bus. @@ -464,8 +465,20 @@ struct clk *devm_clk_get(struct device *dev, const char *id); * @dev: device for clock "consumer" * @id: clock consumer ID * - * Behaves the same as devm_clk_get() except where there is no clock producer. - * In this case, instead of returning -ENOENT, the function returns NULL. + * Context: May sleep. + * + * Return: a struct clk corresponding to the clock producer, or + * valid IS_ERR() condition containing errno. The implementation + * uses @dev and @id to determine the clock consumer, and thereby + * the clock producer. If no such clk is found, it returns NULL + * which serves as a dummy clk. That's the only difference compared + * to devm_clk_get(). + * + * Drivers must assume that the clock source is neither prepared nor + * enabled. + * + * The clock will automatically be freed when the device is unbound + * from the bus. */ struct clk *devm_clk_get_optional(struct device *dev, const char *id); -- cgit v1.2.3 From 7ef9651e9792b08eb310c6beb202cbc947f43cab Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Fri, 20 May 2022 09:57:36 +0200 Subject: clk: Provide new devm_clk helpers for prepared and enabled clocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a driver keeps a clock prepared (or enabled) during the whole lifetime of the driver, these helpers allow to simplify the drivers. Reviewed-by: Jonathan Cameron Reviewed-by: Alexandru Ardelean Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20220520075737.758761-4-u.kleine-koenig@pengutronix.de Signed-off-by: Stephen Boyd --- drivers/clk/clk-devres.c | 27 ++++++++++++ include/linux/clk.h | 109 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) (limited to 'include/linux') diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c index c822f4ef1584..43ccd20e0298 100644 --- a/drivers/clk/clk-devres.c +++ b/drivers/clk/clk-devres.c @@ -66,12 +66,39 @@ struct clk *devm_clk_get(struct device *dev, const char *id) } EXPORT_SYMBOL(devm_clk_get); +struct clk *devm_clk_get_prepared(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get, clk_prepare, clk_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_prepared); + +struct clk *devm_clk_get_enabled(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get, + clk_prepare_enable, clk_disable_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_enabled); + struct clk *devm_clk_get_optional(struct device *dev, const char *id) { return __devm_clk_get(dev, id, clk_get_optional, NULL, NULL); } EXPORT_SYMBOL(devm_clk_get_optional); +struct clk *devm_clk_get_optional_prepared(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get_optional, + clk_prepare, clk_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_optional_prepared); + +struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get_optional, + clk_prepare_enable, clk_disable_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_optional_enabled); + struct clk_bulk_devres { struct clk_bulk_data *clks; int num_clks; diff --git a/include/linux/clk.h b/include/linux/clk.h index c8fc398d2ad7..c13061cabdfc 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -459,6 +459,47 @@ int __must_check devm_clk_bulk_get_all(struct device *dev, */ struct clk *devm_clk_get(struct device *dev, const char *id); +/** + * devm_clk_get_prepared - devm_clk_get() + clk_prepare() + * @dev: device for clock "consumer" + * @id: clock consumer ID + * + * Context: May sleep. + * + * Return: a struct clk corresponding to the clock producer, or + * valid IS_ERR() condition containing errno. The implementation + * uses @dev and @id to determine the clock consumer, and thereby + * the clock producer. (IOW, @id may be identical strings, but + * clk_get may return different clock producers depending on @dev.) + * + * The returned clk (if valid) is prepared. Drivers must however assume + * that the clock is not enabled. + * + * The clock will automatically be unprepared and freed when the device + * is unbound from the bus. + */ +struct clk *devm_clk_get_prepared(struct device *dev, const char *id); + +/** + * devm_clk_get_enabled - devm_clk_get() + clk_prepare_enable() + * @dev: device for clock "consumer" + * @id: clock consumer ID + * + * Context: May sleep. + * + * Return: a struct clk corresponding to the clock producer, or + * valid IS_ERR() condition containing errno. The implementation + * uses @dev and @id to determine the clock consumer, and thereby + * the clock producer. (IOW, @id may be identical strings, but + * clk_get may return different clock producers depending on @dev.) + * + * The returned clk (if valid) is prepared and enabled. + * + * The clock will automatically be disabled, unprepared and freed + * when the device is unbound from the bus. + */ +struct clk *devm_clk_get_enabled(struct device *dev, const char *id); + /** * devm_clk_get_optional - lookup and obtain a managed reference to an optional * clock producer. @@ -482,6 +523,50 @@ struct clk *devm_clk_get(struct device *dev, const char *id); */ struct clk *devm_clk_get_optional(struct device *dev, const char *id); +/** + * devm_clk_get_optional_prepared - devm_clk_get_optional() + clk_prepare() + * @dev: device for clock "consumer" + * @id: clock consumer ID + * + * Context: May sleep. + * + * Return: a struct clk corresponding to the clock producer, or + * valid IS_ERR() condition containing errno. The implementation + * uses @dev and @id to determine the clock consumer, and thereby + * the clock producer. If no such clk is found, it returns NULL + * which serves as a dummy clk. That's the only difference compared + * to devm_clk_get_prepared(). + * + * The returned clk (if valid) is prepared. Drivers must however + * assume that the clock is not enabled. + * + * The clock will automatically be unprepared and freed when the + * device is unbound from the bus. + */ +struct clk *devm_clk_get_optional_prepared(struct device *dev, const char *id); + +/** + * devm_clk_get_optional_enabled - devm_clk_get_optional() + + * clk_prepare_enable() + * @dev: device for clock "consumer" + * @id: clock consumer ID + * + * Context: May sleep. + * + * Return: a struct clk corresponding to the clock producer, or + * valid IS_ERR() condition containing errno. The implementation + * uses @dev and @id to determine the clock consumer, and thereby + * the clock producer. If no such clk is found, it returns NULL + * which serves as a dummy clk. That's the only difference compared + * to devm_clk_get_enabled(). + * + * The returned clk (if valid) is prepared and enabled. + * + * The clock will automatically be disabled, unprepared and freed + * when the device is unbound from the bus. + */ +struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id); + /** * devm_get_clk_from_child - lookup and obtain a managed reference to a * clock producer from child node. @@ -826,12 +911,36 @@ static inline struct clk *devm_clk_get(struct device *dev, const char *id) return NULL; } +static inline struct clk *devm_clk_get_prepared(struct device *dev, + const char *id) +{ + return NULL; +} + +static inline struct clk *devm_clk_get_enabled(struct device *dev, + const char *id) +{ + return NULL; +} + static inline struct clk *devm_clk_get_optional(struct device *dev, const char *id) { return NULL; } +static inline struct clk *devm_clk_get_optional_prepared(struct device *dev, + const char *id) +{ + return NULL; +} + +static inline struct clk *devm_clk_get_optional_enabled(struct device *dev, + const char *id) +{ + return NULL; +} + static inline int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, struct clk_bulk_data *clks) { -- cgit v1.2.3 From 5a0e4529d9aee8ce348f628ad476c9ddb6cf457d Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 24 May 2022 10:21:52 -0500 Subject: dmaengine: dw-edma: Remove unused irq field in struct dw_edma_chip The "irq" field of struct dw_edma_chip was never used. Remove it. Link: https://lore.kernel.org/r/20220524152159.2370739-2-Frank.Li@nxp.com Tested-by: Serge Semin Tested-by: Manivannan Sadhasivam Signed-off-by: Frank Li Signed-off-by: Bjorn Helgaas Reviewed-by: Serge Semin Reviewed-by: Manivannan Sadhasivam Acked-By: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 1 - include/linux/dma/edma.h | 2 -- 2 files changed, 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index cee7aa231d7b..bc07923c3bc0 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -214,7 +214,6 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->dw = dw; chip->dev = dev; chip->id = pdev->devfn; - chip->irq = pdev->irq; dw->mf = vsec_data.mf; dw->nr_irqs = nr_irqs; diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index cab6e18773da..d4333e721588 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -18,13 +18,11 @@ struct dw_edma; * struct dw_edma_chip - representation of DesignWare eDMA controller hardware * @dev: struct device of the eDMA controller * @id: instance ID - * @irq: irq line * @dw: struct dw_edma that is filed by dw_edma_probe() */ struct dw_edma_chip { struct device *dev; int id; - int irq; struct dw_edma *dw; }; -- cgit v1.2.3 From e4a8864f74e9e9e4a7eb93952a4cfa35c165c930 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Fri, 10 Jun 2022 16:21:30 -0700 Subject: iosys-map: Fix typo in documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's one argument, vaddr_iomem, not 2 (vaddr and _iomem). Signed-off-by: Lucas De Marchi Reviewed-by: Christian König Link: https://patchwork.freedesktop.org/patch/msgid/20220610232130.2865479-3-lucas.demarchi@intel.com --- include/linux/iosys-map.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iosys-map.h b/include/linux/iosys-map.h index e69a002d5aa4..4b8406ee8bc4 100644 --- a/include/linux/iosys-map.h +++ b/include/linux/iosys-map.h @@ -23,7 +23,7 @@ * memcpy(vaddr, src, len); * * void *vaddr_iomem = ...; // pointer to I/O memory - * memcpy_toio(vaddr, _iomem, src, len); + * memcpy_toio(vaddr_iomem, src, len); * * The user of such pointer may not have information about the mapping of that * region or may want to have a single code path to handle operations on that -- cgit v1.2.3 From d687f621c518d791b5fffde8add3112d869b0b1b Mon Sep 17 00:00:00 2001 From: Delyan Kratunov Date: Tue, 14 Jun 2022 23:10:42 +0000 Subject: bpf: move bpf_prog to bpf.h In order to add a version of bpf_prog_run_array which accesses the bpf_prog->aux member, bpf_prog needs to be more than a forward declaration inside bpf.h. Given that filter.h already includes bpf.h, this merely reorders the type declarations for filter.h users. bpf.h users now have access to bpf_prog internals. Signed-off-by: Delyan Kratunov Link: https://lore.kernel.org/r/3ed7824e3948f22d84583649ccac0ff0d38b6b58.1655248076.git.delyank@fb.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 36 ++++++++++++++++++++++++++++++++++++ include/linux/filter.h | 34 ---------------------------------- 2 files changed, 36 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 8e6092d0ea95..69106ae46464 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -5,6 +5,7 @@ #define _LINUX_BPF_H 1 #include +#include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include @@ -1084,6 +1086,40 @@ struct bpf_prog_aux { }; }; +struct bpf_prog { + u16 pages; /* Number of allocated pages */ + u16 jited:1, /* Is our filter JIT'ed? */ + jit_requested:1,/* archs need to JIT the prog */ + gpl_compatible:1, /* Is filter GPL compatible? */ + cb_access:1, /* Is control block accessed? */ + dst_needed:1, /* Do we need dst entry? */ + blinding_requested:1, /* needs constant blinding */ + blinded:1, /* Was blinded */ + is_func:1, /* program is a bpf function */ + kprobe_override:1, /* Do we override a kprobe? */ + has_callchain_buf:1, /* callchain buffer allocated? */ + enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */ + call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */ + call_get_func_ip:1, /* Do we call get_func_ip() */ + tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */ + enum bpf_prog_type type; /* Type of BPF program */ + enum bpf_attach_type expected_attach_type; /* For some prog types */ + u32 len; /* Number of filter blocks */ + u32 jited_len; /* Size of jited insns in bytes */ + u8 tag[BPF_TAG_SIZE]; + struct bpf_prog_stats __percpu *stats; + int __percpu *active; + unsigned int (*bpf_func)(const void *ctx, + const struct bpf_insn *insn); + struct bpf_prog_aux *aux; /* Auxiliary fields */ + struct sock_fprog_kern *orig_prog; /* Original BPF program */ + /* Instructions for interpreter */ + union { + DECLARE_FLEX_ARRAY(struct sock_filter, insns); + DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi); + }; +}; + struct bpf_array_aux { /* Programs with direct jumps into programs part of this array. */ struct list_head poke_progs; diff --git a/include/linux/filter.h b/include/linux/filter.h index ed0c0ff42ad5..d0cbb31b1b4d 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -559,40 +559,6 @@ struct bpf_prog_stats { struct u64_stats_sync syncp; } __aligned(2 * sizeof(u64)); -struct bpf_prog { - u16 pages; /* Number of allocated pages */ - u16 jited:1, /* Is our filter JIT'ed? */ - jit_requested:1,/* archs need to JIT the prog */ - gpl_compatible:1, /* Is filter GPL compatible? */ - cb_access:1, /* Is control block accessed? */ - dst_needed:1, /* Do we need dst entry? */ - blinding_requested:1, /* needs constant blinding */ - blinded:1, /* Was blinded */ - is_func:1, /* program is a bpf function */ - kprobe_override:1, /* Do we override a kprobe? */ - has_callchain_buf:1, /* callchain buffer allocated? */ - enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */ - call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */ - call_get_func_ip:1, /* Do we call get_func_ip() */ - tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */ - enum bpf_prog_type type; /* Type of BPF program */ - enum bpf_attach_type expected_attach_type; /* For some prog types */ - u32 len; /* Number of filter blocks */ - u32 jited_len; /* Size of jited insns in bytes */ - u8 tag[BPF_TAG_SIZE]; - struct bpf_prog_stats __percpu *stats; - int __percpu *active; - unsigned int (*bpf_func)(const void *ctx, - const struct bpf_insn *insn); - struct bpf_prog_aux *aux; /* Auxiliary fields */ - struct sock_fprog_kern *orig_prog; /* Original BPF program */ - /* Instructions for interpreter */ - union { - DECLARE_FLEX_ARRAY(struct sock_filter, insns); - DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi); - }; -}; - struct sk_filter { refcount_t refcnt; struct rcu_head rcu; -- cgit v1.2.3 From 8c7dcb84e3b744b2b70baa7a44a9b1881c33a9c9 Mon Sep 17 00:00:00 2001 From: Delyan Kratunov Date: Tue, 14 Jun 2022 23:10:46 +0000 Subject: bpf: implement sleepable uprobes by chaining gps uprobes work by raising a trap, setting a task flag from within the interrupt handler, and processing the actual work for the uprobe on the way back to userspace. As a result, uprobe handlers already execute in a might_fault/_sleep context. The primary obstacle to sleepable bpf uprobe programs is therefore on the bpf side. Namely, the bpf_prog_array attached to the uprobe is protected by normal rcu. In order for uprobe bpf programs to become sleepable, it has to be protected by the tasks_trace rcu flavor instead (and kfree() called after a corresponding grace period). Therefore, the free path for bpf_prog_array now chains a tasks_trace and normal grace periods one after the other. Users who iterate under tasks_trace read section would be safe, as would users who iterate under normal read sections (from non-sleepable locations). The downside is that the tasks_trace latency affects all perf_event-attached bpf programs (and not just uprobe ones). This is deemed safe given the possible attach rates for kprobe/uprobe/tp programs. Separately, non-sleepable programs need access to dynamically sized rcu-protected maps, so bpf_run_prog_array_sleepables now conditionally takes an rcu read section, in addition to the overarching tasks_trace section. Signed-off-by: Delyan Kratunov Link: https://lore.kernel.org/r/ce844d62a2fd0443b08c5ab02e95bc7149f9aeb1.1655248076.git.delyank@fb.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 52 +++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/core.c | 15 +++++++++++++ kernel/trace/bpf_trace.c | 4 ++-- kernel/trace/trace_uprobe.c | 5 ++--- 4 files changed, 71 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 69106ae46464..f3e88afdaffe 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -26,6 +26,7 @@ #include #include #include +#include struct bpf_verifier_env; struct bpf_verifier_log; @@ -1372,6 +1373,8 @@ extern struct bpf_empty_prog_array bpf_empty_prog_array; struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); void bpf_prog_array_free(struct bpf_prog_array *progs); +/* Use when traversal over the bpf_prog_array uses tasks_trace rcu */ +void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs); int bpf_prog_array_length(struct bpf_prog_array *progs); bool bpf_prog_array_is_empty(struct bpf_prog_array *array); int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, @@ -1463,6 +1466,55 @@ bpf_prog_run_array(const struct bpf_prog_array *array, return ret; } +/* Notes on RCU design for bpf_prog_arrays containing sleepable programs: + * + * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array + * overall. As a result, we must use the bpf_prog_array_free_sleepable + * in order to use the tasks_trace rcu grace period. + * + * When a non-sleepable program is inside the array, we take the rcu read + * section and disable preemption for that program alone, so it can access + * rcu-protected dynamically sized maps. + */ +static __always_inline u32 +bpf_prog_run_array_sleepable(const struct bpf_prog_array __rcu *array_rcu, + const void *ctx, bpf_prog_run_fn run_prog) +{ + const struct bpf_prog_array_item *item; + const struct bpf_prog *prog; + const struct bpf_prog_array *array; + struct bpf_run_ctx *old_run_ctx; + struct bpf_trace_run_ctx run_ctx; + u32 ret = 1; + + might_fault(); + + rcu_read_lock_trace(); + migrate_disable(); + + array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held()); + if (unlikely(!array)) + goto out; + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); + item = &array->items[0]; + while ((prog = READ_ONCE(item->prog))) { + if (!prog->aux->sleepable) + rcu_read_lock(); + + run_ctx.bpf_cookie = item->bpf_cookie; + ret &= run_prog(prog, ctx); + item++; + + if (!prog->aux->sleepable) + rcu_read_unlock(); + } + bpf_reset_run_ctx(old_run_ctx); +out: + migrate_enable(); + rcu_read_unlock_trace(); + return ret; +} + #ifdef CONFIG_BPF_SYSCALL DECLARE_PER_CPU(int, bpf_prog_active); extern struct mutex bpf_stats_enabled_mutex; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index e78cc5eea4a5..b5ffebcce6cc 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2279,6 +2279,21 @@ void bpf_prog_array_free(struct bpf_prog_array *progs) kfree_rcu(progs, rcu); } +static void __bpf_prog_array_free_sleepable_cb(struct rcu_head *rcu) +{ + struct bpf_prog_array *progs; + + progs = container_of(rcu, struct bpf_prog_array, rcu); + kfree_rcu(progs, rcu); +} + +void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs) +{ + if (!progs || progs == &bpf_empty_prog_array.hdr) + return; + call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb); +} + int bpf_prog_array_length(struct bpf_prog_array *array) { struct bpf_prog_array_item *item; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 10b157a6d73e..d1c22594dbf9 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1936,7 +1936,7 @@ int perf_event_attach_bpf_prog(struct perf_event *event, event->prog = prog; event->bpf_cookie = bpf_cookie; rcu_assign_pointer(event->tp_event->prog_array, new_array); - bpf_prog_array_free(old_array); + bpf_prog_array_free_sleepable(old_array); unlock: mutex_unlock(&bpf_event_mutex); @@ -1962,7 +1962,7 @@ void perf_event_detach_bpf_prog(struct perf_event *event) bpf_prog_array_delete_safe(old_array, event->prog); } else { rcu_assign_pointer(event->tp_event->prog_array, new_array); - bpf_prog_array_free(old_array); + bpf_prog_array_free_sleepable(old_array); } bpf_prog_put(event->prog); diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 9711589273cd..0282c119b1b2 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "trace_dynevent.h" #include "trace_probe.h" @@ -1346,9 +1347,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, if (bpf_prog_array_valid(call)) { u32 ret; - preempt_disable(); - ret = trace_call_bpf(call, regs); - preempt_enable(); + ret = bpf_prog_run_array_sleepable(call->prog_array, regs, bpf_prog_run); if (!ret) return; } -- cgit v1.2.3 From d92725256b4f22d084b813b37ddc394da79aacab Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 30 May 2022 14:34:50 -0400 Subject: mm: avoid unnecessary page fault retires on shared memory types I observed that for each of the shared file-backed page faults, we're very likely to retry one more time for the 1st write fault upon no page. It's because we'll need to release the mmap lock for dirty rate limit purpose with balance_dirty_pages_ratelimited() (in fault_dirty_shared_page()). Then after that throttling we return VM_FAULT_RETRY. We did that probably because VM_FAULT_RETRY is the only way we can return to the fault handler at that time telling it we've released the mmap lock. However that's not ideal because it's very likely the fault does not need to be retried at all since the pgtable was well installed before the throttling, so the next continuous fault (including taking mmap read lock, walk the pgtable, etc.) could be in most cases unnecessary. It's not only slowing down page faults for shared file-backed, but also add more mmap lock contention which is in most cases not needed at all. To observe this, one could try to write to some shmem page and look at "pgfault" value in /proc/vmstat, then we should expect 2 counts for each shmem write simply because we retried, and vm event "pgfault" will capture that. To make it more efficient, add a new VM_FAULT_COMPLETED return code just to show that we've completed the whole fault and released the lock. It's also a hint that we should very possibly not need another fault immediately on this page because we've just completed it. This patch provides a ~12% perf boost on my aarch64 test VM with a simple program sequentially dirtying 400MB shmem file being mmap()ed and these are the time it needs: Before: 650.980 ms (+-1.94%) After: 569.396 ms (+-1.38%) I believe it could help more than that. We need some special care on GUP and the s390 pgfault handler (for gmap code before returning from pgfault), the rest changes in the page fault handlers should be relatively straightforward. Another thing to mention is that mm_account_fault() does take this new fault as a generic fault to be accounted, unlike VM_FAULT_RETRY. I explicitly didn't touch hmm_vma_fault() and break_ksm() because they do not handle VM_FAULT_RETRY even with existing code, so I'm literally keeping them as-is. Link: https://lkml.kernel.org/r/20220530183450.42886-1-peterx@redhat.com Signed-off-by: Peter Xu Acked-by: Geert Uytterhoeven Acked-by: Peter Zijlstra (Intel) Acked-by: Johannes Weiner Acked-by: Vineet Gupta Acked-by: Guo Ren Acked-by: Max Filippov Acked-by: Christian Borntraeger Acked-by: Michael Ellerman (powerpc) Acked-by: Catalin Marinas Reviewed-by: Alistair Popple Reviewed-by: Ingo Molnar Acked-by: Russell King (Oracle) [arm part] Acked-by: Heiko Carstens Cc: Vasily Gorbik Cc: Stafford Horne Cc: David S. Miller Cc: Johannes Berg Cc: Brian Cain Cc: Richard Henderson Cc: Richard Weinberger Cc: Benjamin Herrenschmidt Cc: Thomas Gleixner Cc: Janosch Frank Cc: Albert Ou Cc: Anton Ivanov Cc: Dave Hansen Cc: Borislav Petkov Cc: Sven Schnelle Cc: Andrea Arcangeli Cc: James Bottomley Cc: Al Viro Cc: Alexander Gordeev Cc: Jonas Bonn Cc: Will Deacon Cc: Vlastimil Babka Cc: Michal Simek Cc: Matt Turner Cc: Paul Mackerras Cc: David Hildenbrand Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Stefan Kristiansson Cc: Paul Walmsley Cc: Ivan Kokshaysky Cc: Chris Zankel Cc: Hugh Dickins Cc: Dinh Nguyen Cc: Rich Felker Cc: H. Peter Anvin Cc: Andy Lutomirski Cc: Thomas Bogendoerfer Cc: Helge Deller Cc: Yoshinori Sato Signed-off-by: Andrew Morton --- arch/alpha/mm/fault.c | 4 ++++ arch/arc/mm/fault.c | 4 ++++ arch/arm/mm/fault.c | 4 ++++ arch/arm64/mm/fault.c | 4 ++++ arch/csky/mm/fault.c | 4 ++++ arch/hexagon/mm/vm_fault.c | 4 ++++ arch/ia64/mm/fault.c | 4 ++++ arch/m68k/mm/fault.c | 4 ++++ arch/microblaze/mm/fault.c | 4 ++++ arch/mips/mm/fault.c | 4 ++++ arch/nios2/mm/fault.c | 4 ++++ arch/openrisc/mm/fault.c | 4 ++++ arch/parisc/mm/fault.c | 4 ++++ arch/powerpc/mm/copro_fault.c | 5 +++++ arch/powerpc/mm/fault.c | 5 +++++ arch/riscv/mm/fault.c | 4 ++++ arch/s390/mm/fault.c | 12 ++++++++++++ arch/sh/mm/fault.c | 4 ++++ arch/sparc/mm/fault_32.c | 4 ++++ arch/sparc/mm/fault_64.c | 5 +++++ arch/um/kernel/trap.c | 4 ++++ arch/x86/mm/fault.c | 4 ++++ arch/xtensa/mm/fault.c | 4 ++++ include/linux/mm_types.h | 2 ++ mm/gup.c | 34 +++++++++++++++++++++++++++++++++- mm/memory.c | 2 +- 26 files changed, 139 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index ec20c1004abf..ef427a6bdd1a 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -155,6 +155,10 @@ retry: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index dad27e4d69ff..5ca59a482632 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -146,6 +146,10 @@ retry: return; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + /* * Fault retry nuances, mmap_lock already relinquished by core mm */ diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index a062e07516dd..46cccd6bf705 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -322,6 +322,10 @@ retry: return 0; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return 0; + if (!(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_RETRY) { flags |= FAULT_FLAG_TRIED; diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index c5e11768e5c1..de166cdeb89a 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -608,6 +608,10 @@ retry: return 0; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return 0; + if (fault & VM_FAULT_RETRY) { mm_flags |= FAULT_FLAG_TRIED; goto retry; diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index 7215a46b6b8e..e15f736cca4b 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -285,6 +285,10 @@ good_area: return; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) { flags |= FAULT_FLAG_TRIED; diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c index 4fac4b9eb316..f73c7cbfe326 100644 --- a/arch/hexagon/mm/vm_fault.c +++ b/arch/hexagon/mm/vm_fault.c @@ -96,6 +96,10 @@ good_area: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + /* The most common case -- we are done. */ if (likely(!(fault & VM_FAULT_ERROR))) { if (fault & VM_FAULT_RETRY) { diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 07379d1a227f..ef78c2d66cdd 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -139,6 +139,10 @@ retry: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { /* * We ran out of memory, or some other thing happened diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index 71aa9f6315dc..4d2837eb3e2a 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -141,6 +141,10 @@ good_area: if (fault_signal_pending(fault, regs)) return 0; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return 0; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index a9626e6a68af..5c40c3ebe52f 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -222,6 +222,10 @@ good_area: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index b08bc556d30d..a27045f5a556 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -162,6 +162,10 @@ good_area: return; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c index a32f14cd72f2..edaca0a6c1c1 100644 --- a/arch/nios2/mm/fault.c +++ b/arch/nios2/mm/fault.c @@ -139,6 +139,10 @@ good_area: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c index 53b760af3bb7..b4762d66e9ef 100644 --- a/arch/openrisc/mm/fault.c +++ b/arch/openrisc/mm/fault.c @@ -165,6 +165,10 @@ good_area: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index 84bc437be5cd..9ad80d4d3389 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -311,6 +311,10 @@ good_area: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { /* * We hit a shared mapping outside of the file, or some diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c index c1cb21a00884..7c507fb48182 100644 --- a/arch/powerpc/mm/copro_fault.c +++ b/arch/powerpc/mm/copro_fault.c @@ -65,6 +65,11 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, ret = 0; *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0, NULL); + + /* The fault is fully completed (including releasing mmap lock) */ + if (*flt & VM_FAULT_COMPLETED) + return 0; + if (unlikely(*flt & VM_FAULT_ERROR)) { if (*flt & VM_FAULT_OOM) { ret = -ENOMEM; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index d53fed4eccbd..014005428687 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -511,6 +511,10 @@ retry: if (fault_signal_pending(fault, regs)) return user_mode(regs) ? 0 : SIGBUS; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + goto out; + /* * Handle the retry right now, the mmap_lock has been released in that * case. @@ -525,6 +529,7 @@ retry: if (unlikely(fault & VM_FAULT_ERROR)) return mm_fault_error(regs, address, fault); +out: /* * Major/minor page fault accounting. */ diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 40694f0cab9e..f2fbd1400b7c 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -326,6 +326,10 @@ good_area: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_RETRY)) { flags |= FAULT_FLAG_TRIED; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e173b6187ad5..973dcd05c293 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -433,6 +433,17 @@ retry: goto out_up; goto out; } + + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) { + if (gmap) { + mmap_read_lock(mm); + goto out_gmap; + } + fault = 0; + goto out; + } + if (unlikely(fault & VM_FAULT_ERROR)) goto out_up; @@ -452,6 +463,7 @@ retry: mmap_read_lock(mm); goto retry; } +out_gmap: if (IS_ENABLED(CONFIG_PGSTE) && gmap) { address = __gmap_link(gmap, current->thread.gmap_addr, address); diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index e175667b1363..acd2f5e50bfc 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -485,6 +485,10 @@ good_area: if (mm_fault_error(regs, error_code, address, fault)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (fault & VM_FAULT_RETRY) { flags |= FAULT_FLAG_TRIED; diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index ad569d9bd124..91259f291c54 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -190,6 +190,10 @@ good_area: if (fault_signal_pending(fault, regs)) return; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 253e07043298..4acc12eafbf5 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -427,6 +427,10 @@ good_area: if (fault_signal_pending(fault, regs)) goto exit_exception; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + goto lock_released; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; @@ -449,6 +453,7 @@ good_area: } mmap_read_unlock(mm); +lock_released: mm_rss = get_mm_rss(mm); #if defined(CONFIG_TRANSPARENT_HUGEPAGE) mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE)); diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index d1d5d0be0308..d3ce21c4ca32 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -76,6 +76,10 @@ good_area: if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) goto out_nosemaphore; + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return 0; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) { goto out_of_memory; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index fad8faa29d04..fe10c6d76bac 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1408,6 +1408,10 @@ good_area: return; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + /* * If we need to retry the mmap_lock has already been released, * and if there is a fatal signal pending there is no guarantee diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index 16f0a5ff5799..8c781b05c0bd 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c @@ -172,6 +172,10 @@ good_area: return; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index c29ab4c0cd5c..6b961a29bf26 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -729,6 +729,7 @@ typedef __bitwise unsigned int vm_fault_t; * @VM_FAULT_NEEDDSYNC: ->fault did not modify page tables and needs * fsync() to complete (for synchronous page faults * in DAX) + * @VM_FAULT_COMPLETED: ->fault completed, meanwhile mmap lock released * @VM_FAULT_HINDEX_MASK: mask HINDEX value * */ @@ -746,6 +747,7 @@ enum vm_fault_reason { VM_FAULT_FALLBACK = (__force vm_fault_t)0x000800, VM_FAULT_DONE_COW = (__force vm_fault_t)0x001000, VM_FAULT_NEEDDSYNC = (__force vm_fault_t)0x002000, + VM_FAULT_COMPLETED = (__force vm_fault_t)0x004000, VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000, }; diff --git a/mm/gup.c b/mm/gup.c index 551264407624..407a81d5ca03 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -951,6 +951,25 @@ static int faultin_page(struct vm_area_struct *vma, } ret = handle_mm_fault(vma, address, fault_flags, NULL); + + if (ret & VM_FAULT_COMPLETED) { + /* + * With FAULT_FLAG_RETRY_NOWAIT we'll never release the + * mmap lock in the page fault handler. Sanity check this. + */ + WARN_ON_ONCE(fault_flags & FAULT_FLAG_RETRY_NOWAIT); + if (locked) + *locked = 0; + /* + * We should do the same as VM_FAULT_RETRY, but let's not + * return -EBUSY since that's not reflecting the reality of + * what has happened - we've just fully completed a page + * fault, with the mmap lock released. Use -EAGAIN to show + * that we want to take the mmap lock _again_. + */ + return -EAGAIN; + } + if (ret & VM_FAULT_ERROR) { int err = vm_fault_to_errno(ret, *flags); @@ -1177,6 +1196,7 @@ retry: case 0: goto retry; case -EBUSY: + case -EAGAIN: ret = 0; fallthrough; case -EFAULT: @@ -1303,6 +1323,18 @@ retry: return -EINTR; ret = handle_mm_fault(vma, address, fault_flags, NULL); + + if (ret & VM_FAULT_COMPLETED) { + /* + * NOTE: it's a pity that we need to retake the lock here + * to pair with the unlock() in the callers. Ideally we + * could tell the callers so they do not need to unlock. + */ + mmap_read_lock(mm); + *unlocked = true; + return 0; + } + if (ret & VM_FAULT_ERROR) { int err = vm_fault_to_errno(ret, 0); @@ -1368,7 +1400,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, /* VM_FAULT_RETRY couldn't trigger, bypass */ return ret; - /* VM_FAULT_RETRY cannot return errors */ + /* VM_FAULT_RETRY or VM_FAULT_COMPLETED cannot return errors */ if (!*locked) { BUG_ON(ret < 0); BUG_ON(ret >= nr_pages); diff --git a/mm/memory.c b/mm/memory.c index 7a089145cad4..580c62febe42 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3020,7 +3020,7 @@ static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf) balance_dirty_pages_ratelimited(mapping); if (fpin) { fput(fpin); - return VM_FAULT_RETRY; + return VM_FAULT_COMPLETED; } } -- cgit v1.2.3 From bcc728eb4f446073e0160671d7d0059a4e9aa300 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Tue, 31 May 2022 10:04:21 +0800 Subject: mm/damon: remove obsolete comments of kdamond_stop Since commit 0f91d13366a4 ("mm/damon: simplify stop mechanism") delete kdamond_stop and change to use kthread stop mechanism, these obsolete comments should be removed accordingly. Link: https://lkml.kernel.org/r/20220531020421.46849-1-zhouchengming@bytedance.com Signed-off-by: Chengming Zhou Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 7c62da31ce4b..2765c7d99beb 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -397,7 +397,6 @@ struct damon_callback { * detail. * * @kdamond: Kernel thread who does the monitoring. - * @kdamond_stop: Notifies whether kdamond should stop. * @kdamond_lock: Mutex for the synchronizations with @kdamond. * * For each monitoring context, one kernel thread for the monitoring is @@ -406,14 +405,14 @@ struct damon_callback { * Once started, the monitoring thread runs until explicitly required to be * terminated or every monitoring target is invalid. The validity of the * targets is checked via the &damon_operations.target_valid of @ops. The - * termination can also be explicitly requested by writing non-zero to - * @kdamond_stop. The thread sets @kdamond to NULL when it terminates. - * Therefore, users can know whether the monitoring is ongoing or terminated by - * reading @kdamond. Reads and writes to @kdamond and @kdamond_stop from - * outside of the monitoring thread must be protected by @kdamond_lock. - * - * Note that the monitoring thread protects only @kdamond and @kdamond_stop via - * @kdamond_lock. Accesses to other fields must be protected by themselves. + * termination can also be explicitly requested by calling damon_stop(). + * The thread sets @kdamond to NULL when it terminates. Therefore, users can + * know whether the monitoring is ongoing or terminated by reading @kdamond. + * Reads and writes to @kdamond from outside of the monitoring thread must + * be protected by @kdamond_lock. + * + * Note that the monitoring thread protects only @kdamond via @kdamond_lock. + * Accesses to other fields must be protected by themselves. * * @ops: Set of monitoring operations for given use cases. * @callback: Set of callbacks for monitoring events notifications. -- cgit v1.2.3 From 9384d79249d04b03572abb7e551a35d99c9268c0 Mon Sep 17 00:00:00 2001 From: "Fabio M. De Francesco" Date: Mon, 6 Jun 2022 16:15:33 +0200 Subject: mm/highmem: delete memmove_page() Matthew Wilcox reported that, while he was looking at memmove_page(), he realized that it can't actually work. The reasons are hidden in its implementation, which makes use of memmove() on logical addresses provided by kmap_local_page(). memmove() does the wrong thing when it tests "if (dest <= src)". Therefore, delete memmove_page(). No need to change any other code because we have no call sites of memmove_page() across the whole kernel. Link: https://lkml.kernel.org/r/20220606141533.555-1-fmdefrancesco@gmail.com Signed-off-by: Fabio M. De Francesco Reported-by: Matthew Wilcox Reviewed-by: Baoquan He Reviewed-by: Ira Weiny Cc: Sebastian Andrzej Siewior Signed-off-by: Andrew Morton --- include/linux/highmem.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 3af34de54330..fee9835e3793 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -336,19 +336,6 @@ static inline void memcpy_page(struct page *dst_page, size_t dst_off, kunmap_local(dst); } -static inline void memmove_page(struct page *dst_page, size_t dst_off, - struct page *src_page, size_t src_off, - size_t len) -{ - char *dst = kmap_local_page(dst_page); - char *src = kmap_local_page(src_page); - - VM_BUG_ON(dst_off + len > PAGE_SIZE || src_off + len > PAGE_SIZE); - memmove(dst + dst_off, src + src_off, len); - kunmap_local(src); - kunmap_local(dst); -} - static inline void memset_page(struct page *page, size_t offset, int val, size_t len) { -- cgit v1.2.3 From c200d90049dbe08fa8b016f74b713fddefca0479 Mon Sep 17 00:00:00 2001 From: Patrick Wang Date: Sat, 11 Jun 2022 11:55:48 +0800 Subject: mm: kmemleak: remove kmemleak_not_leak_phys() and the min_count argument to kmemleak_alloc_phys() Patch series "mm: kmemleak: store objects allocated with physical address separately and check when scan", v4. The kmemleak_*_phys() interface uses "min_low_pfn" and "max_low_pfn" to check address. But on some architectures, kmemleak_*_phys() is called before those two variables initialized. The following steps will be taken: 1) Add OBJECT_PHYS flag and rbtree for the objects allocated with physical address 2) Store physical address in objects if allocated with OBJECT_PHYS 3) Check the boundary when scan instead of in kmemleak_*_phys() This patch set will solve: https://lore.kernel.org/r/20220527032504.30341-1-yee.lee@mediatek.com https://lore.kernel.org/r/9dd08bb5-f39e-53d8-f88d-bec598a08c93@gmail.com v3: https://lore.kernel.org/r/20220609124950.1694394-1-patrick.wang.shcn@gmail.com v2: https://lore.kernel.org/r/20220603035415.1243913-1-patrick.wang.shcn@gmail.com v1: https://lore.kernel.org/r/20220531150823.1004101-1-patrick.wang.shcn@gmail.com This patch (of 4): Remove the unused kmemleak_not_leak_phys() function. And remove the min_count argument to kmemleak_alloc_phys() function, assume it's 0. Link: https://lkml.kernel.org/r/20220611035551.1823303-1-patrick.wang.shcn@gmail.com Link: https://lkml.kernel.org/r/20220611035551.1823303-2-patrick.wang.shcn@gmail.com Signed-off-by: Patrick Wang Suggested-by: Catalin Marinas Reviewed-by: Catalin Marinas Cc: Yee Lee Signed-off-by: Andrew Morton --- Documentation/dev-tools/kmemleak.rst | 1 - drivers/of/fdt.c | 2 +- include/linux/kmemleak.h | 8 ++------ mm/kmemleak.c | 20 +++----------------- mm/memblock.c | 14 +++++++------- tools/testing/memblock/linux/kmemleak.h | 2 +- 6 files changed, 14 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst index 1c935f41cd3a..5483fd39ef29 100644 --- a/Documentation/dev-tools/kmemleak.rst +++ b/Documentation/dev-tools/kmemleak.rst @@ -174,7 +174,6 @@ mapping: - ``kmemleak_alloc_phys`` - ``kmemleak_free_part_phys`` -- ``kmemleak_not_leak_phys`` - ``kmemleak_ignore_phys`` Dealing with false positives/negatives diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index a8f5b6532165..2c677e84c3f5 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -529,7 +529,7 @@ static int __init __reserved_mem_reserve_reg(unsigned long node, pr_debug("Reserved memory: reserved region for node '%s': base %pa, size %lu MiB\n", uname, &base, (unsigned long)(size / SZ_1M)); if (!nomap) - kmemleak_alloc_phys(base, size, 0, 0); + kmemleak_alloc_phys(base, size, 0); } else pr_info("Reserved memory: failed to reserve memory for node '%s': base %pa, size %lu MiB\n", diff --git a/include/linux/kmemleak.h b/include/linux/kmemleak.h index 34684b2026ab..6a3cd1bf4680 100644 --- a/include/linux/kmemleak.h +++ b/include/linux/kmemleak.h @@ -29,10 +29,9 @@ extern void kmemleak_not_leak(const void *ptr) __ref; extern void kmemleak_ignore(const void *ptr) __ref; extern void kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp) __ref; extern void kmemleak_no_scan(const void *ptr) __ref; -extern void kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count, +extern void kmemleak_alloc_phys(phys_addr_t phys, size_t size, gfp_t gfp) __ref; extern void kmemleak_free_part_phys(phys_addr_t phys, size_t size) __ref; -extern void kmemleak_not_leak_phys(phys_addr_t phys) __ref; extern void kmemleak_ignore_phys(phys_addr_t phys) __ref; static inline void kmemleak_alloc_recursive(const void *ptr, size_t size, @@ -107,15 +106,12 @@ static inline void kmemleak_no_scan(const void *ptr) { } static inline void kmemleak_alloc_phys(phys_addr_t phys, size_t size, - int min_count, gfp_t gfp) + gfp_t gfp) { } static inline void kmemleak_free_part_phys(phys_addr_t phys, size_t size) { } -static inline void kmemleak_not_leak_phys(phys_addr_t phys) -{ -} static inline void kmemleak_ignore_phys(phys_addr_t phys) { } diff --git a/mm/kmemleak.c b/mm/kmemleak.c index a182f5ddaf68..156eafafa182 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1125,15 +1125,13 @@ EXPORT_SYMBOL(kmemleak_no_scan); * address argument * @phys: physical address of the object * @size: size of the object - * @min_count: minimum number of references to this object. - * See kmemleak_alloc() * @gfp: kmalloc() flags used for kmemleak internal memory allocations */ -void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count, - gfp_t gfp) +void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, gfp_t gfp) { if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn) - kmemleak_alloc(__va(phys), size, min_count, gfp); + /* assume min_count 0 */ + kmemleak_alloc(__va(phys), size, 0, gfp); } EXPORT_SYMBOL(kmemleak_alloc_phys); @@ -1151,18 +1149,6 @@ void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size) } EXPORT_SYMBOL(kmemleak_free_part_phys); -/** - * kmemleak_not_leak_phys - similar to kmemleak_not_leak but taking a physical - * address argument - * @phys: physical address of the object - */ -void __ref kmemleak_not_leak_phys(phys_addr_t phys) -{ - if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn) - kmemleak_not_leak(__va(phys)); -} -EXPORT_SYMBOL(kmemleak_not_leak_phys); - /** * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical * address argument diff --git a/mm/memblock.c b/mm/memblock.c index e4f03a6e8e56..749abd2685c4 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1345,8 +1345,8 @@ __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, * from the regions with mirroring enabled and then retried from any * memory region. * - * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for - * allocated boot memory block, so that it is never reported as leaks. + * In addition, function using kmemleak_alloc_phys for allocated boot + * memory block, it is never reported as leaks. * * Return: * Physical address of allocated memory block on success, %0 on failure. @@ -1398,12 +1398,12 @@ done: */ if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) /* - * The min_count is set to 0 so that memblock allocated - * blocks are never reported as leaks. This is because many - * of these blocks are only referred via the physical - * address which is not looked up by kmemleak. + * Memblock allocated blocks are never reported as + * leaks. This is because many of these blocks are + * only referred via the physical address which is + * not looked up by kmemleak. */ - kmemleak_alloc_phys(found, size, 0, 0); + kmemleak_alloc_phys(found, size, 0); return found; } diff --git a/tools/testing/memblock/linux/kmemleak.h b/tools/testing/memblock/linux/kmemleak.h index 462f8c5e8aa0..5fed13bb9ec4 100644 --- a/tools/testing/memblock/linux/kmemleak.h +++ b/tools/testing/memblock/linux/kmemleak.h @@ -7,7 +7,7 @@ static inline void kmemleak_free_part_phys(phys_addr_t phys, size_t size) } static inline void kmemleak_alloc_phys(phys_addr_t phys, size_t size, - int min_count, gfp_t gfp) + gfp_t gfp) { } -- cgit v1.2.3 From fc4db90fe71e640e3fe88df346f7cf653b75315d Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Fri, 10 Jun 2022 11:03:10 -0700 Subject: mm: kmem: make mem_cgroup_from_obj() vmalloc()-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently mem_cgroup_from_obj() is not working properly with objects allocated using vmalloc(). It creates problems in some cases, when it's called for static objects belonging to modules or generally allocated using vmalloc(). This patch makes mem_cgroup_from_obj() safe to be called on objects allocated using vmalloc(). It also introduces mem_cgroup_from_slab_obj(), which is a faster version to use in places when we know the object is either a slab object or a generic slab page (e.g. when adding an object to a lru list). Link: https://lkml.kernel.org/r/20220610180310.1725111-1-roman.gushchin@linux.dev Suggested-by: Kefeng Wang Signed-off-by: Roman Gushchin Tested-by: Linux Kernel Functional Testing Acked-by: Shakeel Butt Tested-by: Vasily Averin Acked-by: Michal Hocko Acked-by: Muchun Song Cc: Johannes Weiner Cc: Naresh Kamboju Cc: Qian Cai Cc: Kefeng Wang Cc: David S. Miller Cc: Eric Dumazet Cc: Florian Westphal Cc: Jakub Kicinski Cc: Michal Koutný Cc: Paolo Abeni Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 6 ++++ mm/list_lru.c | 2 +- mm/memcontrol.c | 71 ++++++++++++++++++++++++++++++++-------------- 3 files changed, 57 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 9ecead1042b9..3ce96ce5fe3e 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1740,6 +1740,7 @@ static inline int memcg_kmem_id(struct mem_cgroup *memcg) } struct mem_cgroup *mem_cgroup_from_obj(void *p); +struct mem_cgroup *mem_cgroup_from_slab_obj(void *p); static inline void count_objcg_event(struct obj_cgroup *objcg, enum vm_event_item idx) @@ -1801,6 +1802,11 @@ static inline struct mem_cgroup *mem_cgroup_from_obj(void *p) return NULL; } +static inline struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) +{ + return NULL; +} + static inline void count_objcg_event(struct obj_cgroup *objcg, enum vm_event_item idx) { diff --git a/mm/list_lru.c b/mm/list_lru.c index ba76428ceece..a05e5bef3b40 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -71,7 +71,7 @@ list_lru_from_kmem(struct list_lru *lru, int nid, void *ptr, if (!list_lru_memcg_aware(lru)) goto out; - memcg = mem_cgroup_from_obj(ptr); + memcg = mem_cgroup_from_slab_obj(ptr); if (!memcg) goto out; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 28c1532cc91f..c1ae9b3f8d35 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -783,7 +783,7 @@ void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val) struct lruvec *lruvec; rcu_read_lock(); - memcg = mem_cgroup_from_obj(p); + memcg = mem_cgroup_from_slab_obj(p); /* * Untracked pages have no memcg, no lruvec. Update only the @@ -2841,27 +2841,9 @@ int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, return 0; } -/* - * Returns a pointer to the memory cgroup to which the kernel object is charged. - * - * A passed kernel object can be a slab object or a generic kernel page, so - * different mechanisms for getting the memory cgroup pointer should be used. - * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller - * can not know for sure how the kernel object is implemented. - * mem_cgroup_from_obj() can be safely used in such cases. - * - * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), - * cgroup_mutex, etc. - */ -struct mem_cgroup *mem_cgroup_from_obj(void *p) +static __always_inline +struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) { - struct folio *folio; - - if (mem_cgroup_disabled()) - return NULL; - - folio = virt_to_folio(p); - /* * Slab objects are accounted individually, not per-page. * Memcg membership data for each individual object is saved in @@ -2894,6 +2876,53 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p) return page_memcg_check(folio_page(folio, 0)); } +/* + * Returns a pointer to the memory cgroup to which the kernel object is charged. + * + * A passed kernel object can be a slab object, vmalloc object or a generic + * kernel page, so different mechanisms for getting the memory cgroup pointer + * should be used. + * + * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller + * can not know for sure how the kernel object is implemented. + * mem_cgroup_from_obj() can be safely used in such cases. + * + * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), + * cgroup_mutex, etc. + */ +struct mem_cgroup *mem_cgroup_from_obj(void *p) +{ + struct folio *folio; + + if (mem_cgroup_disabled()) + return NULL; + + if (unlikely(is_vmalloc_addr(p))) + folio = page_folio(vmalloc_to_page(p)); + else + folio = virt_to_folio(p); + + return mem_cgroup_from_obj_folio(folio, p); +} + +/* + * Returns a pointer to the memory cgroup to which the kernel object is charged. + * Similar to mem_cgroup_from_obj(), but faster and not suitable for objects, + * allocated using vmalloc(). + * + * A passed kernel object must be a slab object or a generic kernel page. + * + * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), + * cgroup_mutex, etc. + */ +struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) +{ + if (mem_cgroup_disabled()) + return NULL; + + return mem_cgroup_from_obj_folio(virt_to_folio(p), p); +} + static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) { struct obj_cgroup *objcg = NULL; -- cgit v1.2.3 From 1d0403d20f6c281cb3d14c5f1db5317caeec48e9 Mon Sep 17 00:00:00 2001 From: Vasily Averin Date: Fri, 3 Jun 2022 07:19:43 +0300 Subject: net: set proper memcg for net_init hooks allocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit __register_pernet_operations() executes init hook of registered pernet_operation structure in all existing net namespaces. Typically, these hooks are called by a process associated with the specified net namespace, and all __GFP_ACCOUNT marked allocation are accounted for corresponding container/memcg. However __register_pernet_operations() calls the hooks in the same context, and as a result all marked allocations are accounted to one memcg for all processed net namespaces. This patch adjusts active memcg for each net namespace and helps to account memory allocated inside ops_init() into the proper memcg. Link: https://lkml.kernel.org/r/f9394752-e272-9bf9-645f-a18c56d1c4ec@openvz.org Signed-off-by: Vasily Averin Acked-by: Roman Gushchin Acked-by: Shakeel Butt Cc: Michal Koutný Cc: Vlastimil Babka Cc: Michal Hocko Cc: Florian Westphal Cc: David S. Miller Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Eric Dumazet Cc: Johannes Weiner Cc: Kefeng Wang Cc: Linux Kernel Functional Testing Cc: Muchun Song Cc: Naresh Kamboju Cc: Qian Cai Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 47 +++++++++++++++++++++++++++++++++++++++++++++- net/core/net_namespace.c | 7 +++++++ 2 files changed, 53 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 3ce96ce5fe3e..04f2f33607e9 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1756,6 +1756,42 @@ static inline void count_objcg_event(struct obj_cgroup *objcg, rcu_read_unlock(); } +/** + * get_mem_cgroup_from_obj - get a memcg associated with passed kernel object. + * @p: pointer to object from which memcg should be extracted. It can be NULL. + * + * Retrieves the memory group into which the memory of the pointed kernel + * object is accounted. If memcg is found, its reference is taken. + * If a passed kernel object is uncharged, or if proper memcg cannot be found, + * as well as if mem_cgroup is disabled, NULL is returned. + * + * Return: valid memcg pointer with taken reference or NULL. + */ +static inline struct mem_cgroup *get_mem_cgroup_from_obj(void *p) +{ + struct mem_cgroup *memcg; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_obj(p); + } while (memcg && !css_tryget(&memcg->css)); + rcu_read_unlock(); + return memcg; +} + +/** + * mem_cgroup_or_root - always returns a pointer to a valid memory cgroup. + * @memcg: pointer to a valid memory cgroup or NULL. + * + * If passed argument is not NULL, returns it without any additional checks + * and changes. Otherwise, root_mem_cgroup is returned. + * + * NOTE: root_mem_cgroup can be NULL during early boot. + */ +static inline struct mem_cgroup *mem_cgroup_or_root(struct mem_cgroup *memcg) +{ + return memcg ? memcg : root_mem_cgroup; +} #else static inline bool mem_cgroup_kmem_disabled(void) { @@ -1799,7 +1835,7 @@ static inline int memcg_kmem_id(struct mem_cgroup *memcg) static inline struct mem_cgroup *mem_cgroup_from_obj(void *p) { - return NULL; + return NULL; } static inline struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) @@ -1812,6 +1848,15 @@ static inline void count_objcg_event(struct obj_cgroup *objcg, { } +static inline struct mem_cgroup *get_mem_cgroup_from_obj(void *p) +{ + return NULL; +} + +static inline struct mem_cgroup *mem_cgroup_or_root(struct mem_cgroup *memcg) +{ + return NULL; +} #endif /* CONFIG_MEMCG_KMEM */ #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 0ec2f5906a27..6b9f19122ec1 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -1143,7 +1144,13 @@ static int __register_pernet_operations(struct list_head *list, * setup_net() and cleanup_net() are not possible. */ for_each_net(net) { + struct mem_cgroup *old, *memcg; + + memcg = mem_cgroup_or_root(get_mem_cgroup_from_obj(net)); + old = set_active_memcg(memcg); error = ops_init(ops, net); + set_active_memcg(old); + mem_cgroup_put(memcg); if (error) goto out_undo; list_add_tail(&net->exit_list, &net_exit_list); -- cgit v1.2.3 From 4815a36009044ba69a9b8d781943ec6505c451a2 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 3 Jun 2022 20:10:12 +0300 Subject: include/linux/rbtree.h: replace kernel.h with the necessary inclusions When kernel.h is used in the headers it adds a lot into dependency hell, especially when there are circular dependencies are involved. Replace kernel.h inclusion with the list of what is really being used. Link: https://lkml.kernel.org/r/20220603171012.48880-1-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Matthew Wilcox Signed-off-by: Andrew Morton --- include/linux/rbtree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h index 235047d7a1b5..f7edca369eda 100644 --- a/include/linux/rbtree.h +++ b/include/linux/rbtree.h @@ -17,9 +17,9 @@ #ifndef _LINUX_RBTREE_H #define _LINUX_RBTREE_H +#include #include -#include #include #include -- cgit v1.2.3 From dabba87229411a5e9d20ac03ffc36463c53ae672 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Fri, 27 May 2022 02:55:34 +0000 Subject: fs/kernel_read_file: allow to read files up-to ssize_t Patch series "Allow to kexec with initramfs larger than 2G", v2. Currently, the largest initramfs that is supported by kexec_file_load() syscall is 2G. This is because kernel_read_file() returns int, and is limited to INT_MAX or 2G. On the other hand, there are kexec based boot loaders (i.e. u-root), that may need to boot netboot images that might be larger than 2G. The first patch changes the return type from int to ssize_t in kernel_read_file* functions. The second patch increases the maximum initramfs file size to 4G. Tested: verified that can kexec_file_load() works with 4G initramfs on x86_64. This patch (of 2): Currently, the maximum file size that is supported is 2G. This may be too small in some cases. For example, kexec_file_load() system call loads initramfs. In some netboot cases initramfs can be rather large. Allow to use up-to ssize_t bytes. The callers still can limit the maximum file size via buf_size. Link: https://lkml.kernel.org/r/20220527025535.3953665-1-pasha.tatashin@soleen.com Link: https://lkml.kernel.org/r/20220527025535.3953665-2-pasha.tatashin@soleen.com Signed-off-by: Pasha Tatashin Cc: Al Viro Cc: Baoquan He Cc: "Eric W. Biederman" Cc: Greg Thelen Cc: Sasha Levin Signed-off-by: Andrew Morton --- fs/kernel_read_file.c | 38 +++++++++++++++++++------------------- include/linux/kernel_read_file.h | 32 ++++++++++++++++---------------- include/linux/limits.h | 1 + 3 files changed, 36 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/fs/kernel_read_file.c b/fs/kernel_read_file.c index 1b07550485b9..5d826274570c 100644 --- a/fs/kernel_read_file.c +++ b/fs/kernel_read_file.c @@ -29,15 +29,15 @@ * change between calls to kernel_read_file(). * * Returns number of bytes read (no single read will be bigger - * than INT_MAX), or negative on error. + * than SSIZE_MAX), or negative on error. * */ -int kernel_read_file(struct file *file, loff_t offset, void **buf, - size_t buf_size, size_t *file_size, - enum kernel_read_file_id id) +ssize_t kernel_read_file(struct file *file, loff_t offset, void **buf, + size_t buf_size, size_t *file_size, + enum kernel_read_file_id id) { loff_t i_size, pos; - size_t copied; + ssize_t copied; void *allocated = NULL; bool whole_file; int ret; @@ -58,7 +58,7 @@ int kernel_read_file(struct file *file, loff_t offset, void **buf, goto out; } /* The file is too big for sane activities. */ - if (i_size > INT_MAX) { + if (i_size > SSIZE_MAX) { ret = -EFBIG; goto out; } @@ -124,12 +124,12 @@ out: } EXPORT_SYMBOL_GPL(kernel_read_file); -int kernel_read_file_from_path(const char *path, loff_t offset, void **buf, - size_t buf_size, size_t *file_size, - enum kernel_read_file_id id) +ssize_t kernel_read_file_from_path(const char *path, loff_t offset, void **buf, + size_t buf_size, size_t *file_size, + enum kernel_read_file_id id) { struct file *file; - int ret; + ssize_t ret; if (!path || !*path) return -EINVAL; @@ -144,14 +144,14 @@ int kernel_read_file_from_path(const char *path, loff_t offset, void **buf, } EXPORT_SYMBOL_GPL(kernel_read_file_from_path); -int kernel_read_file_from_path_initns(const char *path, loff_t offset, - void **buf, size_t buf_size, - size_t *file_size, - enum kernel_read_file_id id) +ssize_t kernel_read_file_from_path_initns(const char *path, loff_t offset, + void **buf, size_t buf_size, + size_t *file_size, + enum kernel_read_file_id id) { struct file *file; struct path root; - int ret; + ssize_t ret; if (!path || !*path) return -EINVAL; @@ -171,12 +171,12 @@ int kernel_read_file_from_path_initns(const char *path, loff_t offset, } EXPORT_SYMBOL_GPL(kernel_read_file_from_path_initns); -int kernel_read_file_from_fd(int fd, loff_t offset, void **buf, - size_t buf_size, size_t *file_size, - enum kernel_read_file_id id) +ssize_t kernel_read_file_from_fd(int fd, loff_t offset, void **buf, + size_t buf_size, size_t *file_size, + enum kernel_read_file_id id) { struct fd f = fdget(fd); - int ret = -EBADF; + ssize_t ret = -EBADF; if (!f.file || !(f.file->f_mode & FMODE_READ)) goto out; diff --git a/include/linux/kernel_read_file.h b/include/linux/kernel_read_file.h index 575ffa1031d3..90451e2e12bd 100644 --- a/include/linux/kernel_read_file.h +++ b/include/linux/kernel_read_file.h @@ -35,21 +35,21 @@ static inline const char *kernel_read_file_id_str(enum kernel_read_file_id id) return kernel_read_file_str[id]; } -int kernel_read_file(struct file *file, loff_t offset, - void **buf, size_t buf_size, - size_t *file_size, - enum kernel_read_file_id id); -int kernel_read_file_from_path(const char *path, loff_t offset, - void **buf, size_t buf_size, - size_t *file_size, - enum kernel_read_file_id id); -int kernel_read_file_from_path_initns(const char *path, loff_t offset, - void **buf, size_t buf_size, - size_t *file_size, - enum kernel_read_file_id id); -int kernel_read_file_from_fd(int fd, loff_t offset, - void **buf, size_t buf_size, - size_t *file_size, - enum kernel_read_file_id id); +ssize_t kernel_read_file(struct file *file, loff_t offset, + void **buf, size_t buf_size, + size_t *file_size, + enum kernel_read_file_id id); +ssize_t kernel_read_file_from_path(const char *path, loff_t offset, + void **buf, size_t buf_size, + size_t *file_size, + enum kernel_read_file_id id); +ssize_t kernel_read_file_from_path_initns(const char *path, loff_t offset, + void **buf, size_t buf_size, + size_t *file_size, + enum kernel_read_file_id id); +ssize_t kernel_read_file_from_fd(int fd, loff_t offset, + void **buf, size_t buf_size, + size_t *file_size, + enum kernel_read_file_id id); #endif /* _LINUX_KERNEL_READ_FILE_H */ diff --git a/include/linux/limits.h b/include/linux/limits.h index b568b9c30bbf..f6bcc9369010 100644 --- a/include/linux/limits.h +++ b/include/linux/limits.h @@ -7,6 +7,7 @@ #include #define SIZE_MAX (~(size_t)0) +#define SSIZE_MAX ((ssize_t)(SIZE_MAX >> 1)) #define PHYS_ADDR_MAX (~(phys_addr_t)0) #define U8_MAX ((u8)~0U) -- cgit v1.2.3 From a793679827a87b01f9d973ea30b923cd5a3ff2c5 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Tue, 14 Jun 2022 10:46:11 +0200 Subject: linux/phy.h: add phydev_err_probe() wrapper for dev_err_probe() The dev_err_probe() function is quite useful to avoid boilerplate related to -EPROBE_DEFER handling. Add a phydev_err_probe() helper to simplify making use of that from phy drivers which otherwise use the phydev_* helpers. Signed-off-by: Rasmus Villemoes Signed-off-by: Jakub Kicinski --- include/linux/phy.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/phy.h b/include/linux/phy.h index 508f1149665b..bed9a347481b 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1539,6 +1539,9 @@ static inline void phy_device_reset(struct phy_device *phydev, int value) #define phydev_err(_phydev, format, args...) \ dev_err(&_phydev->mdio.dev, format, ##args) +#define phydev_err_probe(_phydev, err, format, args...) \ + dev_err_probe(&_phydev->mdio.dev, err, format, ##args) + #define phydev_info(_phydev, format, args...) \ dev_info(&_phydev->mdio.dev, format, ##args) -- cgit v1.2.3 From 508362ac66b0478affb4e52cb8da98478312d72d Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Wed, 15 Jun 2022 16:48:43 +0300 Subject: bpf: Allow helpers to accept pointers with a fixed size Before this commit, the BPF verifier required ARG_PTR_TO_MEM arguments to be followed by ARG_CONST_SIZE holding the size of the memory region. The helpers had to check that size in runtime. There are cases where the size expected by a helper is a compile-time constant. Checking it in runtime is an unnecessary overhead and waste of BPF registers. This commit allows helpers to accept pointers to memory without the corresponding ARG_CONST_SIZE, given that they define the memory region size in struct bpf_func_proto and use ARG_PTR_TO_FIXED_SIZE_MEM type. arg_size is unionized with arg_btf_id to reduce the kernel image size, and it's valid because they are used by different argument types. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Link: https://lore.kernel.org/r/20220615134847.3753567-3-maximmi@nvidia.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 13 +++++++++++++ kernel/bpf/verifier.c | 43 ++++++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index f3e88afdaffe..a94531971a7a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -401,6 +401,9 @@ enum bpf_type_flag { /* DYNPTR points to a ringbuf record. */ DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS), + /* Size is known at compile time. */ + MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS), + __BPF_TYPE_FLAG_MAX, __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1, }; @@ -464,6 +467,8 @@ enum bpf_arg_type { * all bytes or clear them in error case. */ ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | ARG_PTR_TO_MEM, + /* Pointer to valid memory of size known at compile time. */ + ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM, /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. @@ -529,6 +534,14 @@ struct bpf_func_proto { u32 *arg5_btf_id; }; u32 *arg_btf_id[5]; + struct { + size_t arg1_size; + size_t arg2_size; + size_t arg3_size; + size_t arg4_size; + size_t arg5_size; + }; + size_t arg_size[5]; }; int *ret_btf_id; /* return value btf_id */ bool (*allowed)(const struct bpf_prog *prog); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index eadc23a8452c..2859901ffbe3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5848,6 +5848,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; enum bpf_arg_type arg_type = fn->arg_type[arg]; enum bpf_reg_type type = reg->type; + u32 *arg_btf_id = NULL; int err = 0; if (arg_type == ARG_DONTCARE) @@ -5884,7 +5885,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, */ goto skip_type_check; - err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg], meta); + /* arg_btf_id and arg_size are in a union. */ + if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) + arg_btf_id = fn->arg_btf_id[arg]; + + err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); if (err) return err; @@ -6011,6 +6016,11 @@ skip_type_check: * next is_mem_size argument below. */ meta->raw_mode = arg_type & MEM_UNINIT; + if (arg_type & MEM_FIXED_SIZE) { + err = check_helper_mem_access(env, regno, + fn->arg_size[arg], false, + meta); + } } else if (arg_type_is_mem_size(arg_type)) { bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); @@ -6400,11 +6410,19 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn) return count <= 1; } -static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, - enum bpf_arg_type arg_next) +static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) { - return (base_type(arg_curr) == ARG_PTR_TO_MEM) != - arg_type_is_mem_size(arg_next); + bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; + bool has_size = fn->arg_size[arg] != 0; + bool is_next_size = false; + + if (arg + 1 < ARRAY_SIZE(fn->arg_type)) + is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]); + + if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) + return is_next_size; + + return has_size == is_next_size || is_next_size == is_fixed; } static bool check_arg_pair_ok(const struct bpf_func_proto *fn) @@ -6415,11 +6433,11 @@ static bool check_arg_pair_ok(const struct bpf_func_proto *fn) * helper function specification. */ if (arg_type_is_mem_size(fn->arg1_type) || - base_type(fn->arg5_type) == ARG_PTR_TO_MEM || - check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || - check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || - check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || - check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) + check_args_pair_invalid(fn, 0) || + check_args_pair_invalid(fn, 1) || + check_args_pair_invalid(fn, 2) || + check_args_pair_invalid(fn, 3) || + check_args_pair_invalid(fn, 4)) return false; return true; @@ -6460,7 +6478,10 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) return false; - if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) + if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && + /* arg_btf_id and arg_size are in a union. */ + (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || + !(fn->arg_type[i] & MEM_FIXED_SIZE))) return false; } -- cgit v1.2.3 From f0a6d77b351c18c122fc1638ac9e58f5e0346f64 Mon Sep 17 00:00:00 2001 From: Sergey Shtylyov Date: Tue, 14 Jun 2022 22:51:47 +0300 Subject: ata: make transfer mode masks *unsigned int* The packed transfer mode masks and also the {pio|mwdma|udma}_mask fields of *struct*s ata_device and ata_port_info are declared as *unsigned long* (which is a 64-bit type on 64-bit architectures) but actually the packed masks occupy only 20 bits (7 PIO modes, 5 MWDMA modes, and 8 UDMA modes) and the PIO/MWDMA/UDMA masks easily fit into just 8 bits each, so we can safely use (always 32-bit) *unsigned int* variables instead. This saves 745 bytes of object code in libata-core.o alone, not to mention LLDDs... Signed-off-by: Sergey Shtylyov Signed-off-by: Damien Le Moal --- drivers/ata/libata-acpi.c | 8 +++---- drivers/ata/libata-core.c | 38 ++++++++++++++++---------------- drivers/ata/pata_acpi.c | 2 +- drivers/ata/pata_ali.c | 2 +- drivers/ata/pata_amd.c | 14 ++++++------ drivers/ata/pata_hpt366.c | 2 +- drivers/ata/pata_hpt37x.c | 6 +++--- drivers/ata/pata_hpt3x2n.c | 2 +- drivers/ata/pata_pdc2027x.c | 4 ++-- drivers/ata/pata_serverworks.c | 4 ++-- drivers/ata/pata_sis.c | 2 +- drivers/ata/pata_via.c | 2 +- include/linux/libata.h | 49 +++++++++++++++++++++--------------------- 13 files changed, 67 insertions(+), 68 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-acpi.c b/drivers/ata/libata-acpi.c index 3d345d173556..61b4ccf88bf1 100644 --- a/drivers/ata/libata-acpi.c +++ b/drivers/ata/libata-acpi.c @@ -480,10 +480,10 @@ static int ata_dev_get_GTF(struct ata_device *dev, struct ata_acpi_gtf **gtf) * RETURNS: * Determined xfermask. */ -unsigned long ata_acpi_gtm_xfermask(struct ata_device *dev, - const struct ata_acpi_gtm *gtm) +unsigned int ata_acpi_gtm_xfermask(struct ata_device *dev, + const struct ata_acpi_gtm *gtm) { - unsigned long xfer_mask = 0; + unsigned int xfer_mask = 0; unsigned int type; int unit; u8 mode; @@ -525,7 +525,7 @@ int ata_acpi_cbl_80wire(struct ata_port *ap, const struct ata_acpi_gtm *gtm) struct ata_device *dev; ata_for_each_dev(dev, &ap->link, ENABLED) { - unsigned long xfer_mask, udma_mask; + unsigned int xfer_mask, udma_mask; xfer_mask = ata_acpi_gtm_xfermask(dev, gtm); ata_unpack_xfermask(xfer_mask, NULL, NULL, &udma_mask); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 980328a4b896..035092184c08 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -93,7 +93,7 @@ struct ata_force_param { const char *name; u8 cbl; u8 spd_limit; - unsigned long xfer_mask; + unsigned int xfer_mask; unsigned int horkage_on; unsigned int horkage_off; u16 lflags_on; @@ -425,7 +425,7 @@ static void ata_force_xfermask(struct ata_device *dev) for (i = ata_force_tbl_size - 1; i >= 0; i--) { const struct ata_force_ent *fe = &ata_force_tbl[i]; - unsigned long pio_mask, mwdma_mask, udma_mask; + unsigned int pio_mask, mwdma_mask, udma_mask; if (fe->port != -1 && fe->port != dev->link->ap->print_id) continue; @@ -803,11 +803,11 @@ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, * RETURNS: * Packed xfer_mask. */ -unsigned long ata_pack_xfermask(unsigned long pio_mask, - unsigned long mwdma_mask, - unsigned long udma_mask) +unsigned int ata_pack_xfermask(unsigned int pio_mask, + unsigned int mwdma_mask, + unsigned int udma_mask) { - return ((pio_mask << ATA_SHIFT_PIO) & ATA_MASK_PIO) | + return ((pio_mask << ATA_SHIFT_PIO) & ATA_MASK_PIO) | ((mwdma_mask << ATA_SHIFT_MWDMA) & ATA_MASK_MWDMA) | ((udma_mask << ATA_SHIFT_UDMA) & ATA_MASK_UDMA); } @@ -823,8 +823,8 @@ EXPORT_SYMBOL_GPL(ata_pack_xfermask); * Unpack @xfer_mask into @pio_mask, @mwdma_mask and @udma_mask. * Any NULL destination masks will be ignored. */ -void ata_unpack_xfermask(unsigned long xfer_mask, unsigned long *pio_mask, - unsigned long *mwdma_mask, unsigned long *udma_mask) +void ata_unpack_xfermask(unsigned int xfer_mask, unsigned int *pio_mask, + unsigned int *mwdma_mask, unsigned int *udma_mask) { if (pio_mask) *pio_mask = (xfer_mask & ATA_MASK_PIO) >> ATA_SHIFT_PIO; @@ -857,7 +857,7 @@ static const struct ata_xfer_ent { * RETURNS: * Matching XFER_* value, 0xff if no match found. */ -u8 ata_xfer_mask2mode(unsigned long xfer_mask) +u8 ata_xfer_mask2mode(unsigned int xfer_mask) { int highbit = fls(xfer_mask) - 1; const struct ata_xfer_ent *ent; @@ -881,7 +881,7 @@ EXPORT_SYMBOL_GPL(ata_xfer_mask2mode); * RETURNS: * Matching xfer_mask, 0 if no match found. */ -unsigned long ata_xfer_mode2mask(u8 xfer_mode) +unsigned int ata_xfer_mode2mask(u8 xfer_mode) { const struct ata_xfer_ent *ent; @@ -930,7 +930,7 @@ EXPORT_SYMBOL_GPL(ata_xfer_mode2shift); * Constant C string representing highest speed listed in * @mode_mask, or the constant C string "". */ -const char *ata_mode_string(unsigned long xfer_mask) +const char *ata_mode_string(unsigned int xfer_mask) { static const char * const xfer_mode_str[] = { "PIO0", @@ -1383,9 +1383,9 @@ static inline void ata_dump_id(struct ata_device *dev, const u16 *id) * RETURNS: * Computed xfermask */ -unsigned long ata_id_xfermask(const u16 *id) +unsigned int ata_id_xfermask(const u16 *id) { - unsigned long pio_mask, mwdma_mask, udma_mask; + unsigned int pio_mask, mwdma_mask, udma_mask; /* Usual case. Word 53 indicates word 64 is valid */ if (id[ATA_ID_FIELD_VALID] & (1 << 1)) { @@ -2534,7 +2534,7 @@ int ata_dev_configure(struct ata_device *dev) struct ata_port *ap = dev->link->ap; bool print_info = ata_dev_print_info(dev); const u16 *id = dev->id; - unsigned long xfer_mask; + unsigned int xfer_mask; unsigned int err_mask; char revbuf[7]; /* XYZ-99\0 */ char fwrevbuf[ATA_ID_FW_REV_LEN+1]; @@ -3202,8 +3202,8 @@ u8 ata_timing_cycle2mode(unsigned int xfer_shift, int cycle) int ata_down_xfermask_limit(struct ata_device *dev, unsigned int sel) { char buf[32]; - unsigned long orig_mask, xfer_mask; - unsigned long pio_mask, mwdma_mask, udma_mask; + unsigned int orig_mask, xfer_mask; + unsigned int pio_mask, mwdma_mask, udma_mask; int quiet, highbit; quiet = !!(sel & ATA_DNXFER_QUIET); @@ -3381,7 +3381,7 @@ int ata_do_set_mode(struct ata_link *link, struct ata_device **r_failed_dev) /* step 1: calculate xfer_mask */ ata_for_each_dev(dev, link, ENABLED) { - unsigned long pio_mask, dma_mask; + unsigned int pio_mask, dma_mask; unsigned int mode_mask; mode_mask = ATA_DMA_MASK_ATA; @@ -4217,7 +4217,7 @@ static void ata_dev_xfermask(struct ata_device *dev) struct ata_link *link = dev->link; struct ata_port *ap = link->ap; struct ata_host *host = ap->host; - unsigned long xfer_mask; + unsigned int xfer_mask; /* controller modes available */ xfer_mask = ata_pack_xfermask(ap->pio_mask, @@ -5776,7 +5776,7 @@ int ata_host_register(struct ata_host *host, struct scsi_host_template *sht) /* set cable, sata_spd_limit and report */ for (i = 0; i < host->n_ports; i++) { struct ata_port *ap = host->ports[i]; - unsigned long xfer_mask; + unsigned int xfer_mask; /* set SATA cable type if still unset */ if (ap->cbl == ATA_CBL_NONE && (ap->flags & ATA_FLAG_SATA)) diff --git a/drivers/ata/pata_acpi.c b/drivers/ata/pata_acpi.c index ade4c3eee230..f8706ee427d2 100644 --- a/drivers/ata/pata_acpi.c +++ b/drivers/ata/pata_acpi.c @@ -97,7 +97,7 @@ static unsigned long pacpi_discover_modes(struct ata_port *ap, struct ata_device * this case the list of discovered valid modes obtained by ACPI probing */ -static unsigned long pacpi_mode_filter(struct ata_device *adev, unsigned long mask) +static unsigned int pacpi_mode_filter(struct ata_device *adev, unsigned int mask) { struct pata_acpi *acpi = adev->link->ap->private_data; return mask & acpi->mask[adev->devno]; diff --git a/drivers/ata/pata_ali.c b/drivers/ata/pata_ali.c index 1b90cda27246..76ad0e73fe2a 100644 --- a/drivers/ata/pata_ali.c +++ b/drivers/ata/pata_ali.c @@ -115,7 +115,7 @@ static int ali_c2_cable_detect(struct ata_port *ap) * fix that later on. Also ensure we do not do UDMA on WDC drives */ -static unsigned long ali_20_filter(struct ata_device *adev, unsigned long mask) +static unsigned int ali_20_filter(struct ata_device *adev, unsigned int mask) { char model_num[ATA_ID_PROD_LEN + 1]; /* No DMA on anything but a disk for now */ diff --git a/drivers/ata/pata_amd.c b/drivers/ata/pata_amd.c index 154748cfcc79..f216f9d7b9ec 100644 --- a/drivers/ata/pata_amd.c +++ b/drivers/ata/pata_amd.c @@ -264,8 +264,8 @@ static void amd133_set_dmamode(struct ata_port *ap, struct ata_device *adev) * cached during driver attach and are consulted to select transfer * mode. */ -static unsigned long nv_mode_filter(struct ata_device *dev, - unsigned long xfer_mask) +static unsigned int nv_mode_filter(struct ata_device *dev, + unsigned int xfer_mask) { static const unsigned int udma_mask_map[] = { ATA_UDMA2, ATA_UDMA1, ATA_UDMA0, 0, @@ -274,7 +274,7 @@ static unsigned long nv_mode_filter(struct ata_device *dev, char acpi_str[32] = ""; u32 saved_udma, udma; const struct ata_acpi_gtm *gtm; - unsigned long bios_limit = 0, acpi_limit = 0, limit; + unsigned int bios_limit = 0, acpi_limit = 0, limit; /* find out what BIOS configured */ udma = saved_udma = (unsigned long)ap->host->private_data; @@ -310,10 +310,10 @@ static unsigned long nv_mode_filter(struct ata_device *dev, cable detection result */ limit |= ata_pack_xfermask(ATA_PIO4, ATA_MWDMA2, ATA_UDMA2); - ata_port_dbg(ap, "nv_mode_filter: 0x%lx&0x%lx->0x%lx, " - "BIOS=0x%lx (0x%x) ACPI=0x%lx%s\n", - xfer_mask, limit, xfer_mask & limit, bios_limit, - saved_udma, acpi_limit, acpi_str); + ata_port_dbg(ap, + "nv_mode_filter: 0x%x&0x%x->0x%x, BIOS=0x%x (0x%x) ACPI=0x%x%s\n", + xfer_mask, limit, xfer_mask & limit, bios_limit, + saved_udma, acpi_limit, acpi_str); return xfer_mask & limit; } diff --git a/drivers/ata/pata_hpt366.c b/drivers/ata/pata_hpt366.c index c99e8f0708b3..7e441fb304d3 100644 --- a/drivers/ata/pata_hpt366.c +++ b/drivers/ata/pata_hpt366.c @@ -194,7 +194,7 @@ static int hpt_dma_blacklisted(const struct ata_device *dev, char *modestr, * Block UDMA on devices that cause trouble with this controller. */ -static unsigned long hpt366_filter(struct ata_device *adev, unsigned long mask) +static unsigned int hpt366_filter(struct ata_device *adev, unsigned int mask) { if (adev->class == ATA_DEV_ATA) { if (hpt_dma_blacklisted(adev, "UDMA", bad_ata33)) diff --git a/drivers/ata/pata_hpt37x.c b/drivers/ata/pata_hpt37x.c index d1a3d99d5d0a..ce3c5eaa7e76 100644 --- a/drivers/ata/pata_hpt37x.c +++ b/drivers/ata/pata_hpt37x.c @@ -278,7 +278,7 @@ static const char * const bad_ata100_5[] = { * Block UDMA on devices that cause trouble with this controller. */ -static unsigned long hpt370_filter(struct ata_device *adev, unsigned long mask) +static unsigned int hpt370_filter(struct ata_device *adev, unsigned int mask) { if (adev->class == ATA_DEV_ATA) { if (hpt_dma_blacklisted(adev, "UDMA", bad_ata33)) @@ -297,7 +297,7 @@ static unsigned long hpt370_filter(struct ata_device *adev, unsigned long mask) * Block UDMA on devices that cause trouble with this controller. */ -static unsigned long hpt370a_filter(struct ata_device *adev, unsigned long mask) +static unsigned int hpt370a_filter(struct ata_device *adev, unsigned int mask) { if (adev->class == ATA_DEV_ATA) { if (hpt_dma_blacklisted(adev, "UDMA100", bad_ata100_5)) @@ -314,7 +314,7 @@ static unsigned long hpt370a_filter(struct ata_device *adev, unsigned long mask) * The Marvell bridge chips used on the HighPoint SATA cards do not seem * to support the UltraDMA modes 1, 2, and 3 as well as any MWDMA modes... */ -static unsigned long hpt372_filter(struct ata_device *adev, unsigned long mask) +static unsigned int hpt372_filter(struct ata_device *adev, unsigned int mask) { if (ata_id_is_sata(adev->id)) mask &= ~((0xE << ATA_SHIFT_UDMA) | ATA_MASK_MWDMA); diff --git a/drivers/ata/pata_hpt3x2n.c b/drivers/ata/pata_hpt3x2n.c index d1595e17dca2..617c95522f43 100644 --- a/drivers/ata/pata_hpt3x2n.c +++ b/drivers/ata/pata_hpt3x2n.c @@ -113,7 +113,7 @@ static u32 hpt3x2n_find_mode(struct ata_port *ap, int speed) * The Marvell bridge chips used on the HighPoint SATA cards do not seem * to support the UltraDMA modes 1, 2, and 3 as well as any MWDMA modes... */ -static unsigned long hpt372n_filter(struct ata_device *adev, unsigned long mask) +static unsigned int hpt372n_filter(struct ata_device *adev, unsigned int mask) { if (ata_id_is_sata(adev->id)) mask &= ~((0xE << ATA_SHIFT_UDMA) | ATA_MASK_MWDMA); diff --git a/drivers/ata/pata_pdc2027x.c b/drivers/ata/pata_pdc2027x.c index 4fbb3eed8b0b..4191aa61c8e4 100644 --- a/drivers/ata/pata_pdc2027x.c +++ b/drivers/ata/pata_pdc2027x.c @@ -57,7 +57,7 @@ static int pdc2027x_prereset(struct ata_link *link, unsigned long deadline); static void pdc2027x_set_piomode(struct ata_port *ap, struct ata_device *adev); static void pdc2027x_set_dmamode(struct ata_port *ap, struct ata_device *adev); static int pdc2027x_check_atapi_dma(struct ata_queued_cmd *qc); -static unsigned long pdc2027x_mode_filter(struct ata_device *adev, unsigned long mask); +static unsigned int pdc2027x_mode_filter(struct ata_device *adev, unsigned int mask); static int pdc2027x_cable_detect(struct ata_port *ap); static int pdc2027x_set_mode(struct ata_link *link, struct ata_device **r_failed); @@ -251,7 +251,7 @@ static int pdc2027x_prereset(struct ata_link *link, unsigned long deadline) * Block UDMA on devices that cause trouble with this controller. */ -static unsigned long pdc2027x_mode_filter(struct ata_device *adev, unsigned long mask) +static unsigned int pdc2027x_mode_filter(struct ata_device *adev, unsigned int mask) { unsigned char model_num[ATA_ID_PROD_LEN + 1]; struct ata_device *pair = ata_dev_pair(adev); diff --git a/drivers/ata/pata_serverworks.c b/drivers/ata/pata_serverworks.c index e410fe44177f..c0bc4af0d196 100644 --- a/drivers/ata/pata_serverworks.c +++ b/drivers/ata/pata_serverworks.c @@ -150,7 +150,7 @@ static u8 serverworks_is_csb(struct pci_dev *pdev) * bug we hit. */ -static unsigned long serverworks_osb4_filter(struct ata_device *adev, unsigned long mask) +static unsigned int serverworks_osb4_filter(struct ata_device *adev, unsigned int mask) { if (adev->class == ATA_DEV_ATA) mask &= ~ATA_MASK_UDMA; @@ -166,7 +166,7 @@ static unsigned long serverworks_osb4_filter(struct ata_device *adev, unsigned l * Check the blacklist and disable UDMA5 if matched */ -static unsigned long serverworks_csb_filter(struct ata_device *adev, unsigned long mask) +static unsigned int serverworks_csb_filter(struct ata_device *adev, unsigned int mask) { const char *p; char model_num[ATA_ID_PROD_LEN + 1]; diff --git a/drivers/ata/pata_sis.c b/drivers/ata/pata_sis.c index b5b764e18adf..92e4cf05de2c 100644 --- a/drivers/ata/pata_sis.c +++ b/drivers/ata/pata_sis.c @@ -525,7 +525,7 @@ static void sis_133_set_dmamode (struct ata_port *ap, struct ata_device *adev) * Block UDMA6 on devices that do not support it. */ -static unsigned long sis_133_mode_filter(struct ata_device *adev, unsigned long mask) +static unsigned int sis_133_mode_filter(struct ata_device *adev, unsigned int mask) { struct ata_port *ap = adev->link->ap; struct pci_dev *pdev = to_pci_dev(ap->host->dev); diff --git a/drivers/ata/pata_via.c b/drivers/ata/pata_via.c index 215c02d4056a..34f00f389932 100644 --- a/drivers/ata/pata_via.c +++ b/drivers/ata/pata_via.c @@ -352,7 +352,7 @@ static void via_set_dmamode(struct ata_port *ap, struct ata_device *adev) * one breed of Transcend SSD. Return the updated mask. */ -static unsigned long via_mode_filter(struct ata_device *dev, unsigned long mask) +static unsigned int via_mode_filter(struct ata_device *dev, unsigned int mask) { struct ata_host *host = dev->link->ap->host; const struct via_isa_bridge *config = host->private_data; diff --git a/include/linux/libata.h b/include/linux/libata.h index 0f2a59c9c735..a8bc88b4fe07 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -275,7 +275,7 @@ enum { PORT_DISABLED = 2, /* encoding various smaller bitmaps into a single - * unsigned long bitmap + * unsigned int bitmap */ ATA_NR_PIO_MODES = 7, ATA_NR_MWDMA_MODES = 5, @@ -426,12 +426,9 @@ enum { }; enum ata_xfer_mask { - ATA_MASK_PIO = ((1LU << ATA_NR_PIO_MODES) - 1) - << ATA_SHIFT_PIO, - ATA_MASK_MWDMA = ((1LU << ATA_NR_MWDMA_MODES) - 1) - << ATA_SHIFT_MWDMA, - ATA_MASK_UDMA = ((1LU << ATA_NR_UDMA_MODES) - 1) - << ATA_SHIFT_UDMA, + ATA_MASK_PIO = ((1U << ATA_NR_PIO_MODES) - 1) << ATA_SHIFT_PIO, + ATA_MASK_MWDMA = ((1U << ATA_NR_MWDMA_MODES) - 1) << ATA_SHIFT_MWDMA, + ATA_MASK_UDMA = ((1U << ATA_NR_UDMA_MODES) - 1) << ATA_SHIFT_UDMA, }; enum hsm_task_states { @@ -680,9 +677,9 @@ struct ata_device { unsigned int cdb_len; /* per-dev xfer mask */ - unsigned long pio_mask; - unsigned long mwdma_mask; - unsigned long udma_mask; + unsigned int pio_mask; + unsigned int mwdma_mask; + unsigned int udma_mask; /* for CHS addressing */ u16 cylinders; /* Number of cylinders */ @@ -885,7 +882,7 @@ struct ata_port_operations { * Configuration and exception handling */ int (*cable_detect)(struct ata_port *ap); - unsigned long (*mode_filter)(struct ata_device *dev, unsigned long xfer_mask); + unsigned int (*mode_filter)(struct ata_device *dev, unsigned int xfer_mask); void (*set_piomode)(struct ata_port *ap, struct ata_device *dev); void (*set_dmamode)(struct ata_port *ap, struct ata_device *dev); int (*set_mode)(struct ata_link *link, struct ata_device **r_failed_dev); @@ -981,9 +978,9 @@ struct ata_port_operations { struct ata_port_info { unsigned long flags; unsigned long link_flags; - unsigned long pio_mask; - unsigned long mwdma_mask; - unsigned long udma_mask; + unsigned int pio_mask; + unsigned int mwdma_mask; + unsigned int udma_mask; struct ata_port_operations *port_ops; void *private_data; }; @@ -1102,16 +1099,18 @@ extern void ata_msleep(struct ata_port *ap, unsigned int msecs); extern u32 ata_wait_register(struct ata_port *ap, void __iomem *reg, u32 mask, u32 val, unsigned long interval, unsigned long timeout); extern int atapi_cmd_type(u8 opcode); -extern unsigned long ata_pack_xfermask(unsigned long pio_mask, - unsigned long mwdma_mask, unsigned long udma_mask); -extern void ata_unpack_xfermask(unsigned long xfer_mask, - unsigned long *pio_mask, unsigned long *mwdma_mask, - unsigned long *udma_mask); -extern u8 ata_xfer_mask2mode(unsigned long xfer_mask); -extern unsigned long ata_xfer_mode2mask(u8 xfer_mode); +extern unsigned int ata_pack_xfermask(unsigned int pio_mask, + unsigned int mwdma_mask, + unsigned int udma_mask); +extern void ata_unpack_xfermask(unsigned int xfer_mask, + unsigned int *pio_mask, + unsigned int *mwdma_mask, + unsigned int *udma_mask); +extern u8 ata_xfer_mask2mode(unsigned int xfer_mask); +extern unsigned int ata_xfer_mode2mask(u8 xfer_mode); extern int ata_xfer_mode2shift(u8 xfer_mode); -extern const char *ata_mode_string(unsigned long xfer_mask); -extern unsigned long ata_id_xfermask(const u16 *id); +extern const char *ata_mode_string(unsigned int xfer_mask); +extern unsigned int ata_id_xfermask(const u16 *id); extern int ata_std_qc_defer(struct ata_queued_cmd *qc); extern enum ata_completion_errors ata_noop_qc_prep(struct ata_queued_cmd *qc); extern void ata_sg_init(struct ata_queued_cmd *qc, struct scatterlist *sg, @@ -1283,8 +1282,8 @@ static inline const struct ata_acpi_gtm *ata_acpi_init_gtm(struct ata_port *ap) } int ata_acpi_stm(struct ata_port *ap, const struct ata_acpi_gtm *stm); int ata_acpi_gtm(struct ata_port *ap, struct ata_acpi_gtm *stm); -unsigned long ata_acpi_gtm_xfermask(struct ata_device *dev, - const struct ata_acpi_gtm *gtm); +unsigned int ata_acpi_gtm_xfermask(struct ata_device *dev, + const struct ata_acpi_gtm *gtm); int ata_acpi_cbl_80wire(struct ata_port *ap, const struct ata_acpi_gtm *gtm); #else static inline const struct ata_acpi_gtm *ata_acpi_init_gtm(struct ata_port *ap) -- cgit v1.2.3 From d64de9773c18409d2161228242968ff3ebe3707e Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Thu, 9 Jun 2022 20:31:19 +0800 Subject: crypto: hisilicon/qm - modify event irq processing When the driver receives an event interrupt, the driver will enable the event interrupt after handling all completed tasks on the function, tasks on the function are parsed through only one thread. If the task's user callback takes time, other tasks on the function will be blocked. Therefore, the event irq processing is modified as follows: 1. Obtain the ID of the queue that completes the task. 2. Enable event interrupt. 3. Parse the completed tasks in the queue and call the user callback. Enabling event interrupt in advance can quickly report pending event interrupts and process tasks in multiple threads. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 142 +++++++++++++++++++++++++++--------------- include/linux/hisi_acc_qm.h | 8 ++- 2 files changed, 99 insertions(+), 51 deletions(-) (limited to 'include/linux') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index f8d36b68494e..ad83c194d664 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -877,13 +877,6 @@ static void qm_pm_put_sync(struct hisi_qm *qm) pm_runtime_put_autosuspend(dev); } -static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe) -{ - u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; - - return &qm->qp_array[cqn]; -} - static void qm_cq_head_update(struct hisi_qp *qp) { if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) { @@ -894,47 +887,37 @@ static void qm_cq_head_update(struct hisi_qp *qp) } } -static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) +static void qm_poll_req_cb(struct hisi_qp *qp) { - if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) - return; - - if (qp->event_cb) { - qp->event_cb(qp); - return; - } - - if (qp->req_cb) { - struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; - - while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { - dma_rmb(); - qp->req_cb(qp, qp->sqe + qm->sqe_size * - le16_to_cpu(cqe->sq_head)); - qm_cq_head_update(qp); - cqe = qp->cqe + qp->qp_status.cq_head; - qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, - qp->qp_status.cq_head, 0); - atomic_dec(&qp->qp_status.used); - } + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + struct hisi_qm *qm = qp->qm; - /* set c_flag */ + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { + dma_rmb(); + qp->req_cb(qp, qp->sqe + qm->sqe_size * + le16_to_cpu(cqe->sq_head)); + qm_cq_head_update(qp); + cqe = qp->cqe + qp->qp_status.cq_head; qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, - qp->qp_status.cq_head, 1); + qp->qp_status.cq_head, 0); + atomic_dec(&qp->qp_status.used); } + + /* set c_flag */ + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1); } -static void qm_work_process(struct work_struct *work) +static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) { - struct hisi_qm *qm = container_of(work, struct hisi_qm, work); + struct hisi_qm *qm = poll_data->qm; struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; - struct hisi_qp *qp; int eqe_num = 0; + u16 cqn; while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { + cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; + poll_data->qp_finish_id[eqe_num] = cqn; eqe_num++; - qp = qm_to_hisi_qp(qm, eqe); - qm_poll_qp(qp, qm); if (qm->status.eq_head == QM_EQ_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; @@ -945,34 +928,70 @@ static void qm_work_process(struct work_struct *work) qm->status.eq_head++; } - if (eqe_num == QM_EQ_DEPTH / 2 - 1) { - eqe_num = 0; - qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); - } + if (eqe_num == (QM_EQ_DEPTH >> 1) - 1) + break; } qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); + + return eqe_num; } -static irqreturn_t do_qm_irq(int irq, void *data) +static void qm_work_process(struct work_struct *work) { - struct hisi_qm *qm = (struct hisi_qm *)data; + struct hisi_qm_poll_data *poll_data = + container_of(work, struct hisi_qm_poll_data, work); + struct hisi_qm *qm = poll_data->qm; + struct hisi_qp *qp; + int eqe_num, i; - /* the workqueue created by device driver of QM */ - queue_work(qm->wq, &qm->work); + /* Get qp id of completed tasks and re-enable the interrupt. */ + eqe_num = qm_get_complete_eqe_num(poll_data); + for (i = eqe_num - 1; i >= 0; i--) { + qp = &qm->qp_array[poll_data->qp_finish_id[i]]; + if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) + continue; - return IRQ_HANDLED; + if (qp->event_cb) { + qp->event_cb(qp); + continue; + } + + if (likely(qp->req_cb)) + qm_poll_req_cb(qp); + } +} + +static bool do_qm_irq(struct hisi_qm *qm) +{ + struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; + struct hisi_qm_poll_data *poll_data; + u16 cqn; + + if (!readl(qm->io_base + QM_VF_EQ_INT_SOURCE)) + return false; + + if (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { + cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; + poll_data = &qm->poll_data[cqn]; + queue_work(qm->wq, &poll_data->work); + + return true; + } + + return false; } static irqreturn_t qm_irq(int irq, void *data) { struct hisi_qm *qm = data; + bool ret; - if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE)) - return do_qm_irq(irq, data); + ret = do_qm_irq(qm); + if (ret) + return IRQ_HANDLED; atomic64_inc(&qm->debug.dfx.err_irq_cnt); - dev_err(&qm->pdev->dev, "invalid int source\n"); qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); return IRQ_NONE; @@ -3551,8 +3570,10 @@ static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) for (i = num - 1; i >= 0; i--) { qdma = &qm->qp_array[i].qdma; dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); + kfree(qm->poll_data[i].qp_finish_id); } + kfree(qm->poll_data); kfree(qm->qp_array); } @@ -3561,12 +3582,18 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) struct device *dev = &qm->pdev->dev; size_t off = qm->sqe_size * QM_Q_DEPTH; struct hisi_qp *qp; + int ret = -ENOMEM; + + qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), + GFP_KERNEL); + if (!qm->poll_data[id].qp_finish_id) + return -ENOMEM; qp = &qm->qp_array[id]; qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, GFP_KERNEL); if (!qp->qdma.va) - return -ENOMEM; + goto err_free_qp_finish_id; qp->sqe = qp->qdma.va; qp->sqe_dma = qp->qdma.dma; @@ -3577,6 +3604,10 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) qp->qp_id = id; return 0; + +err_free_qp_finish_id: + kfree(qm->poll_data[id].qp_finish_id); + return ret; } static void hisi_qm_pre_init(struct hisi_qm *qm) @@ -6024,7 +6055,11 @@ err_disable_pcidev: static int hisi_qm_init_work(struct hisi_qm *qm) { - INIT_WORK(&qm->work, qm_work_process); + int i; + + for (i = 0; i < qm->qp_num; i++) + INIT_WORK(&qm->poll_data[i].work, qm_work_process); + if (qm->fun_type == QM_HW_PF) INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); @@ -6052,11 +6087,18 @@ static int hisi_qp_alloc_memory(struct hisi_qm *qm) if (!qm->qp_array) return -ENOMEM; + qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL); + if (!qm->poll_data) { + kfree(qm->qp_array); + return -ENOMEM; + } + /* one more page for device or qp statuses */ qp_dma_size = qm->sqe_size * QM_Q_DEPTH + sizeof(struct qm_cqe) * QM_Q_DEPTH; qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; for (i = 0; i < qm->qp_num; i++) { + qm->poll_data[i].qm = qm; ret = hisi_qp_memory_init(qm, qp_dma_size, i); if (ret) goto err_init_qp_mem; diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 6cabafffd0dd..116e8bd68c99 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -265,6 +265,12 @@ struct hisi_qm_list { void (*unregister_from_crypto)(struct hisi_qm *qm); }; +struct hisi_qm_poll_data { + struct hisi_qm *qm; + struct work_struct work; + u16 *qp_finish_id; +}; + struct hisi_qm { enum qm_hw_ver ver; enum qm_fun_type fun_type; @@ -302,6 +308,7 @@ struct hisi_qm { struct rw_semaphore qps_lock; struct idr qp_idr; struct hisi_qp *qp_array; + struct hisi_qm_poll_data *poll_data; struct mutex mailbox_lock; @@ -312,7 +319,6 @@ struct hisi_qm { u32 error_mask; struct workqueue_struct *wq; - struct work_struct work; struct work_struct rst_work; struct work_struct cmd_process; -- cgit v1.2.3 From fa9c562f9735d24c3253747eb21f3f0c0f6de48e Mon Sep 17 00:00:00 2001 From: Ong Boon Leong Date: Wed, 15 Jun 2022 16:39:04 +0800 Subject: net: make xpcs_do_config to accept advertising for pcs-xpcs and sja1105 xpcs_config() has 'advertising' input that is required for C37 1000BASE-X AN in later patch series. So, we prepare xpcs_do_config() for it. For sja1105, xpcs_do_config() is used for xpcs configuration without depending on advertising input, so set to NULL. Reported-by: kernel test robot Signed-off-by: Ong Boon Leong Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105_main.c | 2 +- drivers/net/pcs/pcs-xpcs.c | 6 +++--- include/linux/pcs/pcs-xpcs.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 72b6fc1932b5..b253e27bcfb4 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -2330,7 +2330,7 @@ int sja1105_static_config_reload(struct sja1105_private *priv, else mode = MLO_AN_PHY; - rc = xpcs_do_config(xpcs, priv->phy_mode[i], mode); + rc = xpcs_do_config(xpcs, priv->phy_mode[i], mode, NULL); if (rc < 0) goto out; diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c index 4cfd05c15aee..48d81c40aab7 100644 --- a/drivers/net/pcs/pcs-xpcs.c +++ b/drivers/net/pcs/pcs-xpcs.c @@ -795,7 +795,7 @@ static int xpcs_config_2500basex(struct dw_xpcs *xpcs) } int xpcs_do_config(struct dw_xpcs *xpcs, phy_interface_t interface, - unsigned int mode) + unsigned int mode, const unsigned long *advertising) { const struct xpcs_compat *compat; int ret; @@ -843,7 +843,7 @@ static int xpcs_config(struct phylink_pcs *pcs, unsigned int mode, { struct dw_xpcs *xpcs = phylink_pcs_to_xpcs(pcs); - return xpcs_do_config(xpcs, interface, mode); + return xpcs_do_config(xpcs, interface, mode, advertising); } static int xpcs_get_state_c73(struct dw_xpcs *xpcs, @@ -864,7 +864,7 @@ static int xpcs_get_state_c73(struct dw_xpcs *xpcs, state->link = 0; - return xpcs_do_config(xpcs, state->interface, MLO_AN_INBAND); + return xpcs_do_config(xpcs, state->interface, MLO_AN_INBAND, NULL); } if (state->an_enabled && xpcs_aneg_done_c73(xpcs, state, compat)) { diff --git a/include/linux/pcs/pcs-xpcs.h b/include/linux/pcs/pcs-xpcs.h index 266eb26fb029..37eb97cc2283 100644 --- a/include/linux/pcs/pcs-xpcs.h +++ b/include/linux/pcs/pcs-xpcs.h @@ -30,7 +30,7 @@ int xpcs_get_an_mode(struct dw_xpcs *xpcs, phy_interface_t interface); void xpcs_link_up(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, int speed, int duplex); int xpcs_do_config(struct dw_xpcs *xpcs, phy_interface_t interface, - unsigned int mode); + unsigned int mode, const unsigned long *advertising); void xpcs_get_interfaces(struct dw_xpcs *xpcs, unsigned long *interfaces); int xpcs_config_eee(struct dw_xpcs *xpcs, int mult_fact_100ns, int enable); -- cgit v1.2.3 From b47aec885bcd672ebca2108a8b7e9ce3e3982775 Mon Sep 17 00:00:00 2001 From: Ong Boon Leong Date: Wed, 15 Jun 2022 16:39:06 +0800 Subject: net: pcs: xpcs: add CL37 1000BASE-X AN support For CL37 1000BASE-X AN, DW xPCS does not support C22 method but offers C45 vendor-specific MII MMD for programming. We also add the ability to disable Autoneg (through ethtool for certain network switch that supports 1000BASE-X (1000Mbps and Full-Duplex) but not Autoneg capability. v4: Fixes to comment from Russell King. Thanks! https://patchwork.kernel.org/comment/24894239/ Make xpcs_modify_changed() as private, change to use mdiodev_modify_changed() for cleaner code. v3: Fixes to issues spotted by Russell King. Thanks! https://patchwork.kernel.org/comment/24890210/ Use phylink_mii_c22_pcs_decode_state(), remove unnecessary interrupt clearing and skip speed & duplex setting if AN is enabled. v2: Fixes to issues spotted by Russell King in v1. Thanks! https://patchwork.kernel.org/comment/24826650/ Use phylink_mii_c22_pcs_encode_advertisement() and implement C45 MII ADV handling since IP only support C45 access. Tested-by: Emilio Riva Signed-off-by: Ong Boon Leong Reviewed-by: Russell King (Oracle) Signed-off-by: David S. Miller --- drivers/net/pcs/pcs-xpcs.c | 170 +++++++++++++++++++++++++++++++++++++++++++ drivers/net/pcs/pcs-xpcs.h | 1 - include/linux/pcs/pcs-xpcs.h | 1 + 3 files changed, 171 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c index 48d81c40aab7..a5d520e34ea3 100644 --- a/drivers/net/pcs/pcs-xpcs.c +++ b/drivers/net/pcs/pcs-xpcs.c @@ -77,6 +77,14 @@ static const int xpcs_sgmii_features[] = { __ETHTOOL_LINK_MODE_MASK_NBITS, }; +static const int xpcs_1000basex_features[] = { + ETHTOOL_LINK_MODE_Pause_BIT, + ETHTOOL_LINK_MODE_Asym_Pause_BIT, + ETHTOOL_LINK_MODE_Autoneg_BIT, + ETHTOOL_LINK_MODE_1000baseX_Full_BIT, + __ETHTOOL_LINK_MODE_MASK_NBITS, +}; + static const int xpcs_2500basex_features[] = { ETHTOOL_LINK_MODE_Pause_BIT, ETHTOOL_LINK_MODE_Asym_Pause_BIT, @@ -102,6 +110,10 @@ static const phy_interface_t xpcs_sgmii_interfaces[] = { PHY_INTERFACE_MODE_SGMII, }; +static const phy_interface_t xpcs_1000basex_interfaces[] = { + PHY_INTERFACE_MODE_1000BASEX, +}; + static const phy_interface_t xpcs_2500basex_interfaces[] = { PHY_INTERFACE_MODE_2500BASEX, PHY_INTERFACE_MODE_MAX, @@ -112,6 +124,7 @@ enum { DW_XPCS_10GKR, DW_XPCS_XLGMII, DW_XPCS_SGMII, + DW_XPCS_1000BASEX, DW_XPCS_2500BASEX, DW_XPCS_INTERFACE_MAX, }; @@ -189,6 +202,14 @@ int xpcs_write(struct dw_xpcs *xpcs, int dev, u32 reg, u16 val) return mdiobus_c45_write(bus, addr, dev, reg, val); } +static int xpcs_modify_changed(struct dw_xpcs *xpcs, int dev, u32 reg, + u16 mask, u16 set) +{ + u32 reg_addr = mdiobus_c45_addr(dev, reg); + + return mdiodev_modify_changed(xpcs->mdiodev, reg_addr, mask, set); +} + static int xpcs_read_vendor(struct dw_xpcs *xpcs, int dev, u32 reg) { return xpcs_read(xpcs, dev, DW_VENDOR | reg); @@ -237,6 +258,7 @@ static int xpcs_soft_reset(struct dw_xpcs *xpcs, break; case DW_AN_C37_SGMII: case DW_2500BASEX: + case DW_AN_C37_1000BASEX: dev = MDIO_MMD_VEND2; break; default: @@ -772,6 +794,68 @@ static int xpcs_config_aneg_c37_sgmii(struct dw_xpcs *xpcs, unsigned int mode) return ret; } +static int xpcs_config_aneg_c37_1000basex(struct dw_xpcs *xpcs, unsigned int mode, + const unsigned long *advertising) +{ + phy_interface_t interface = PHY_INTERFACE_MODE_1000BASEX; + int ret, mdio_ctrl, adv; + bool changed = 0; + + /* According to Chap 7.12, to set 1000BASE-X C37 AN, AN must + * be disabled first:- + * 1) VR_MII_MMD_CTRL Bit(12)[AN_ENABLE] = 0b + * 2) VR_MII_AN_CTRL Bit(2:1)[PCS_MODE] = 00b (1000BASE-X C37) + */ + mdio_ctrl = xpcs_read(xpcs, MDIO_MMD_VEND2, DW_VR_MII_MMD_CTRL); + if (mdio_ctrl < 0) + return mdio_ctrl; + + if (mdio_ctrl & AN_CL37_EN) { + ret = xpcs_write(xpcs, MDIO_MMD_VEND2, DW_VR_MII_MMD_CTRL, + mdio_ctrl & ~AN_CL37_EN); + if (ret < 0) + return ret; + } + + ret = xpcs_read(xpcs, MDIO_MMD_VEND2, DW_VR_MII_AN_CTRL); + if (ret < 0) + return ret; + + ret &= ~DW_VR_MII_PCS_MODE_MASK; + ret = xpcs_write(xpcs, MDIO_MMD_VEND2, DW_VR_MII_AN_CTRL, ret); + if (ret < 0) + return ret; + + /* Check for advertising changes and update the C45 MII ADV + * register accordingly. + */ + adv = phylink_mii_c22_pcs_encode_advertisement(interface, + advertising); + if (adv >= 0) { + ret = xpcs_modify_changed(xpcs, MDIO_MMD_VEND2, + MII_ADVERTISE, 0xffff, adv); + if (ret < 0) + return ret; + + changed = ret; + } + + /* Clear CL37 AN complete status */ + ret = xpcs_write(xpcs, MDIO_MMD_VEND2, DW_VR_MII_AN_INTR_STS, 0); + if (ret < 0) + return ret; + + if (phylink_autoneg_inband(mode) && + linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, advertising)) { + ret = xpcs_write(xpcs, MDIO_MMD_VEND2, DW_VR_MII_MMD_CTRL, + mdio_ctrl | AN_CL37_EN); + if (ret < 0) + return ret; + } + + return changed; +} + static int xpcs_config_2500basex(struct dw_xpcs *xpcs) { int ret; @@ -817,6 +901,12 @@ int xpcs_do_config(struct dw_xpcs *xpcs, phy_interface_t interface, if (ret) return ret; break; + case DW_AN_C37_1000BASEX: + ret = xpcs_config_aneg_c37_1000basex(xpcs, mode, + advertising); + if (ret) + return ret; + break; case DW_2500BASEX: ret = xpcs_config_2500basex(xpcs); if (ret) @@ -921,6 +1011,29 @@ static int xpcs_get_state_c37_sgmii(struct dw_xpcs *xpcs, return 0; } +static int xpcs_get_state_c37_1000basex(struct dw_xpcs *xpcs, + struct phylink_link_state *state) +{ + int lpa, bmsr; + + if (state->an_enabled) { + /* Reset link state */ + state->link = false; + + lpa = xpcs_read(xpcs, MDIO_MMD_VEND2, MII_LPA); + if (lpa < 0 || lpa & LPA_RFAULT) + return lpa; + + bmsr = xpcs_read(xpcs, MDIO_MMD_VEND2, MII_BMSR); + if (bmsr < 0) + return bmsr; + + phylink_mii_c22_pcs_decode_state(state, bmsr, lpa); + } + + return 0; +} + static void xpcs_get_state(struct phylink_pcs *pcs, struct phylink_link_state *state) { @@ -948,6 +1061,13 @@ static void xpcs_get_state(struct phylink_pcs *pcs, ERR_PTR(ret)); } break; + case DW_AN_C37_1000BASEX: + ret = xpcs_get_state_c37_1000basex(xpcs, state); + if (ret) { + pr_err("xpcs_get_state_c37_1000basex returned %pe\n", + ERR_PTR(ret)); + } + break; default: return; } @@ -983,6 +1103,35 @@ static void xpcs_link_up_sgmii(struct dw_xpcs *xpcs, unsigned int mode, pr_err("%s: xpcs_write returned %pe\n", __func__, ERR_PTR(ret)); } +static void xpcs_link_up_1000basex(struct dw_xpcs *xpcs, unsigned int mode, + int speed, int duplex) +{ + int val, ret; + + if (phylink_autoneg_inband(mode)) + return; + + switch (speed) { + case SPEED_1000: + val = BMCR_SPEED1000; + break; + case SPEED_100: + case SPEED_10: + default: + pr_err("%s: speed = %d\n", __func__, speed); + return; + } + + if (duplex == DUPLEX_FULL) + val |= BMCR_FULLDPLX; + else + pr_err("%s: half duplex not supported\n", __func__); + + ret = xpcs_write(xpcs, MDIO_MMD_VEND2, MDIO_CTRL1, val); + if (ret) + pr_err("%s: xpcs_write returned %pe\n", __func__, ERR_PTR(ret)); +} + void xpcs_link_up(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, int speed, int duplex) { @@ -992,9 +1141,23 @@ void xpcs_link_up(struct phylink_pcs *pcs, unsigned int mode, return xpcs_config_usxgmii(xpcs, speed); if (interface == PHY_INTERFACE_MODE_SGMII) return xpcs_link_up_sgmii(xpcs, mode, speed, duplex); + if (interface == PHY_INTERFACE_MODE_1000BASEX) + return xpcs_link_up_1000basex(xpcs, mode, speed, duplex); } EXPORT_SYMBOL_GPL(xpcs_link_up); +static void xpcs_an_restart(struct phylink_pcs *pcs) +{ + struct dw_xpcs *xpcs = phylink_pcs_to_xpcs(pcs); + int ret; + + ret = xpcs_read(xpcs, MDIO_MMD_VEND2, MDIO_CTRL1); + if (ret >= 0) { + ret |= BMCR_ANRESTART; + xpcs_write(xpcs, MDIO_MMD_VEND2, MDIO_CTRL1, ret); + } +} + static u32 xpcs_get_id(struct dw_xpcs *xpcs) { int ret; @@ -1060,6 +1223,12 @@ static const struct xpcs_compat synopsys_xpcs_compat[DW_XPCS_INTERFACE_MAX] = { .num_interfaces = ARRAY_SIZE(xpcs_sgmii_interfaces), .an_mode = DW_AN_C37_SGMII, }, + [DW_XPCS_1000BASEX] = { + .supported = xpcs_1000basex_features, + .interface = xpcs_1000basex_interfaces, + .num_interfaces = ARRAY_SIZE(xpcs_1000basex_interfaces), + .an_mode = DW_AN_C37_1000BASEX, + }, [DW_XPCS_2500BASEX] = { .supported = xpcs_2500basex_features, .interface = xpcs_2500basex_interfaces, @@ -1115,6 +1284,7 @@ static const struct phylink_pcs_ops xpcs_phylink_ops = { .pcs_validate = xpcs_validate, .pcs_config = xpcs_config, .pcs_get_state = xpcs_get_state, + .pcs_an_restart = xpcs_an_restart, .pcs_link_up = xpcs_link_up, }; diff --git a/drivers/net/pcs/pcs-xpcs.h b/drivers/net/pcs/pcs-xpcs.h index 35651d32a224..770df50323a0 100644 --- a/drivers/net/pcs/pcs-xpcs.h +++ b/drivers/net/pcs/pcs-xpcs.h @@ -109,7 +109,6 @@ int xpcs_read(struct dw_xpcs *xpcs, int dev, u32 reg); int xpcs_write(struct dw_xpcs *xpcs, int dev, u32 reg, u16 val); - int nxp_sja1105_sgmii_pma_config(struct dw_xpcs *xpcs); int nxp_sja1110_sgmii_pma_config(struct dw_xpcs *xpcs); int nxp_sja1110_2500basex_pma_config(struct dw_xpcs *xpcs); diff --git a/include/linux/pcs/pcs-xpcs.h b/include/linux/pcs/pcs-xpcs.h index 37eb97cc2283..d2da1e0b4a92 100644 --- a/include/linux/pcs/pcs-xpcs.h +++ b/include/linux/pcs/pcs-xpcs.h @@ -17,6 +17,7 @@ #define DW_AN_C73 1 #define DW_AN_C37_SGMII 2 #define DW_2500BASEX 3 +#define DW_AN_C37_1000BASEX 4 struct xpcs_id; -- cgit v1.2.3 From d0804085c5a7feb557bd3219777b51f5598dfdc6 Mon Sep 17 00:00:00 2001 From: Moudy Ho Date: Fri, 10 Jun 2022 14:34:18 +0800 Subject: soc: mediatek: mutex: add common interface for modules setting In order to allow multiple modules to operate MUTEX hardware through a common interfrace, two flexible indexes "mtk_mutex_mod_index" and "mtk_mutex_sof_index" need to be added to replace original component ID so that like DDP and MDP can add their own MOD table or SOF settings independently. In addition, 2 generic interface "mtk_mutex_write_mod" and "mtk_mutex_write_sof" have been added, which is expected to replace the "mtk_mutex_add_comp" and "mtk_mutex_remove_comp" pair originally dedicated to DDP in the future. Signed-off-by: Moudy Ho Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Rex-BC Chen Reviewed-by: CK Hu Link: https://lore.kernel.org/r/20220610063424.7800-2-moudy.ho@mediatek.com Signed-off-by: Matthias Brugger --- drivers/soc/mediatek/mtk-mutex.c | 53 ++++++++++++++++++++++++++++++++++ include/linux/soc/mediatek/mtk-mutex.h | 25 ++++++++++++++++ 2 files changed, 78 insertions(+) (limited to 'include/linux') diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c index b8d5c4a62542..56987e2ff906 100644 --- a/drivers/soc/mediatek/mtk-mutex.c +++ b/drivers/soc/mediatek/mtk-mutex.c @@ -199,6 +199,7 @@ struct mtk_mutex_data { const unsigned int *mutex_sof; const unsigned int mutex_mod_reg; const unsigned int mutex_sof_reg; + const unsigned int *mutex_table_mod; const bool no_clk; }; @@ -644,6 +645,58 @@ void mtk_mutex_release(struct mtk_mutex *mutex) } EXPORT_SYMBOL_GPL(mtk_mutex_release); +int mtk_mutex_write_mod(struct mtk_mutex *mutex, + enum mtk_mutex_mod_index idx, bool clear) +{ + struct mtk_mutex_ctx *mtx = container_of(mutex, struct mtk_mutex_ctx, + mutex[mutex->id]); + unsigned int reg; + unsigned int offset; + + WARN_ON(&mtx->mutex[mutex->id] != mutex); + + if (idx < MUTEX_MOD_IDX_MDP_RDMA0 || + idx >= MUTEX_MOD_IDX_MAX) { + dev_err(mtx->dev, "Not supported MOD table index : %d", idx); + return -EINVAL; + } + + offset = DISP_REG_MUTEX_MOD(mtx->data->mutex_mod_reg, + mutex->id); + reg = readl_relaxed(mtx->regs + offset); + + if (clear) + reg &= ~BIT(mtx->data->mutex_table_mod[idx]); + else + reg |= BIT(mtx->data->mutex_table_mod[idx]); + + writel_relaxed(reg, mtx->regs + offset); + + return 0; +} +EXPORT_SYMBOL_GPL(mtk_mutex_write_mod); + +int mtk_mutex_write_sof(struct mtk_mutex *mutex, + enum mtk_mutex_sof_index idx) +{ + struct mtk_mutex_ctx *mtx = container_of(mutex, struct mtk_mutex_ctx, + mutex[mutex->id]); + + WARN_ON(&mtx->mutex[mutex->id] != mutex); + + if (idx < MUTEX_SOF_IDX_SINGLE_MODE || + idx >= MUTEX_SOF_IDX_MAX) { + dev_err(mtx->dev, "Not supported SOF index : %d", idx); + return -EINVAL; + } + + writel_relaxed(idx, mtx->regs + + DISP_REG_MUTEX_SOF(mtx->data->mutex_sof_reg, mutex->id)); + + return 0; +} +EXPORT_SYMBOL_GPL(mtk_mutex_write_sof); + static int mtk_mutex_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; diff --git a/include/linux/soc/mediatek/mtk-mutex.h b/include/linux/soc/mediatek/mtk-mutex.h index 6fe4ffbde290..2ddab9d2b85d 100644 --- a/include/linux/soc/mediatek/mtk-mutex.h +++ b/include/linux/soc/mediatek/mtk-mutex.h @@ -10,6 +10,26 @@ struct regmap; struct device; struct mtk_mutex; +enum mtk_mutex_mod_index { + /* MDP table index */ + MUTEX_MOD_IDX_MDP_RDMA0, + MUTEX_MOD_IDX_MDP_RSZ0, + MUTEX_MOD_IDX_MDP_RSZ1, + MUTEX_MOD_IDX_MDP_TDSHP0, + MUTEX_MOD_IDX_MDP_WROT0, + MUTEX_MOD_IDX_MDP_WDMA, + MUTEX_MOD_IDX_MDP_AAL0, + MUTEX_MOD_IDX_MDP_CCORR0, + + MUTEX_MOD_IDX_MAX /* ALWAYS keep at the end */ +}; + +enum mtk_mutex_sof_index { + MUTEX_SOF_IDX_SINGLE_MODE, + + MUTEX_SOF_IDX_MAX /* ALWAYS keep at the end */ +}; + struct mtk_mutex *mtk_mutex_get(struct device *dev); int mtk_mutex_prepare(struct mtk_mutex *mutex); void mtk_mutex_add_comp(struct mtk_mutex *mutex, @@ -22,5 +42,10 @@ void mtk_mutex_unprepare(struct mtk_mutex *mutex); void mtk_mutex_put(struct mtk_mutex *mutex); void mtk_mutex_acquire(struct mtk_mutex *mutex); void mtk_mutex_release(struct mtk_mutex *mutex); +int mtk_mutex_write_mod(struct mtk_mutex *mutex, + enum mtk_mutex_mod_index idx, + bool clear); +int mtk_mutex_write_sof(struct mtk_mutex *mutex, + enum mtk_mutex_sof_index idx); #endif /* MTK_MUTEX_H */ -- cgit v1.2.3 From e5758850c2ea448dd750a280d128a3590d68b899 Mon Sep 17 00:00:00 2001 From: Moudy Ho Date: Fri, 10 Jun 2022 14:34:23 +0800 Subject: soc: mediatek: mutex: add functions that operate registers by CMDQ Due to HW limitations, MDP3 is necessary to enable MUTEX in each frame for SOF triggering and cooperate with CMDQ control to reduce the amount of interrupts generated(also, reduce frame latency). In response to the above situation, a new interface "mtk_mutex_enable_by_cmdq" has been added to achieve the purpose. Signed-off-by: Moudy Ho Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Rex-BC Chen Reviewed-by: CK Hu Link: https://lore.kernel.org/r/20220610063424.7800-7-moudy.ho@mediatek.com Signed-off-by: Matthias Brugger --- drivers/soc/mediatek/mtk-mutex.c | 45 +++++++++++++++++++++++++++++++++- include/linux/soc/mediatek/mtk-mutex.h | 2 ++ 2 files changed, 46 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c index a566f4cf6372..fa8e0ba38803 100644 --- a/drivers/soc/mediatek/mtk-mutex.c +++ b/drivers/soc/mediatek/mtk-mutex.c @@ -7,10 +7,12 @@ #include #include #include +#include #include #include #include #include +#include #define MT2701_MUTEX0_MOD0 0x2c #define MT2701_MUTEX0_SOF0 0x30 @@ -218,6 +220,8 @@ struct mtk_mutex_ctx { void __iomem *regs; struct mtk_mutex mutex[10]; const struct mtk_mutex_data *data; + phys_addr_t addr; + struct cmdq_client_reg cmdq_reg; }; static const unsigned int mt2701_mutex_mod[DDP_COMPONENT_ID_MAX] = { @@ -632,6 +636,30 @@ void mtk_mutex_enable(struct mtk_mutex *mutex) } EXPORT_SYMBOL_GPL(mtk_mutex_enable); +int mtk_mutex_enable_by_cmdq(struct mtk_mutex *mutex, void *pkt) +{ + struct mtk_mutex_ctx *mtx = container_of(mutex, struct mtk_mutex_ctx, + mutex[mutex->id]); +#if IS_REACHABLE(CONFIG_MTK_CMDQ) + struct cmdq_pkt *cmdq_pkt = (struct cmdq_pkt *)pkt; + + WARN_ON(&mtx->mutex[mutex->id] != mutex); + + if (!mtx->cmdq_reg.size) { + dev_err(mtx->dev, "mediatek,gce-client-reg hasn't been set"); + return -EINVAL; + } + + cmdq_pkt_write(cmdq_pkt, mtx->cmdq_reg.subsys, + mtx->addr + DISP_REG_MUTEX_EN(mutex->id), 1); + return 0; +#else + dev_err(mtx->dev, "Not support for enable MUTEX by CMDQ"); + return -ENODEV; +#endif +} +EXPORT_SYMBOL_GPL(mtk_mutex_enable_by_cmdq); + void mtk_mutex_disable(struct mtk_mutex *mutex) { struct mtk_mutex_ctx *mtx = container_of(mutex, struct mtk_mutex_ctx, @@ -722,8 +750,11 @@ static int mtk_mutex_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct mtk_mutex_ctx *mtx; - struct resource *regs; + struct resource *regs, addr; int i; +#if IS_REACHABLE(CONFIG_MTK_CMDQ) + int ret; +#endif mtx = devm_kzalloc(dev, sizeof(*mtx), GFP_KERNEL); if (!mtx) @@ -743,6 +774,18 @@ static int mtk_mutex_probe(struct platform_device *pdev) } } + if (of_address_to_resource(dev->of_node, 0, &addr) < 0) { + dev_err(dev, "Failed to get addr\n"); + return -EINVAL; + } + mtx->addr = addr.start; + +#if IS_REACHABLE(CONFIG_MTK_CMDQ) + ret = cmdq_dev_get_client_reg(dev, &mtx->cmdq_reg, 0); + if (ret) + dev_dbg(dev, "No mediatek,gce-client-reg!\n"); +#endif + regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); mtx->regs = devm_ioremap_resource(dev, regs); if (IS_ERR(mtx->regs)) { diff --git a/include/linux/soc/mediatek/mtk-mutex.h b/include/linux/soc/mediatek/mtk-mutex.h index 2ddab9d2b85d..a0f4f51a3b45 100644 --- a/include/linux/soc/mediatek/mtk-mutex.h +++ b/include/linux/soc/mediatek/mtk-mutex.h @@ -35,6 +35,8 @@ int mtk_mutex_prepare(struct mtk_mutex *mutex); void mtk_mutex_add_comp(struct mtk_mutex *mutex, enum mtk_ddp_comp_id id); void mtk_mutex_enable(struct mtk_mutex *mutex); +int mtk_mutex_enable_by_cmdq(struct mtk_mutex *mutex, + void *pkt); void mtk_mutex_disable(struct mtk_mutex *mutex); void mtk_mutex_remove_comp(struct mtk_mutex *mutex, enum mtk_ddp_comp_id id); -- cgit v1.2.3 From dc368e1c658e4f478a45e8d1d5b0c8392ca87506 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jun 2022 15:54:07 -0700 Subject: bpf: Fix non-static bpf_func_proto struct definitions This patch does two things: 1) Marks the dynptr bpf_func_proto structs that were added in [1] as static, as pointed out by the kernel test robot in [2]. 2) There are some bpf_func_proto structs marked as extern which can instead be statically defined. [1] https://lore.kernel.org/bpf/20220523210712.3641569-1-joannelkoong@gmail.com/ [2] https://lore.kernel.org/bpf/62ab89f2.Pko7sI08RAKdF8R6%25lkp@intel.com/ Reported-by: kernel test robot Signed-off-by: Joanne Koong Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220616225407.1878436-1-joannelkoong@gmail.com --- include/linux/bpf.h | 3 --- kernel/bpf/helpers.c | 12 ++++++------ kernel/bpf/syscall.c | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a94531971a7a..0edd7d2c0064 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2363,12 +2363,9 @@ extern const struct bpf_func_proto bpf_for_each_map_elem_proto; extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; extern const struct bpf_func_proto bpf_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_sk_getsockopt_proto; -extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto; extern const struct bpf_func_proto bpf_find_vma_proto; extern const struct bpf_func_proto bpf_loop_proto; -extern const struct bpf_func_proto bpf_strncmp_proto; extern const struct bpf_func_proto bpf_copy_from_user_task_proto; -extern const struct bpf_func_proto bpf_kptr_xchg_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 225806a02efb..a1c84d256f83 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -584,7 +584,7 @@ BPF_CALL_3(bpf_strncmp, const char *, s1, u32, s1_sz, const char *, s2) return strncmp(s1, s2, s1_sz); } -const struct bpf_func_proto bpf_strncmp_proto = { +static const struct bpf_func_proto bpf_strncmp_proto = { .func = bpf_strncmp, .gpl_only = false, .ret_type = RET_INTEGER, @@ -1402,7 +1402,7 @@ BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr) */ #define BPF_PTR_POISON ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) -const struct bpf_func_proto bpf_kptr_xchg_proto = { +static const struct bpf_func_proto bpf_kptr_xchg_proto = { .func = bpf_kptr_xchg, .gpl_only = false, .ret_type = RET_PTR_TO_BTF_ID_OR_NULL, @@ -1487,7 +1487,7 @@ error: return err; } -const struct bpf_func_proto bpf_dynptr_from_mem_proto = { +static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { .func = bpf_dynptr_from_mem, .gpl_only = false, .ret_type = RET_INTEGER, @@ -1513,7 +1513,7 @@ BPF_CALL_4(bpf_dynptr_read, void *, dst, u32, len, struct bpf_dynptr_kern *, src return 0; } -const struct bpf_func_proto bpf_dynptr_read_proto = { +static const struct bpf_func_proto bpf_dynptr_read_proto = { .func = bpf_dynptr_read, .gpl_only = false, .ret_type = RET_INTEGER, @@ -1539,7 +1539,7 @@ BPF_CALL_4(bpf_dynptr_write, struct bpf_dynptr_kern *, dst, u32, offset, void *, return 0; } -const struct bpf_func_proto bpf_dynptr_write_proto = { +static const struct bpf_func_proto bpf_dynptr_write_proto = { .func = bpf_dynptr_write, .gpl_only = false, .ret_type = RET_INTEGER, @@ -1566,7 +1566,7 @@ BPF_CALL_3(bpf_dynptr_data, struct bpf_dynptr_kern *, ptr, u32, offset, u32, len return (unsigned long)(ptr->data + ptr->offset + offset); } -const struct bpf_func_proto bpf_dynptr_data_proto = { +static const struct bpf_func_proto bpf_dynptr_data_proto = { .func = bpf_dynptr_data, .gpl_only = false, .ret_type = RET_PTR_TO_DYNPTR_MEM_OR_NULL, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index aeb31137b2ed..7d5af5b99f0d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5131,7 +5131,7 @@ BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flag return *res ? 0 : -ENOENT; } -const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { +static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .func = bpf_kallsyms_lookup_name, .gpl_only = false, .ret_type = RET_INTEGER, -- cgit v1.2.3 From 9a2139c2912ea64e288749c21452930dc752d4fd Mon Sep 17 00:00:00 2001 From: Caleb Connolly Date: Fri, 29 Apr 2022 23:08:56 +0100 Subject: spmi: add a helper to look up an SPMI device from a device node The helper function spmi_device_from_of() takes a device node and returns the SPMI device associated with it. This is like of_find_device_by_node but for SPMI devices. Signed-off-by: Caleb Connolly Acked-by: Stephen Boyd Link: https://lore.kernel.org/r/20220429220904.137297-2-caleb.connolly@linaro.org Signed-off-by: Jonathan Cameron --- drivers/spmi/spmi.c | 17 +++++++++++++++++ include/linux/spmi.h | 3 +++ 2 files changed, 20 insertions(+) (limited to 'include/linux') diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c index b37ead9e2fad..a456ce5141e1 100644 --- a/drivers/spmi/spmi.c +++ b/drivers/spmi/spmi.c @@ -386,6 +386,23 @@ static struct bus_type spmi_bus_type = { .uevent = spmi_drv_uevent, }; +/** + * spmi_device_from_of() - get the associated SPMI device from a device node + * + * @np: device node + * + * Returns the struct spmi_device associated with a device node or NULL. + */ +struct spmi_device *spmi_device_from_of(struct device_node *np) +{ + struct device *dev = bus_find_device_by_of_node(&spmi_bus_type, np); + + if (dev) + return to_spmi_device(dev); + return NULL; +} +EXPORT_SYMBOL_GPL(spmi_device_from_of); + /** * spmi_controller_alloc() - Allocate a new SPMI device * @ctrl: associated controller diff --git a/include/linux/spmi.h b/include/linux/spmi.h index 729bcbf9f5ad..eac1956a8727 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -164,6 +164,9 @@ static inline void spmi_driver_unregister(struct spmi_driver *sdrv) module_driver(__spmi_driver, spmi_driver_register, \ spmi_driver_unregister) +struct device_node; + +struct spmi_device *spmi_device_from_of(struct device_node *np); int spmi_register_read(struct spmi_device *sdev, u8 addr, u8 *buf); int spmi_ext_register_read(struct spmi_device *sdev, u8 addr, u8 *buf, size_t len); -- cgit v1.2.3 From 4a08069461ac9822855116d24fbf11d5fb223cd1 Mon Sep 17 00:00:00 2001 From: Dmitry Rokosov Date: Tue, 7 Jun 2022 18:39:18 +0000 Subject: iio: trigger: warn about non-registered iio trigger getting attempt As a part of patch series about wrong trigger register() and get() calls order in the some IIO drivers trigger initialization path: https://lore.kernel.org/all/20220524181150.9240-1-ddrokosov@sberdevices.ru/ runtime WARN_ONCE() is added to alarm IIO driver authors who make such a mistake. When an IIO driver allocates a new IIO trigger, it should register it before calling the get() operation. In other words, each IIO driver must abide by IIO trigger alloc()/register()/get() calls order. Signed-off-by: Dmitry Rokosov Link: https://lore.kernel.org/r/20220607183907.20017-1-ddrokosov@sberdevices.ru Signed-off-by: Jonathan Cameron --- drivers/iio/industrialio-trigger.c | 2 ++ include/linux/iio/trigger.h | 5 +++++ 2 files changed, 7 insertions(+) (limited to 'include/linux') diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index d89bf90650d5..26d610d4cbb8 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -576,6 +576,8 @@ struct iio_trigger *viio_trigger_alloc(struct device *parent, if (trig->name == NULL) goto free_descs; + INIT_LIST_HEAD(&trig->list); + trig->subirq_chip.name = trig->name; trig->subirq_chip.irq_mask = &iio_trig_subirqmask; trig->subirq_chip.irq_unmask = &iio_trig_subirqunmask; diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h index 4c69b144677b..03b1d6863436 100644 --- a/include/linux/iio/trigger.h +++ b/include/linux/iio/trigger.h @@ -93,6 +93,11 @@ static inline void iio_trigger_put(struct iio_trigger *trig) static inline struct iio_trigger *iio_trigger_get(struct iio_trigger *trig) { get_device(&trig->dev); + + WARN_ONCE(list_empty(&trig->list), + "Getting non-registered iio trigger %s is prohibited\n", + trig->name); + __module_get(trig->owner); return trig; -- cgit v1.2.3 From bdb6cfe7512f7a214815a3092f0be50963dcacbc Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Sat, 18 Jun 2022 11:28:32 +0100 Subject: net: mii: add mii_bmcr_encode_fixed() Add a function to encode a fixed speed/duplex to a BMCR value. Signed-off-by: Russell King (Oracle) Signed-off-by: David S. Miller --- include/linux/mii.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mii.h b/include/linux/mii.h index 5ee13083cec7..d5a959ce4877 100644 --- a/include/linux/mii.h +++ b/include/linux/mii.h @@ -545,4 +545,39 @@ static inline u8 mii_resolve_flowctrl_fdx(u16 lcladv, u16 rmtadv) return cap; } +/** + * mii_bmcr_encode_fixed - encode fixed speed/duplex settings to a BMCR value + * @speed: a SPEED_* value + * @duplex: a DUPLEX_* value + * + * Encode the speed and duplex to a BMCR value. 2500, 1000, 100 and 10 Mbps are + * supported. 2500Mbps is encoded to 1000Mbps. Other speeds are encoded as 10 + * Mbps. Unknown duplex values are encoded to half-duplex. + */ +static inline u16 mii_bmcr_encode_fixed(int speed, int duplex) +{ + u16 bmcr; + + switch (speed) { + case SPEED_2500: + case SPEED_1000: + bmcr = BMCR_SPEED1000; + break; + + case SPEED_100: + bmcr = BMCR_SPEED100; + break; + + case SPEED_10: + default: + bmcr = BMCR_SPEED10; + break; + } + + if (duplex == DUPLEX_FULL) + bmcr |= BMCR_FULLDPLX; + + return bmcr; +} + #endif /* __LINUX_MII_H__ */ -- cgit v1.2.3 From 2e0aee8f0a22c60a1ae0876f7233e70ad9d026b8 Mon Sep 17 00:00:00 2001 From: Sergey Shtylyov Date: Thu, 16 Jun 2022 23:51:49 +0300 Subject: ata: make ata_port::fastdrain_cnt *unsigned int* *unsigned long* ata_port::fastdrain_cnt (64-bit value in a 64-bit kernel) is always assigned from the 32-bit *unsigned int* variables, thus could also be made just *unsigned int*... Signed-off-by: Sergey Shtylyov Signed-off-by: Damien Le Moal --- include/linux/libata.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/libata.h b/include/linux/libata.h index a8bc88b4fe07..0269ff114f5a 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -847,7 +847,7 @@ struct ata_port { enum ata_lpm_policy target_lpm_policy; struct timer_list fastdrain_timer; - unsigned long fastdrain_cnt; + unsigned int fastdrain_cnt; async_cookie_t cookie; -- cgit v1.2.3 From 8e1c69149f27189cff93a0cfe9402e576d89ce29 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 29 Apr 2022 01:04:10 +0000 Subject: KVM: Avoid pfn_to_page() and vice versa when releasing pages Invert the order of KVM's page/pfn release helpers so that the "inner" helper operates on a page instead of a pfn. As pointed out by Linus[*], converting between struct page and a pfn isn't necessarily cheap, and that's not even counting the overhead of is_error_noslot_pfn() and kvm_is_reserved_pfn(). Even if the checks were dirt cheap, there's no reason to convert from a page to a pfn and back to a page, just to mark the page dirty/accessed or to put a reference to the page. Opportunistically drop a stale declaration of kvm_set_page_accessed() from kvm_host.h (there was no implementation). No functional change intended. [*] https://lore.kernel.org/all/CAHk-=wifQimj2d6npq-wCi5onYPjzQg4vyO4tFcPJJZr268cRw@mail.gmail.com Signed-off-by: Sean Christopherson Message-Id: <20220429010416.2788472-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 1 - virt/kvm/kvm_main.c | 64 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c20f2d55840c..af4b5c0bf04e 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1139,7 +1139,6 @@ unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn, bool *writable); void kvm_release_page_clean(struct page *page); void kvm_release_page_dirty(struct page *page); -void kvm_set_page_accessed(struct page *page); kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn); kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 351cbd121cf5..4732a99935f9 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2820,18 +2820,40 @@ struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) } EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); +static bool kvm_is_ad_tracked_page(struct page *page) +{ + /* + * Per page-flags.h, pages tagged PG_reserved "should in general not be + * touched (e.g. set dirty) except by its owner". + */ + return !PageReserved(page); +} + +static void kvm_set_page_dirty(struct page *page) +{ + if (kvm_is_ad_tracked_page(page)) + SetPageDirty(page); +} + +static void kvm_set_page_accessed(struct page *page) +{ + if (kvm_is_ad_tracked_page(page)) + mark_page_accessed(page); +} + void kvm_release_page_clean(struct page *page) { WARN_ON(is_error_page(page)); - kvm_release_pfn_clean(page_to_pfn(page)); + kvm_set_page_accessed(page); + put_page(page); } EXPORT_SYMBOL_GPL(kvm_release_page_clean); void kvm_release_pfn_clean(kvm_pfn_t pfn) { if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) - put_page(pfn_to_page(pfn)); + kvm_release_page_clean(pfn_to_page(pfn)); } EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); @@ -2839,40 +2861,40 @@ void kvm_release_page_dirty(struct page *page) { WARN_ON(is_error_page(page)); - kvm_release_pfn_dirty(page_to_pfn(page)); + kvm_set_page_dirty(page); + kvm_release_page_clean(page); } EXPORT_SYMBOL_GPL(kvm_release_page_dirty); void kvm_release_pfn_dirty(kvm_pfn_t pfn) { - kvm_set_pfn_dirty(pfn); - kvm_release_pfn_clean(pfn); + if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) + kvm_release_page_dirty(pfn_to_page(pfn)); } EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); -static bool kvm_is_ad_tracked_pfn(kvm_pfn_t pfn) -{ - if (!pfn_valid(pfn)) - return false; - - /* - * Per page-flags.h, pages tagged PG_reserved "should in general not be - * touched (e.g. set dirty) except by its owner". - */ - return !PageReserved(pfn_to_page(pfn)); -} - +/* + * Note, checking for an error/noslot pfn is the caller's responsibility when + * directly marking a page dirty/accessed. Unlike the "release" helpers, the + * "set" helpers are not to be used when the pfn might point at garbage. + */ void kvm_set_pfn_dirty(kvm_pfn_t pfn) { - if (kvm_is_ad_tracked_pfn(pfn)) - SetPageDirty(pfn_to_page(pfn)); + if (WARN_ON(is_error_noslot_pfn(pfn))) + return; + + if (pfn_valid(pfn)) + kvm_set_page_dirty(pfn_to_page(pfn)); } EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); void kvm_set_pfn_accessed(kvm_pfn_t pfn) { - if (kvm_is_ad_tracked_pfn(pfn)) - mark_page_accessed(pfn_to_page(pfn)); + if (WARN_ON(is_error_noslot_pfn(pfn))) + return; + + if (pfn_valid(pfn)) + kvm_set_page_accessed(pfn_to_page(pfn)); } EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); -- cgit v1.2.3 From b1624f99aa8fedafcabf1b92fa51ed88dde14acb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 29 Apr 2022 01:04:13 +0000 Subject: KVM: Remove kvm_vcpu_gfn_to_page() and kvm_vcpu_gpa_to_page() Drop helpers to convert a gfn/gpa to a 'struct page' in the context of a vCPU. KVM doesn't require that guests be backed by 'struct page' memory, thus any use of helpers that assume 'struct page' is bound to be flawed, as was the case for the recently removed last user in x86's nested VMX. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20220429010416.2788472-8-seanjc@google.com> Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 7 ------- virt/kvm/kvm_main.c | 31 +++++++++++-------------------- 2 files changed, 11 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index af4b5c0bf04e..4334789409c0 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1230,7 +1230,6 @@ struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn); kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn); int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map); -struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn); void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty); unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); @@ -1718,12 +1717,6 @@ static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn) return (hpa_t)pfn << PAGE_SHIFT; } -static inline struct page *kvm_vcpu_gpa_to_page(struct kvm_vcpu *vcpu, - gpa_t gpa) -{ - return kvm_vcpu_gfn_to_page(vcpu, gpa_to_gfn(gpa)); -} - static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa) { unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 78d280e119b3..7cd0e3d67f9f 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2720,8 +2720,18 @@ int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, } EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); -static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) +/* + * Do not use this helper unless you are absolutely certain the gfn _must_ be + * backed by 'struct page'. A valid example is if the backing memslot is + * controlled by KVM. Note, if the returned page is valid, it's refcount has + * been elevated by gfn_to_pfn(). + */ +struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) { + kvm_pfn_t pfn; + + pfn = gfn_to_pfn(kvm, gfn); + if (is_error_noslot_pfn(pfn)) return KVM_ERR_PTR_BAD_PAGE; @@ -2730,15 +2740,6 @@ static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) return pfn_to_page(pfn); } - -struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) -{ - kvm_pfn_t pfn; - - pfn = gfn_to_pfn(kvm, gfn); - - return kvm_pfn_to_page(pfn); -} EXPORT_SYMBOL_GPL(gfn_to_page); void kvm_release_pfn(kvm_pfn_t pfn, bool dirty) @@ -2808,16 +2809,6 @@ void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty) } EXPORT_SYMBOL_GPL(kvm_vcpu_unmap); -struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) -{ - kvm_pfn_t pfn; - - pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); - - return kvm_pfn_to_page(pfn); -} -EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); - static bool kvm_is_ad_tracked_page(struct page *page) { /* -- cgit v1.2.3 From 284dc49307738d2a897fd375431f741213cd0f27 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 29 Apr 2022 01:04:14 +0000 Subject: KVM: Take a 'struct page', not a pfn in kvm_is_zone_device_page() Operate on a 'struct page' instead of a pfn when checking if a page is a ZONE_DEVICE page, and rename the helper accordingly. Generally speaking, KVM doesn't actually care about ZONE_DEVICE memory, i.e. shouldn't do anything special for ZONE_DEVICE memory. Rather, KVM wants to treat ZONE_DEVICE memory like regular memory, and the need to identify ZONE_DEVICE memory only arises as an exception to PG_reserved pages. In other words, KVM should only ever check for ZONE_DEVICE memory after KVM has already verified that there is a struct page associated with the pfn. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20220429010416.2788472-9-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 5 +++-- include/linux/kvm_host.h | 2 +- virt/kvm/kvm_main.c | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 2dcedf04ef85..b209aaf096df 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2788,15 +2788,16 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, const struct kvm_memory_slot *slot) { + struct page *page = pfn_to_page(pfn); + int level = PG_LEVEL_4K; unsigned long hva; unsigned long flags; - int level = PG_LEVEL_4K; pgd_t pgd; p4d_t p4d; pud_t pud; pmd_t pmd; - if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn)) + if (!PageCompound(page) && !kvm_is_zone_device_page(page)) return PG_LEVEL_4K; /* diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 4334789409c0..6fb433ac2ba1 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1571,7 +1571,7 @@ void kvm_arch_sync_events(struct kvm *kvm); int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); bool kvm_is_reserved_pfn(kvm_pfn_t pfn); -bool kvm_is_zone_device_pfn(kvm_pfn_t pfn); +bool kvm_is_zone_device_page(struct page *page); struct kvm_irq_ack_notifier { struct hlist_node link; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 7cd0e3d67f9f..aa1bcd5f140d 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -168,7 +168,7 @@ __weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) { } -bool kvm_is_zone_device_pfn(kvm_pfn_t pfn) +bool kvm_is_zone_device_page(struct page *page) { /* * The metadata used by is_zone_device_page() to determine whether or @@ -176,10 +176,10 @@ bool kvm_is_zone_device_pfn(kvm_pfn_t pfn) * the device has been pinned, e.g. by get_user_pages(). WARN if the * page_count() is zero to help detect bad usage of this helper. */ - if (!pfn_valid(pfn) || WARN_ON_ONCE(!page_count(pfn_to_page(pfn)))) + if (WARN_ON_ONCE(!page_count(page))) return false; - return is_zone_device_page(pfn_to_page(pfn)); + return is_zone_device_page(page); } bool kvm_is_reserved_pfn(kvm_pfn_t pfn) @@ -192,7 +192,7 @@ bool kvm_is_reserved_pfn(kvm_pfn_t pfn) if (pfn_valid(pfn)) return PageReserved(pfn_to_page(pfn)) && !is_zero_pfn(pfn) && - !kvm_is_zone_device_pfn(pfn); + !kvm_is_zone_device_page(pfn_to_page(pfn)); return true; } -- cgit v1.2.3 From b14b2690c50e02145bb867dfcde8845eb17aa8a4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 29 Apr 2022 01:04:15 +0000 Subject: KVM: Rename/refactor kvm_is_reserved_pfn() to kvm_pfn_to_refcounted_page() Rename and refactor kvm_is_reserved_pfn() to kvm_pfn_to_refcounted_page() to better reflect what KVM is actually checking, and to eliminate extra pfn_to_page() lookups. The kvm_release_pfn_*() an kvm_try_get_pfn() helpers in particular benefit from "refouncted" nomenclature, as it's not all that obvious why KVM needs to get/put refcounts for some PG_reserved pages (ZERO_PAGE and ZONE_DEVICE). Add a comment to call out that the list of exceptions to PG_reserved is all but guaranteed to be incomplete. The list has mostly been compiled by people throwing noodles at KVM and finding out they stick a little too well, e.g. the ZERO_PAGE's refcount overflowed and ZONE_DEVICE pages didn't get freed. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20220429010416.2788472-10-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 15 ++++++----- arch/x86/kvm/mmu/tdp_mmu.c | 2 +- include/linux/kvm_host.h | 2 +- virt/kvm/kvm_main.c | 66 ++++++++++++++++++++++++++++++++++++---------- 4 files changed, 63 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b209aaf096df..7ebb97c95da7 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -534,6 +534,7 @@ static int mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep) kvm_pfn_t pfn; u64 old_spte = *sptep; int level = sptep_to_sp(sptep)->role.level; + struct page *page; if (!is_shadow_present_pte(old_spte) || !spte_has_volatile_bits(old_spte)) @@ -549,11 +550,13 @@ static int mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep) pfn = spte_to_pfn(old_spte); /* - * KVM does not hold the refcount of the page used by - * kvm mmu, before reclaiming the page, we should - * unmap it from mmu first. + * KVM doesn't hold a reference to any pages mapped into the guest, and + * instead uses the mmu_notifier to ensure that KVM unmaps any pages + * before they are reclaimed. Sanity check that, if the pfn is backed + * by a refcounted page, the refcount is elevated. */ - WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn))); + page = kvm_pfn_to_refcounted_page(pfn); + WARN_ON(page && !page_count(page)); if (is_accessed_spte(old_spte)) kvm_set_pfn_accessed(pfn); @@ -2881,7 +2884,7 @@ void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault if (unlikely(fault->max_level == PG_LEVEL_4K)) return; - if (is_error_noslot_pfn(fault->pfn) || kvm_is_reserved_pfn(fault->pfn)) + if (is_error_noslot_pfn(fault->pfn) || !kvm_pfn_to_refcounted_page(fault->pfn)) return; if (kvm_slot_dirty_track_enabled(slot)) @@ -5993,7 +5996,7 @@ restart: * the guest, and the guest page table is using 4K page size * mapping if the indirect sp has level = 1. */ - if (sp->role.direct && !kvm_is_reserved_pfn(pfn) && + if (sp->role.direct && kvm_pfn_to_refcounted_page(pfn) && sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn, pfn, PG_LEVEL_NUM)) { pte_list_remove(kvm, rmap_head, sptep); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 8e8d412fb65c..1e777b739ac4 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1751,7 +1751,7 @@ static void zap_collapsible_spte_range(struct kvm *kvm, */ pfn = spte_to_pfn(iter.old_spte); - if (kvm_is_reserved_pfn(pfn)) + if (!kvm_pfn_to_refcounted_page(pfn)) continue; max_mapping_level = kvm_mmu_max_mapping_level(kvm, slot, diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 6fb433ac2ba1..a2bbdf3ab086 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1570,7 +1570,7 @@ void kvm_arch_sync_events(struct kvm *kvm); int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); -bool kvm_is_reserved_pfn(kvm_pfn_t pfn); +struct page *kvm_pfn_to_refcounted_page(kvm_pfn_t pfn); bool kvm_is_zone_device_page(struct page *page); struct kvm_irq_ack_notifier { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index aa1bcd5f140d..04196096f9e4 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -182,19 +182,36 @@ bool kvm_is_zone_device_page(struct page *page) return is_zone_device_page(page); } -bool kvm_is_reserved_pfn(kvm_pfn_t pfn) +/* + * Returns a 'struct page' if the pfn is "valid" and backed by a refcounted + * page, NULL otherwise. Note, the list of refcounted PG_reserved page types + * is likely incomplete, it has been compiled purely through people wanting to + * back guest with a certain type of memory and encountering issues. + */ +struct page *kvm_pfn_to_refcounted_page(kvm_pfn_t pfn) { + struct page *page; + + if (!pfn_valid(pfn)) + return NULL; + + page = pfn_to_page(pfn); + if (!PageReserved(page)) + return page; + + /* The ZERO_PAGE(s) is marked PG_reserved, but is refcounted. */ + if (is_zero_pfn(pfn)) + return page; + /* * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting * perspective they are "normal" pages, albeit with slightly different * usage rules. */ - if (pfn_valid(pfn)) - return PageReserved(pfn_to_page(pfn)) && - !is_zero_pfn(pfn) && - !kvm_is_zone_device_page(pfn_to_page(pfn)); + if (kvm_is_zone_device_page(page)) + return page; - return true; + return NULL; } /* @@ -2501,9 +2518,12 @@ static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) static int kvm_try_get_pfn(kvm_pfn_t pfn) { - if (kvm_is_reserved_pfn(pfn)) + struct page *page = kvm_pfn_to_refcounted_page(pfn); + + if (!page) return 1; - return get_page_unless_zero(pfn_to_page(pfn)); + + return get_page_unless_zero(page); } static int hva_to_pfn_remapped(struct vm_area_struct *vma, @@ -2728,6 +2748,7 @@ EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); */ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) { + struct page *page; kvm_pfn_t pfn; pfn = gfn_to_pfn(kvm, gfn); @@ -2735,10 +2756,11 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) if (is_error_noslot_pfn(pfn)) return KVM_ERR_PTR_BAD_PAGE; - if (kvm_is_reserved_pfn(pfn)) + page = kvm_pfn_to_refcounted_page(pfn); + if (!page) return KVM_ERR_PTR_BAD_PAGE; - return pfn_to_page(pfn); + return page; } EXPORT_SYMBOL_GPL(gfn_to_page); @@ -2841,8 +2863,16 @@ EXPORT_SYMBOL_GPL(kvm_release_page_clean); void kvm_release_pfn_clean(kvm_pfn_t pfn) { - if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) - kvm_release_page_clean(pfn_to_page(pfn)); + struct page *page; + + if (is_error_noslot_pfn(pfn)) + return; + + page = kvm_pfn_to_refcounted_page(pfn); + if (!page) + return; + + kvm_release_page_clean(page); } EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); @@ -2857,8 +2887,16 @@ EXPORT_SYMBOL_GPL(kvm_release_page_dirty); void kvm_release_pfn_dirty(kvm_pfn_t pfn) { - if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) - kvm_release_page_dirty(pfn_to_page(pfn)); + struct page *page; + + if (is_error_noslot_pfn(pfn)) + return; + + page = kvm_pfn_to_refcounted_page(pfn); + if (!page) + return; + + kvm_release_page_dirty(page); } EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); -- cgit v1.2.3 From 7b0a0e3c3a88260b6fcb017e49f198463aa62ed1 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Thu, 14 Apr 2022 16:50:57 +0200 Subject: wifi: cfg80211: do some rework towards MLO link APIs In order to support multi-link operation with multiple links, start adding some APIs. The notable addition here is to have the link ID in a new nl80211 attribute, that will be used to differentiate the links in many nl80211 operations. So far, this patch adds the netlink NL80211_ATTR_MLO_LINK_ID attribute (as well as the NL80211_ATTR_MLO_LINKS attribute) and plugs it through the system in some places, checking the validity etc. along with other infrastructure needed for it. For now, I've decided to include only the over-the-air link ID in the API. I know we discussed that we eventually need to have to have other ways of identifying a link, but for local AP mode and auth/assoc commands as well as set_key etc. we'll use the OTA ID. Also included in this patch is some refactoring of the data structures in struct wireless_dev, splitting for the first time the data into type dependent pieces, to make reasoning about these things easier. Signed-off-by: Johannes Berg --- drivers/net/wireless/ath/ath6kl/cfg80211.c | 6 +- drivers/net/wireless/ath/wil6210/cfg80211.c | 9 +- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 4 +- drivers/net/wireless/marvell/libertas/mesh.c | 10 +- drivers/net/wireless/marvell/mwifiex/11h.c | 2 +- drivers/net/wireless/marvell/mwifiex/cfg80211.c | 18 +- drivers/net/wireless/microchip/wilc1000/cfg80211.c | 3 +- drivers/net/wireless/quantenna/qtnfmac/cfg80211.c | 14 +- drivers/net/wireless/quantenna/qtnfmac/commands.c | 2 +- drivers/net/wireless/quantenna/qtnfmac/event.c | 15 +- drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c | 4 +- include/linux/ieee80211.h | 3 + include/net/cfg80211.h | 99 +++- include/uapi/linux/nl80211.h | 28 + net/mac80211/cfg.c | 8 +- net/mac80211/mlme.c | 2 +- net/wireless/ap.c | 46 +- net/wireless/chan.c | 196 +++++-- net/wireless/core.c | 28 +- net/wireless/core.h | 13 +- net/wireless/ibss.c | 57 +- net/wireless/mesh.c | 31 +- net/wireless/mlme.c | 74 +-- net/wireless/nl80211.c | 623 +++++++++++++++------ net/wireless/ocb.c | 5 +- net/wireless/rdev-ops.h | 32 +- net/wireless/reg.c | 139 +++-- net/wireless/scan.c | 8 +- net/wireless/sme.c | 102 ++-- net/wireless/trace.h | 86 ++- net/wireless/util.c | 44 +- net/wireless/wext-compat.c | 48 +- net/wireless/wext-sme.c | 29 +- 33 files changed, 1255 insertions(+), 533 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c index bd1183830e91..33ed54738d47 100644 --- a/drivers/net/wireless/ath/ath6kl/cfg80211.c +++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c @@ -1119,7 +1119,7 @@ void ath6kl_cfg80211_ch_switch_notify(struct ath6kl_vif *vif, int freq, NL80211_CHAN_HT20 : NL80211_CHAN_NO_HT); mutex_lock(&vif->wdev.mtx); - cfg80211_ch_switch_notify(vif->ndev, &chandef); + cfg80211_ch_switch_notify(vif->ndev, &chandef, 0); mutex_unlock(&vif->wdev.mtx); } @@ -2967,7 +2967,8 @@ static int ath6kl_change_beacon(struct wiphy *wiphy, struct net_device *dev, return ath6kl_set_ies(vif, beacon); } -static int ath6kl_stop_ap(struct wiphy *wiphy, struct net_device *dev) +static int ath6kl_stop_ap(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id) { struct ath6kl *ar = ath6kl_priv(dev); struct ath6kl_vif *vif = netdev_priv(dev); @@ -3368,6 +3369,7 @@ static int ath6kl_cfg80211_sscan_stop(struct wiphy *wiphy, static int ath6kl_cfg80211_set_bitrate(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id, const u8 *addr, const struct cfg80211_bitrate_mask *mask) { diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c index 8f2638f5b87b..f93bdffa4d1d 100644 --- a/drivers/net/wireless/ath/wil6210/cfg80211.c +++ b/drivers/net/wireless/ath/wil6210/cfg80211.c @@ -2098,8 +2098,8 @@ static int wil_cfg80211_change_beacon(struct wiphy *wiphy, bcon->tail_len)) privacy = 1; - memcpy(vif->ssid, wdev->ssid, wdev->ssid_len); - vif->ssid_len = wdev->ssid_len; + memcpy(vif->ssid, wdev->u.ap.ssid, wdev->u.ap.ssid_len); + vif->ssid_len = wdev->u.ap.ssid_len; /* in case privacy has changed, need to restart the AP */ if (vif->privacy != privacy) { @@ -2108,7 +2108,7 @@ static int wil_cfg80211_change_beacon(struct wiphy *wiphy, rc = _wil_cfg80211_start_ap(wiphy, ndev, vif->ssid, vif->ssid_len, privacy, - wdev->beacon_interval, + wdev->links[0].ap.beacon_interval, vif->channel, vif->wmi_edmg_channel, bcon, vif->hidden_ssid, @@ -2186,7 +2186,8 @@ static int wil_cfg80211_start_ap(struct wiphy *wiphy, } static int wil_cfg80211_stop_ap(struct wiphy *wiphy, - struct net_device *ndev) + struct net_device *ndev, + unsigned int link_id) { struct wil6210_priv *wil = wiphy_to_wil(wiphy); struct wil6210_vif *vif = ndev_to_vif(ndev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 605206abe424..11e1f07f83e0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4965,7 +4965,8 @@ exit: return err; } -static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) +static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev, + unsigned int link_id) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); @@ -5302,6 +5303,7 @@ exit: static int brcmf_cfg80211_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); diff --git a/drivers/net/wireless/marvell/libertas/mesh.c b/drivers/net/wireless/marvell/libertas/mesh.c index a58c1e141f2c..90ffe8d1e0e8 100644 --- a/drivers/net/wireless/marvell/libertas/mesh.c +++ b/drivers/net/wireless/marvell/libertas/mesh.c @@ -109,9 +109,9 @@ static int lbs_mesh_config(struct lbs_private *priv, uint16_t action, if (priv->mesh_dev) { mesh_wdev = priv->mesh_dev->ieee80211_ptr; - ie->val.mesh_id_len = mesh_wdev->mesh_id_up_len; - memcpy(ie->val.mesh_id, mesh_wdev->ssid, - mesh_wdev->mesh_id_up_len); + ie->val.mesh_id_len = mesh_wdev->u.mesh.id_up_len; + memcpy(ie->val.mesh_id, mesh_wdev->u.mesh.id, + mesh_wdev->u.mesh.id_up_len); } ie->len = sizeof(struct mrvl_meshie_val) - @@ -986,8 +986,8 @@ static int lbs_add_mesh(struct lbs_private *priv) mesh_wdev->wiphy = priv->wdev->wiphy; if (priv->mesh_tlv) { - sprintf(mesh_wdev->ssid, "mesh"); - mesh_wdev->mesh_id_up_len = 4; + sprintf(mesh_wdev->u.mesh.id, "mesh"); + mesh_wdev->u.mesh.id_up_len = 4; } mesh_wdev->netdev = mesh_dev; diff --git a/drivers/net/wireless/marvell/mwifiex/11h.c b/drivers/net/wireless/marvell/mwifiex/11h.c index 3fa25cd64cda..4ca8d0135708 100644 --- a/drivers/net/wireless/marvell/mwifiex/11h.c +++ b/drivers/net/wireless/marvell/mwifiex/11h.c @@ -304,6 +304,6 @@ void mwifiex_dfs_chan_sw_work_queue(struct work_struct *work) mwifiex_dbg(priv->adapter, MSG, "indicating channel switch completion to kernel\n"); mutex_lock(&priv->wdev.mtx); - cfg80211_ch_switch_notify(priv->netdev, &priv->dfs_chandef); + cfg80211_ch_switch_notify(priv->netdev, &priv->dfs_chandef, 0); mutex_unlock(&priv->wdev.mtx); } diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c index 6f23ec34e2e2..d68c40e0e122 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -1753,10 +1753,12 @@ mwifiex_mgmt_stypes[NUM_NL80211_IFTYPES] = { * Function configures data rates to firmware using bitrate mask * provided by cfg80211. */ -static int mwifiex_cfg80211_set_bitrate_mask(struct wiphy *wiphy, - struct net_device *dev, - const u8 *peer, - const struct cfg80211_bitrate_mask *mask) +static int +mwifiex_cfg80211_set_bitrate_mask(struct wiphy *wiphy, + struct net_device *dev, + unsigned int link_id, + const u8 *peer, + const struct cfg80211_bitrate_mask *mask) { struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev); u16 bitmap_rates[MAX_BITMAP_RATES_SIZE]; @@ -1998,7 +2000,8 @@ mwifiex_cfg80211_get_antenna(struct wiphy *wiphy, u32 *tx_ant, u32 *rx_ant) /* cfg80211 operation handler for stop ap. * Function stops BSS running at uAP interface. */ -static int mwifiex_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *dev) +static int mwifiex_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id) { struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev); @@ -2421,7 +2424,7 @@ mwifiex_cfg80211_connect(struct wiphy *wiphy, struct net_device *dev, return -EINVAL; } - if (priv->wdev.current_bss) { + if (priv->wdev.connected) { mwifiex_dbg(adapter, ERROR, "%s: already connected\n", dev->name); return -EALREADY; @@ -2649,7 +2652,7 @@ mwifiex_cfg80211_scan(struct wiphy *wiphy, return -EBUSY; } - if (!priv->wdev.current_bss && priv->scan_block) + if (!priv->wdev.connected && priv->scan_block) priv->scan_block = false; if (!mwifiex_stop_bg_scan(priv)) @@ -4025,6 +4028,7 @@ mwifiex_cfg80211_channel_switch(struct wiphy *wiphy, struct net_device *dev, static int mwifiex_cfg80211_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) { struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev); diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c index 1ac4684fab25..5c2c7f1dbffd 100644 --- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c +++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c @@ -1426,7 +1426,8 @@ static int change_beacon(struct wiphy *wiphy, struct net_device *dev, return wilc_add_beacon(vif, 0, 0, beacon); } -static int stop_ap(struct wiphy *wiphy, struct net_device *dev) +static int stop_ap(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id) { int ret; struct wilc_vif *vif = netdev_priv(dev); diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c index 84b15a655eab..1593e810b3ca 100644 --- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c +++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c @@ -352,7 +352,8 @@ static int qtnf_start_ap(struct wiphy *wiphy, struct net_device *dev, return ret; } -static int qtnf_stop_ap(struct wiphy *wiphy, struct net_device *dev) +static int qtnf_stop_ap(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id) { struct qtnf_vif *vif = qtnf_netdev_get_priv(dev); int ret; @@ -500,7 +501,7 @@ qtnf_dump_station(struct wiphy *wiphy, struct net_device *dev, switch (vif->wdev.iftype) { case NL80211_IFTYPE_STATION: - if (idx != 0 || !vif->wdev.current_bss) + if (idx != 0 || !vif->wdev.connected) return -ENOENT; ether_addr_copy(mac, vif->bssid); @@ -729,7 +730,7 @@ qtnf_disconnect(struct wiphy *wiphy, struct net_device *dev, pr_err("VIF%u.%u: failed to disconnect\n", mac->macid, vif->vifid); - if (vif->wdev.current_bss) { + if (vif->wdev.connected) { netif_carrier_off(vif->netdev); cfg80211_disconnected(vif->netdev, reason_code, NULL, 0, true, GFP_KERNEL); @@ -745,10 +746,11 @@ qtnf_dump_survey(struct wiphy *wiphy, struct net_device *dev, struct qtnf_wmac *mac = wiphy_priv(wiphy); struct wireless_dev *wdev = dev->ieee80211_ptr; struct ieee80211_supported_band *sband; - const struct cfg80211_chan_def *chandef = &wdev->chandef; + const struct cfg80211_chan_def *chandef = wdev_chandef(wdev, 0); struct ieee80211_channel *chan; int ret; + sband = wiphy->bands[NL80211_BAND_2GHZ]; if (sband && idx >= sband->n_channels) { idx -= sband->n_channels; @@ -765,7 +767,7 @@ qtnf_dump_survey(struct wiphy *wiphy, struct net_device *dev, survey->channel = chan; survey->filled = 0x0; - if (chan == chandef->chan) + if (chandef && chan == chandef->chan) survey->filled = SURVEY_INFO_IN_USE; ret = qtnf_cmd_get_chan_stats(mac, chan->center_freq, survey); @@ -778,7 +780,7 @@ qtnf_dump_survey(struct wiphy *wiphy, struct net_device *dev, static int qtnf_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, - struct cfg80211_chan_def *chandef) + unsigned int link_id, struct cfg80211_chan_def *chandef) { struct net_device *ndev = wdev->netdev; struct qtnf_vif *vif; diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c index c68563c83098..3d734a7a5ba8 100644 --- a/drivers/net/wireless/quantenna/qtnfmac/commands.c +++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c @@ -2005,7 +2005,7 @@ int qtnf_cmd_send_scan(struct qtnf_wmac *mac) dwell_active = scan_req->duration; dwell_passive = scan_req->duration; } else if (wdev->iftype == NL80211_IFTYPE_STATION && - wdev->current_bss) { + wdev->connected) { /* let device select dwell based on traffic conditions */ dwell_active = QTNF_SCAN_TIME_AUTO; dwell_passive = QTNF_SCAN_TIME_AUTO; diff --git a/drivers/net/wireless/quantenna/qtnfmac/event.c b/drivers/net/wireless/quantenna/qtnfmac/event.c index 8dc80574d08d..4fafe370101a 100644 --- a/drivers/net/wireless/quantenna/qtnfmac/event.c +++ b/drivers/net/wireless/quantenna/qtnfmac/event.c @@ -189,7 +189,7 @@ qtnf_event_handle_bss_join(struct qtnf_vif *vif, vif->mac->macid, vif->vifid, join_info->bssid, chandef.chan->hw_value); - if (!vif->wdev.ssid_len) { + if (!vif->wdev.u.client.ssid_len) { pr_warn("VIF%u.%u: SSID unknown for BSS:%pM\n", vif->mac->macid, vif->vifid, join_info->bssid); @@ -197,7 +197,7 @@ qtnf_event_handle_bss_join(struct qtnf_vif *vif, goto done; } - ie = kzalloc(2 + vif->wdev.ssid_len, GFP_KERNEL); + ie = kzalloc(2 + vif->wdev.u.client.ssid_len, GFP_KERNEL); if (!ie) { pr_warn("VIF%u.%u: IE alloc failed for BSS:%pM\n", vif->mac->macid, vif->vifid, @@ -207,14 +207,15 @@ qtnf_event_handle_bss_join(struct qtnf_vif *vif, } ie[0] = WLAN_EID_SSID; - ie[1] = vif->wdev.ssid_len; - memcpy(ie + 2, vif->wdev.ssid, vif->wdev.ssid_len); + ie[1] = vif->wdev.u.client.ssid_len; + memcpy(ie + 2, vif->wdev.u.client.ssid, + vif->wdev.u.client.ssid_len); bss = cfg80211_inform_bss(wiphy, chandef.chan, CFG80211_BSS_FTYPE_UNKNOWN, join_info->bssid, 0, WLAN_CAPABILITY_ESS, 100, - ie, 2 + vif->wdev.ssid_len, + ie, 2 + vif->wdev.u.client.ssid_len, 0, GFP_KERNEL); if (!bss) { pr_warn("VIF%u.%u: can't connect to unknown BSS: %pM\n", @@ -470,14 +471,14 @@ qtnf_event_handle_freq_change(struct qtnf_wmac *mac, continue; if (vif->wdev.iftype == NL80211_IFTYPE_STATION && - !vif->wdev.current_bss) + !vif->wdev.connected) continue; if (!vif->netdev) continue; mutex_lock(&vif->wdev.mtx); - cfg80211_ch_switch_notify(vif->netdev, &chandef); + cfg80211_ch_switch_notify(vif->netdev, &chandef, 0); mutex_unlock(&vif->wdev.mtx); } diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c index 43b5604c0bca..349aa3c4b668 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c @@ -2086,6 +2086,7 @@ static u8 rtw_get_chan_type(struct adapter *adapter) } static int cfg80211_rtw_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) { struct adapter *adapter = wiphy_to_adapter(wiphy); @@ -2446,7 +2447,8 @@ static int cfg80211_rtw_change_beacon(struct wiphy *wiphy, struct net_device *nd return rtw_add_beacon(adapter, info->head, info->head_len, info->tail, info->tail_len); } -static int cfg80211_rtw_stop_ap(struct wiphy *wiphy, struct net_device *ndev) +static int cfg80211_rtw_stop_ap(struct wiphy *wiphy, struct net_device *ndev, + unsigned int link_id) { return 0; } diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 5c65ae6b8154..e15771965916 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4376,4 +4376,7 @@ enum ieee80211_range_params_max_total_ltf { IEEE80211_RANGE_PARAMS_MAX_TOTAL_LTF_UNSPECIFIED, }; +/* multi-link device */ +#define IEEE80211_MLD_MAX_NUM_LINKS 15 + #endif /* LINUX_IEEE80211_H */ diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 6d02e12e4702..772e099fc932 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -1158,6 +1158,7 @@ struct cfg80211_mbssid_elems { /** * struct cfg80211_beacon_data - beacon data + * @link_id: the link ID for the AP MLD link sending this beacon * @head: head portion of beacon (before TIM IE) * or %NULL if not changed * @tail: tail portion of beacon (after TIM IE) @@ -1188,6 +1189,8 @@ struct cfg80211_mbssid_elems { * attribute is present in beacon data or not. */ struct cfg80211_beacon_data { + unsigned int link_id; + const u8 *head, *tail; const u8 *beacon_ies; const u8 *proberesp_ies; @@ -4201,7 +4204,8 @@ struct cfg80211_ops { struct cfg80211_ap_settings *settings); int (*change_beacon)(struct wiphy *wiphy, struct net_device *dev, struct cfg80211_beacon_data *info); - int (*stop_ap)(struct wiphy *wiphy, struct net_device *dev); + int (*stop_ap)(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id); int (*add_station)(struct wiphy *wiphy, struct net_device *dev, @@ -4309,6 +4313,7 @@ struct cfg80211_ops { int (*set_bitrate_mask)(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id, const u8 *peer, const struct cfg80211_bitrate_mask *mask); @@ -4384,6 +4389,7 @@ struct cfg80211_ops { int (*get_channel)(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef); int (*start_p2p_device)(struct wiphy *wiphy, @@ -4420,6 +4426,7 @@ struct cfg80211_ops { struct cfg80211_qos_map *qos_map); int (*set_ap_chanwidth)(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id, struct cfg80211_chan_def *chandef); int (*add_tx_ts)(struct wiphy *wiphy, struct net_device *dev, @@ -4545,10 +4552,14 @@ struct cfg80211_ops { * @WIPHY_FLAG_HAS_STATIC_WEP: The device supports static WEP key installation * before connection. * @WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK: The device supports bigger kek and kck keys + * @WIPHY_FLAG_SUPPORTS_MLO: This is a temporary flag gating the MLO APIs, + * in order to not have them reachable in normal drivers, until we have + * complete feature/interface combinations/etc. advertisement. No driver + * should set this flag for now. */ enum wiphy_flags { WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK = BIT(0), - /* use hole at 1 */ + WIPHY_FLAG_SUPPORTS_MLO = BIT(1), WIPHY_FLAG_SPLIT_SCAN_6GHZ = BIT(2), WIPHY_FLAG_NETNS_OK = BIT(3), WIPHY_FLAG_PS_ON_BY_DEFAULT = BIT(4), @@ -5505,6 +5516,8 @@ static inline void wiphy_unlock(struct wiphy *wiphy) * @netdev: (private) Used to reference back to the netdev, may be %NULL * @identifier: (private) Identifier used in nl80211 to identify this * wireless device if it has no netdev + * @connected_addr: (private) BSSID or AP MLD address if connected + * @connected: indicates if connected or not (STA mode) * @current_bss: (private) Used by the internal configuration code * @chandef: (private) Used by the internal configuration code to track * the user-set channel definition. @@ -5585,8 +5598,6 @@ struct wireless_dev { u8 address[ETH_ALEN] __aligned(sizeof(u16)); /* currently used for IBSS and SME - might be rearranged later */ - u8 ssid[IEEE80211_MAX_SSID_LEN]; - u8 ssid_len, mesh_id_len, mesh_id_up_len; struct cfg80211_conn *conn; struct cfg80211_cached_keys *connect_keys; enum ieee80211_bss_type conn_bss_type; @@ -5598,20 +5609,17 @@ struct wireless_dev { struct list_head event_list; spinlock_t event_lock; - struct cfg80211_internal_bss *current_bss; /* associated / joined */ - struct cfg80211_chan_def preset_chandef; - struct cfg80211_chan_def chandef; + u8 connected:1; bool ps; int ps_timeout; - int beacon_interval; - u32 ap_unexpected_nlportid; u32 owner_nlportid; bool nl_owner_dead; + /* FIXME: need to rework radar detection for MLO */ bool cac_started; unsigned long cac_start_time; unsigned int cac_time_ms; @@ -5639,6 +5647,50 @@ struct wireless_dev { struct work_struct pmsr_free_wk; unsigned long unprot_beacon_reported; + + union { + struct { + u8 connected_addr[ETH_ALEN] __aligned(2); + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + } client; + struct { + int beacon_interval; + struct cfg80211_chan_def preset_chandef; + struct cfg80211_chan_def chandef; + u8 id[IEEE80211_MAX_SSID_LEN]; + u8 id_len, id_up_len; + } mesh; + struct { + struct cfg80211_chan_def preset_chandef; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + } ap; + struct { + struct cfg80211_internal_bss *current_bss; + struct cfg80211_chan_def chandef; + int beacon_interval; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + } ibss; + struct { + struct cfg80211_chan_def chandef; + } ocb; + } u; + + struct { + u8 addr[ETH_ALEN] __aligned(2); + union { + struct { + unsigned int beacon_interval; + struct cfg80211_chan_def chandef; + } ap; + struct { + struct cfg80211_internal_bss *current_bss; + } client; + }; + } links[IEEE80211_MLD_MAX_NUM_LINKS]; + u16 valid_links; }; static inline const u8 *wdev_address(struct wireless_dev *wdev) @@ -5667,6 +5719,31 @@ static inline void *wdev_priv(struct wireless_dev *wdev) return wiphy_priv(wdev->wiphy); } +/** + * wdev_chandef - return chandef pointer from wireless_dev + * @wdev: the wdev + * @link_id: the link ID for MLO + * + * Return: The chandef depending on the mode, or %NULL. + */ +struct cfg80211_chan_def *wdev_chandef(struct wireless_dev *wdev, + unsigned int link_id); + +static inline void WARN_INVALID_LINK_ID(struct wireless_dev *wdev, + unsigned int link_id) +{ + WARN_ON(link_id && !wdev->valid_links); + WARN_ON(wdev->valid_links && + !(wdev->valid_links & BIT(link_id))); +} + +#define for_each_valid_link(wdev, link_id) \ + for (link_id = 0; \ + link_id < ((wdev)->valid_links ? ARRAY_SIZE((wdev)->links) : 1); \ + link_id++) \ + if (!(wdev)->valid_links || \ + ((wdev)->valid_links & BIT(link_id))) + /** * DOC: Utility functions * @@ -7882,12 +7959,14 @@ bool cfg80211_reg_can_beacon_relax(struct wiphy *wiphy, * cfg80211_ch_switch_notify - update wdev channel and notify userspace * @dev: the device which switched channels * @chandef: the new channel definition + * @link_id: the link ID for MLO, must be 0 for non-MLO * * Caller must acquire wdev_lock, therefore must only be called from sleepable * driver context! */ void cfg80211_ch_switch_notify(struct net_device *dev, - struct cfg80211_chan_def *chandef); + struct cfg80211_chan_def *chandef, + unsigned int link_id); /* * cfg80211_ch_switch_started_notify - notify channel switch start diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 98f905f16411..a9a2c9fef295 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -323,6 +323,17 @@ * Once the association is done, the driver cleans the FILS AAD data. */ +/** + * DOC: Multi-Link Operation + * + * In Multi-Link Operation, a connection between to MLDs utilizes multiple + * links. To use this in nl80211, various commands and responses now need + * to or will include the new %NL80211_ATTR_MLO_LINKS attribute. + * Additionally, various commands that need to operate on a specific link + * now need to be given the %NL80211_ATTR_MLO_LINK_ID attribute, e.g. to + * use %NL80211_CMD_START_AP or similar functions. + */ + /** * enum nl80211_commands - supported nl80211 commands * @@ -1237,6 +1248,12 @@ * to describe the BSSID address of the AP and %NL80211_ATTR_TIMEOUT to * specify the timeout value. * + * @NL80211_CMD_ADD_LINK: Add a new link to an interface. The + * %NL80211_ATTR_MLO_LINK_ID attribute is used for the new link. + * @NL80211_CMD_REMOVE_LINK: Remove a link from an interface. This may come + * without %NL80211_ATTR_MLO_LINK_ID as an easy way to remove all links + * in preparation for e.g. roaming to a regular (non-MLO) AP. + * * @NL80211_CMD_MAX: highest used command number * @__NL80211_CMD_AFTER_LAST: internal use */ @@ -1481,6 +1498,9 @@ enum nl80211_commands { NL80211_CMD_ASSOC_COMEBACK, + NL80211_CMD_ADD_LINK, + NL80211_CMD_REMOVE_LINK, + /* add new commands above here */ /* used to define NL80211_CMD_MAX below */ @@ -2663,6 +2683,11 @@ enum nl80211_commands { * association request when used with NL80211_CMD_NEW_STATION). Can be set * only if %NL80211_STA_FLAG_WME is set. * + * @NL80211_ATTR_MLO_LINK_ID: A (u8) link ID for use with MLO, to be used with + * various commands that need a link ID to operate. + * @NL80211_ATTR_MLO_LINKS: A nested array of links, each containing some + * per-link information and a link ID. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -3177,6 +3202,9 @@ enum nl80211_attrs { NL80211_ATTR_DISABLE_EHT, + NL80211_ATTR_MLO_LINKS, + NL80211_ATTR_MLO_LINK_ID, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index 881efbfb96f6..362cac9e2135 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1342,7 +1342,8 @@ static void ieee80211_free_next_beacon(struct ieee80211_sub_if_data *sdata) sdata->u.ap.next_beacon = NULL; } -static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev) +static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); struct ieee80211_sub_if_data *vlan; @@ -3049,6 +3050,7 @@ static int ieee80211_set_cqm_rssi_range_config(struct wiphy *wiphy, static int ieee80211_set_bitrate_mask(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id, const u8 *addr, const struct cfg80211_bitrate_mask *mask) { @@ -3390,7 +3392,7 @@ static int __ieee80211_csa_finalize(struct ieee80211_sub_if_data *sdata) if (err) return err; - cfg80211_ch_switch_notify(sdata->dev, &sdata->csa_chandef); + cfg80211_ch_switch_notify(sdata->dev, &sdata->csa_chandef, 0); return 0; } @@ -3898,6 +3900,7 @@ unlock: static int ieee80211_cfg_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) { struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev); @@ -3958,6 +3961,7 @@ static int ieee80211_set_qos_map(struct wiphy *wiphy, static int ieee80211_set_ap_chanwidth(struct wiphy *wiphy, struct net_device *dev, + unsigned int link_id, struct cfg80211_chan_def *chandef) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 6d5ad71ef02c..e0a9b7d63071 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -1314,7 +1314,7 @@ static void ieee80211_chswitch_post_beacon(struct ieee80211_sub_if_data *sdata) return; } - cfg80211_ch_switch_notify(sdata->dev, &sdata->reserved_chandef); + cfg80211_ch_switch_notify(sdata->dev, &sdata->reserved_chandef, 0); } void ieee80211_chswitch_done(struct ieee80211_vif *vif, bool success) diff --git a/net/wireless/ap.c b/net/wireless/ap.c index 550ac9d827fe..e68923200018 100644 --- a/net/wireless/ap.c +++ b/net/wireless/ap.c @@ -1,4 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 +/* + * Parts of this file are + * Copyright (C) 2022 Intel Corporation + */ #include #include #include @@ -7,8 +11,9 @@ #include "rdev-ops.h" -int __cfg80211_stop_ap(struct cfg80211_registered_device *rdev, - struct net_device *dev, bool notify) +static int ___cfg80211_stop_ap(struct cfg80211_registered_device *rdev, + struct net_device *dev, unsigned int link_id, + bool notify) { struct wireless_dev *wdev = dev->ieee80211_ptr; int err; @@ -22,15 +27,16 @@ int __cfg80211_stop_ap(struct cfg80211_registered_device *rdev, dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO) return -EOPNOTSUPP; - if (!wdev->beacon_interval) + if (!wdev->links[link_id].ap.beacon_interval) return -ENOENT; - err = rdev_stop_ap(rdev, dev); + err = rdev_stop_ap(rdev, dev, link_id); if (!err) { wdev->conn_owner_nlportid = 0; - wdev->beacon_interval = 0; - memset(&wdev->chandef, 0, sizeof(wdev->chandef)); - wdev->ssid_len = 0; + wdev->links[link_id].ap.beacon_interval = 0; + memset(&wdev->links[link_id].ap.chandef, 0, + sizeof(wdev->links[link_id].ap.chandef)); + wdev->u.ap.ssid_len = 0; rdev_set_qos_map(rdev, dev, NULL); if (notify) nl80211_send_ap_stopped(wdev); @@ -46,14 +52,36 @@ int __cfg80211_stop_ap(struct cfg80211_registered_device *rdev, return err; } +int __cfg80211_stop_ap(struct cfg80211_registered_device *rdev, + struct net_device *dev, int link_id, + bool notify) +{ + unsigned int link; + int ret = 0; + + if (link_id >= 0) + return ___cfg80211_stop_ap(rdev, dev, link_id, notify); + + for_each_valid_link(dev->ieee80211_ptr, link) { + int ret1 = ___cfg80211_stop_ap(rdev, dev, link, notify); + + if (ret1) + ret = ret1; + /* try the next one also if one errored */ + } + + return ret; +} + int cfg80211_stop_ap(struct cfg80211_registered_device *rdev, - struct net_device *dev, bool notify) + struct net_device *dev, int link_id, + bool notify) { struct wireless_dev *wdev = dev->ieee80211_ptr; int err; wdev_lock(wdev); - err = __cfg80211_stop_ap(rdev, dev, notify); + err = __cfg80211_stop_ap(rdev, dev, link_id, notify); wdev_unlock(wdev); return err; diff --git a/net/wireless/chan.c b/net/wireless/chan.c index f74f176e0d9d..efc2de4bab57 100644 --- a/net/wireless/chan.c +++ b/net/wireless/chan.c @@ -672,14 +672,21 @@ bool cfg80211_chandef_dfs_usable(struct wiphy *wiphy, * range of chandef. */ bool cfg80211_is_sub_chan(struct cfg80211_chan_def *chandef, - struct ieee80211_channel *chan) + struct ieee80211_channel *chan, + bool primary_only) { int width; u32 freq; + if (!chandef->chan) + return false; + if (chandef->chan->center_freq == chan->center_freq) return true; + if (primary_only) + return false; + width = cfg80211_chandef_get_width(chandef); if (width <= 20) return false; @@ -704,23 +711,25 @@ bool cfg80211_is_sub_chan(struct cfg80211_chan_def *chandef, bool cfg80211_beaconing_iface_active(struct wireless_dev *wdev) { - bool active = false; + unsigned int link; ASSERT_WDEV_LOCK(wdev); - if (!wdev->chandef.chan) - return false; - switch (wdev->iftype) { case NL80211_IFTYPE_AP: case NL80211_IFTYPE_P2P_GO: - active = wdev->beacon_interval != 0; + for_each_valid_link(wdev, link) { + if (wdev->links[link].ap.beacon_interval) + return true; + } break; case NL80211_IFTYPE_ADHOC: - active = wdev->ssid_len != 0; + if (wdev->u.ibss.ssid_len) + return true; break; case NL80211_IFTYPE_MESH_POINT: - active = wdev->mesh_id_len != 0; + if (wdev->u.mesh.id_len) + return true; break; case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_OCB: @@ -737,7 +746,35 @@ bool cfg80211_beaconing_iface_active(struct wireless_dev *wdev) WARN_ON(1); } - return active; + return false; +} + +bool cfg80211_wdev_on_sub_chan(struct wireless_dev *wdev, + struct ieee80211_channel *chan, + bool primary_only) +{ + unsigned int link; + + switch (wdev->iftype) { + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + for_each_valid_link(wdev, link) { + if (cfg80211_is_sub_chan(&wdev->links[link].ap.chandef, + chan, primary_only)) + return true; + } + break; + case NL80211_IFTYPE_ADHOC: + return cfg80211_is_sub_chan(&wdev->u.ibss.chandef, chan, + primary_only); + case NL80211_IFTYPE_MESH_POINT: + return cfg80211_is_sub_chan(&wdev->u.mesh.chandef, chan, + primary_only); + default: + break; + } + + return false; } static bool cfg80211_is_wiphy_oper_chan(struct wiphy *wiphy, @@ -752,7 +789,7 @@ static bool cfg80211_is_wiphy_oper_chan(struct wiphy *wiphy, continue; } - if (cfg80211_is_sub_chan(&wdev->chandef, chan)) { + if (cfg80211_wdev_on_sub_chan(wdev, chan, false)) { wdev_unlock(wdev); return true; } @@ -772,7 +809,8 @@ cfg80211_offchan_chain_is_active(struct cfg80211_registered_device *rdev, if (!cfg80211_chandef_valid(&rdev->background_radar_chandef)) return false; - return cfg80211_is_sub_chan(&rdev->background_radar_chandef, channel); + return cfg80211_is_sub_chan(&rdev->background_radar_chandef, channel, + false); } bool cfg80211_any_wiphy_oper_chan(struct wiphy *wiphy, @@ -1176,6 +1214,68 @@ bool cfg80211_chandef_usable(struct wiphy *wiphy, } EXPORT_SYMBOL(cfg80211_chandef_usable); +static bool cfg80211_ir_permissive_check_wdev(enum nl80211_iftype iftype, + struct wireless_dev *wdev, + struct ieee80211_channel *chan) +{ + struct ieee80211_channel *other_chan = NULL; + unsigned int link_id; + int r1, r2; + + for_each_valid_link(wdev, link_id) { + if (wdev->iftype == NL80211_IFTYPE_STATION && + wdev->links[link_id].client.current_bss) + other_chan = wdev->links[link_id].client.current_bss->pub.channel; + + /* + * If a GO already operates on the same GO_CONCURRENT channel, + * this one (maybe the same one) can beacon as well. We allow + * the operation even if the station we relied on with + * GO_CONCURRENT is disconnected now. But then we must make sure + * we're not outdoor on an indoor-only channel. + */ + if (iftype == NL80211_IFTYPE_P2P_GO && + wdev->iftype == NL80211_IFTYPE_P2P_GO && + wdev->links[link_id].ap.beacon_interval && + !(chan->flags & IEEE80211_CHAN_INDOOR_ONLY)) + other_chan = wdev->links[link_id].ap.chandef.chan; + + if (!other_chan) + continue; + + if (chan == other_chan) + return true; + + if (chan->band != NL80211_BAND_5GHZ && + chan->band != NL80211_BAND_6GHZ) + continue; + + r1 = cfg80211_get_unii(chan->center_freq); + r2 = cfg80211_get_unii(other_chan->center_freq); + + if (r1 != -EINVAL && r1 == r2) { + /* + * At some locations channels 149-165 are considered a + * bundle, but at other locations, e.g., Indonesia, + * channels 149-161 are considered a bundle while + * channel 165 is left out and considered to be in a + * different bundle. Thus, in case that there is a + * station interface connected to an AP on channel 165, + * it is assumed that channels 149-161 are allowed for + * GO operations. However, having a station interface + * connected to an AP on channels 149-161, does not + * allow GO operation on channel 165. + */ + if (chan->center_freq == 5825 && + other_chan->center_freq != 5825) + continue; + return true; + } + } + + return false; +} + /* * Check if the channel can be used under permissive conditions mandated by * some regulatory bodies, i.e., the channel is marked with @@ -1219,59 +1319,14 @@ static bool cfg80211_ir_permissive_chan(struct wiphy *wiphy, * the current registered device. */ list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { - struct ieee80211_channel *other_chan = NULL; - int r1, r2; + bool ret; wdev_lock(wdev); - if (wdev->iftype == NL80211_IFTYPE_STATION && - wdev->current_bss) - other_chan = wdev->current_bss->pub.channel; - - /* - * If a GO already operates on the same GO_CONCURRENT channel, - * this one (maybe the same one) can beacon as well. We allow - * the operation even if the station we relied on with - * GO_CONCURRENT is disconnected now. But then we must make sure - * we're not outdoor on an indoor-only channel. - */ - if (iftype == NL80211_IFTYPE_P2P_GO && - wdev->iftype == NL80211_IFTYPE_P2P_GO && - wdev->beacon_interval && - !(chan->flags & IEEE80211_CHAN_INDOOR_ONLY)) - other_chan = wdev->chandef.chan; + ret = cfg80211_ir_permissive_check_wdev(iftype, wdev, chan); wdev_unlock(wdev); - if (!other_chan) - continue; - - if (chan == other_chan) - return true; - - if (chan->band != NL80211_BAND_5GHZ && - chan->band != NL80211_BAND_6GHZ) - continue; - - r1 = cfg80211_get_unii(chan->center_freq); - r2 = cfg80211_get_unii(other_chan->center_freq); - - if (r1 != -EINVAL && r1 == r2) { - /* - * At some locations channels 149-165 are considered a - * bundle, but at other locations, e.g., Indonesia, - * channels 149-161 are considered a bundle while - * channel 165 is left out and considered to be in a - * different bundle. Thus, in case that there is a - * station interface connected to an AP on channel 165, - * it is assumed that channels 149-161 are allowed for - * GO operations. However, having a station interface - * connected to an AP on channels 149-161, does not - * allow GO operation on channel 165. - */ - if (chan->center_freq == 5825 && - other_chan->center_freq != 5825) - continue; - return true; - } + if (ret) + return ret; } return false; @@ -1374,3 +1429,24 @@ bool cfg80211_any_usable_channels(struct wiphy *wiphy, return false; } EXPORT_SYMBOL(cfg80211_any_usable_channels); + +struct cfg80211_chan_def *wdev_chandef(struct wireless_dev *wdev, + unsigned int link_id) +{ + ASSERT_WDEV_LOCK(wdev); + + switch (wdev->iftype) { + case NL80211_IFTYPE_MESH_POINT: + return &wdev->u.mesh.chandef; + case NL80211_IFTYPE_ADHOC: + return &wdev->u.ibss.chandef; + case NL80211_IFTYPE_OCB: + return &wdev->u.ocb.chandef; + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + return &wdev->links[link_id].ap.chandef; + default: + return NULL; + } +} +EXPORT_SYMBOL(wdev_chandef); diff --git a/net/wireless/core.c b/net/wireless/core.c index f08d4b3bb148..3e5d12040726 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -1118,6 +1118,7 @@ static void _cfg80211_unregister_wdev(struct wireless_dev *wdev, bool unregister_netdev) { struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy); + unsigned int link_id; ASSERT_RTNL(); lockdep_assert_held(&rdev->wiphy.mtx); @@ -1167,11 +1168,22 @@ static void _cfg80211_unregister_wdev(struct wireless_dev *wdev, */ cfg80211_process_wdev_events(wdev); - if (WARN_ON(wdev->current_bss)) { - cfg80211_unhold_bss(wdev->current_bss); - cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub); - wdev->current_bss = NULL; + if (wdev->iftype == NL80211_IFTYPE_STATION || + wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) { + for (link_id = 0; link_id < ARRAY_SIZE(wdev->links); link_id++) { + struct cfg80211_internal_bss *curbss; + + curbss = wdev->links[link_id].client.current_bss; + + if (WARN_ON(curbss)) { + cfg80211_unhold_bss(curbss); + cfg80211_put_bss(wdev->wiphy, &curbss->pub); + wdev->links[link_id].client.current_bss = NULL; + } + } } + + wdev->connected = false; } void cfg80211_unregister_wdev(struct wireless_dev *wdev) @@ -1233,7 +1245,7 @@ void __cfg80211_leave(struct cfg80211_registered_device *rdev, break; case NL80211_IFTYPE_AP: case NL80211_IFTYPE_P2P_GO: - __cfg80211_stop_ap(rdev, dev, true); + __cfg80211_stop_ap(rdev, dev, -1, true); break; case NL80211_IFTYPE_OCB: __cfg80211_leave_ocb(rdev, dev); @@ -1463,9 +1475,9 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb, memcpy(&setup, &default_mesh_setup, sizeof(setup)); /* back compat only needed for mesh_id */ - setup.mesh_id = wdev->ssid; - setup.mesh_id_len = wdev->mesh_id_up_len; - if (wdev->mesh_id_up_len) + setup.mesh_id = wdev->u.mesh.id; + setup.mesh_id_len = wdev->u.mesh.id_up_len; + if (wdev->u.mesh.id_up_len) __cfg80211_join_mesh(rdev, dev, &setup, &default_mesh_config); diff --git a/net/wireless/core.h b/net/wireless/core.h index 5436ada91b1a..2c195067ddff 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -307,6 +307,7 @@ void cfg80211_bss_expire(struct cfg80211_registered_device *rdev); void cfg80211_bss_age(struct cfg80211_registered_device *rdev, unsigned long age_secs); void cfg80211_update_assoc_bss_entry(struct wireless_dev *wdev, + unsigned int link, struct ieee80211_channel *channel); /* IBSS */ @@ -353,9 +354,11 @@ int cfg80211_leave_ocb(struct cfg80211_registered_device *rdev, /* AP */ int __cfg80211_stop_ap(struct cfg80211_registered_device *rdev, - struct net_device *dev, bool notify); + struct net_device *dev, int link, + bool notify); int cfg80211_stop_ap(struct cfg80211_registered_device *rdev, - struct net_device *dev, bool notify); + struct net_device *dev, int link, + bool notify); /* MLME */ int cfg80211_mlme_auth(struct cfg80211_registered_device *rdev, @@ -507,7 +510,11 @@ bool cfg80211_any_wiphy_oper_chan(struct wiphy *wiphy, bool cfg80211_beaconing_iface_active(struct wireless_dev *wdev); bool cfg80211_is_sub_chan(struct cfg80211_chan_def *chandef, - struct ieee80211_channel *chan); + struct ieee80211_channel *chan, + bool primary_only); +bool cfg80211_wdev_on_sub_chan(struct wireless_dev *wdev, + struct ieee80211_channel *chan, + bool primary_only); static inline unsigned int elapsed_jiffies_msecs(unsigned long start) { diff --git a/net/wireless/ibss.c b/net/wireless/ibss.c index 5d89eec2869a..4935f94d1acc 100644 --- a/net/wireless/ibss.c +++ b/net/wireless/ibss.c @@ -28,7 +28,7 @@ void __cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC)) return; - if (!wdev->ssid_len) + if (!wdev->u.ibss.ssid_len) return; bss = cfg80211_get_bss(wdev->wiphy, channel, bssid, NULL, 0, @@ -37,13 +37,13 @@ void __cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, if (WARN_ON(!bss)) return; - if (wdev->current_bss) { - cfg80211_unhold_bss(wdev->current_bss); - cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub); + if (wdev->u.ibss.current_bss) { + cfg80211_unhold_bss(wdev->u.ibss.current_bss); + cfg80211_put_bss(wdev->wiphy, &wdev->u.ibss.current_bss->pub); } cfg80211_hold_bss(bss_from_pub(bss)); - wdev->current_bss = bss_from_pub(bss); + wdev->u.ibss.current_bss = bss_from_pub(bss); if (!(wdev->wiphy->flags & WIPHY_FLAG_HAS_STATIC_WEP)) cfg80211_upload_connect_keys(wdev); @@ -96,7 +96,7 @@ int __cfg80211_join_ibss(struct cfg80211_registered_device *rdev, lockdep_assert_held(&rdev->wiphy.mtx); ASSERT_WDEV_LOCK(wdev); - if (wdev->ssid_len) + if (wdev->u.ibss.ssid_len) return -EALREADY; if (!params->basic_rates) { @@ -131,7 +131,7 @@ int __cfg80211_join_ibss(struct cfg80211_registered_device *rdev, kfree_sensitive(wdev->connect_keys); wdev->connect_keys = connkeys; - wdev->chandef = params->chandef; + wdev->u.ibss.chandef = params->chandef; if (connkeys) { params->wep_keys = connkeys->params; params->wep_tx_key = connkeys->def; @@ -146,8 +146,8 @@ int __cfg80211_join_ibss(struct cfg80211_registered_device *rdev, return err; } - memcpy(wdev->ssid, params->ssid, params->ssid_len); - wdev->ssid_len = params->ssid_len; + memcpy(wdev->u.ibss.ssid, params->ssid, params->ssid_len); + wdev->u.ibss.ssid_len = params->ssid_len; return 0; } @@ -173,14 +173,14 @@ static void __cfg80211_clear_ibss(struct net_device *dev, bool nowext) for (i = 0; i < 6; i++) rdev_del_key(rdev, dev, i, false, NULL); - if (wdev->current_bss) { - cfg80211_unhold_bss(wdev->current_bss); - cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub); + if (wdev->u.ibss.current_bss) { + cfg80211_unhold_bss(wdev->u.ibss.current_bss); + cfg80211_put_bss(wdev->wiphy, &wdev->u.ibss.current_bss->pub); } - wdev->current_bss = NULL; - wdev->ssid_len = 0; - memset(&wdev->chandef, 0, sizeof(wdev->chandef)); + wdev->u.ibss.current_bss = NULL; + wdev->u.ibss.ssid_len = 0; + memset(&wdev->u.ibss.chandef, 0, sizeof(wdev->u.ibss.chandef)); #ifdef CONFIG_CFG80211_WEXT if (!nowext) wdev->wext.ibss.ssid_len = 0; @@ -205,7 +205,7 @@ int __cfg80211_leave_ibss(struct cfg80211_registered_device *rdev, ASSERT_WDEV_LOCK(wdev); - if (!wdev->ssid_len) + if (!wdev->u.ibss.ssid_len) return -ENOLINK; err = rdev_leave_ibss(rdev, dev); @@ -339,7 +339,7 @@ int cfg80211_ibss_wext_siwfreq(struct net_device *dev, wdev_lock(wdev); err = 0; - if (wdev->ssid_len) + if (wdev->u.ibss.ssid_len) err = __cfg80211_leave_ibss(rdev, dev, true); wdev_unlock(wdev); @@ -374,8 +374,8 @@ int cfg80211_ibss_wext_giwfreq(struct net_device *dev, return -EINVAL; wdev_lock(wdev); - if (wdev->current_bss) - chan = wdev->current_bss->pub.channel; + if (wdev->u.ibss.current_bss) + chan = wdev->u.ibss.current_bss->pub.channel; else if (wdev->wext.ibss.chandef.chan) chan = wdev->wext.ibss.chandef.chan; wdev_unlock(wdev); @@ -408,7 +408,7 @@ int cfg80211_ibss_wext_siwessid(struct net_device *dev, wdev_lock(wdev); err = 0; - if (wdev->ssid_len) + if (wdev->u.ibss.ssid_len) err = __cfg80211_leave_ibss(rdev, dev, true); wdev_unlock(wdev); @@ -419,8 +419,8 @@ int cfg80211_ibss_wext_siwessid(struct net_device *dev, if (len > 0 && ssid[len - 1] == '\0') len--; - memcpy(wdev->ssid, ssid, len); - wdev->wext.ibss.ssid = wdev->ssid; + memcpy(wdev->u.ibss.ssid, ssid, len); + wdev->wext.ibss.ssid = wdev->u.ibss.ssid; wdev->wext.ibss.ssid_len = len; wdev_lock(wdev); @@ -443,10 +443,10 @@ int cfg80211_ibss_wext_giwessid(struct net_device *dev, data->flags = 0; wdev_lock(wdev); - if (wdev->ssid_len) { + if (wdev->u.ibss.ssid_len) { data->flags = 1; - data->length = wdev->ssid_len; - memcpy(ssid, wdev->ssid, data->length); + data->length = wdev->u.ibss.ssid_len; + memcpy(ssid, wdev->u.ibss.ssid, data->length); } else if (wdev->wext.ibss.ssid && wdev->wext.ibss.ssid_len) { data->flags = 1; data->length = wdev->wext.ibss.ssid_len; @@ -494,7 +494,7 @@ int cfg80211_ibss_wext_siwap(struct net_device *dev, wdev_lock(wdev); err = 0; - if (wdev->ssid_len) + if (wdev->u.ibss.ssid_len) err = __cfg80211_leave_ibss(rdev, dev, true); wdev_unlock(wdev); @@ -527,8 +527,9 @@ int cfg80211_ibss_wext_giwap(struct net_device *dev, ap_addr->sa_family = ARPHRD_ETHER; wdev_lock(wdev); - if (wdev->current_bss) - memcpy(ap_addr->sa_data, wdev->current_bss->pub.bssid, ETH_ALEN); + if (wdev->u.ibss.current_bss) + memcpy(ap_addr->sa_data, wdev->u.ibss.current_bss->pub.bssid, + ETH_ALEN); else if (wdev->wext.ibss.bssid) memcpy(ap_addr->sa_data, wdev->wext.ibss.bssid, ETH_ALEN); else diff --git a/net/wireless/mesh.c b/net/wireless/mesh.c index e4e363138279..59a3c5c092b1 100644 --- a/net/wireless/mesh.c +++ b/net/wireless/mesh.c @@ -1,4 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 +/* + * Portions + * Copyright (C) 2022 Intel Corporation + */ #include #include #include @@ -114,7 +118,7 @@ int __cfg80211_join_mesh(struct cfg80211_registered_device *rdev, setup->is_secure) return -EOPNOTSUPP; - if (wdev->mesh_id_len) + if (wdev->u.mesh.id_len) return -EALREADY; if (!setup->mesh_id_len) @@ -125,7 +129,7 @@ int __cfg80211_join_mesh(struct cfg80211_registered_device *rdev, if (!setup->chandef.chan) { /* if no channel explicitly given, use preset channel */ - setup->chandef = wdev->preset_chandef; + setup->chandef = wdev->u.mesh.preset_chandef; } if (!setup->chandef.chan) { @@ -209,10 +213,10 @@ int __cfg80211_join_mesh(struct cfg80211_registered_device *rdev, err = rdev_join_mesh(rdev, dev, conf, setup); if (!err) { - memcpy(wdev->ssid, setup->mesh_id, setup->mesh_id_len); - wdev->mesh_id_len = setup->mesh_id_len; - wdev->chandef = setup->chandef; - wdev->beacon_interval = setup->beacon_interval; + memcpy(wdev->u.mesh.id, setup->mesh_id, setup->mesh_id_len); + wdev->u.mesh.id_len = setup->mesh_id_len; + wdev->u.mesh.chandef = setup->chandef; + wdev->u.mesh.beacon_interval = setup->beacon_interval; } return err; @@ -241,15 +245,15 @@ int cfg80211_set_mesh_channel(struct cfg80211_registered_device *rdev, err = rdev_libertas_set_mesh_channel(rdev, wdev->netdev, chandef->chan); if (!err) - wdev->chandef = *chandef; + wdev->u.mesh.chandef = *chandef; return err; } - if (wdev->mesh_id_len) + if (wdev->u.mesh.id_len) return -EBUSY; - wdev->preset_chandef = *chandef; + wdev->u.mesh.preset_chandef = *chandef; return 0; } @@ -267,15 +271,16 @@ int __cfg80211_leave_mesh(struct cfg80211_registered_device *rdev, if (!rdev->ops->leave_mesh) return -EOPNOTSUPP; - if (!wdev->mesh_id_len) + if (!wdev->u.mesh.id_len) return -ENOTCONN; err = rdev_leave_mesh(rdev, dev); if (!err) { wdev->conn_owner_nlportid = 0; - wdev->mesh_id_len = 0; - wdev->beacon_interval = 0; - memset(&wdev->chandef, 0, sizeof(wdev->chandef)); + wdev->u.mesh.id_len = 0; + wdev->u.mesh.beacon_interval = 0; + memset(&wdev->u.mesh.chandef, 0, + sizeof(wdev->u.mesh.chandef)); rdev_set_qos_map(rdev, dev, NULL); cfg80211_sched_dfs_chan_update(rdev); } diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index c8155a483ec2..fab2d6206cdd 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -92,8 +92,7 @@ static void cfg80211_process_deauth(struct wireless_dev *wdev, nl80211_send_deauth(rdev, wdev->netdev, buf, len, reconnect, GFP_KERNEL); - if (!wdev->current_bss || - !ether_addr_equal(wdev->current_bss->pub.bssid, bssid)) + if (!wdev->connected || !ether_addr_equal(wdev->u.client.connected_addr, bssid)) return; __cfg80211_disconnected(wdev->netdev, NULL, 0, reason_code, from_ap); @@ -113,8 +112,8 @@ static void cfg80211_process_disassoc(struct wireless_dev *wdev, nl80211_send_disassoc(rdev, wdev->netdev, buf, len, reconnect, GFP_KERNEL); - if (WARN_ON(!wdev->current_bss || - !ether_addr_equal(wdev->current_bss->pub.bssid, bssid))) + if (WARN_ON(!wdev->connected || + !ether_addr_equal(wdev->u.client.connected_addr, bssid))) return; __cfg80211_disconnected(wdev->netdev, NULL, 0, reason_code, from_ap); @@ -260,8 +259,8 @@ int cfg80211_mlme_auth(struct cfg80211_registered_device *rdev, if (!key || !key_len || key_idx < 0 || key_idx > 3) return -EINVAL; - if (wdev->current_bss && - ether_addr_equal(bssid, wdev->current_bss->pub.bssid)) + if (wdev->connected && + ether_addr_equal(bssid, wdev->u.client.connected_addr)) return -EALREADY; req.bss = cfg80211_get_bss(&rdev->wiphy, chan, bssid, ssid, ssid_len, @@ -322,9 +321,9 @@ int cfg80211_mlme_assoc(struct cfg80211_registered_device *rdev, ASSERT_WDEV_LOCK(wdev); - if (wdev->current_bss && - (!req->prev_bssid || !ether_addr_equal(wdev->current_bss->pub.bssid, - req->prev_bssid))) + if (wdev->connected && + (!req->prev_bssid || + !ether_addr_equal(wdev->u.client.connected_addr, req->prev_bssid))) return -EALREADY; cfg80211_oper_and_ht_capa(&req->ht_capa_mask, @@ -364,13 +363,13 @@ int cfg80211_mlme_deauth(struct cfg80211_registered_device *rdev, ASSERT_WDEV_LOCK(wdev); if (local_state_change && - (!wdev->current_bss || - !ether_addr_equal(wdev->current_bss->pub.bssid, bssid))) + (!wdev->connected || + !ether_addr_equal(wdev->u.client.connected_addr, bssid))) return 0; if (ether_addr_equal(wdev->disconnect_bssid, bssid) || - (wdev->current_bss && - ether_addr_equal(wdev->current_bss->pub.bssid, bssid))) + (wdev->connected && + ether_addr_equal(wdev->u.client.connected_addr, bssid))) wdev->conn_owner_nlportid = 0; return rdev_deauth(rdev, dev, &req); @@ -392,11 +391,12 @@ int cfg80211_mlme_disassoc(struct cfg80211_registered_device *rdev, ASSERT_WDEV_LOCK(wdev); - if (!wdev->current_bss) + if (!wdev->connected) return -ENOTCONN; - if (ether_addr_equal(wdev->current_bss->pub.bssid, bssid)) - req.bss = &wdev->current_bss->pub; + if (ether_addr_equal(wdev->links[0].client.current_bss->pub.bssid, + bssid)) + req.bss = &wdev->links[0].client.current_bss->pub; else return -ENOTCONN; @@ -405,7 +405,7 @@ int cfg80211_mlme_disassoc(struct cfg80211_registered_device *rdev, return err; /* driver should have reported the disassoc */ - WARN_ON(wdev->current_bss); + WARN_ON(wdev->connected); return 0; } @@ -420,10 +420,10 @@ void cfg80211_mlme_down(struct cfg80211_registered_device *rdev, if (!rdev->ops->deauth) return; - if (!wdev->current_bss) + if (!wdev->connected) return; - memcpy(bssid, wdev->current_bss->pub.bssid, ETH_ALEN); + memcpy(bssid, wdev->u.client.connected_addr, ETH_ALEN); cfg80211_mlme_deauth(rdev, dev, bssid, NULL, 0, WLAN_REASON_DEAUTH_LEAVING, false); } @@ -676,28 +676,34 @@ int cfg80211_mlme_mgmt_tx(struct cfg80211_registered_device *rdev, switch (wdev->iftype) { case NL80211_IFTYPE_ADHOC: + /* + * check for IBSS DA must be done by driver as + * cfg80211 doesn't track the stations + */ + if (!wdev->u.ibss.current_bss || + !ether_addr_equal(wdev->u.ibss.current_bss->pub.bssid, + mgmt->bssid)) { + err = -ENOTCONN; + break; + } + break; case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: - if (!wdev->current_bss) { + if (!wdev->connected) { err = -ENOTCONN; break; } - if (!ether_addr_equal(wdev->current_bss->pub.bssid, + /* FIXME: MLD may address this differently */ + + if (!ether_addr_equal(wdev->u.client.connected_addr, mgmt->bssid)) { err = -ENOTCONN; break; } - /* - * check for IBSS DA must be done by driver as - * cfg80211 doesn't track the stations - */ - if (wdev->iftype == NL80211_IFTYPE_ADHOC) - break; - /* for station, check that DA is the AP */ - if (!ether_addr_equal(wdev->current_bss->pub.bssid, + if (!ether_addr_equal(wdev->u.client.connected_addr, mgmt->da)) { err = -ENOTCONN; break; @@ -743,12 +749,12 @@ int cfg80211_mlme_mgmt_tx(struct cfg80211_registered_device *rdev, if (!ieee80211_is_action(mgmt->frame_control) || mgmt->u.action.category != WLAN_CATEGORY_PUBLIC) return -EINVAL; - if (!wdev->current_bss && + if (!wdev->connected && !wiphy_ext_feature_isset( &rdev->wiphy, NL80211_EXT_FEATURE_MGMT_TX_RANDOM_TA)) return -EINVAL; - if (wdev->current_bss && + if (wdev->connected && !wiphy_ext_feature_isset( &rdev->wiphy, NL80211_EXT_FEATURE_MGMT_TX_RANDOM_TA_CONNECTED)) @@ -940,12 +946,16 @@ void cfg80211_cac_event(struct net_device *netdev, struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); unsigned long timeout; + /* not yet supported */ + if (wdev->valid_links) + return; + trace_cfg80211_cac_event(netdev, event); if (WARN_ON(!wdev->cac_started && event != NL80211_RADAR_CAC_STARTED)) return; - if (WARN_ON(!wdev->chandef.chan)) + if (WARN_ON(!wdev->links[0].ap.chandef.chan)) return; switch (event) { diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 740b29481bc6..af31978fc9cc 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -792,6 +792,10 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { NL80211_EHT_MIN_CAPABILITY_LEN, NL80211_EHT_MAX_CAPABILITY_LEN), [NL80211_ATTR_DISABLE_EHT] = { .type = NLA_FLAG }, + [NL80211_ATTR_MLO_LINKS] = + NLA_POLICY_NESTED_ARRAY(nl80211_policy), + [NL80211_ATTR_MLO_LINK_ID] = + NLA_POLICY_RANGE(NLA_U8, 0, IEEE80211_MLD_MAX_NUM_LINKS), }; /* policy for the key attributes */ @@ -1225,6 +1229,37 @@ static bool nl80211_put_txq_stats(struct sk_buff *msg, /* netlink command implementations */ +/** + * nl80211_link_id - return link ID + * @attrs: attributes to look at + * + * Returns: the link ID or 0 if not given + * + * Note this function doesn't do any validation of the link + * ID validity wrt. links that were actually added, so it must + * be called only from ops with %NL80211_FLAG_MLO_VALID_LINK_ID + * or if additional validation is done. + */ +static unsigned int nl80211_link_id(struct nlattr **attrs) +{ + struct nlattr *linkid = attrs[NL80211_ATTR_MLO_LINK_ID]; + + if (!linkid) + return 0; + + return nla_get_u8(linkid); +} + +static int nl80211_link_id_or_invalid(struct nlattr **attrs) +{ + struct nlattr *linkid = attrs[NL80211_ATTR_MLO_LINK_ID]; + + if (!linkid) + return -1; + + return nla_get_u8(linkid); +} + struct key_parse { struct key_params p; int idx; @@ -1496,11 +1531,15 @@ static int nl80211_key_allowed(struct wireless_dev *wdev) case NL80211_IFTYPE_MESH_POINT: break; case NL80211_IFTYPE_ADHOC: + if (wdev->u.ibss.current_bss) + return 0; + return -ENOLINK; case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: - if (!wdev->current_bss) - return -ENOLINK; - break; + /* for MLO, require driver validation of the link ID */ + if (wdev->connected) + return 0; + return -ENOLINK; case NL80211_IFTYPE_UNSPECIFIED: case NL80211_IFTYPE_OCB: case NL80211_IFTYPE_MONITOR: @@ -3232,12 +3271,14 @@ int nl80211_parse_chandef(struct cfg80211_registered_device *rdev, static int __nl80211_set_channel(struct cfg80211_registered_device *rdev, struct net_device *dev, - struct genl_info *info) + struct genl_info *info, + int _link_id) { struct cfg80211_chan_def chandef; int result; enum nl80211_iftype iftype = NL80211_IFTYPE_MONITOR; struct wireless_dev *wdev = NULL; + int link_id = _link_id; if (dev) wdev = dev->ieee80211_ptr; @@ -3246,6 +3287,12 @@ static int __nl80211_set_channel(struct cfg80211_registered_device *rdev, if (wdev) iftype = wdev->iftype; + if (link_id < 0) { + if (wdev && wdev->valid_links) + return -EINVAL; + link_id = 0; + } + result = nl80211_parse_chandef(rdev, info, &chandef); if (result) return result; @@ -3254,49 +3301,48 @@ static int __nl80211_set_channel(struct cfg80211_registered_device *rdev, case NL80211_IFTYPE_AP: case NL80211_IFTYPE_P2P_GO: if (!cfg80211_reg_can_beacon_relax(&rdev->wiphy, &chandef, - iftype)) { - result = -EINVAL; - break; - } - if (wdev->beacon_interval) { + iftype)) + return -EINVAL; + if (wdev->links[link_id].ap.beacon_interval) { + struct ieee80211_channel *cur_chan; + if (!dev || !rdev->ops->set_ap_chanwidth || !(rdev->wiphy.features & - NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE)) { - result = -EBUSY; - break; - } + NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE)) + return -EBUSY; /* Only allow dynamic channel width changes */ - if (chandef.chan != wdev->preset_chandef.chan) { - result = -EBUSY; - break; - } - result = rdev_set_ap_chanwidth(rdev, dev, &chandef); + cur_chan = wdev->links[link_id].ap.chandef.chan; + if (chandef.chan != cur_chan) + return -EBUSY; + + result = rdev_set_ap_chanwidth(rdev, dev, link_id, + &chandef); if (result) - break; + return result; + wdev->links[link_id].ap.chandef = chandef; + } else { + wdev->u.ap.preset_chandef = chandef; } - wdev->preset_chandef = chandef; - result = 0; - break; + return 0; case NL80211_IFTYPE_MESH_POINT: - result = cfg80211_set_mesh_channel(rdev, wdev, &chandef); - break; + return cfg80211_set_mesh_channel(rdev, wdev, &chandef); case NL80211_IFTYPE_MONITOR: - result = cfg80211_set_monitor_channel(rdev, &chandef); - break; + return cfg80211_set_monitor_channel(rdev, &chandef); default: - result = -EINVAL; + break; } - return result; + return -EINVAL; } static int nl80211_set_channel(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; + int link_id = nl80211_link_id_or_invalid(info->attrs); struct net_device *netdev = info->user_ptr[1]; - return __nl80211_set_channel(rdev, netdev, info); + return __nl80211_set_channel(rdev, netdev, info, link_id); } static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info) @@ -3411,7 +3457,7 @@ static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info) result = __nl80211_set_channel( rdev, nl80211_can_set_dev_channel(wdev) ? netdev : NULL, - info); + info, -1); if (result) goto out; } @@ -3696,15 +3742,13 @@ static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flag nla_put_u8(msg, NL80211_ATTR_4ADDR, wdev->use_4addr)) goto nla_put_failure; - if (rdev->ops->get_channel) { - int ret; + if (rdev->ops->get_channel && !wdev->valid_links) { struct cfg80211_chan_def chandef = {}; + int ret; - ret = rdev_get_channel(rdev, wdev, &chandef); - if (ret == 0) { - if (nl80211_send_chandef(msg, &chandef)) - goto nla_put_failure; - } + ret = rdev_get_channel(rdev, wdev, 0, &chandef); + if (ret == 0 && nl80211_send_chandef(msg, &chandef)) + goto nla_put_failure; } if (rdev->ops->get_tx_power) { @@ -3721,27 +3765,24 @@ static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flag switch (wdev->iftype) { case NL80211_IFTYPE_AP: case NL80211_IFTYPE_P2P_GO: - if (wdev->ssid_len && - nla_put(msg, NL80211_ATTR_SSID, wdev->ssid_len, wdev->ssid)) + if (wdev->u.ap.ssid_len && + nla_put(msg, NL80211_ATTR_SSID, wdev->u.ap.ssid_len, + wdev->u.ap.ssid)) goto nla_put_failure_locked; break; case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: - case NL80211_IFTYPE_ADHOC: { - const struct element *ssid_elem; - - if (!wdev->current_bss) - break; - rcu_read_lock(); - ssid_elem = ieee80211_bss_get_elem(&wdev->current_bss->pub, - WLAN_EID_SSID); - if (ssid_elem && - nla_put(msg, NL80211_ATTR_SSID, ssid_elem->datalen, - ssid_elem->data)) - goto nla_put_failure_rcu_locked; - rcu_read_unlock(); + if (wdev->u.client.ssid_len && + nla_put(msg, NL80211_ATTR_SSID, wdev->u.client.ssid_len, + wdev->u.client.ssid)) + goto nla_put_failure_locked; + break; + case NL80211_IFTYPE_ADHOC: + if (wdev->u.ibss.ssid_len && + nla_put(msg, NL80211_ATTR_SSID, wdev->u.ibss.ssid_len, + wdev->u.ibss.ssid)) + goto nla_put_failure_locked; break; - } default: /* nothing */ break; @@ -3761,8 +3802,6 @@ static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flag genlmsg_end(msg, hdr); return 0; - nla_put_failure_rcu_locked: - rcu_read_unlock(); nla_put_failure_locked: wdev_unlock(wdev); nla_put_failure: @@ -4014,10 +4053,11 @@ static int nl80211_set_interface(struct sk_buff *skb, struct genl_info *info) wdev_lock(wdev); BUILD_BUG_ON(IEEE80211_MAX_SSID_LEN != IEEE80211_MAX_MESH_ID_LEN); - wdev->mesh_id_up_len = + wdev->u.mesh.id_up_len = nla_len(info->attrs[NL80211_ATTR_MESH_ID]); - memcpy(wdev->ssid, nla_data(info->attrs[NL80211_ATTR_MESH_ID]), - wdev->mesh_id_up_len); + memcpy(wdev->u.mesh.id, + nla_data(info->attrs[NL80211_ATTR_MESH_ID]), + wdev->u.mesh.id_up_len); wdev_unlock(wdev); } @@ -4122,10 +4162,11 @@ static int _nl80211_new_interface(struct sk_buff *skb, struct genl_info *info) wdev_lock(wdev); BUILD_BUG_ON(IEEE80211_MAX_SSID_LEN != IEEE80211_MAX_MESH_ID_LEN); - wdev->mesh_id_up_len = + wdev->u.mesh.id_up_len = nla_len(info->attrs[NL80211_ATTR_MESH_ID]); - memcpy(wdev->ssid, nla_data(info->attrs[NL80211_ATTR_MESH_ID]), - wdev->mesh_id_up_len); + memcpy(wdev->u.mesh.id, + nla_data(info->attrs[NL80211_ATTR_MESH_ID]), + wdev->u.mesh.id_up_len); wdev_unlock(wdev); break; case NL80211_IFTYPE_NAN: @@ -4662,7 +4703,7 @@ static int nl80211_set_mac_acl(struct sk_buff *skb, struct genl_info *info) dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO) return -EOPNOTSUPP; - if (!dev->ieee80211_ptr->beacon_interval) + if (!dev->ieee80211_ptr->links[0].ap.beacon_interval) return -EINVAL; acl = parse_acl_data(&rdev->wiphy, info); @@ -4818,14 +4859,24 @@ static void he_build_mcs_mask(u16 he_mcs_map, } } -static u16 he_get_txmcsmap(struct genl_info *info, +static u16 he_get_txmcsmap(struct genl_info *info, unsigned int link_id, const struct ieee80211_sta_he_cap *he_cap) { struct net_device *dev = info->user_ptr[1]; struct wireless_dev *wdev = dev->ieee80211_ptr; - __le16 tx_mcs; + struct cfg80211_chan_def *chandef; + __le16 tx_mcs; - switch (wdev->chandef.width) { + chandef = wdev_chandef(wdev, link_id); + if (!chandef) { + /* + * This is probably broken, but we never maintained + * a chandef in these cases, so it always was. + */ + return le16_to_cpu(he_cap->he_mcs_nss_supp.tx_mcs_80); + } + + switch (chandef->width) { case NL80211_CHAN_WIDTH_80P80: tx_mcs = he_cap->he_mcs_nss_supp.tx_mcs_80p80; break; @@ -4836,6 +4887,7 @@ static u16 he_get_txmcsmap(struct genl_info *info, tx_mcs = he_cap->he_mcs_nss_supp.tx_mcs_80; break; } + return le16_to_cpu(tx_mcs); } @@ -4843,7 +4895,8 @@ static bool he_set_mcs_mask(struct genl_info *info, struct wireless_dev *wdev, struct ieee80211_supported_band *sband, struct nl80211_txrate_he *txrate, - u16 mcs[NL80211_HE_NSS_MAX]) + u16 mcs[NL80211_HE_NSS_MAX], + unsigned int link_id) { const struct ieee80211_sta_he_cap *he_cap; u16 tx_mcs_mask[NL80211_HE_NSS_MAX] = {}; @@ -4856,7 +4909,7 @@ static bool he_set_mcs_mask(struct genl_info *info, memset(mcs, 0, sizeof(u16) * NL80211_HE_NSS_MAX); - tx_mcs_map = he_get_txmcsmap(info, he_cap); + tx_mcs_map = he_get_txmcsmap(info, link_id, he_cap); /* Build he_mcs_mask from HE capabilities */ he_build_mcs_mask(tx_mcs_map, tx_mcs_mask); @@ -4876,7 +4929,8 @@ static int nl80211_parse_tx_bitrate_mask(struct genl_info *info, enum nl80211_attrs attr, struct cfg80211_bitrate_mask *mask, struct net_device *dev, - bool default_all_enabled) + bool default_all_enabled, + unsigned int link_id) { struct nlattr *tb[NL80211_TXRATE_MAX + 1]; struct cfg80211_registered_device *rdev = info->user_ptr[0]; @@ -4913,7 +4967,7 @@ static int nl80211_parse_tx_bitrate_mask(struct genl_info *info, if (!he_cap) continue; - he_tx_mcs_map = he_get_txmcsmap(info, he_cap); + he_tx_mcs_map = he_get_txmcsmap(info, link_id, he_cap); he_build_mcs_mask(he_tx_mcs_map, mask->control[i].he_mcs); mask->control[i].he_gi = 0xFF; @@ -4978,7 +5032,8 @@ static int nl80211_parse_tx_bitrate_mask(struct genl_info *info, if (tb[NL80211_TXRATE_HE] && !he_set_mcs_mask(info, wdev, sband, nla_data(tb[NL80211_TXRATE_HE]), - mask->control[band].he_mcs)) + mask->control[band].he_mcs, + link_id)) return -EINVAL; if (tb[NL80211_TXRATE_HE_GI]) @@ -5215,6 +5270,8 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev, memset(bcn, 0, sizeof(*bcn)); + bcn->link_id = nl80211_link_id(attrs); + if (attrs[NL80211_ATTR_BEACON_HEAD]) { bcn->head = nla_data(attrs[NL80211_ATTR_BEACON_HEAD]); bcn->head_len = nla_len(attrs[NL80211_ATTR_BEACON_HEAD]); @@ -5468,22 +5525,20 @@ static bool nl80211_get_ap_channel(struct cfg80211_registered_device *rdev, struct cfg80211_ap_settings *params) { struct wireless_dev *wdev; - bool ret = false; list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { if (wdev->iftype != NL80211_IFTYPE_AP && wdev->iftype != NL80211_IFTYPE_P2P_GO) continue; - if (!wdev->preset_chandef.chan) + if (!wdev->u.ap.preset_chandef.chan) continue; - params->chandef = wdev->preset_chandef; - ret = true; - break; + params->chandef = wdev->u.ap.preset_chandef; + return true; } - return ret; + return false; } static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, @@ -5541,6 +5596,7 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; + unsigned int link_id = nl80211_link_id(info->attrs); struct net_device *dev = info->user_ptr[1]; struct wireless_dev *wdev = dev->ieee80211_ptr; struct cfg80211_ap_settings *params; @@ -5553,7 +5609,7 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) if (!rdev->ops->start_ap) return -EOPNOTSUPP; - if (wdev->beacon_interval) + if (wdev->links[link_id].ap.beacon_interval) return -EALREADY; /* these are required for START_AP */ @@ -5595,6 +5651,18 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) err = -EINVAL; goto out; } + + if (wdev->u.ap.ssid_len && + (wdev->u.ap.ssid_len != params->ssid_len || + memcmp(wdev->u.ap.ssid, params->ssid, params->ssid_len))) { + /* require identical SSID for MLO */ + err = -EINVAL; + goto out; + } + } else if (wdev->valid_links) { + /* require SSID for MLO */ + err = -EINVAL; + goto out; } if (info->attrs[NL80211_ATTR_HIDDEN_SSID]) @@ -5662,8 +5730,12 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) err = nl80211_parse_chandef(rdev, info, ¶ms->chandef); if (err) goto out; - } else if (wdev->preset_chandef.chan) { - params->chandef = wdev->preset_chandef; + } else if (wdev->valid_links) { + /* with MLD need to specify the channel configuration */ + err = -EINVAL; + goto out; + } else if (wdev->u.ap.preset_chandef.chan) { + params->chandef = wdev->u.ap.preset_chandef; } else if (!nl80211_get_ap_channel(rdev, params)) { err = -EINVAL; goto out; @@ -5679,7 +5751,7 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) err = nl80211_parse_tx_bitrate_mask(info, info->attrs, NL80211_ATTR_TX_RATES, ¶ms->beacon_rate, - dev, false); + dev, false, link_id); if (err) goto out; @@ -5779,19 +5851,28 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) params->flags |= NL80211_AP_SETTINGS_EXTERNAL_AUTH_SUPPORT; wdev_lock(wdev); + if (wdev->conn_owner_nlportid && + info->attrs[NL80211_ATTR_SOCKET_OWNER] && + wdev->conn_owner_nlportid != info->snd_portid) { + err = -EINVAL; + goto out_unlock; + } + + /* FIXME: validate MLO/link-id against driver capabilities */ + err = rdev_start_ap(rdev, dev, params); if (!err) { - wdev->preset_chandef = params->chandef; - wdev->beacon_interval = params->beacon_interval; - wdev->chandef = params->chandef; - wdev->ssid_len = params->ssid_len; - memcpy(wdev->ssid, params->ssid, wdev->ssid_len); + wdev->links[link_id].ap.beacon_interval = params->beacon_interval; + wdev->links[link_id].ap.chandef = params->chandef; + wdev->u.ap.ssid_len = params->ssid_len; + memcpy(wdev->u.ap.ssid, params->ssid, + params->ssid_len); if (info->attrs[NL80211_ATTR_SOCKET_OWNER]) wdev->conn_owner_nlportid = info->snd_portid; } +out_unlock: wdev_unlock(wdev); - out: kfree(params->acl); kfree(params->beacon.mbssid_ies); @@ -5807,6 +5888,7 @@ out: static int nl80211_set_beacon(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; + unsigned int link_id = nl80211_link_id(info->attrs); struct net_device *dev = info->user_ptr[1]; struct wireless_dev *wdev = dev->ieee80211_ptr; struct cfg80211_beacon_data params; @@ -5819,7 +5901,7 @@ static int nl80211_set_beacon(struct sk_buff *skb, struct genl_info *info) if (!rdev->ops->change_beacon) return -EOPNOTSUPP; - if (!wdev->beacon_interval) + if (!wdev->links[link_id].ap.beacon_interval) return -EINVAL; err = nl80211_parse_beacon(rdev, info->attrs, ¶ms); @@ -5838,9 +5920,10 @@ out: static int nl80211_stop_ap(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; + unsigned int link_id = nl80211_link_id(info->attrs); struct net_device *dev = info->user_ptr[1]; - return cfg80211_stop_ap(rdev, dev, false); + return cfg80211_stop_ap(rdev, dev, link_id, false); } static const struct nla_policy sta_flags_policy[NL80211_STA_FLAG_MAX + 1] = { @@ -7590,7 +7673,7 @@ static int nl80211_get_mesh_config(struct sk_buff *skb, wdev_lock(wdev); /* If not connected, get default parameters */ - if (!wdev->mesh_id_len) + if (!wdev->u.mesh.id_len) memcpy(&cur_params, &default_mesh_config, sizeof(cur_params)); else err = rdev_get_mesh_config(rdev, dev, &cur_params); @@ -7971,7 +8054,7 @@ static int nl80211_update_mesh_config(struct sk_buff *skb, return err; wdev_lock(wdev); - if (!wdev->mesh_id_len) + if (!wdev->u.mesh.id_len) err = -ENOLINK; if (!err) @@ -8463,14 +8546,44 @@ int nl80211_parse_random_mac(struct nlattr **attrs, return 0; } -static bool cfg80211_off_channel_oper_allowed(struct wireless_dev *wdev) +static bool cfg80211_off_channel_oper_allowed(struct wireless_dev *wdev, + struct ieee80211_channel *chan) { + unsigned int link_id; + bool all_ok = true; + ASSERT_WDEV_LOCK(wdev); if (!cfg80211_beaconing_iface_active(wdev)) return true; - if (!(wdev->chandef.chan->flags & IEEE80211_CHAN_RADAR)) + /* + * FIXME: check if we have a free HW resource/link for chan + * + * This, as well as the FIXME below, requires knowing the link + * capabilities of the hardware. + */ + + /* we cannot leave radar channels */ + for_each_valid_link(wdev, link_id) { + struct cfg80211_chan_def *chandef; + + chandef = wdev_chandef(wdev, link_id); + if (!chandef) + continue; + + /* + * FIXME: don't require all_ok, but rather check only the + * correct HW resource/link onto which 'chan' falls, + * as only that link leaves the channel for doing + * the off-channel operation. + */ + + if (chandef->chan->flags & IEEE80211_CHAN_RADAR) + all_ok = false; + } + + if (all_ok) return true; return regulatory_pre_cac_allowed(wdev->wiphy); @@ -8553,7 +8666,7 @@ nl80211_check_scan_flags(struct wiphy *wiphy, struct wireless_dev *wdev, int err; if (!(wiphy->features & randomness_flag) || - (wdev && wdev->current_bss)) + (wdev && wdev->connected)) return -EOPNOTSUPP; err = nl80211_parse_random_mac(attrs, mac_addr, mac_addr_mask); @@ -8690,17 +8803,14 @@ static int nl80211_trigger_scan(struct sk_buff *skb, struct genl_info *info) request->n_channels = i; wdev_lock(wdev); - if (!cfg80211_off_channel_oper_allowed(wdev)) { - struct ieee80211_channel *chan; + for (i = 0; i < request->n_channels; i++) { + struct ieee80211_channel *chan = request->channels[i]; - if (request->n_channels != 1) { - wdev_unlock(wdev); - err = -EBUSY; - goto out_free; - } + /* if we can go off-channel to the target channel we're good */ + if (cfg80211_off_channel_oper_allowed(wdev, chan)) + continue; - chan = request->channels[0]; - if (chan->center_freq != wdev->chandef.chan->center_freq) { + if (!cfg80211_wdev_on_sub_chan(wdev, chan, true)) { wdev_unlock(wdev); err = -EBUSY; goto out_free; @@ -9445,7 +9555,7 @@ static int nl80211_start_radar_detection(struct sk_buff *skb, err = rdev_start_radar_detection(rdev, dev, &chandef, cac_time_ms); if (!err) { - wdev->chandef = chandef; + wdev->links[0].ap.chandef = chandef; wdev->cac_started = true; wdev->cac_start_time = jiffies; wdev->cac_time_ms = cac_time_ms; @@ -9513,6 +9623,7 @@ static int nl80211_notify_radar_detection(struct sk_buff *skb, static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; + unsigned int link_id = nl80211_link_id(info->attrs); struct net_device *dev = info->user_ptr[1]; struct wireless_dev *wdev = dev->ieee80211_ptr; struct cfg80211_csa_settings params; @@ -9539,15 +9650,15 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info) need_handle_dfs_flag = false; /* useless if AP is not running */ - if (!wdev->beacon_interval) + if (!wdev->links[link_id].ap.beacon_interval) return -ENOTCONN; break; case NL80211_IFTYPE_ADHOC: - if (!wdev->ssid_len) + if (!wdev->u.ibss.ssid_len) return -ENOTCONN; break; case NL80211_IFTYPE_MESH_POINT: - if (!wdev->mesh_id_len) + if (!wdev->u.mesh.id_len) return -ENOTCONN; break; default: @@ -9718,6 +9829,7 @@ static int nl80211_send_bss(struct sk_buff *msg, struct netlink_callback *cb, { struct cfg80211_bss *res = &intbss->pub; const struct cfg80211_bss_ies *ies; + unsigned int link_id; void *hdr; struct nlattr *bss; @@ -9822,13 +9934,15 @@ static int nl80211_send_bss(struct sk_buff *msg, struct netlink_callback *cb, switch (wdev->iftype) { case NL80211_IFTYPE_P2P_CLIENT: case NL80211_IFTYPE_STATION: - if (intbss == wdev->current_bss && - nla_put_u32(msg, NL80211_BSS_STATUS, - NL80211_BSS_STATUS_ASSOCIATED)) - goto nla_put_failure; + for_each_valid_link(wdev, link_id) { + if (intbss == wdev->links[link_id].client.current_bss && + nla_put_u32(msg, NL80211_BSS_STATUS, + NL80211_BSS_STATUS_ASSOCIATED)) + goto nla_put_failure; + } break; case NL80211_IFTYPE_ADHOC: - if (intbss == wdev->current_bss && + if (intbss == wdev->u.ibss.current_bss && nla_put_u32(msg, NL80211_BSS_STATUS, NL80211_BSS_STATUS_IBSS_JOINED)) goto nla_put_failure; @@ -11362,7 +11476,7 @@ static int nl80211_update_connect_params(struct sk_buff *skb, } wdev_lock(dev->ieee80211_ptr); - if (!wdev->current_bss) + if (!wdev->connected) ret = -ENOLINK; else ret = rdev_update_connect_params(rdev, dev, &connect, changed); @@ -11575,9 +11689,9 @@ static int nl80211_remain_on_channel(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; + unsigned int link_id = nl80211_link_id(info->attrs); struct wireless_dev *wdev = info->user_ptr[1]; struct cfg80211_chan_def chandef; - const struct cfg80211_chan_def *compat_chandef; struct sk_buff *msg; void *hdr; u64 cookie; @@ -11607,10 +11721,22 @@ static int nl80211_remain_on_channel(struct sk_buff *skb, return err; wdev_lock(wdev); - if (!cfg80211_off_channel_oper_allowed(wdev) && - !cfg80211_chandef_identical(&wdev->chandef, &chandef)) { - compat_chandef = cfg80211_chandef_compatible(&wdev->chandef, - &chandef); + if (!cfg80211_off_channel_oper_allowed(wdev, chandef.chan)) { + const struct cfg80211_chan_def *oper_chandef, *compat_chandef; + + oper_chandef = wdev_chandef(wdev, link_id); + + if (WARN_ON(!oper_chandef)) { + /* cannot happen since we must beacon to get here */ + WARN_ON(1); + wdev_unlock(wdev); + return -EBUSY; + } + + /* note: returns first one if identical chandefs */ + compat_chandef = cfg80211_chandef_compatible(&chandef, + oper_chandef); + if (compat_chandef != &chandef) { wdev_unlock(wdev); return -EBUSY; @@ -11672,6 +11798,7 @@ static int nl80211_set_tx_bitrate_mask(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_bitrate_mask mask; + unsigned int link_id = nl80211_link_id(info->attrs); struct cfg80211_registered_device *rdev = info->user_ptr[0]; struct net_device *dev = info->user_ptr[1]; struct wireless_dev *wdev = dev->ieee80211_ptr; @@ -11683,11 +11810,11 @@ static int nl80211_set_tx_bitrate_mask(struct sk_buff *skb, wdev_lock(wdev); err = nl80211_parse_tx_bitrate_mask(info, info->attrs, NL80211_ATTR_TX_RATES, &mask, - dev, true); + dev, true, link_id); if (err) goto out; - err = rdev_set_bitrate_mask(rdev, dev, NULL, &mask); + err = rdev_set_bitrate_mask(rdev, dev, link_id, NULL, &mask); out: wdev_unlock(wdev); return err; @@ -11812,7 +11939,8 @@ static int nl80211_tx_mgmt(struct sk_buff *skb, struct genl_info *info) return -EINVAL; wdev_lock(wdev); - if (params.offchan && !cfg80211_off_channel_oper_allowed(wdev)) { + if (params.offchan && + !cfg80211_off_channel_oper_allowed(wdev, chandef.chan)) { wdev_unlock(wdev); return -EBUSY; } @@ -12030,12 +12158,13 @@ static int cfg80211_cqm_rssi_update(struct cfg80211_registered_device *rdev, * connection is established and enough beacons received to calculate * the average. */ - if (!wdev->cqm_config->last_rssi_event_value && wdev->current_bss && + if (!wdev->cqm_config->last_rssi_event_value && + wdev->links[0].client.current_bss && rdev->ops->get_station) { struct station_info sinfo = {}; u8 *mac_addr; - mac_addr = wdev->current_bss->pub.bssid; + mac_addr = wdev->links[0].client.current_bss->pub.bssid; err = rdev_get_station(rdev, dev, mac_addr, &sinfo); if (err) @@ -12298,7 +12427,7 @@ static int nl80211_join_mesh(struct sk_buff *skb, struct genl_info *info) err = nl80211_parse_tx_bitrate_mask(info, info->attrs, NL80211_ATTR_TX_RATES, &setup.beacon_rate, - dev, false); + dev, false, 0); if (err) return err; @@ -13268,7 +13397,7 @@ static int nl80211_set_rekey_data(struct sk_buff *skb, struct genl_info *info) rekey_data.akm = nla_get_u32(tb[NL80211_REKEY_DATA_AKM]); wdev_lock(wdev); - if (!wdev->current_bss) { + if (!wdev->connected) { err = -ENOTCONN; goto out; } @@ -14537,7 +14666,7 @@ static int nl80211_add_tx_ts(struct sk_buff *skb, struct genl_info *info) switch (wdev->iftype) { case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: - if (wdev->current_bss) + if (wdev->connected) break; err = -ENOTCONN; goto out; @@ -14710,13 +14839,13 @@ static int nl80211_set_pmk(struct sk_buff *skb, struct genl_info *info) return -EINVAL; wdev_lock(wdev); - if (!wdev->current_bss) { + if (!wdev->connected) { ret = -ENOTCONN; goto out; } pmk_conf.aa = nla_data(info->attrs[NL80211_ATTR_MAC]); - if (memcmp(pmk_conf.aa, wdev->current_bss->pub.bssid, ETH_ALEN)) { + if (memcmp(pmk_conf.aa, wdev->u.client.connected_addr, ETH_ALEN)) { ret = -EINVAL; goto out; } @@ -14844,9 +14973,13 @@ static int nl80211_tx_control_port(struct sk_buff *skb, struct genl_info *info) case NL80211_IFTYPE_MESH_POINT: break; case NL80211_IFTYPE_ADHOC: + if (wdev->u.ibss.current_bss) + break; + err = -ENOTCONN; + goto out; case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: - if (wdev->current_bss) + if (wdev->connected) break; err = -ENOTCONN; goto out; @@ -14882,12 +15015,14 @@ static int nl80211_get_ftm_responder_stats(struct sk_buff *skb, struct net_device *dev = info->user_ptr[1]; struct wireless_dev *wdev = dev->ieee80211_ptr; struct cfg80211_ftm_responder_stats ftm_stats = {}; + unsigned int link_id = nl80211_link_id(info->attrs); struct sk_buff *msg; void *hdr; struct nlattr *ftm_stats_attr; int err; - if (wdev->iftype != NL80211_IFTYPE_AP || !wdev->beacon_interval) + if (wdev->iftype != NL80211_IFTYPE_AP || + !wdev->links[link_id].ap.beacon_interval) return -EOPNOTSUPP; err = rdev_get_ftm_responder_stats(rdev, dev, &ftm_stats); @@ -15017,7 +15152,8 @@ static int nl80211_probe_mesh_link(struct sk_buff *skb, struct genl_info *info) static int parse_tid_conf(struct cfg80211_registered_device *rdev, struct nlattr *attrs[], struct net_device *dev, struct cfg80211_tid_cfg *tid_conf, - struct genl_info *info, const u8 *peer) + struct genl_info *info, const u8 *peer, + unsigned int link_id) { struct netlink_ext_ack *extack = info->extack; u64 mask; @@ -15092,7 +15228,7 @@ static int parse_tid_conf(struct cfg80211_registered_device *rdev, attr = NL80211_TID_CONFIG_ATTR_TX_RATE; err = nl80211_parse_tx_bitrate_mask(info, attrs, attr, &tid_conf->txrate_mask, dev, - true); + true, link_id); if (err) return err; @@ -15119,6 +15255,7 @@ static int nl80211_set_tid_config(struct sk_buff *skb, { struct cfg80211_registered_device *rdev = info->user_ptr[0]; struct nlattr *attrs[NL80211_TID_CONFIG_ATTR_MAX + 1]; + unsigned int link_id = nl80211_link_id(info->attrs); struct net_device *dev = info->user_ptr[1]; struct cfg80211_tid_config *tid_config; struct nlattr *tid; @@ -15156,7 +15293,7 @@ static int nl80211_set_tid_config(struct sk_buff *skb, ret = parse_tid_conf(rdev, attrs, dev, &tid_config->tid_conf[conf_idx], - info, tid_config->peer); + info, tid_config->peer, link_id); if (ret) goto bad_tid_conf; @@ -15295,6 +15432,62 @@ static int nl80211_set_fils_aad(struct sk_buff *skb, return rdev_set_fils_aad(rdev, dev, &fils_aad); } +static int nl80211_add_link(struct sk_buff *skb, struct genl_info *info) +{ + unsigned int link_id = nl80211_link_id(info->attrs); + struct net_device *dev = info->user_ptr[1]; + struct wireless_dev *wdev = dev->ieee80211_ptr; + + if (!(wdev->wiphy->flags & WIPHY_FLAG_SUPPORTS_MLO)) + return -EINVAL; + + switch (wdev->iftype) { + case NL80211_IFTYPE_AP: + break; + default: + return -EINVAL; + } + + if (!info->attrs[NL80211_ATTR_MAC] || + !is_valid_ether_addr(nla_data(info->attrs[NL80211_ATTR_MAC]))) + return -EINVAL; + + wdev_lock(wdev); + wdev->valid_links |= BIT(link_id); + ether_addr_copy(wdev->links[link_id].addr, + nla_data(info->attrs[NL80211_ATTR_MAC])); + wdev_unlock(wdev); + + return 0; +} + +static int nl80211_remove_link(struct sk_buff *skb, struct genl_info *info) +{ + unsigned int link_id = nl80211_link_id(info->attrs); + struct net_device *dev = info->user_ptr[1]; + struct wireless_dev *wdev = dev->ieee80211_ptr; + + /* cannot remove if there's no link */ + if (!info->attrs[NL80211_ATTR_MLO_LINK_ID]) + return -EINVAL; + + switch (wdev->iftype) { + case NL80211_IFTYPE_AP: + break; + default: + return -EINVAL; + } + + /* FIXME: stop the link operations first */ + + wdev_lock(wdev); + wdev->valid_links &= ~BIT(link_id); + eth_zero_addr(wdev->links[link_id].addr); + wdev_unlock(wdev); + + return 0; +} + #define NL80211_FLAG_NEED_WIPHY 0x01 #define NL80211_FLAG_NEED_NETDEV 0x02 #define NL80211_FLAG_NEED_RTNL 0x04 @@ -15307,6 +15500,8 @@ static int nl80211_set_fils_aad(struct sk_buff *skb, NL80211_FLAG_CHECK_NETDEV_UP) #define NL80211_FLAG_CLEAR_SKB 0x20 #define NL80211_FLAG_NO_WIPHY_MTX 0x40 +#define NL80211_FLAG_MLO_VALID_LINK_ID 0x80 +#define NL80211_FLAG_MLO_UNSUPPORTED 0x100 #define INTERNAL_FLAG_SELECTORS(__sel) \ SELECTOR(__sel, NONE, 0) /* must be first */ \ @@ -15316,6 +15511,12 @@ static int nl80211_set_fils_aad(struct sk_buff *skb, NL80211_FLAG_NEED_WDEV) \ SELECTOR(__sel, NETDEV, \ NL80211_FLAG_NEED_NETDEV) \ + SELECTOR(__sel, NETDEV_LINK, \ + NL80211_FLAG_NEED_NETDEV | \ + NL80211_FLAG_MLO_VALID_LINK_ID) \ + SELECTOR(__sel, NETDEV_NO_MLO, \ + NL80211_FLAG_NEED_NETDEV | \ + NL80211_FLAG_MLO_UNSUPPORTED) \ SELECTOR(__sel, WIPHY_RTNL, \ NL80211_FLAG_NEED_WIPHY | \ NL80211_FLAG_NEED_RTNL) \ @@ -15331,14 +15532,31 @@ static int nl80211_set_fils_aad(struct sk_buff *skb, NL80211_FLAG_NEED_RTNL) \ SELECTOR(__sel, NETDEV_UP, \ NL80211_FLAG_NEED_NETDEV_UP) \ + SELECTOR(__sel, NETDEV_UP_LINK, \ + NL80211_FLAG_NEED_NETDEV_UP | \ + NL80211_FLAG_MLO_VALID_LINK_ID) \ + SELECTOR(__sel, NETDEV_UP_NO_MLO, \ + NL80211_FLAG_NEED_NETDEV_UP | \ + NL80211_FLAG_MLO_UNSUPPORTED) \ + SELECTOR(__sel, NETDEV_UP_NO_MLO_CLEAR, \ + NL80211_FLAG_NEED_NETDEV_UP | \ + NL80211_FLAG_CLEAR_SKB | \ + NL80211_FLAG_MLO_UNSUPPORTED) \ SELECTOR(__sel, NETDEV_UP_NOTMX, \ NL80211_FLAG_NEED_NETDEV_UP | \ NL80211_FLAG_NO_WIPHY_MTX) \ + SELECTOR(__sel, NETDEV_UP_NOTMX_NOMLO, \ + NL80211_FLAG_NEED_NETDEV_UP | \ + NL80211_FLAG_NO_WIPHY_MTX | \ + NL80211_FLAG_MLO_UNSUPPORTED) \ SELECTOR(__sel, NETDEV_UP_CLEAR, \ NL80211_FLAG_NEED_NETDEV_UP | \ NL80211_FLAG_CLEAR_SKB) \ SELECTOR(__sel, WDEV_UP, \ NL80211_FLAG_NEED_WDEV_UP) \ + SELECTOR(__sel, WDEV_UP_LINK, \ + NL80211_FLAG_NEED_WDEV_UP | \ + NL80211_FLAG_MLO_VALID_LINK_ID) \ SELECTOR(__sel, WDEV_UP_RTNL, \ NL80211_FLAG_NEED_WDEV_UP | \ NL80211_FLAG_NEED_RTNL) \ @@ -15362,9 +15580,10 @@ static int nl80211_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = NULL; - struct wireless_dev *wdev; - struct net_device *dev; + struct wireless_dev *wdev = NULL; + struct net_device *dev = NULL; u32 internal_flags; + int err; if (WARN_ON(ops->internal_flags >= ARRAY_SIZE(nl80211_internal_flags))) return -EINVAL; @@ -15375,8 +15594,8 @@ static int nl80211_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, if (internal_flags & NL80211_FLAG_NEED_WIPHY) { rdev = cfg80211_get_dev_from_info(genl_info_net(info), info); if (IS_ERR(rdev)) { - rtnl_unlock(); - return PTR_ERR(rdev); + err = PTR_ERR(rdev); + goto out_unlock; } info->user_ptr[0] = rdev; } else if (internal_flags & NL80211_FLAG_NEED_NETDEV || @@ -15384,17 +15603,18 @@ static int nl80211_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, wdev = __cfg80211_wdev_from_attrs(NULL, genl_info_net(info), info->attrs); if (IS_ERR(wdev)) { - rtnl_unlock(); - return PTR_ERR(wdev); + err = PTR_ERR(wdev); + goto out_unlock; } dev = wdev->netdev; + dev_hold(dev); rdev = wiphy_to_rdev(wdev->wiphy); if (internal_flags & NL80211_FLAG_NEED_NETDEV) { if (!dev) { - rtnl_unlock(); - return -EINVAL; + err = -EINVAL; + goto out_unlock; } info->user_ptr[1] = dev; @@ -15404,14 +15624,44 @@ static int nl80211_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, if (internal_flags & NL80211_FLAG_CHECK_NETDEV_UP && !wdev_running(wdev)) { - rtnl_unlock(); - return -ENETDOWN; + err = -ENETDOWN; + goto out_unlock; } - dev_hold(dev); info->user_ptr[0] = rdev; } + if (internal_flags & NL80211_FLAG_MLO_VALID_LINK_ID) { + struct nlattr *link_id = info->attrs[NL80211_ATTR_MLO_LINK_ID]; + + if (!wdev) { + err = -EINVAL; + goto out_unlock; + } + + /* MLO -> require valid link ID */ + if (wdev->valid_links && + (!link_id || + !(wdev->valid_links & BIT(nla_get_u16(link_id))))) { + err = -EINVAL; + goto out_unlock; + } + + /* non-MLO -> no link ID attribute accepted */ + if (!wdev->valid_links && link_id) { + err = -EINVAL; + goto out_unlock; + } + } + + if (internal_flags & NL80211_FLAG_MLO_UNSUPPORTED) { + if (info->attrs[NL80211_ATTR_MLO_LINK_ID] || + (wdev && wdev->valid_links)) { + err = -EINVAL; + goto out_unlock; + } + } + if (rdev && !(internal_flags & NL80211_FLAG_NO_WIPHY_MTX)) { wiphy_lock(&rdev->wiphy); /* we keep the mutex locked until post_doit */ @@ -15421,6 +15671,10 @@ static int nl80211_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, rtnl_unlock(); return 0; +out_unlock: + rtnl_unlock(); + dev_put(dev); + return err; } static void nl80211_post_doit(const struct genl_ops *ops, struct sk_buff *skb, @@ -15636,6 +15890,7 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_set_key, .flags = GENL_UNS_ADMIN_PERM, + /* cannot use NL80211_FLAG_MLO_VALID_LINK_ID, depends on key */ .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | NL80211_FLAG_CLEAR_SKB), }, @@ -15659,21 +15914,24 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .flags = GENL_UNS_ADMIN_PERM, .doit = nl80211_set_beacon, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_START_AP, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .flags = GENL_UNS_ADMIN_PERM, .doit = nl80211_start_ap, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_STOP_AP, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .flags = GENL_UNS_ADMIN_PERM, .doit = nl80211_stop_ap, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_GET_STATION, @@ -15939,7 +16197,9 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_remain_on_channel, .flags = GENL_UNS_ADMIN_PERM, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV_UP), + /* FIXME: requiring a link ID here is probably not good */ + .internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV_UP | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_CANCEL_REMAIN_ON_CHANNEL, @@ -15953,7 +16213,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_set_tx_bitrate_mask, .flags = GENL_UNS_ADMIN_PERM, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_REGISTER_FRAME, @@ -16002,7 +16263,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_set_channel, .flags = GENL_UNS_ADMIN_PERM, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_JOIN_MESH, @@ -16163,7 +16425,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_set_mac_acl, .flags = GENL_UNS_ADMIN_PERM, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV | + NL80211_FLAG_MLO_UNSUPPORTED), }, { .cmd = NL80211_CMD_RADAR_DETECT, @@ -16171,7 +16434,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .doit = nl80211_start_radar_detection, .flags = GENL_UNS_ADMIN_PERM, .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | - NL80211_FLAG_NO_WIPHY_MTX), + NL80211_FLAG_NO_WIPHY_MTX | + NL80211_FLAG_MLO_UNSUPPORTED), }, { .cmd = NL80211_CMD_GET_PROTOCOL_FEATURES, @@ -16217,7 +16481,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_channel_switch, .flags = GENL_UNS_ADMIN_PERM, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_VENDOR, @@ -16240,7 +16505,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_add_tx_ts, .flags = GENL_UNS_ADMIN_PERM, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | + NL80211_FLAG_MLO_UNSUPPORTED), }, { .cmd = NL80211_CMD_DEL_TX_TS, @@ -16301,7 +16567,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .cmd = NL80211_CMD_GET_FTM_RESPONDER_STATS, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = nl80211_get_ftm_responder_stats, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_PEER_MEASUREMENT_START, @@ -16333,7 +16600,8 @@ static const struct genl_small_ops nl80211_small_ops[] = { .cmd = NL80211_CMD_SET_TID_CONFIG, .doit = nl80211_set_tid_config, .flags = GENL_UNS_ADMIN_PERM, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV), + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV | + NL80211_FLAG_MLO_VALID_LINK_ID), }, { .cmd = NL80211_CMD_SET_SAR_SPECS, @@ -16357,6 +16625,19 @@ static const struct genl_small_ops nl80211_small_ops[] = { .flags = GENL_UNS_ADMIN_PERM, .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), }, + { + .cmd = NL80211_CMD_ADD_LINK, + .doit = nl80211_add_link, + .flags = GENL_UNS_ADMIN_PERM, + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), + }, + { + .cmd = NL80211_CMD_REMOVE_LINK, + .doit = nl80211_remove_link, + .flags = GENL_UNS_ADMIN_PERM, + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | + NL80211_FLAG_MLO_VALID_LINK_ID), + }, }; static struct genl_family nl80211_fam __ro_after_init = { @@ -17984,23 +18265,37 @@ static void nl80211_ch_switch_notify(struct cfg80211_registered_device *rdev, } void cfg80211_ch_switch_notify(struct net_device *dev, - struct cfg80211_chan_def *chandef) + struct cfg80211_chan_def *chandef, + unsigned int link_id) { struct wireless_dev *wdev = dev->ieee80211_ptr; struct wiphy *wiphy = wdev->wiphy; struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); ASSERT_WDEV_LOCK(wdev); + WARN_INVALID_LINK_ID(wdev, link_id); - trace_cfg80211_ch_switch_notify(dev, chandef); - - wdev->chandef = *chandef; - wdev->preset_chandef = *chandef; + trace_cfg80211_ch_switch_notify(dev, chandef, link_id); - if ((wdev->iftype == NL80211_IFTYPE_STATION || - wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) && - !WARN_ON(!wdev->current_bss)) - cfg80211_update_assoc_bss_entry(wdev, chandef->chan); + switch (wdev->iftype) { + case NL80211_IFTYPE_STATION: + case NL80211_IFTYPE_P2P_CLIENT: + if (!WARN_ON(!wdev->links[link_id].client.current_bss)) + cfg80211_update_assoc_bss_entry(wdev, link_id, + chandef->chan); + break; + case NL80211_IFTYPE_MESH_POINT: + wdev->u.mesh.chandef = *chandef; + wdev->u.mesh.preset_chandef = *chandef; + break; + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + wdev->links[link_id].ap.chandef = *chandef; + break; + default: + WARN_ON(1); + break; + } cfg80211_sched_dfs_chan_update(rdev); diff --git a/net/wireless/ocb.c b/net/wireless/ocb.c index 2d26a6d980bf..27a1732264f9 100644 --- a/net/wireless/ocb.c +++ b/net/wireless/ocb.c @@ -4,6 +4,7 @@ * * Copyright: (c) 2014 Czech Technical University in Prague * (c) 2014 Volkswagen Group Research + * Copyright (C) 2022 Intel Corporation * Author: Rostislav Lisovy * Funded by: Volkswagen Group Research */ @@ -34,7 +35,7 @@ int __cfg80211_join_ocb(struct cfg80211_registered_device *rdev, err = rdev_join_ocb(rdev, dev, setup); if (!err) - wdev->chandef = setup->chandef; + wdev->u.ocb.chandef = setup->chandef; return err; } @@ -69,7 +70,7 @@ int __cfg80211_leave_ocb(struct cfg80211_registered_device *rdev, err = rdev_leave_ocb(rdev, dev); if (!err) - memset(&wdev->chandef, 0, sizeof(wdev->chandef)); + memset(&wdev->u.ocb.chandef, 0, sizeof(wdev->u.ocb.chandef)); return err; } diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h index 439bcf52369c..d2300eff03ae 100644 --- a/net/wireless/rdev-ops.h +++ b/net/wireless/rdev-ops.h @@ -1,4 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ +/* + * Portions of this file + * Copyright(c) 2016-2017 Intel Deutschland GmbH + * Copyright (C) 2018, 2021-2022 Intel Corporation + */ #ifndef __CFG80211_RDEV_OPS #define __CFG80211_RDEV_OPS @@ -172,11 +177,11 @@ static inline int rdev_change_beacon(struct cfg80211_registered_device *rdev, } static inline int rdev_stop_ap(struct cfg80211_registered_device *rdev, - struct net_device *dev) + struct net_device *dev, unsigned int link_id) { int ret; - trace_rdev_stop_ap(&rdev->wiphy, dev); - ret = rdev->ops->stop_ap(&rdev->wiphy, dev); + trace_rdev_stop_ap(&rdev->wiphy, dev, link_id); + ret = rdev->ops->stop_ap(&rdev->wiphy, dev, link_id); trace_rdev_return_int(&rdev->wiphy, ret); return ret; } @@ -651,12 +656,14 @@ static inline int rdev_testmode_dump(struct cfg80211_registered_device *rdev, static inline int rdev_set_bitrate_mask(struct cfg80211_registered_device *rdev, - struct net_device *dev, const u8 *peer, + struct net_device *dev, unsigned int link_id, + const u8 *peer, const struct cfg80211_bitrate_mask *mask) { int ret; - trace_rdev_set_bitrate_mask(&rdev->wiphy, dev, peer, mask); - ret = rdev->ops->set_bitrate_mask(&rdev->wiphy, dev, peer, mask); + trace_rdev_set_bitrate_mask(&rdev->wiphy, dev, link_id, peer, mask); + ret = rdev->ops->set_bitrate_mask(&rdev->wiphy, dev, link_id, + peer, mask); trace_rdev_return_int(&rdev->wiphy, ret); return ret; } @@ -944,12 +951,13 @@ static inline int rdev_set_noack_map(struct cfg80211_registered_device *rdev, static inline int rdev_get_channel(struct cfg80211_registered_device *rdev, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) { int ret; - trace_rdev_get_channel(&rdev->wiphy, wdev); - ret = rdev->ops->get_channel(&rdev->wiphy, wdev, chandef); + trace_rdev_get_channel(&rdev->wiphy, wdev, link_id); + ret = rdev->ops->get_channel(&rdev->wiphy, wdev, link_id, chandef); trace_rdev_return_chandef(&rdev->wiphy, ret, chandef); return ret; @@ -1107,12 +1115,14 @@ static inline int rdev_set_qos_map(struct cfg80211_registered_device *rdev, static inline int rdev_set_ap_chanwidth(struct cfg80211_registered_device *rdev, - struct net_device *dev, struct cfg80211_chan_def *chandef) + struct net_device *dev, + unsigned int link_id, + struct cfg80211_chan_def *chandef) { int ret; - trace_rdev_set_ap_chanwidth(&rdev->wiphy, dev, chandef); - ret = rdev->ops->set_ap_chanwidth(&rdev->wiphy, dev, chandef); + trace_rdev_set_ap_chanwidth(&rdev->wiphy, dev, link_id, chandef); + ret = rdev->ops->set_ap_chanwidth(&rdev->wiphy, dev, link_id, chandef); trace_rdev_return_int(&rdev->wiphy, ret); return ret; diff --git a/net/wireless/reg.c b/net/wireless/reg.c index 58e83ce642ad..c7383ede794f 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -5,7 +5,7 @@ * Copyright 2008-2011 Luis R. Rodriguez * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2017 Intel Deutschland GmbH - * Copyright (C) 2018 - 2021 Intel Corporation + * Copyright (C) 2018 - 2022 Intel Corporation * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -2370,6 +2370,7 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev) struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); enum nl80211_iftype iftype; bool ret; + int link; wdev_lock(wdev); iftype = wdev->iftype; @@ -2378,62 +2379,83 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev) if (!wdev->netdev || !netif_running(wdev->netdev)) goto wdev_inactive_unlock; - switch (iftype) { - case NL80211_IFTYPE_AP: - case NL80211_IFTYPE_P2P_GO: - case NL80211_IFTYPE_MESH_POINT: - if (!wdev->beacon_interval) - goto wdev_inactive_unlock; - chandef = wdev->chandef; - break; - case NL80211_IFTYPE_ADHOC: - if (!wdev->ssid_len) - goto wdev_inactive_unlock; - chandef = wdev->chandef; - break; - case NL80211_IFTYPE_STATION: - case NL80211_IFTYPE_P2P_CLIENT: - if (!wdev->current_bss || - !wdev->current_bss->pub.channel) - goto wdev_inactive_unlock; - - if (!rdev->ops->get_channel || - rdev_get_channel(rdev, wdev, &chandef)) - cfg80211_chandef_create(&chandef, - wdev->current_bss->pub.channel, - NL80211_CHAN_NO_HT); - break; - case NL80211_IFTYPE_MONITOR: - case NL80211_IFTYPE_AP_VLAN: - case NL80211_IFTYPE_P2P_DEVICE: - /* no enforcement required */ - break; - default: - /* others not implemented for now */ - WARN_ON(1); - break; - } + for (link = 0; link < ARRAY_SIZE(wdev->links); link++) { + struct ieee80211_channel *chan; - wdev_unlock(wdev); + if (!wdev->valid_links && link > 0) + break; + if (!(wdev->valid_links & BIT(link))) + continue; + switch (iftype) { + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + case NL80211_IFTYPE_MESH_POINT: + if (!wdev->u.mesh.beacon_interval) + continue; + chandef = wdev->u.mesh.chandef; + break; + case NL80211_IFTYPE_ADHOC: + if (!wdev->u.ibss.ssid_len) + continue; + chandef = wdev->u.ibss.chandef; + break; + case NL80211_IFTYPE_STATION: + case NL80211_IFTYPE_P2P_CLIENT: + /* Maybe we could consider disabling that link only? */ + if (!wdev->links[link].client.current_bss) + continue; - switch (iftype) { - case NL80211_IFTYPE_AP: - case NL80211_IFTYPE_P2P_GO: - case NL80211_IFTYPE_ADHOC: - case NL80211_IFTYPE_MESH_POINT: - wiphy_lock(wiphy); - ret = cfg80211_reg_can_beacon_relax(wiphy, &chandef, iftype); - wiphy_unlock(wiphy); + chan = wdev->links[link].client.current_bss->pub.channel; + if (!chan) + continue; - return ret; - case NL80211_IFTYPE_STATION: - case NL80211_IFTYPE_P2P_CLIENT: - return cfg80211_chandef_usable(wiphy, &chandef, - IEEE80211_CHAN_DISABLED); - default: - break; + if (!rdev->ops->get_channel || + rdev_get_channel(rdev, wdev, link, &chandef)) + cfg80211_chandef_create(&chandef, chan, + NL80211_CHAN_NO_HT); + break; + case NL80211_IFTYPE_MONITOR: + case NL80211_IFTYPE_AP_VLAN: + case NL80211_IFTYPE_P2P_DEVICE: + /* no enforcement required */ + break; + default: + /* others not implemented for now */ + WARN_ON(1); + break; + } + + wdev_unlock(wdev); + + switch (iftype) { + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + case NL80211_IFTYPE_ADHOC: + case NL80211_IFTYPE_MESH_POINT: + wiphy_lock(wiphy); + ret = cfg80211_reg_can_beacon_relax(wiphy, &chandef, + iftype); + wiphy_unlock(wiphy); + + if (!ret) + return ret; + break; + case NL80211_IFTYPE_STATION: + case NL80211_IFTYPE_P2P_CLIENT: + ret = cfg80211_chandef_usable(wiphy, &chandef, + IEEE80211_CHAN_DISABLED); + if (!ret) + return ret; + break; + default: + break; + } + + wdev_lock(wdev); } + wdev_unlock(wdev); + return true; wdev_inactive_unlock: @@ -4215,8 +4237,17 @@ static void cfg80211_check_and_end_cac(struct cfg80211_registered_device *rdev) * In both cases we should end the CAC on the wdev. */ list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { - if (wdev->cac_started && - !cfg80211_chandef_dfs_usable(&rdev->wiphy, &wdev->chandef)) + struct cfg80211_chan_def *chandef; + + if (!wdev->cac_started) + continue; + + /* FIXME: radar detection is tied to link 0 for now */ + chandef = wdev_chandef(wdev, 0); + if (!chandef) + continue; + + if (!cfg80211_chandef_dfs_usable(&rdev->wiphy, chandef)) rdev_end_cac(rdev, wdev->netdev); } } diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 6d82bd9eaf8c..0134e5d5c81a 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -5,7 +5,7 @@ * Copyright 2008 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2016 Intel Deutschland GmbH - * Copyright (C) 2018-2021 Intel Corporation + * Copyright (C) 2018-2022 Intel Corporation */ #include #include @@ -2617,7 +2617,8 @@ void cfg80211_bss_iter(struct wiphy *wiphy, spin_lock_bh(&rdev->bss_lock); list_for_each_entry(bss, &rdev->bss_list, list) { - if (!chandef || cfg80211_is_sub_chan(chandef, bss->pub.channel)) + if (!chandef || cfg80211_is_sub_chan(chandef, bss->pub.channel, + false)) iter(wiphy, &bss->pub, iter_data); } @@ -2626,11 +2627,12 @@ void cfg80211_bss_iter(struct wiphy *wiphy, EXPORT_SYMBOL(cfg80211_bss_iter); void cfg80211_update_assoc_bss_entry(struct wireless_dev *wdev, + unsigned int link_id, struct ieee80211_channel *chan) { struct wiphy *wiphy = wdev->wiphy; struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); - struct cfg80211_internal_bss *cbss = wdev->current_bss; + struct cfg80211_internal_bss *cbss = wdev->links[link_id].client.current_bss; struct cfg80211_internal_bss *new = NULL; struct cfg80211_internal_bss *bss; struct cfg80211_bss *nontrans_bss; diff --git a/net/wireless/sme.c b/net/wireless/sme.c index ff4d48fcbfb2..35602201057b 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -5,7 +5,7 @@ * (for nl80211's connect() and wext) * * Copyright 2009 Johannes Berg - * Copyright (C) 2009, 2020 Intel Corporation. All rights reserved. + * Copyright (C) 2009, 2020, 2022 Intel Corporation. All rights reserved. * Copyright 2017 Intel Deutschland GmbH */ @@ -454,6 +454,20 @@ void cfg80211_sme_abandon_assoc(struct wireless_dev *wdev) schedule_work(&rdev->conn_work); } +static void cfg80211_wdev_release_bsses(struct wireless_dev *wdev) +{ + unsigned int link; + + for_each_valid_link(wdev, link) { + if (!wdev->links[link].client.current_bss) + continue; + cfg80211_unhold_bss(wdev->links[link].client.current_bss); + cfg80211_put_bss(wdev->wiphy, + &wdev->links[link].client.current_bss->pub); + wdev->links[link].client.current_bss = NULL; + } +} + static int cfg80211_sme_get_conn_ies(struct wireless_dev *wdev, const u8 *ies, size_t ies_len, const u8 **out_ies, size_t *out_ies_len) @@ -521,12 +535,11 @@ static int cfg80211_sme_connect(struct wireless_dev *wdev, if (!rdev->ops->auth || !rdev->ops->assoc) return -EOPNOTSUPP; - if (wdev->current_bss) { - cfg80211_unhold_bss(wdev->current_bss); - cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub); - wdev->current_bss = NULL; + cfg80211_wdev_release_bsses(wdev); + if (wdev->connected) { cfg80211_sme_free(wdev); + wdev->connected = false; } if (wdev->conn) @@ -563,8 +576,8 @@ static int cfg80211_sme_connect(struct wireless_dev *wdev, wdev->conn->auto_auth = false; } - wdev->conn->params.ssid = wdev->ssid; - wdev->conn->params.ssid_len = wdev->ssid_len; + wdev->conn->params.ssid = wdev->u.client.ssid; + wdev->conn->params.ssid_len = wdev->u.client.ssid_len; /* see if we have the bss already */ bss = cfg80211_get_conn_bss(wdev); @@ -648,7 +661,7 @@ static bool cfg80211_is_all_idle(void) list_for_each_entry(rdev, &cfg80211_rdev_list, list) { list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { wdev_lock(wdev); - if (wdev->conn || wdev->current_bss || + if (wdev->conn || wdev->connected || cfg80211_beaconing_iface_active(wdev)) is_all_idle = false; wdev_unlock(wdev); @@ -668,7 +681,6 @@ static void disconnect_work(struct work_struct *work) DECLARE_WORK(cfg80211_disconnect_work, disconnect_work); - /* * API calls for drivers implementing connect/disconnect and * SME event handling @@ -729,23 +741,19 @@ void __cfg80211_connect_result(struct net_device *dev, if (!cr->bss && (cr->status == WLAN_STATUS_SUCCESS)) { WARN_ON_ONCE(!wiphy_to_rdev(wdev->wiphy)->ops->connect); cr->bss = cfg80211_get_bss(wdev->wiphy, NULL, cr->bssid, - wdev->ssid, wdev->ssid_len, + wdev->u.client.ssid, wdev->u.client.ssid_len, wdev->conn_bss_type, IEEE80211_PRIVACY_ANY); if (cr->bss) cfg80211_hold_bss(bss_from_pub(cr->bss)); } - if (wdev->current_bss) { - cfg80211_unhold_bss(wdev->current_bss); - cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub); - wdev->current_bss = NULL; - } + cfg80211_wdev_release_bsses(wdev); if (cr->status != WLAN_STATUS_SUCCESS) { kfree_sensitive(wdev->connect_keys); wdev->connect_keys = NULL; - wdev->ssid_len = 0; + wdev->u.client.ssid_len = 0; wdev->conn_owner_nlportid = 0; if (cr->bss) { cfg80211_unhold_bss(bss_from_pub(cr->bss)); @@ -758,7 +766,9 @@ void __cfg80211_connect_result(struct net_device *dev, if (WARN_ON(!cr->bss)) return; - wdev->current_bss = bss_from_pub(cr->bss); + wdev->links[0].client.current_bss = bss_from_pub(cr->bss); + wdev->connected = true; + ether_addr_copy(wdev->u.client.connected_addr, cr->bss->bssid); if (!(wdev->wiphy->flags & WIPHY_FLAG_HAS_STATIC_WEP)) cfg80211_upload_connect_keys(wdev); @@ -801,7 +811,7 @@ void cfg80211_connect_done(struct net_device *dev, found = cfg80211_get_bss(wdev->wiphy, NULL, params->bss->bssid, - wdev->ssid, wdev->ssid_len, + wdev->u.client.ssid, wdev->u.client.ssid_len, wdev->conn_bss_type, IEEE80211_PRIVACY_ANY); if (found) { @@ -906,18 +916,17 @@ void __cfg80211_roamed(struct wireless_dev *wdev, wdev->iftype != NL80211_IFTYPE_P2P_CLIENT)) goto out; - if (WARN_ON(!wdev->current_bss)) + if (WARN_ON(!wdev->connected)) goto out; - cfg80211_unhold_bss(wdev->current_bss); - cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub); - wdev->current_bss = NULL; + cfg80211_wdev_release_bsses(wdev); if (WARN_ON(!info->bss)) return; cfg80211_hold_bss(bss_from_pub(info->bss)); - wdev->current_bss = bss_from_pub(info->bss); + wdev->links[0].client.current_bss = bss_from_pub(info->bss); + ether_addr_copy(wdev->u.client.connected_addr, info->bss->bssid); wdev->unprot_beacon_reported = 0; nl80211_send_roamed(wiphy_to_rdev(wdev->wiphy), @@ -963,8 +972,8 @@ void cfg80211_roamed(struct net_device *dev, struct cfg80211_roam_info *info, if (!info->bss) { info->bss = cfg80211_get_bss(wdev->wiphy, info->channel, - info->bssid, wdev->ssid, - wdev->ssid_len, + info->bssid, wdev->u.client.ssid, + wdev->u.client.ssid_len, wdev->conn_bss_type, IEEE80211_PRIVACY_ANY); } @@ -1034,8 +1043,8 @@ void __cfg80211_port_authorized(struct wireless_dev *wdev, const u8 *bssid) if (WARN_ON(wdev->iftype != NL80211_IFTYPE_STATION)) return; - if (WARN_ON(!wdev->current_bss) || - WARN_ON(!ether_addr_equal(wdev->current_bss->pub.bssid, bssid))) + if (WARN_ON(!wdev->connected) || + WARN_ON(!ether_addr_equal(wdev->u.client.connected_addr, bssid))) return; nl80211_send_port_authorized(wiphy_to_rdev(wdev->wiphy), wdev->netdev, @@ -1087,13 +1096,9 @@ void __cfg80211_disconnected(struct net_device *dev, const u8 *ie, wdev->iftype != NL80211_IFTYPE_P2P_CLIENT)) return; - if (wdev->current_bss) { - cfg80211_unhold_bss(wdev->current_bss); - cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub); - } - - wdev->current_bss = NULL; - wdev->ssid_len = 0; + cfg80211_wdev_release_bsses(wdev); + wdev->connected = false; + wdev->u.client.ssid_len = 0; wdev->conn_owner_nlportid = 0; kfree_sensitive(wdev->connect_keys); wdev->connect_keys = NULL; @@ -1182,19 +1187,20 @@ int cfg80211_connect(struct cfg80211_registered_device *rdev, * already connected, so reject a new SSID unless it's the * same (which is the case for re-association.) */ - if (wdev->ssid_len && - (wdev->ssid_len != connect->ssid_len || - memcmp(wdev->ssid, connect->ssid, wdev->ssid_len))) + if (wdev->u.client.ssid_len && + (wdev->u.client.ssid_len != connect->ssid_len || + memcmp(wdev->u.client.ssid, connect->ssid, wdev->u.client.ssid_len))) return -EALREADY; /* * If connected, reject (re-)association unless prev_bssid * matches the current BSSID. */ - if (wdev->current_bss) { + if (wdev->connected) { if (!prev_bssid) return -EALREADY; - if (!ether_addr_equal(prev_bssid, wdev->current_bss->pub.bssid)) + if (!ether_addr_equal(prev_bssid, + wdev->u.client.connected_addr)) return -ENOTCONN; } @@ -1245,8 +1251,8 @@ int cfg80211_connect(struct cfg80211_registered_device *rdev, } wdev->connect_keys = connkeys; - memcpy(wdev->ssid, connect->ssid, connect->ssid_len); - wdev->ssid_len = connect->ssid_len; + memcpy(wdev->u.client.ssid, connect->ssid, connect->ssid_len); + wdev->u.client.ssid_len = connect->ssid_len; wdev->conn_bss_type = connect->pbss ? IEEE80211_BSS_TYPE_PBSS : IEEE80211_BSS_TYPE_ESS; @@ -1262,8 +1268,8 @@ int cfg80211_connect(struct cfg80211_registered_device *rdev, * This could be reassoc getting refused, don't clear * ssid_len in that case. */ - if (!wdev->current_bss) - wdev->ssid_len = 0; + if (!wdev->connected) + wdev->u.client.ssid_len = 0; return err; } @@ -1287,7 +1293,7 @@ int cfg80211_disconnect(struct cfg80211_registered_device *rdev, err = cfg80211_sme_disconnect(wdev, reason); else if (!rdev->ops->disconnect) cfg80211_mlme_down(rdev, dev); - else if (wdev->ssid_len) + else if (wdev->u.client.ssid_len) err = rdev_disconnect(rdev, dev, reason); /* @@ -1295,8 +1301,8 @@ int cfg80211_disconnect(struct cfg80211_registered_device *rdev, * in which case cfg80211_disconnected() will take care of * this later. */ - if (!wdev->current_bss) - wdev->ssid_len = 0; + if (!wdev->connected) + wdev->u.client.ssid_len = 0; return err; } @@ -1320,7 +1326,7 @@ void cfg80211_autodisconnect_wk(struct work_struct *work) break; case NL80211_IFTYPE_AP: case NL80211_IFTYPE_P2P_GO: - __cfg80211_stop_ap(rdev, wdev->netdev, false); + __cfg80211_stop_ap(rdev, wdev->netdev, -1, false); break; case NL80211_IFTYPE_MESH_POINT: __cfg80211_leave_mesh(rdev, wdev->netdev); @@ -1332,7 +1338,7 @@ void cfg80211_autodisconnect_wk(struct work_struct *work) * ops->disconnect not implemented. Otherwise we can * use cfg80211_disconnect. */ - if (rdev->ops->disconnect || wdev->current_bss) + if (rdev->ops->disconnect || wdev->connected) cfg80211_disconnect(rdev, wdev->netdev, WLAN_REASON_DEAUTH_LEAVING, true); diff --git a/net/wireless/trace.h b/net/wireless/trace.h index 228079d7690a..3b2c956b8d78 100644 --- a/net/wireless/trace.h +++ b/net/wireless/trace.h @@ -569,6 +569,7 @@ TRACE_EVENT(rdev_start_ap, __field(bool, privacy) __field(enum nl80211_auth_type, auth_type) __field(int, inactivity_timeout) + __field(unsigned int, link_id) ), TP_fast_assign( WIPHY_ASSIGN; @@ -583,16 +584,17 @@ TRACE_EVENT(rdev_start_ap, __entry->inactivity_timeout = settings->inactivity_timeout; memset(__entry->ssid, 0, IEEE80211_MAX_SSID_LEN + 1); memcpy(__entry->ssid, settings->ssid, settings->ssid_len); + __entry->link_id = settings->beacon.link_id; ), TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", AP settings - ssid: %s, " CHAN_DEF_PR_FMT ", beacon interval: %d, dtim period: %d, " "hidden ssid: %d, wpa versions: %u, privacy: %s, " - "auth type: %d, inactivity timeout: %d", + "auth type: %d, inactivity timeout: %d, link_id: %d", WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->ssid, CHAN_DEF_PR_ARG, __entry->beacon_interval, __entry->dtim_period, __entry->hidden_ssid, __entry->wpa_ver, BOOL_TO_STR(__entry->privacy), __entry->auth_type, - __entry->inactivity_timeout) + __entry->inactivity_timeout, __entry->link_id) ); TRACE_EVENT(rdev_change_beacon, @@ -602,6 +604,7 @@ TRACE_EVENT(rdev_change_beacon, TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY + __field(int, link_id) __dynamic_array(u8, head, info ? info->head_len : 0) __dynamic_array(u8, tail, info ? info->tail_len : 0) __dynamic_array(u8, beacon_ies, info ? info->beacon_ies_len : 0) @@ -615,6 +618,7 @@ TRACE_EVENT(rdev_change_beacon, WIPHY_ASSIGN; NETDEV_ASSIGN; if (info) { + __entry->link_id = info->link_id; if (info->head) memcpy(__get_dynamic_array(head), info->head, info->head_len); @@ -635,9 +639,30 @@ TRACE_EVENT(rdev_change_beacon, if (info->probe_resp) memcpy(__get_dynamic_array(probe_resp), info->probe_resp, info->probe_resp_len); + } else { + __entry->link_id = -1; } ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT, WIPHY_PR_ARG, NETDEV_PR_ARG) + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id:%d", + WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id) +); + +TRACE_EVENT(rdev_stop_ap, + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, + unsigned int link_id), + TP_ARGS(wiphy, netdev, link_id), + TP_STRUCT__entry( + WIPHY_ENTRY + NETDEV_ENTRY + __field(unsigned int, link_id) + ), + TP_fast_assign( + WIPHY_ASSIGN; + NETDEV_ASSIGN; + __entry->link_id = link_id; + ), + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d", + WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id) ); DECLARE_EVENT_CLASS(wiphy_netdev_evt, @@ -654,11 +679,6 @@ DECLARE_EVENT_CLASS(wiphy_netdev_evt, TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT, WIPHY_PR_ARG, NETDEV_PR_ARG) ); -DEFINE_EVENT(wiphy_netdev_evt, rdev_stop_ap, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev), - TP_ARGS(wiphy, netdev) -); - DEFINE_EVENT(wiphy_netdev_evt, rdev_set_rekey_data, TP_PROTO(struct wiphy *wiphy, struct net_device *netdev), TP_ARGS(wiphy, netdev) @@ -1619,20 +1639,24 @@ TRACE_EVENT(rdev_testmode_dump, TRACE_EVENT(rdev_set_bitrate_mask, TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, + unsigned int link_id, const u8 *peer, const struct cfg80211_bitrate_mask *mask), - TP_ARGS(wiphy, netdev, peer, mask), + TP_ARGS(wiphy, netdev, link_id, peer, mask), TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY + __field(unsigned int, link_id) MAC_ENTRY(peer) ), TP_fast_assign( WIPHY_ASSIGN; NETDEV_ASSIGN; + __entry->link_id = link_id; MAC_ASSIGN(peer, peer); ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", peer: " MAC_PR_FMT, - WIPHY_PR_ARG, NETDEV_PR_ARG, MAC_PR_ARG(peer)) + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, peer: " MAC_PR_FMT, + WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id, + MAC_PR_ARG(peer)) ); TRACE_EVENT(rdev_update_mgmt_frame_registrations, @@ -2040,9 +2064,22 @@ TRACE_EVENT(rdev_set_noack_map, WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->noack_map) ); -DEFINE_EVENT(wiphy_wdev_evt, rdev_get_channel, - TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev), - TP_ARGS(wiphy, wdev) +TRACE_EVENT(rdev_get_channel, + TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id), + TP_ARGS(wiphy, wdev, link_id), + TP_STRUCT__entry( + WIPHY_ENTRY + WDEV_ENTRY + __field(unsigned int, link_id) + ), + TP_fast_assign( + WIPHY_ASSIGN; + WDEV_ASSIGN; + __entry->link_id = link_id; + ), + TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT ", link_id: %u", + WIPHY_PR_ARG, WDEV_PR_ARG, __entry->link_id) ); TRACE_EVENT(rdev_return_chandef, @@ -2296,20 +2333,24 @@ TRACE_EVENT(rdev_set_qos_map, TRACE_EVENT(rdev_set_ap_chanwidth, TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, + unsigned int link_id, struct cfg80211_chan_def *chandef), - TP_ARGS(wiphy, netdev, chandef), + TP_ARGS(wiphy, netdev, link_id, chandef), TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY CHAN_DEF_ENTRY + __field(unsigned int, link_id) ), TP_fast_assign( WIPHY_ASSIGN; NETDEV_ASSIGN; CHAN_DEF_ASSIGN(chandef); + __entry->link_id = link_id; ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT, - WIPHY_PR_ARG, NETDEV_PR_ARG, CHAN_DEF_PR_ARG) + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT ", link:%d", + WIPHY_PR_ARG, NETDEV_PR_ARG, CHAN_DEF_PR_ARG, + __entry->link_id) ); TRACE_EVENT(rdev_add_tx_ts, @@ -3022,18 +3063,21 @@ TRACE_EVENT(cfg80211_chandef_dfs_required, TRACE_EVENT(cfg80211_ch_switch_notify, TP_PROTO(struct net_device *netdev, - struct cfg80211_chan_def *chandef), - TP_ARGS(netdev, chandef), + struct cfg80211_chan_def *chandef, + unsigned int link_id), + TP_ARGS(netdev, chandef, link_id), TP_STRUCT__entry( NETDEV_ENTRY CHAN_DEF_ENTRY + __field(unsigned int, link_id) ), TP_fast_assign( NETDEV_ASSIGN; CHAN_DEF_ASSIGN(chandef); + __entry->link_id = link_id; ), - TP_printk(NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT, - NETDEV_PR_ARG, CHAN_DEF_PR_ARG) + TP_printk(NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT ", link:%d", + NETDEV_PR_ARG, CHAN_DEF_PR_ARG, __entry->link_id) ); TRACE_EVENT(cfg80211_ch_switch_started_notify, diff --git a/net/wireless/util.c b/net/wireless/util.c index a60d7d638e72..b7257862e0fe 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -5,7 +5,7 @@ * Copyright 2007-2009 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2017 Intel Deutschland GmbH - * Copyright (C) 2018-2021 Intel Corporation + * Copyright (C) 2018-2022 Intel Corporation */ #include #include @@ -1041,7 +1041,6 @@ int cfg80211_change_iface(struct cfg80211_registered_device *rdev, return -EBUSY; dev->ieee80211_ptr->use_4addr = false; - dev->ieee80211_ptr->mesh_id_up_len = 0; wdev_lock(dev->ieee80211_ptr); rdev_set_qos_map(rdev, dev, NULL); wdev_unlock(dev->ieee80211_ptr); @@ -1049,7 +1048,7 @@ int cfg80211_change_iface(struct cfg80211_registered_device *rdev, switch (otype) { case NL80211_IFTYPE_AP: case NL80211_IFTYPE_P2P_GO: - cfg80211_stop_ap(rdev, dev, true); + cfg80211_stop_ap(rdev, dev, -1, true); break; case NL80211_IFTYPE_ADHOC: cfg80211_leave_ibss(rdev, dev, false); @@ -1073,6 +1072,11 @@ int cfg80211_change_iface(struct cfg80211_registered_device *rdev, cfg80211_process_rdev_events(rdev); cfg80211_mlme_purge_registrations(dev->ieee80211_ptr); + + memset(&dev->ieee80211_ptr->u, 0, + sizeof(dev->ieee80211_ptr->u)); + memset(&dev->ieee80211_ptr->links, 0, + sizeof(dev->ieee80211_ptr->links)); } err = rdev_change_virtual_intf(rdev, dev, ntype, params); @@ -1930,6 +1934,24 @@ bool ieee80211_chandef_to_operating_class(struct cfg80211_chan_def *chandef, } EXPORT_SYMBOL(ieee80211_chandef_to_operating_class); +static int cfg80211_wdev_bi(struct wireless_dev *wdev) +{ + switch (wdev->iftype) { + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + WARN_ON(wdev->valid_links); + return wdev->links[0].ap.beacon_interval; + case NL80211_IFTYPE_MESH_POINT: + return wdev->u.mesh.beacon_interval; + case NL80211_IFTYPE_ADHOC: + return wdev->u.ibss.beacon_interval; + default: + break; + } + + return 0; +} + static void cfg80211_calculate_bi_data(struct wiphy *wiphy, u32 new_beacon_int, u32 *beacon_int_gcd, bool *beacon_int_different) @@ -1940,19 +1962,27 @@ static void cfg80211_calculate_bi_data(struct wiphy *wiphy, u32 new_beacon_int, *beacon_int_different = false; list_for_each_entry(wdev, &wiphy->wdev_list, list) { - if (!wdev->beacon_interval) + int wdev_bi; + + /* this feature isn't supported with MLO */ + if (wdev->valid_links) + continue; + + wdev_bi = cfg80211_wdev_bi(wdev); + + if (!wdev_bi) continue; if (!*beacon_int_gcd) { - *beacon_int_gcd = wdev->beacon_interval; + *beacon_int_gcd = wdev_bi; continue; } - if (wdev->beacon_interval == *beacon_int_gcd) + if (wdev_bi == *beacon_int_gcd) continue; *beacon_int_different = true; - *beacon_int_gcd = gcd(*beacon_int_gcd, wdev->beacon_interval); + *beacon_int_gcd = gcd(*beacon_int_gcd, wdev_bi); } if (new_beacon_int && *beacon_int_gcd != new_beacon_int) { diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c index a32065d600a1..a9767bfe7330 100644 --- a/net/wireless/wext-compat.c +++ b/net/wireless/wext-compat.c @@ -7,7 +7,7 @@ * we directly assign the wireless handlers of wireless interfaces. * * Copyright 2008-2009 Johannes Berg - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2022 Intel Corporation */ #include @@ -415,6 +415,9 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, int err, i; bool rejoin = false; + if (wdev->valid_links) + return -EINVAL; + if (pairwise && !addr) return -EINVAL; @@ -437,7 +440,7 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, return -EOPNOTSUPP; if (params->cipher == WLAN_CIPHER_SUITE_AES_CMAC) { - if (!wdev->current_bss) + if (!wdev->connected) return -ENOLINK; if (!rdev->ops->set_default_mgmt_key) @@ -450,7 +453,9 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, if (remove) { err = 0; - if (wdev->current_bss) { + if (wdev->connected || + (wdev->iftype == NL80211_IFTYPE_ADHOC && + wdev->u.ibss.current_bss)) { /* * If removing the current TX key, we will need to * join a new IBSS without the privacy bit clear. @@ -501,7 +506,9 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, return -EINVAL; err = 0; - if (wdev->current_bss) + if (wdev->connected || + (wdev->iftype == NL80211_IFTYPE_ADHOC && + wdev->u.ibss.current_bss)) err = rdev_add_key(rdev, dev, idx, pairwise, addr, params); else if (params->cipher != WLAN_CIPHER_SUITE_WEP40 && params->cipher != WLAN_CIPHER_SUITE_WEP104) @@ -526,7 +533,9 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, if ((params->cipher == WLAN_CIPHER_SUITE_WEP40 || params->cipher == WLAN_CIPHER_SUITE_WEP104) && (tx_key || (!addr && wdev->wext.default_key == -1))) { - if (wdev->current_bss) { + if (wdev->connected || + (wdev->iftype == NL80211_IFTYPE_ADHOC && + wdev->u.ibss.current_bss)) { /* * If we are getting a new TX key from not having * had one before we need to join a new IBSS with @@ -549,7 +558,9 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, if (params->cipher == WLAN_CIPHER_SUITE_AES_CMAC && (tx_key || (!addr && wdev->wext.default_mgmt_key == -1))) { - if (wdev->current_bss) + if (wdev->connected || + (wdev->iftype == NL80211_IFTYPE_ADHOC && + wdev->u.ibss.current_bss)) err = rdev_set_default_mgmt_key(rdev, dev, idx); if (!err) wdev->wext.default_mgmt_key = idx; @@ -595,6 +606,11 @@ static int cfg80211_wext_siwencode(struct net_device *dev, return -EOPNOTSUPP; wiphy_lock(&rdev->wiphy); + if (wdev->valid_links) { + err = -EOPNOTSUPP; + goto out; + } + idx = erq->flags & IW_ENCODE_INDEX; if (idx == 0) { idx = wdev->wext.default_key; @@ -613,7 +629,9 @@ static int cfg80211_wext_siwencode(struct net_device *dev, /* No key data - just set the default TX key index */ err = 0; wdev_lock(wdev); - if (wdev->current_bss) + if (wdev->connected || + (wdev->iftype == NL80211_IFTYPE_ADHOC && + wdev->u.ibss.current_bss)) err = rdev_set_default_key(rdev, dev, idx, true, true); if (!err) @@ -865,7 +883,7 @@ static int cfg80211_wext_giwfreq(struct net_device *dev, break; } - ret = rdev_get_channel(rdev, wdev, &chandef); + ret = rdev_get_channel(rdev, wdev, 0, &chandef); if (ret) break; freq->m = chandef.chan->center_freq; @@ -1270,7 +1288,10 @@ static int cfg80211_wext_siwrate(struct net_device *dev, return -EINVAL; wiphy_lock(&rdev->wiphy); - ret = rdev_set_bitrate_mask(rdev, dev, NULL, &mask); + if (dev->ieee80211_ptr->valid_links) + ret = -EOPNOTSUPP; + else + ret = rdev_set_bitrate_mask(rdev, dev, 0, NULL, &mask); wiphy_unlock(&rdev->wiphy); return ret; @@ -1294,8 +1315,9 @@ static int cfg80211_wext_giwrate(struct net_device *dev, err = 0; wdev_lock(wdev); - if (wdev->current_bss) - memcpy(addr, wdev->current_bss->pub.bssid, ETH_ALEN); + if (!wdev->valid_links && wdev->links[0].client.current_bss) + memcpy(addr, wdev->links[0].client.current_bss->pub.bssid, + ETH_ALEN); else err = -EOPNOTSUPP; wdev_unlock(wdev); @@ -1339,11 +1361,11 @@ static struct iw_statistics *cfg80211_wireless_stats(struct net_device *dev) /* Grab BSSID of current BSS, if any */ wdev_lock(wdev); - if (!wdev->current_bss) { + if (wdev->valid_links || !wdev->links[0].client.current_bss) { wdev_unlock(wdev); return NULL; } - memcpy(bssid, wdev->current_bss->pub.bssid, ETH_ALEN); + memcpy(bssid, wdev->links[0].client.current_bss->pub.bssid, ETH_ALEN); wdev_unlock(wdev); memset(&sinfo, 0, sizeof(sinfo)); diff --git a/net/wireless/wext-sme.c b/net/wireless/wext-sme.c index cd09a9042261..68f45afc352d 100644 --- a/net/wireless/wext-sme.c +++ b/net/wireless/wext-sme.c @@ -3,7 +3,7 @@ * cfg80211 wext compat for managed mode. * * Copyright 2009 Johannes Berg - * Copyright (C) 2009, 2020-2021 Intel Corporation. + * Copyright (C) 2009, 2020-2022 Intel Corporation */ #include @@ -124,9 +124,12 @@ int cfg80211_mgd_wext_giwfreq(struct net_device *dev, if (WARN_ON(wdev->iftype != NL80211_IFTYPE_STATION)) return -EINVAL; + if (wdev->valid_links) + return -EOPNOTSUPP; + wdev_lock(wdev); - if (wdev->current_bss) - chan = wdev->current_bss->pub.channel; + if (wdev->links[0].client.current_bss) + chan = wdev->links[0].client.current_bss->pub.channel; else if (wdev->wext.connect.channel) chan = wdev->wext.connect.channel; wdev_unlock(wdev); @@ -208,15 +211,19 @@ int cfg80211_mgd_wext_giwessid(struct net_device *dev, if (WARN_ON(wdev->iftype != NL80211_IFTYPE_STATION)) return -EINVAL; + if (wdev->valid_links) + return -EINVAL; + data->flags = 0; wdev_lock(wdev); - if (wdev->current_bss) { + if (wdev->links[0].client.current_bss) { const struct element *ssid_elem; rcu_read_lock(); - ssid_elem = ieee80211_bss_get_elem(&wdev->current_bss->pub, - WLAN_EID_SSID); + ssid_elem = ieee80211_bss_get_elem( + &wdev->links[0].client.current_bss->pub, + WLAN_EID_SSID); if (ssid_elem) { data->flags = 1; data->length = ssid_elem->datalen; @@ -300,8 +307,14 @@ int cfg80211_mgd_wext_giwap(struct net_device *dev, ap_addr->sa_family = ARPHRD_ETHER; wdev_lock(wdev); - if (wdev->current_bss) - memcpy(ap_addr->sa_data, wdev->current_bss->pub.bssid, ETH_ALEN); + if (wdev->valid_links) { + wdev_unlock(wdev); + return -EOPNOTSUPP; + } + if (wdev->links[0].client.current_bss) + memcpy(ap_addr->sa_data, + wdev->links[0].client.current_bss->pub.bssid, + ETH_ALEN); else eth_zero_addr(ap_addr->sa_data); wdev_unlock(wdev); -- cgit v1.2.3 From 0f48b8b88aa9ed7b65d7cb55dbc57ec914ddada1 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Tue, 31 May 2022 14:03:38 +0200 Subject: wifi: ieee80211: add definitions for multi-link element Add the definitions necessary to build and parse some of the multi-link element, the per-STA profile isn't fully included. Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 222 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index e15771965916..870f659087d6 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4379,4 +4379,226 @@ enum ieee80211_range_params_max_total_ltf { /* multi-link device */ #define IEEE80211_MLD_MAX_NUM_LINKS 15 +#define IEEE80211_ML_CONTROL_TYPE 0x0007 +#define IEEE80211_ML_CONTROL_TYPE_BASIC 0 +#define IEEE80211_ML_CONTROL_TYPE_PREQ 1 +#define IEEE80211_ML_CONTROL_TYPE_RECONF 2 +#define IEEE80211_ML_CONTROL_TYPE_TDLS 3 +#define IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS 4 +#define IEEE80211_ML_CONTROL_PRESENCE_MASK 0xfff0 + +struct ieee80211_multi_link_elem { + __le16 control; + u8 variable[]; +} __packed; + +#define IEEE80211_MLC_BASIC_PRES_LINK_ID 0x0010 +#define IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT 0x0020 +#define IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY 0x0040 +#define IEEE80211_MLC_BASIC_PRES_EML_CAPA 0x0080 +#define IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP 0x0100 +#define IEEE80211_MLC_BASIC_PRES_MLD_ID 0x0200 + +#define IEEE80211_MED_SYNC_DELAY_DURATION 0x00ff +#define IEEE80211_MED_SYNC_DELAY_SYNC_OFDM_ED_THRESH 0x0f00 +#define IEEE80211_MED_SYNC_DELAY_SYNC_MAX_NUM_TXOPS 0xf000 + +#define IEEE80211_EML_CAP_EMLSR_SUPP 0x0001 +#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY 0x000e +#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_0US 0 +#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US 1 +#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_64US 2 +#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_128US 3 +#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US 4 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY 0x0070 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_0US 0 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_16US 1 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_32US 2 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US 3 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_128US 4 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US 5 +#define IEEE80211_EML_CAP_EMLMR_SUPPORT 0x0080 +#define IEEE80211_EML_CAP_EMLMR_DELAY 0x0700 +#define IEEE80211_EML_CAP_EMLMR_DELAY_0US 0 +#define IEEE80211_EML_CAP_EMLMR_DELAY_32US 1 +#define IEEE80211_EML_CAP_EMLMR_DELAY_64US 2 +#define IEEE80211_EML_CAP_EMLMR_DELAY_128US 3 +#define IEEE80211_EML_CAP_EMLMR_DELAY_256US 4 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT 0x7800 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_0 0 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128US 1 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_256US 2 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_512US 3 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_1TU 4 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_2TU 5 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_4TU 6 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_8TU 7 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_16TU 8 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_32TU 9 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU 10 +#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU 11 + +#define IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS 0x000f +#define IEEE80211_MLD_CAP_OP_SRS_SUPPORT 0x0010 +#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP 0x0060 +#define IEEE80211_MLD_CAP_OP_FREQ_SEP_TYPE_IND 0x0f80 +#define IEEE80211_MLD_CAP_OP_AAR_SUPPORT 0x1000 + +struct ieee80211_mle_basic_common_info { + u8 len; + u8 mld_mac_addr[ETH_ALEN]; + u8 variable[]; +} __packed; + +#define IEEE80211_MLC_PREQ_PRES_MLD_ID 0x0010 + +struct ieee80211_mle_preq_common_info { + u8 len; + u8 variable[]; +} __packed; + +#define IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR 0x0010 + +/* no fixed fields in RECONF */ + +struct ieee80211_mle_tdls_common_info { + u8 len; + u8 ap_mld_mac_addr[ETH_ALEN]; +} __packed; + +#define IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR 0x0010 + +/* no fixed fields in PRIO_ACCESS */ + +/** + * ieee80211_mle_common_size - check multi-link element common size + * @data: multi-link element, must already be checked for size using + * ieee80211_mle_size_ok() + */ +static inline u8 ieee80211_mle_common_size(const u8 *data) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control = le16_to_cpu(mle->control); + u8 common = 0; + + switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) { + case IEEE80211_ML_CONTROL_TYPE_BASIC: + common += sizeof(struct ieee80211_mle_basic_common_info); + break; + case IEEE80211_ML_CONTROL_TYPE_PREQ: + common += sizeof(struct ieee80211_mle_preq_common_info); + break; + case IEEE80211_ML_CONTROL_TYPE_RECONF: + if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR) + common += ETH_ALEN; + return common; + case IEEE80211_ML_CONTROL_TYPE_TDLS: + common += sizeof(struct ieee80211_mle_tdls_common_info); + break; + case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS: + if (control & IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR) + common += ETH_ALEN; + return common; + default: + WARN_ON(1); + return 0; + } + + return common + mle->variable[0]; +} + +/** + * ieee80211_mle_size_ok - validate multi-link element size + * @data: pointer to the element data + * @len: length of the containing element + */ +static inline bool ieee80211_mle_size_ok(const u8 *data, u8 len) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u8 fixed = sizeof(*mle); + u8 common = 0; + bool check_common_len = false; + u16 control; + + if (len < fixed) + return false; + + control = le16_to_cpu(mle->control); + + switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) { + case IEEE80211_ML_CONTROL_TYPE_BASIC: + common += sizeof(struct ieee80211_mle_basic_common_info); + check_common_len = true; + if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_MLD_ID) + common += 1; + break; + case IEEE80211_ML_CONTROL_TYPE_PREQ: + common += sizeof(struct ieee80211_mle_preq_common_info); + if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID) + common += 1; + check_common_len = true; + break; + case IEEE80211_ML_CONTROL_TYPE_RECONF: + if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR) + common += ETH_ALEN; + break; + case IEEE80211_ML_CONTROL_TYPE_TDLS: + common += sizeof(struct ieee80211_mle_tdls_common_info); + check_common_len = true; + break; + case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS: + if (control & IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR) + common += ETH_ALEN; + break; + default: + /* we don't know this type */ + return true; + } + + if (len < fixed + common) + return false; + + if (!check_common_len) + return true; + + /* if present, common length is the first octet there */ + return mle->variable[0] >= common; +} + +enum ieee80211_mle_subelems { + IEEE80211_MLE_SUBELEM_PER_STA_PROFILE = 0, +}; + +#define IEEE80211_MLE_STA_CONTROL_LINK_ID 0x000f +#define IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE 0x0010 +#define IEEE80211_MLE_STA_CONTROL_STA_MAC_ADDR_PRESENT 0x0020 +#define IEEE80211_MLE_STA_CONTROL_BEACON_INT_PRESENT 0x0040 +#define IEEE80211_MLE_STA_CONTROL_TSF_OFFS_PRESENT 0x0080 +#define IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT 0x0100 +#define IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT 0x0200 +#define IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE 0x0400 +#define IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT 0x0800 + +struct ieee80211_mle_per_sta_profile { + __le16 control; + u8 sta_info_len; + u8 variable[]; +} __packed; + +#define for_each_mle_subelement(_elem, _data, _len) \ + if (ieee80211_mle_size_ok(_data, _len)) \ + for_each_element(_elem, \ + _data + ieee80211_mle_common_size(_data),\ + _len - ieee80211_mle_common_size(_data)) + #endif /* LINUX_IEEE80211_H */ -- cgit v1.2.3 From 965b57b469a589d64d81b1688b38dcb537011bb0 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Wed, 15 Jun 2022 09:20:12 -0700 Subject: net: Introduce a new proto_ops ->read_skb() Currently both splice() and sockmap use ->read_sock() to read skb from receive queue, but for sockmap we only read one entire skb at a time, so ->read_sock() is too conservative to use. Introduce a new proto_ops ->read_skb() which supports this sematic, with this we can finally pass the ownership of skb to recv actors. For non-TCP protocols, all ->read_sock() can be simply converted to ->read_skb(). Signed-off-by: Cong Wang Signed-off-by: Daniel Borkmann Reviewed-by: John Fastabend Link: https://lore.kernel.org/bpf/20220615162014.89193-3-xiyou.wangcong@gmail.com --- include/linux/net.h | 4 ++++ include/net/tcp.h | 3 +-- include/net/udp.h | 3 +-- net/core/skmsg.c | 20 +++++--------------- net/ipv4/af_inet.c | 3 ++- net/ipv4/tcp.c | 9 +++------ net/ipv4/udp.c | 10 ++++------ net/ipv6/af_inet6.c | 3 ++- net/unix/af_unix.c | 23 +++++++++-------------- 9 files changed, 31 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/include/linux/net.h b/include/linux/net.h index 12093f4db50c..a03485e8cbb2 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -152,6 +152,8 @@ struct module; struct sk_buff; typedef int (*sk_read_actor_t)(read_descriptor_t *, struct sk_buff *, unsigned int, size_t); +typedef int (*skb_read_actor_t)(struct sock *, struct sk_buff *); + struct proto_ops { int family; @@ -214,6 +216,8 @@ struct proto_ops { */ int (*read_sock)(struct sock *sk, read_descriptor_t *desc, sk_read_actor_t recv_actor); + /* This is different from read_sock(), it reads an entire skb at a time. */ + int (*read_skb)(struct sock *sk, skb_read_actor_t recv_actor); int (*sendpage_locked)(struct sock *sk, struct page *page, int offset, size_t size, int flags); int (*sendmsg_locked)(struct sock *sk, struct msghdr *msg, diff --git a/include/net/tcp.h b/include/net/tcp.h index 7547d90fbb57..8e48dc56837b 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -672,8 +672,7 @@ void tcp_get_info(struct sock *, struct tcp_info *); /* Read 'sendfile()'-style from a TCP socket */ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, sk_read_actor_t recv_actor); -int tcp_read_skb(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor); +int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); void tcp_initialize_rcv_mss(struct sock *sk); diff --git a/include/net/udp.h b/include/net/udp.h index b60eea2e3fae..987f7fc7c0aa 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -306,8 +306,7 @@ struct sock *__udp6_lib_lookup(struct net *net, struct sk_buff *skb); struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb, __be16 sport, __be16 dport); -int udp_read_sock(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor); +int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); /* UDP uses skb->dev_scratch to cache as much information as possible and avoid * possibly multiple cache miss on dequeue() diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 7e03f96e441b..f7f63b7d990c 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1160,21 +1160,17 @@ static void sk_psock_done_strp(struct sk_psock *psock) } #endif /* CONFIG_BPF_STREAM_PARSER */ -static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb, - unsigned int offset, size_t orig_len) +static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb) { - struct sock *sk = (struct sock *)desc->arg.data; struct sk_psock *psock; struct bpf_prog *prog; int ret = __SK_DROP; - int len = orig_len; + int len = skb->len; /* clone here so sk_eat_skb() in tcp_read_sock does not drop our data */ skb = skb_clone(skb, GFP_ATOMIC); - if (!skb) { - desc->error = -ENOMEM; + if (!skb) return 0; - } rcu_read_lock(); psock = sk_psock(sk); @@ -1204,16 +1200,10 @@ out: static void sk_psock_verdict_data_ready(struct sock *sk) { struct socket *sock = sk->sk_socket; - read_descriptor_t desc; - if (unlikely(!sock || !sock->ops || !sock->ops->read_sock)) + if (unlikely(!sock || !sock->ops || !sock->ops->read_skb)) return; - - desc.arg.data = sk; - desc.error = 0; - desc.count = 1; - - sock->ops->read_sock(sk, &desc, sk_psock_verdict_recv); + sock->ops->read_skb(sk, sk_psock_verdict_recv); } void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index da81f56fdd1c..7abd652a558f 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1040,6 +1040,7 @@ const struct proto_ops inet_stream_ops = { .sendpage = inet_sendpage, .splice_read = tcp_splice_read, .read_sock = tcp_read_sock, + .read_skb = tcp_read_skb, .sendmsg_locked = tcp_sendmsg_locked, .sendpage_locked = tcp_sendpage_locked, .peek_len = tcp_peek_len, @@ -1067,7 +1068,7 @@ const struct proto_ops inet_dgram_ops = { .setsockopt = sock_common_setsockopt, .getsockopt = sock_common_getsockopt, .sendmsg = inet_sendmsg, - .read_sock = udp_read_sock, + .read_skb = udp_read_skb, .recvmsg = inet_recvmsg, .mmap = sock_no_mmap, .sendpage = inet_sendpage, diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 124f384f8695..9d2fd3ced21b 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1734,8 +1734,7 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, } EXPORT_SYMBOL(tcp_read_sock); -int tcp_read_skb(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor) +int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor) { struct tcp_sock *tp = tcp_sk(sk); u32 seq = tp->copied_seq; @@ -1750,7 +1749,7 @@ int tcp_read_skb(struct sock *sk, read_descriptor_t *desc, int used; __skb_unlink(skb, &sk->sk_receive_queue); - used = recv_actor(desc, skb, 0, skb->len); + used = recv_actor(sk, skb); if (used <= 0) { if (!copied) copied = used; @@ -1765,9 +1764,7 @@ int tcp_read_skb(struct sock *sk, read_descriptor_t *desc, break; } consume_skb(skb); - if (!desc->count) - break; - WRITE_ONCE(tp->copied_seq, seq); + break; } WRITE_ONCE(tp->copied_seq, seq); diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 6172b4750a88..c660b0bc4d14 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1797,8 +1797,7 @@ busy_check: } EXPORT_SYMBOL(__skb_recv_udp); -int udp_read_sock(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor) +int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor) { int copied = 0; @@ -1820,7 +1819,7 @@ int udp_read_sock(struct sock *sk, read_descriptor_t *desc, continue; } - used = recv_actor(desc, skb, 0, skb->len); + used = recv_actor(sk, skb); if (used <= 0) { if (!copied) copied = used; @@ -1831,13 +1830,12 @@ int udp_read_sock(struct sock *sk, read_descriptor_t *desc, } kfree_skb(skb); - if (!desc->count) - break; + break; } return copied; } -EXPORT_SYMBOL(udp_read_sock); +EXPORT_SYMBOL(udp_read_skb); /* * This should be easy, if there is something there we diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 658823e91eca..0ee0770e79aa 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -702,6 +702,7 @@ const struct proto_ops inet6_stream_ops = { .sendpage_locked = tcp_sendpage_locked, .splice_read = tcp_splice_read, .read_sock = tcp_read_sock, + .read_skb = tcp_read_skb, .peek_len = tcp_peek_len, #ifdef CONFIG_COMPAT .compat_ioctl = inet6_compat_ioctl, @@ -727,7 +728,7 @@ const struct proto_ops inet6_dgram_ops = { .getsockopt = sock_common_getsockopt, /* ok */ .sendmsg = inet6_sendmsg, /* retpoline's sake */ .recvmsg = inet6_recvmsg, /* retpoline's sake */ - .read_sock = udp_read_sock, + .read_skb = udp_read_skb, .mmap = sock_no_mmap, .sendpage = sock_no_sendpage, .set_peek_off = sk_set_peek_off, diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 3453e0053f76..1bed3739768c 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -741,10 +741,8 @@ static ssize_t unix_stream_splice_read(struct socket *, loff_t *ppos, unsigned int flags); static int unix_dgram_sendmsg(struct socket *, struct msghdr *, size_t); static int unix_dgram_recvmsg(struct socket *, struct msghdr *, size_t, int); -static int unix_read_sock(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor); -static int unix_stream_read_sock(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor); +static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor); +static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor); static int unix_dgram_connect(struct socket *, struct sockaddr *, int, int); static int unix_seqpacket_sendmsg(struct socket *, struct msghdr *, size_t); @@ -798,7 +796,7 @@ static const struct proto_ops unix_stream_ops = { .shutdown = unix_shutdown, .sendmsg = unix_stream_sendmsg, .recvmsg = unix_stream_recvmsg, - .read_sock = unix_stream_read_sock, + .read_skb = unix_stream_read_skb, .mmap = sock_no_mmap, .sendpage = unix_stream_sendpage, .splice_read = unix_stream_splice_read, @@ -823,7 +821,7 @@ static const struct proto_ops unix_dgram_ops = { .listen = sock_no_listen, .shutdown = unix_shutdown, .sendmsg = unix_dgram_sendmsg, - .read_sock = unix_read_sock, + .read_skb = unix_read_skb, .recvmsg = unix_dgram_recvmsg, .mmap = sock_no_mmap, .sendpage = sock_no_sendpage, @@ -2487,8 +2485,7 @@ static int unix_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t si return __unix_dgram_recvmsg(sk, msg, size, flags); } -static int unix_read_sock(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor) +static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor) { int copied = 0; @@ -2503,7 +2500,7 @@ static int unix_read_sock(struct sock *sk, read_descriptor_t *desc, if (!skb) return err; - used = recv_actor(desc, skb, 0, skb->len); + used = recv_actor(sk, skb); if (used <= 0) { if (!copied) copied = used; @@ -2514,8 +2511,7 @@ static int unix_read_sock(struct sock *sk, read_descriptor_t *desc, } kfree_skb(skb); - if (!desc->count) - break; + break; } return copied; @@ -2650,13 +2646,12 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, } #endif -static int unix_stream_read_sock(struct sock *sk, read_descriptor_t *desc, - sk_read_actor_t recv_actor) +static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor) { if (unlikely(sk->sk_state != TCP_ESTABLISHED)) return -ENOTCONN; - return unix_read_sock(sk, desc, recv_actor); + return unix_read_skb(sk, recv_actor); } static int unix_stream_read_generic(struct unix_stream_read_state *state, -- cgit v1.2.3 From 414c12385d4741e35d88670c6cc2f40a77809734 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 13 Apr 2022 15:17:25 -0700 Subject: rcu: Provide a get_completed_synchronize_rcu() function It is currently up to the caller to handle stale return values from get_state_synchronize_rcu(). If poll_state_synchronize_rcu() returned true once, a grace period has elapsed, regardless of the fact that counter wrap might cause some future poll_state_synchronize_rcu() invocation to return false. For example, the caller might store a separate flag that indicates whether some previous call to poll_state_synchronize_rcu() determined that the relevant grace period had already ended. This approach works, but it requires extra storage and is easy to get wrong. This commit therefore introduces a get_completed_synchronize_rcu() that returns a cookie that causes poll_state_synchronize_rcu() to always return true. This already-completed cookie can be stored in place of the cookie that previously caused poll_state_synchronize_rcu() to return true. It can also be used to flag a given structure as not having been exposed to readers, and thus not requiring a grace period to elapse. This commit is in preparation for polled expedited grace periods. Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 1 + kernel/rcu/rcu.h | 3 +++ kernel/rcu/tiny.c | 4 ++-- kernel/rcu/tree.c | 3 ++- kernel/rcu/update.c | 13 +++++++++++++ 5 files changed, 21 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1a32036c918c..7f12daa4708b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -41,6 +41,7 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void rcu_barrier_tasks_rude(void); void synchronize_rcu(void); +unsigned long get_completed_synchronize_rcu(void); #ifdef CONFIG_PREEMPT_RCU diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 0adb55941aeb..32291f4eefde 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -23,6 +23,9 @@ #define RCU_SEQ_CTR_SHIFT 2 #define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) +/* Low-order bit definition for polled grace-period APIs. */ +#define RCU_GET_STATE_COMPLETED 0x1 + extern int sysctl_sched_rt_runtime; /* diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 340b3f8b090d..dbee6bea6726 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -58,7 +58,7 @@ void rcu_qs(void) rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; raise_softirq_irqoff(RCU_SOFTIRQ); } - WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 1); + WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 2); local_irq_restore(flags); } @@ -213,7 +213,7 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); */ bool poll_state_synchronize_rcu(unsigned long oldstate) { - return READ_ONCE(rcu_ctrlblk.gp_seq) != oldstate; + return oldstate == RCU_GET_STATE_COMPLETED || READ_ONCE(rcu_ctrlblk.gp_seq) != oldstate; } EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index ec28e259774e..46cfceea8784 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3924,7 +3924,8 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); */ bool poll_state_synchronize_rcu(unsigned long oldstate) { - if (rcu_seq_done_exact(&rcu_state.gp_seq, oldstate)) { + if (oldstate == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rcu_state.gp_seq, oldstate)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index fc7fef575606..2e93acad1e31 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -516,6 +516,19 @@ int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls. EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot); module_param(rcu_cpu_stall_suppress_at_boot, int, 0444); +/** + * get_completed_synchronize_rcu - Return a pre-completed polled state cookie + * + * Returns a value that will always be treated by functions like + * poll_state_synchronize_rcu() as a cookie whose grace period has already + * completed. + */ +unsigned long get_completed_synchronize_rcu(void) +{ + return RCU_GET_STATE_COMPLETED; +} +EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu); + #ifdef CONFIG_PROVE_RCU /* -- cgit v1.2.3 From 3847b64570b1753e9863a4106aec03f4d68e7b17 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 23 May 2022 20:50:11 -0700 Subject: rcu-tasks: Merge state into .b.need_qs and atomically update This commit gets rid of the task_struct structure's ->trc_reader_checked field, making it instead be a bit within the task_struct structure's existing ->trc_reader_special.b.need_qs field. This commit also atomically loads, stores, and checks the resulting combination of the reader-checked and need-quiescent state flags. This will in turn allow significant simplification of the rcu_tasks_trace_postgp() function as well as elimination of the trc_n_readers_need_end counter in later commits. These changes will in turn simplify later elimination of the RCU Tasks Trace scan of the task list, which will make RCU Tasks Trace grace periods less CPU-intensive. [ paulmck: Apply kernel test robot feedback. ] Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/rcupdate.h | 18 +++++---- include/linux/sched.h | 1 - kernel/rcu/tasks.h | 103 ++++++++++++++++++++++++++++++++--------------- 3 files changed, 82 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1a32036c918c..1e728d544fc1 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -169,13 +169,17 @@ void synchronize_rcu_tasks(void); # endif # ifdef CONFIG_TASKS_TRACE_RCU -# define rcu_tasks_trace_qs(t) \ - do { \ - if (!likely(READ_ONCE((t)->trc_reader_checked)) && \ - !unlikely(READ_ONCE((t)->trc_reader_nesting))) { \ - smp_store_release(&(t)->trc_reader_checked, true); \ - smp_mb(); /* Readers partitioned by store. */ \ - } \ +// Bits for ->trc_reader_special.b.need_qs field. +#define TRC_NEED_QS 0x1 // Task needs a quiescent state. +#define TRC_NEED_QS_CHECKED 0x2 // Task has been checked for needing quiescent state. + +u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new); + +# define rcu_tasks_trace_qs(t) \ + do { \ + if (likely(!READ_ONCE((t)->trc_reader_special.b.need_qs)) && \ + likely(!READ_ONCE((t)->trc_reader_nesting))) \ + rcu_trc_cmpxchg_need_qs((t), 0, TRC_NEED_QS_CHECKED); \ } while (0) # else # define rcu_tasks_trace_qs(t) do { } while (0) diff --git a/include/linux/sched.h b/include/linux/sched.h index c46f3a63b758..e6eb5871593e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -843,7 +843,6 @@ struct task_struct { int trc_reader_nesting; int trc_ipi_to_cpu; union rcu_special trc_reader_special; - bool trc_reader_checked; struct list_head trc_holdout_list; #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index bd9f2e24f5c7..7bdc62606816 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1208,6 +1208,39 @@ void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace, "RCU Tasks Trace"); +/* Load from ->trc_reader_special.b.need_qs with proper ordering. */ +static u8 rcu_ld_need_qs(struct task_struct *t) +{ + smp_mb(); // Enforce full grace-period ordering. + return smp_load_acquire(&t->trc_reader_special.b.need_qs); +} + +/* Store to ->trc_reader_special.b.need_qs with proper ordering. */ +static void rcu_st_need_qs(struct task_struct *t, u8 v) +{ + smp_store_release(&t->trc_reader_special.b.need_qs, v); + smp_mb(); // Enforce full grace-period ordering. +} + +/* + * Do a cmpxchg() on ->trc_reader_special.b.need_qs, allowing for + * the four-byte operand-size restriction of some platforms. + * Returns the old value, which is often ignored. + */ +u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new) +{ + union rcu_special ret; + union rcu_special trs_old = READ_ONCE(t->trc_reader_special); + union rcu_special trs_new = trs_old; + + if (trs_old.b.need_qs != old) + return trs_old.b.need_qs; + trs_new.b.need_qs = new; + ret.s = cmpxchg(&t->trc_reader_special.s, trs_old.s, trs_new.s); + return ret.b.need_qs; +} +EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs); + /* * This irq_work handler allows rcu_read_unlock_trace() to be invoked * while the scheduler locks are held. @@ -1221,16 +1254,20 @@ static DEFINE_IRQ_WORK(rcu_tasks_trace_iw, rcu_read_unlock_iw); /* If we are the last reader, wake up the grace-period kthread. */ void rcu_read_unlock_trace_special(struct task_struct *t) { - int nq = READ_ONCE(t->trc_reader_special.b.need_qs); + int nqs = (rcu_ld_need_qs(t) == (TRC_NEED_QS_CHECKED | TRC_NEED_QS)); - if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && - t->trc_reader_special.b.need_mb) + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && t->trc_reader_special.b.need_mb) smp_mb(); // Pairs with update-side barriers. // Update .need_qs before ->trc_reader_nesting for irq/NMI handlers. - if (nq) - WRITE_ONCE(t->trc_reader_special.b.need_qs, false); + if (nqs) { + u8 result = rcu_trc_cmpxchg_need_qs(t, TRC_NEED_QS_CHECKED | TRC_NEED_QS, + TRC_NEED_QS_CHECKED); + + WARN_ONCE(result != (TRC_NEED_QS_CHECKED | TRC_NEED_QS), + "%s: result = %d", __func__, result); + } WRITE_ONCE(t->trc_reader_nesting, 0); - if (nq && atomic_dec_and_test(&trc_n_readers_need_end)) + if (nqs && atomic_dec_and_test(&trc_n_readers_need_end)) irq_work_queue(&rcu_tasks_trace_iw); } EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special); @@ -1260,27 +1297,24 @@ static void trc_read_check_handler(void *t_in) struct task_struct *texp = t_in; // If the task is no longer running on this CPU, leave. - if (unlikely(texp != t)) { + if (unlikely(texp != t)) goto reset_ipi; // Already on holdout list, so will check later. - } // If the task is not in a read-side critical section, and // if this is the last reader, awaken the grace-period kthread. if (likely(!READ_ONCE(t->trc_reader_nesting))) { - WRITE_ONCE(t->trc_reader_checked, true); + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); goto reset_ipi; } // If we are racing with an rcu_read_unlock_trace(), try again later. if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) goto reset_ipi; - WRITE_ONCE(t->trc_reader_checked, true); // Get here if the task is in a read-side critical section. Set // its state so that it will awaken the grace-period kthread upon // exit from that critical section. - atomic_inc(&trc_n_readers_need_end); // One more to wait on. - WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); - WRITE_ONCE(t->trc_reader_special.b.need_qs, true); + if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) + atomic_inc(&trc_n_readers_need_end); // One more to wait on. reset_ipi: // Allow future IPIs to be sent on CPU and for task. @@ -1291,8 +1325,9 @@ reset_ipi: } /* Callback function for scheduler to check locked-down task. */ -static int trc_inspect_reader(struct task_struct *t, void *arg) +static int trc_inspect_reader(struct task_struct *t, void *bhp_in) { + struct list_head *bhp = bhp_in; int cpu = task_cpu(t); int nesting; bool ofl = cpu_is_offline(cpu); @@ -1323,16 +1358,19 @@ static int trc_inspect_reader(struct task_struct *t, void *arg) // If not exiting a read-side critical section, mark as checked // so that the grace-period kthread will remove it from the // holdout list. - t->trc_reader_checked = nesting >= 0; - if (nesting <= 0) + if (nesting <= 0) { + if (!nesting) + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); return nesting ? -EINVAL : 0; // If in QS, done, otherwise try again later. + } // The task is in a read-side critical section, so set up its // state so that it will awaken the grace-period kthread upon exit // from that critical section. - atomic_inc(&trc_n_readers_need_end); // One more to wait on. - WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); - WRITE_ONCE(t->trc_reader_special.b.need_qs, true); + if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) { + atomic_inc(&trc_n_readers_need_end); // One more to wait on. + trc_add_holdout(t, bhp); + } return 0; } @@ -1348,14 +1386,14 @@ static void trc_wait_for_one_reader(struct task_struct *t, // The current task had better be in a quiescent state. if (t == current) { - t->trc_reader_checked = true; + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); return; } // Attempt to nail down the task for inspection. get_task_struct(t); - if (!task_call_func(t, trc_inspect_reader, NULL)) { + if (!task_call_func(t, trc_inspect_reader, bhp)) { put_task_struct(t); return; } @@ -1419,8 +1457,7 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, if (unlikely(t == NULL)) return; - WRITE_ONCE(t->trc_reader_special.b.need_qs, false); - WRITE_ONCE(t->trc_reader_checked, false); + rcu_st_need_qs(t, 0); t->trc_ipi_to_cpu = -1; trc_wait_for_one_reader(t, hop); } @@ -1442,7 +1479,8 @@ static void rcu_tasks_trace_postscan(struct list_head *hop) // Wait for late-stage exiting tasks to finish exiting. // These might have passed the call to exit_tasks_rcu_finish(). synchronize_rcu(); - // Any tasks that exit after this point will set ->trc_reader_checked. + // Any tasks that exit after this point will set + // TRC_NEED_QS_CHECKED in ->trc_reader_special.b.need_qs. } /* Communicate task state back to the RCU tasks trace stall warning request. */ @@ -1460,7 +1498,7 @@ static int trc_check_slow_task(struct task_struct *t, void *arg) return false; // It is running, so decline to inspect it. trc_rdrp->nesting = READ_ONCE(t->trc_reader_nesting); trc_rdrp->ipi_to_cpu = READ_ONCE(t->trc_ipi_to_cpu); - trc_rdrp->needqs = READ_ONCE(t->trc_reader_special.b.need_qs); + trc_rdrp->needqs = rcu_ld_need_qs(t); return true; } @@ -1514,12 +1552,12 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, list_for_each_entry_safe(t, g, hop, trc_holdout_list) { // If safe and needed, try to check the current task. if (READ_ONCE(t->trc_ipi_to_cpu) == -1 && - !READ_ONCE(t->trc_reader_checked)) + !(rcu_ld_need_qs(t) & TRC_NEED_QS_CHECKED)) trc_wait_for_one_reader(t, hop); // If check succeeded, remove this task from the list. if (smp_load_acquire(&t->trc_ipi_to_cpu) == -1 && - READ_ONCE(t->trc_reader_checked)) + rcu_ld_need_qs(t) == TRC_NEED_QS_CHECKED) trc_del_holdout(t); else if (needreport) show_stalled_task_trace(t, firstreport); @@ -1574,12 +1612,12 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) // Stall warning time, so make a list of the offenders. rcu_read_lock(); for_each_process_thread(g, t) - if (READ_ONCE(t->trc_reader_special.b.need_qs)) + if (rcu_ld_need_qs(t) & TRC_NEED_QS) trc_add_holdout(t, &holdouts); rcu_read_unlock(); firstreport = true; list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) { - if (READ_ONCE(t->trc_reader_special.b.need_qs)) + if (rcu_ld_need_qs(t) & TRC_NEED_QS) show_stalled_task_trace(t, &firstreport); trc_del_holdout(t); // Release task_struct reference. } @@ -1595,11 +1633,12 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) /* Report any needed quiescent state for this exiting task. */ static void exit_tasks_rcu_finish_trace(struct task_struct *t) { - WRITE_ONCE(t->trc_reader_checked, true); + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); - WRITE_ONCE(t->trc_reader_nesting, 0); - if (WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs))) + if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS)) rcu_read_unlock_trace_special(t); + else + WRITE_ONCE(t->trc_reader_nesting, 0); } /** -- cgit v1.2.3 From 434c9eefb959c36331a93617ea95df903469b99f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 16 May 2022 17:56:16 -0700 Subject: rcu-tasks: Add data structures for lightweight grace periods This commit adds fields to task_struct and to rcu_tasks_percpu that will be used to avoid the task-list scan for RCU Tasks Trace grace periods, and also initializes these fields. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/sched.h | 2 ++ init/init_task.c | 1 + kernel/fork.c | 1 + kernel/rcu/tasks.h | 4 ++++ 4 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index e6eb5871593e..b88caf54e168 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -844,6 +844,8 @@ struct task_struct { int trc_ipi_to_cpu; union rcu_special trc_reader_special; struct list_head trc_holdout_list; + struct list_head trc_blkd_node; + int trc_blkd_cpu; #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ struct sched_info sched_info; diff --git a/init/init_task.c b/init/init_task.c index 73cc8f03511a..ff6c4b9bfe6b 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -157,6 +157,7 @@ struct task_struct init_task .trc_reader_nesting = 0, .trc_reader_special.s = 0, .trc_holdout_list = LIST_HEAD_INIT(init_task.trc_holdout_list), + .trc_blkd_node = LIST_HEAD_INIT(init_task.trc_blkd_node), #endif #ifdef CONFIG_CPUSETS .mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq, diff --git a/kernel/fork.c b/kernel/fork.c index 9d44f2d46c69..1950eb870244 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1814,6 +1814,7 @@ static inline void rcu_copy_process(struct task_struct *p) p->trc_reader_nesting = 0; p->trc_reader_special.s = 0; INIT_LIST_HEAD(&p->trc_holdout_list); + INIT_LIST_HEAD(&p->trc_blkd_node); #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ } diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 64eb4d7b142e..fd4508af055e 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -29,6 +29,7 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); * @rtp_work: Work queue for invoking callbacks. * @rtp_irq_work: IRQ work queue for deferred wakeups. * @barrier_q_head: RCU callback for barrier operation. + * @rtp_blkd_tasks: List of tasks blocked as readers. * @cpu: CPU number corresponding to this entry. * @rtpp: Pointer to the rcu_tasks structure. */ @@ -40,6 +41,7 @@ struct rcu_tasks_percpu { struct work_struct rtp_work; struct irq_work rtp_irq_work; struct rcu_head barrier_q_head; + struct list_head rtp_blkd_tasks; int cpu; struct rcu_tasks *rtpp; }; @@ -256,6 +258,8 @@ static void cblist_init_generic(struct rcu_tasks *rtp) INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); rtpcp->cpu = cpu; rtpcp->rtpp = rtp; + if (!rtpcp->rtp_blkd_tasks.next) + INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); -- cgit v1.2.3 From 0356d4e66214569de674ab2684f2e0b440a466ab Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 11:30:32 -0700 Subject: rcu-tasks: Track blocked RCU Tasks Trace readers This commit places any task that has ever blocked within its current RCU Tasks Trace read-side critical section on a per-CPU list within the rcu_tasks_percpu structure. Tasks are removed from this list when they exit by the exit_tasks_rcu_finish_trace() function. The purpose of this commit is to provide the information needed to eliminate the current scan of the full task list. This commit offsets the INT_MIN value for ->trc_reader_nesting with the new nesting level in order to avoid queueing tasks that are exiting their read-side critical sections. [ paulmck: Apply kernel test robot feedback. ] [ paulmck: Apply feedback from syzbot+9bb26e7c5e8e4fa7e641@syzkaller.appspotmail.com ] Signed-off-by: Paul E. McKenney Tested-by: syzbot Tested-by: "Zhang, Qiang1" Cc: Peter Zijlstra Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/rcupdate.h | 11 +++++++++-- include/linux/rcupdate_trace.h | 2 +- kernel/rcu/tasks.h | 22 +++++++++++++++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1e728d544fc1..ebdfeead44e5 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -174,12 +174,19 @@ void synchronize_rcu_tasks(void); #define TRC_NEED_QS_CHECKED 0x2 // Task has been checked for needing quiescent state. u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new); +void rcu_tasks_trace_qs_blkd(struct task_struct *t); # define rcu_tasks_trace_qs(t) \ do { \ + int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting); \ + \ if (likely(!READ_ONCE((t)->trc_reader_special.b.need_qs)) && \ - likely(!READ_ONCE((t)->trc_reader_nesting))) \ + likely(!___rttq_nesting)) { \ rcu_trc_cmpxchg_need_qs((t), 0, TRC_NEED_QS_CHECKED); \ + } else if (___rttq_nesting && ___rttq_nesting != INT_MIN && \ + !READ_ONCE((t)->trc_reader_special.b.blocked)) { \ + rcu_tasks_trace_qs_blkd(t); \ + } \ } while (0) # else # define rcu_tasks_trace_qs(t) do { } while (0) @@ -188,7 +195,7 @@ u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new); #define rcu_tasks_qs(t, preempt) \ do { \ rcu_tasks_classic_qs((t), (preempt)); \ - rcu_tasks_trace_qs((t)); \ + rcu_tasks_trace_qs(t); \ } while (0) # ifdef CONFIG_TASKS_RUDE_RCU diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index 6f9c35817398..9bc8cbb33340 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -75,7 +75,7 @@ static inline void rcu_read_unlock_trace(void) nesting = READ_ONCE(t->trc_reader_nesting) - 1; barrier(); // Critical section before disabling. // Disable IPI-based setting of .need_qs. - WRITE_ONCE(t->trc_reader_nesting, INT_MIN); + WRITE_ONCE(t->trc_reader_nesting, INT_MIN + nesting); if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) { WRITE_ONCE(t->trc_reader_nesting, nesting); return; // We assume shallow reader nesting. diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fd4508af055e..bab75ec26bdb 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1261,6 +1261,24 @@ void rcu_read_unlock_trace_special(struct task_struct *t) } EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special); +/* Add a newly blocked reader task to its CPU's list. */ +void rcu_tasks_trace_qs_blkd(struct task_struct *t) +{ + unsigned long flags; + struct rcu_tasks_percpu *rtpcp; + + local_irq_save(flags); + rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu); + raw_spin_lock_rcu_node(rtpcp); // irqs already disabled + t->trc_blkd_cpu = smp_processor_id(); + if (!rtpcp->rtp_blkd_tasks.next) + INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); + list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); + t->trc_reader_special.b.blocked = true; + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd); + /* Add a task to the holdout list, if it is not already on the list. */ static void trc_add_holdout(struct task_struct *t, struct list_head *bhp) { @@ -1586,9 +1604,11 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) /* Report any needed quiescent state for this exiting task. */ static void exit_tasks_rcu_finish_trace(struct task_struct *t) { + union rcu_special trs = READ_ONCE(t->trc_reader_special); + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); - if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS)) + if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS) || trs.b.blocked) rcu_read_unlock_trace_special(t); else WRITE_ONCE(t->trc_reader_nesting, 0); -- cgit v1.2.3 From 5f8a62af527ad8a615d68889d816b72c3f5f227f Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:18 +0200 Subject: context_tracking: Remove unused context_tracking_in_user() This function is not used and CT_WARN_ON() coupled with ct_state() is the preferred way to assert context tracking state values. Reported-by: Nicolas Saenz Julienne Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/context_tracking_state.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index ae1e63e26947..edc7b46376a6 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -41,12 +41,7 @@ static inline bool context_tracking_enabled_this_cpu(void) return context_tracking_enabled() && __this_cpu_read(context_tracking.active); } -static __always_inline bool context_tracking_in_user(void) -{ - return __this_cpu_read(context_tracking.state) == CONTEXT_USER; -} #else -static __always_inline bool context_tracking_in_user(void) { return false; } static __always_inline bool context_tracking_enabled(void) { return false; } static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; } static __always_inline bool context_tracking_enabled_this_cpu(void) { return false; } -- cgit v1.2.3 From 1ade23711971b0eececf0d7fedc29d3c1d2fce01 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 21 Jun 2022 02:53:42 +0300 Subject: bpf: Inline calls to bpf_loop when callback is known Calls to `bpf_loop` are replaced with direct loops to avoid indirection. E.g. the following: bpf_loop(10, foo, NULL, 0); Is replaced by equivalent of the following: for (int i = 0; i < 10; ++i) foo(i, NULL); This transformation could be applied when: - callback is known and does not change during program execution; - flags passed to `bpf_loop` are always zero. Inlining logic works as follows: - During execution simulation function `update_loop_inline_state` tracks the following information for each `bpf_loop` call instruction: - is callback known and constant? - are flags constant and zero? - Function `optimize_bpf_loop` increases stack depth for functions where `bpf_loop` calls can be inlined and invokes `inline_bpf_loop` to apply the inlining. The additional stack space is used to spill registers R6, R7 and R8. These registers are used as loop counter, loop maximal bound and callback context parameter; Measurements using `benchs/run_bench_bpf_loop.sh` inside QEMU / KVM on i7-4710HQ CPU show a drop in latency from 14 ns/op to 2 ns/op. Signed-off-by: Eduard Zingerman Acked-by: Song Liu Link: https://lore.kernel.org/r/20220620235344.569325-4-eddyz87@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 3 + include/linux/bpf_verifier.h | 12 +++ kernel/bpf/bpf_iter.c | 9 ++- kernel/bpf/verifier.c | 180 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 195 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0edd7d2c0064..d05e1495a06e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1286,6 +1286,9 @@ struct bpf_array { #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ #define MAX_TAIL_CALL_CNT 33 +/* Maximum number of loops for bpf_loop */ +#define BPF_MAX_LOOPS BIT(23) + #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ BPF_F_RDONLY_PROG | \ BPF_F_WRONLY | \ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 3930c963fa67..81b19669efba 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -344,6 +344,14 @@ struct bpf_verifier_state_list { int miss_cnt, hit_cnt; }; +struct bpf_loop_inline_state { + int initialized:1; /* set to true upon first entry */ + int fit_for_inline:1; /* true if callback function is the same + * at each call and flags are always zero + */ + u32 callback_subprogno; /* valid when fit_for_inline is true */ +}; + /* Possible states for alu_state member. */ #define BPF_ALU_SANITIZE_SRC (1U << 0) #define BPF_ALU_SANITIZE_DST (1U << 1) @@ -373,6 +381,10 @@ struct bpf_insn_aux_data { u32 mem_size; /* mem_size for non-struct typed var */ }; } btf_var; + /* if instruction is a call to bpf_loop this field tracks + * the state of the relevant registers to make decision about inlining + */ + struct bpf_loop_inline_state loop_inline_state; }; u64 map_key_state; /* constant (32 bit) key tracking for maps */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index d5d96ceca105..7e8fd49406f6 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -723,9 +723,6 @@ const struct bpf_func_proto bpf_for_each_map_elem_proto = { .arg4_type = ARG_ANYTHING, }; -/* maximum number of loops */ -#define MAX_LOOPS BIT(23) - BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, u64, flags) { @@ -733,9 +730,13 @@ BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, u64 ret; u32 i; + /* Note: these safety checks are also verified when bpf_loop + * is inlined, be careful to modify this code in sync. See + * function verifier.c:inline_bpf_loop. + */ if (flags) return -EINVAL; - if (nr_loops > MAX_LOOPS) + if (nr_loops > BPF_MAX_LOOPS) return -E2BIG; for (i = 0; i < nr_loops; i++) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2859901ffbe3..bf72dc511df6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7124,6 +7124,41 @@ static int check_get_func_ip(struct bpf_verifier_env *env) return -ENOTSUPP; } +static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) +{ + return &env->insn_aux_data[env->insn_idx]; +} + +static bool loop_flag_is_zero(struct bpf_verifier_env *env) +{ + struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state *reg = ®s[BPF_REG_4]; + bool reg_is_null = register_is_null(reg); + + if (reg_is_null) + mark_chain_precision(env, BPF_REG_4); + + return reg_is_null; +} + +static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) +{ + struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; + + if (!state->initialized) { + state->initialized = 1; + state->fit_for_inline = loop_flag_is_zero(env); + state->callback_subprogno = subprogno; + return; + } + + if (!state->fit_for_inline) + return; + + state->fit_for_inline = (loop_flag_is_zero(env) && + state->callback_subprogno == subprogno); +} + static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) { @@ -7276,6 +7311,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn err = check_bpf_snprintf_call(env, regs); break; case BPF_FUNC_loop: + update_loop_inline_state(env, meta.subprogno); err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, set_loop_callback_state); break; @@ -7682,11 +7718,6 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env, return true; } -static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) -{ - return &env->insn_aux_data[env->insn_idx]; -} - enum { REASON_BOUNDS = -1, REASON_TYPE = -2, @@ -14315,6 +14346,142 @@ patch_call_imm: return 0; } +static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, + int position, + s32 stack_base, + u32 callback_subprogno, + u32 *cnt) +{ + s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; + s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; + s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; + int reg_loop_max = BPF_REG_6; + int reg_loop_cnt = BPF_REG_7; + int reg_loop_ctx = BPF_REG_8; + + struct bpf_prog *new_prog; + u32 callback_start; + u32 call_insn_offset; + s32 callback_offset; + + /* This represents an inlined version of bpf_iter.c:bpf_loop, + * be careful to modify this code in sync. + */ + struct bpf_insn insn_buf[] = { + /* Return error and jump to the end of the patch if + * expected number of iterations is too big. + */ + BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2), + BPF_MOV32_IMM(BPF_REG_0, -E2BIG), + BPF_JMP_IMM(BPF_JA, 0, 0, 16), + /* spill R6, R7, R8 to use these as loop vars */ + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset), + /* initialize loop vars */ + BPF_MOV64_REG(reg_loop_max, BPF_REG_1), + BPF_MOV32_IMM(reg_loop_cnt, 0), + BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3), + /* loop header, + * if reg_loop_cnt >= reg_loop_max skip the loop body + */ + BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5), + /* callback call, + * correct callback offset would be set after patching + */ + BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt), + BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx), + BPF_CALL_REL(0), + /* increment loop counter */ + BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1), + /* jump to loop header if callback returned 0 */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6), + /* return value of bpf_loop, + * set R0 to the number of iterations + */ + BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt), + /* restore original values of R6, R7, R8 */ + BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset), + BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset), + BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset), + }; + + *cnt = ARRAY_SIZE(insn_buf); + new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); + if (!new_prog) + return new_prog; + + /* callback start is known only after patching */ + callback_start = env->subprog_info[callback_subprogno].start; + /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ + call_insn_offset = position + 12; + callback_offset = callback_start - call_insn_offset - 1; + env->prog->insnsi[call_insn_offset].imm = callback_offset; + + return new_prog; +} + +static bool is_bpf_loop_call(struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL) && + insn->src_reg == 0 && + insn->imm == BPF_FUNC_loop; +} + +/* For all sub-programs in the program (including main) check + * insn_aux_data to see if there are bpf_loop calls that require + * inlining. If such calls are found the calls are replaced with a + * sequence of instructions produced by `inline_bpf_loop` function and + * subprog stack_depth is increased by the size of 3 registers. + * This stack space is used to spill values of the R6, R7, R8. These + * registers are used to store the loop bound, counter and context + * variables. + */ +static int optimize_bpf_loop(struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprogs = env->subprog_info; + int i, cur_subprog = 0, cnt, delta = 0; + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; + u16 stack_depth = subprogs[cur_subprog].stack_depth; + u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; + u16 stack_depth_extra = 0; + + for (i = 0; i < insn_cnt; i++, insn++) { + struct bpf_loop_inline_state *inline_state = + &env->insn_aux_data[i + delta].loop_inline_state; + + if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { + struct bpf_prog *new_prog; + + stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; + new_prog = inline_bpf_loop(env, + i + delta, + -(stack_depth + stack_depth_extra), + inline_state->callback_subprogno, + &cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; + } + + if (subprogs[cur_subprog + 1].start == i + delta + 1) { + subprogs[cur_subprog].stack_depth += stack_depth_extra; + cur_subprog++; + stack_depth = subprogs[cur_subprog].stack_depth; + stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; + stack_depth_extra = 0; + } + } + + env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; + + return 0; +} + static void free_states(struct bpf_verifier_env *env) { struct bpf_verifier_state_list *sl, *sln; @@ -15052,6 +15219,9 @@ skip_full_check: ret = check_max_stack_depth(env); /* instruction rewrites happen after this point */ + if (ret == 0) + ret = optimize_bpf_loop(env); + if (is_priv) { if (ret == 0) opt_hard_wire_dead_code_branches(env); -- cgit v1.2.3 From f9aefd6b2aa38b9787d2705f0f1161dfd23cdb8f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 20 Jun 2022 02:30:17 -0700 Subject: net: warn if mac header was not set Make sure skb_mac_header(), skb_mac_offset() and skb_mac_header_len() uses are not fooled if the mac header has not been set. These checks are enabled if CONFIG_DEBUG_NET=y This commit will likely expose existing bugs in linux networking stacks. Signed-off-by: Eric Dumazet Link: https://lore.kernel.org/r/20220620093017.3366713-1-eric.dumazet@gmail.com Signed-off-by: Paolo Abeni --- include/linux/skbuff.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 82edf0359ab3..cd4a8268894a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2763,8 +2763,14 @@ static inline void skb_set_network_header(struct sk_buff *skb, const int offset) skb->network_header += offset; } +static inline int skb_mac_header_was_set(const struct sk_buff *skb) +{ + return skb->mac_header != (typeof(skb->mac_header))~0U; +} + static inline unsigned char *skb_mac_header(const struct sk_buff *skb) { + DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb)); return skb->head + skb->mac_header; } @@ -2775,14 +2781,10 @@ static inline int skb_mac_offset(const struct sk_buff *skb) static inline u32 skb_mac_header_len(const struct sk_buff *skb) { + DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb)); return skb->network_header - skb->mac_header; } -static inline int skb_mac_header_was_set(const struct sk_buff *skb) -{ - return skb->mac_header != (typeof(skb->mac_header))~0U; -} - static inline void skb_unset_mac_header(struct sk_buff *skb) { skb->mac_header = (typeof(skb->mac_header))~0U; -- cgit v1.2.3 From a37599ebfb656c2af4ca119de556eba29b6926d6 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Wed, 15 Jun 2022 17:20:18 +0000 Subject: usb: typec: mux: Add CONFIG guards for functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are some drivers that can use the Type C mux API, but don't have to. Introduce CONFIG guards for the mux functions so that drivers can include the header file and not run into compilation errors on systems which don't have CONFIG_TYPEC enabled. When CONFIG_TYPEC is not enabled, the Type C mux functions will be stub versions of the original calls. Reported-by: kernel test robot Reviewed-by: Nícolas F. R. A. Prado Reviewed-by: Heikki Krogerus Reviewed-by: AngeloGioacchino Del Regno Tested-by: Nícolas F. R. A. Prado Signed-off-by: Prashant Malani Link: https://lore.kernel.org/r/20220615172129.1314056-3-pmalani@chromium.org Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/typec_mux.h | 44 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index ee57781dcf28..9292f0e07846 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -58,17 +58,13 @@ struct typec_mux_desc { void *drvdata; }; +#if IS_ENABLED(CONFIG_TYPEC) + struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode, const struct typec_altmode_desc *desc); void typec_mux_put(struct typec_mux *mux); int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state); -static inline struct typec_mux * -typec_mux_get(struct device *dev, const struct typec_altmode_desc *desc) -{ - return fwnode_typec_mux_get(dev_fwnode(dev), desc); -} - struct typec_mux_dev * typec_mux_register(struct device *parent, const struct typec_mux_desc *desc); void typec_mux_unregister(struct typec_mux_dev *mux); @@ -76,4 +72,40 @@ void typec_mux_unregister(struct typec_mux_dev *mux); void typec_mux_set_drvdata(struct typec_mux_dev *mux, void *data); void *typec_mux_get_drvdata(struct typec_mux_dev *mux); +#else + +static inline struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode, + const struct typec_altmode_desc *desc) +{ + return NULL; +} + +static inline void typec_mux_put(struct typec_mux *mux) {} + +static inline int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) +{ + return 0; +} + +static inline struct typec_mux_dev * +typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline void typec_mux_unregister(struct typec_mux_dev *mux) {} + +static inline void typec_mux_set_drvdata(struct typec_mux_dev *mux, void *data) {} +static inline void *typec_mux_get_drvdata(struct typec_mux_dev *mux) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +#endif /* CONFIG_TYPEC */ + +static inline struct typec_mux * +typec_mux_get(struct device *dev, const struct typec_altmode_desc *desc) +{ + return fwnode_typec_mux_get(dev_fwnode(dev), desc); +} + #endif /* __USB_TYPEC_MUX */ -- cgit v1.2.3 From 95acd8817e66d031d2e6ee7def3f1e1874819317 Mon Sep 17 00:00:00 2001 From: Tony Ambardar Date: Fri, 17 Jun 2022 12:57:34 +0200 Subject: bpf, x64: Add predicate for bpf2bpf with tailcalls support in JIT The BPF core/verifier is hard-coded to permit mixing bpf2bpf and tail calls for only x86-64. Change the logic to instead rely on a new weak function 'bool bpf_jit_supports_subprog_tailcalls(void)', which a capable JIT backend can override. Update the x86-64 eBPF JIT to reflect this. Signed-off-by: Tony Ambardar [jakub: drop MIPS bits and tweak patch subject] Signed-off-by: Jakub Sitnicki Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220617105735.733938-2-jakub@cloudflare.com --- arch/x86/net/bpf_jit_comp.c | 6 ++++++ include/linux/filter.h | 1 + kernel/bpf/core.c | 6 ++++++ kernel/bpf/verifier.c | 3 ++- 4 files changed, 15 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index f298b18a9a3d..2c51ca9f7cec 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2491,3 +2491,9 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len) return ERR_PTR(-EINVAL); return dst; } + +/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return true; +} diff --git a/include/linux/filter.h b/include/linux/filter.h index d0cbb31b1b4d..4c1a8b247545 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -914,6 +914,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog); void bpf_jit_compile(struct bpf_prog *prog); bool bpf_jit_needs_zext(void); +bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_kfunc_call(void); bool bpf_helper_changes_pkt_data(void *func); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index b5ffebcce6cc..f023cb399e3f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2729,6 +2729,12 @@ bool __weak bpf_jit_needs_zext(void) return false; } +/* Return TRUE if the JIT backend supports mixing bpf2bpf and tailcalls. */ +bool __weak bpf_jit_supports_subprog_tailcalls(void) +{ + return false; +} + bool __weak bpf_jit_supports_kfunc_call(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bf72dc511df6..a20d7736a5b2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6154,7 +6154,8 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) { - return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); + return env->prog->jit_requested && + bpf_jit_supports_subprog_tailcalls(); } static int check_map_func_compatibility(struct bpf_verifier_env *env, -- cgit v1.2.3 From 77515ebaf01920e2db49e04672ef669a7c2907f2 Mon Sep 17 00:00:00 2001 From: Duoming Zhou Date: Tue, 7 Jun 2022 11:26:25 +0800 Subject: devcoredump: remove the useless gfp_t parameter in dev_coredumpv and dev_coredumpm The dev_coredumpv() and dev_coredumpm() could not be used in atomic context, because they call kvasprintf_const() and kstrdup() with GFP_KERNEL parameter. The process is shown below: dev_coredumpv(.., gfp_t gfp) dev_coredumpm(.., gfp_t gfp) dev_set_name kobject_set_name_vargs kvasprintf_const(GFP_KERNEL, ...); //may sleep kstrdup(s, GFP_KERNEL); //may sleep This patch removes gfp_t parameter of dev_coredumpv() and dev_coredumpm() and changes the gfp_t parameter of kzalloc() in dev_coredumpm() to GFP_KERNEL in order to show they could not be used in atomic context. Fixes: 833c95456a70 ("device coredump: add new device coredump class") Reviewed-by: Brian Norris Reviewed-by: Johannes Berg Signed-off-by: Duoming Zhou Link: https://lore.kernel.org/r/df72af3b1862bac7d8e793d1f3931857d3779dfd.1654569290.git.duoming@zju.edu.cn Signed-off-by: Greg Kroah-Hartman --- drivers/base/devcoredump.c | 16 ++++++---------- drivers/bluetooth/btmrvl_sdio.c | 2 +- drivers/bluetooth/hci_qca.c | 2 +- drivers/gpu/drm/etnaviv/etnaviv_dump.c | 2 +- drivers/gpu/drm/msm/disp/msm_disp_snapshot.c | 4 ++-- drivers/gpu/drm/msm/msm_gpu.c | 4 ++-- drivers/media/platform/qcom/venus/core.c | 2 +- drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c | 2 +- drivers/net/wireless/ath/ath10k/coredump.c | 2 +- drivers/net/wireless/ath/wil6210/wil_crash_dump.c | 2 +- drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c | 2 +- drivers/net/wireless/intel/iwlwifi/fw/dbg.c | 6 ++---- drivers/net/wireless/marvell/mwifiex/main.c | 3 +-- drivers/net/wireless/mediatek/mt76/mt7615/mac.c | 3 +-- drivers/net/wireless/mediatek/mt76/mt7921/mac.c | 3 +-- drivers/net/wireless/realtek/rtw88/main.c | 2 +- drivers/net/wireless/realtek/rtw89/ser.c | 2 +- drivers/remoteproc/qcom_q6v5_mss.c | 2 +- drivers/remoteproc/remoteproc_coredump.c | 8 ++++---- include/drm/drm_print.h | 2 +- include/linux/devcoredump.h | 13 ++++++------- sound/soc/intel/avs/apl.c | 2 +- sound/soc/intel/avs/skl.c | 2 +- sound/soc/intel/catpt/dsp.c | 2 +- 24 files changed, 40 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c index f4d794d6bb85..8535f0bd5dfb 100644 --- a/drivers/base/devcoredump.c +++ b/drivers/base/devcoredump.c @@ -173,15 +173,13 @@ static void devcd_freev(void *data) * @dev: the struct device for the crashed device * @data: vmalloc data containing the device coredump * @datalen: length of the data - * @gfp: allocation flags * * This function takes ownership of the vmalloc'ed data and will free * it when it is no longer used. See dev_coredumpm() for more information. */ -void dev_coredumpv(struct device *dev, void *data, size_t datalen, - gfp_t gfp) +void dev_coredumpv(struct device *dev, void *data, size_t datalen) { - dev_coredumpm(dev, NULL, data, datalen, gfp, devcd_readv, devcd_freev); + dev_coredumpm(dev, NULL, data, datalen, devcd_readv, devcd_freev); } EXPORT_SYMBOL_GPL(dev_coredumpv); @@ -236,7 +234,6 @@ static ssize_t devcd_read_from_sgtable(char *buffer, loff_t offset, * @owner: the module that contains the read/free functions, use %THIS_MODULE * @data: data cookie for the @read/@free functions * @datalen: length of the data - * @gfp: allocation flags * @read: function to read from the given buffer * @free: function to free the given buffer * @@ -246,7 +243,7 @@ static ssize_t devcd_read_from_sgtable(char *buffer, loff_t offset, * function will be called to free the data. */ void dev_coredumpm(struct device *dev, struct module *owner, - void *data, size_t datalen, gfp_t gfp, + void *data, size_t datalen, ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen), void (*free)(void *data)) @@ -268,7 +265,7 @@ void dev_coredumpm(struct device *dev, struct module *owner, if (!try_module_get(owner)) goto free; - devcd = kzalloc(sizeof(*devcd), gfp); + devcd = kzalloc(sizeof(*devcd), GFP_KERNEL); if (!devcd) goto put_module; @@ -318,7 +315,6 @@ EXPORT_SYMBOL_GPL(dev_coredumpm); * @dev: the struct device for the crashed device * @table: the dump data * @datalen: length of the data - * @gfp: allocation flags * * Creates a new device coredump for the given device. If a previous one hasn't * been read yet, the new coredump is discarded. The data lifetime is determined @@ -326,9 +322,9 @@ EXPORT_SYMBOL_GPL(dev_coredumpm); * it will free the data. */ void dev_coredumpsg(struct device *dev, struct scatterlist *table, - size_t datalen, gfp_t gfp) + size_t datalen) { - dev_coredumpm(dev, NULL, table, datalen, gfp, devcd_read_from_sgtable, + dev_coredumpm(dev, NULL, table, datalen, devcd_read_from_sgtable, devcd_free_sgtable); } EXPORT_SYMBOL_GPL(dev_coredumpsg); diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c index b8ef66f89fc1..9b9728719db2 100644 --- a/drivers/bluetooth/btmrvl_sdio.c +++ b/drivers/bluetooth/btmrvl_sdio.c @@ -1515,7 +1515,7 @@ done: /* fw_dump_data will be free in device coredump release function * after 5 min */ - dev_coredumpv(&card->func->dev, fw_dump_data, fw_dump_len, GFP_KERNEL); + dev_coredumpv(&card->func->dev, fw_dump_data, fw_dump_len); BT_INFO("== btmrvl firmware dump to /sys/class/devcoredump end"); } diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index eab34e24d944..2e4074211ae9 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -1120,7 +1120,7 @@ static void qca_controller_memdump(struct work_struct *work) qca_memdump->ram_dump_size); memdump_buf = qca_memdump->memdump_buf_head; dev_coredumpv(&hu->serdev->dev, memdump_buf, - qca_memdump->received_dump, GFP_KERNEL); + qca_memdump->received_dump); cancel_delayed_work(&qca->ctrl_memdump_timeout); kfree(qca->qca_memdump); qca->qca_memdump = NULL; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index f418e0b75772..519fcb234b3e 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -225,5 +225,5 @@ void etnaviv_core_dump(struct etnaviv_gem_submit *submit) etnaviv_core_dump_header(&iter, ETDUMP_BUF_END, iter.data); - dev_coredumpv(gpu->dev, iter.start, iter.data - iter.start, GFP_KERNEL); + dev_coredumpv(gpu->dev, iter.start, iter.data - iter.start); } diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c index e75b97127c0d..f057d294c30b 100644 --- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c +++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c @@ -74,8 +74,8 @@ static void _msm_disp_snapshot_work(struct kthread_work *work) * If there is a codedump pending for the device, the dev_coredumpm() * will also free new coredump state. */ - dev_coredumpm(disp_state->dev, THIS_MODULE, disp_state, 0, GFP_KERNEL, - disp_devcoredump_read, msm_disp_state_free); + dev_coredumpm(disp_state->dev, THIS_MODULE, disp_state, 0, + disp_devcoredump_read, msm_disp_state_free); } void msm_disp_snapshot_state(struct drm_device *drm_dev) diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index eb8a6663f309..30576ced0a0a 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -317,8 +317,8 @@ static void msm_gpu_crashstate_capture(struct msm_gpu *gpu, gpu->crashstate = state; /* FIXME: Release the crashstate if this errors out? */ - dev_coredumpm(gpu->dev->dev, THIS_MODULE, gpu, 0, GFP_KERNEL, - msm_gpu_devcoredump_read, msm_gpu_devcoredump_free); + dev_coredumpm(gpu->dev->dev, THIS_MODULE, gpu, 0, + msm_gpu_devcoredump_read, msm_gpu_devcoredump_free); } #else static void msm_gpu_crashstate_capture(struct msm_gpu *gpu, diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index 877eca125803..db84dfb3fb11 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -49,7 +49,7 @@ static void venus_coredump(struct venus_core *core) memcpy(data, mem_va, mem_size); memunmap(mem_va); - dev_coredumpv(dev, data, mem_size, GFP_KERNEL); + dev_coredumpv(dev, data, mem_size); } static void venus_event_notify(struct venus_core *core, u32 event) diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c index c991b30bc9f0..fa520ab7c960 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c @@ -281,5 +281,5 @@ void mcp251xfd_dump(const struct mcp251xfd_priv *priv) mcp251xfd_dump_end(priv, &iter); dev_coredumpv(&priv->spi->dev, iter.start, - iter.data - iter.start, GFP_KERNEL); + iter.data - iter.start); } diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c index fe6b6f97a916..dc9237069921 100644 --- a/drivers/net/wireless/ath/ath10k/coredump.c +++ b/drivers/net/wireless/ath/ath10k/coredump.c @@ -1607,7 +1607,7 @@ int ath10k_coredump_submit(struct ath10k *ar) return -ENODATA; } - dev_coredumpv(ar->dev, dump, le32_to_cpu(dump->len), GFP_KERNEL); + dev_coredumpv(ar->dev, dump, le32_to_cpu(dump->len)); return 0; } diff --git a/drivers/net/wireless/ath/wil6210/wil_crash_dump.c b/drivers/net/wireless/ath/wil6210/wil_crash_dump.c index 89c12cb2aaab..79299609dd62 100644 --- a/drivers/net/wireless/ath/wil6210/wil_crash_dump.c +++ b/drivers/net/wireless/ath/wil6210/wil_crash_dump.c @@ -117,6 +117,6 @@ void wil_fw_core_dump(struct wil6210_priv *wil) /* fw_dump_data will be free in device coredump release function * after 5 min */ - dev_coredumpv(wil_to_dev(wil), fw_dump_data, fw_dump_size, GFP_KERNEL); + dev_coredumpv(wil_to_dev(wil), fw_dump_data, fw_dump_size); wil_info(wil, "fw core dumped, size %d bytes\n", fw_dump_size); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c index eecf8a38d94a..87f3652ef3bd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c @@ -37,7 +37,7 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, return err; } - dev_coredumpv(bus->dev, dump, len + ramsize, GFP_KERNEL); + dev_coredumpv(bus->dev, dump, len + ramsize); return 0; } diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c index abf49022edbe..f2f7cf494a8c 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c @@ -2601,8 +2601,7 @@ static void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt, fw_error_dump.trans_ptr->data, fw_error_dump.trans_ptr->len, fw_error_dump.fwrt_len); - dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len, - GFP_KERNEL); + dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len); } vfree(fw_error_dump.fwrt_ptr); vfree(fw_error_dump.trans_ptr); @@ -2647,8 +2646,7 @@ static void iwl_fw_error_ini_dump(struct iwl_fw_runtime *fwrt, entry->data, entry->size, offs); offs += entry->size; } - dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len, - GFP_KERNEL); + dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len); } iwl_dump_ini_list_free(&dump_list); } diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c index ace7371c4773..26fef0ab1b0d 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.c +++ b/drivers/net/wireless/marvell/mwifiex/main.c @@ -1115,8 +1115,7 @@ void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter) */ mwifiex_dbg(adapter, MSG, "== mwifiex dump information to /sys/class/devcoredump start\n"); - dev_coredumpv(adapter->dev, adapter->devdump_data, adapter->devdump_len, - GFP_KERNEL); + dev_coredumpv(adapter->dev, adapter->devdump_data, adapter->devdump_len); mwifiex_dbg(adapter, MSG, "== mwifiex dump information to /sys/class/devcoredump end\n"); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c index bd687f7de628..5336fe8c668d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c @@ -2421,6 +2421,5 @@ void mt7615_coredump_work(struct work_struct *work) dev_kfree_skb(skb); } - dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ, - GFP_KERNEL); + dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index a630ddbf19e5..cac284f95ce0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -1630,8 +1630,7 @@ void mt7921_coredump_work(struct work_struct *work) } if (dump) - dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ, - GFP_KERNEL); + dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ); mt7921_reset(&dev->mt76); } diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index efabd5b1bf5b..a276544cecdd 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -414,7 +414,7 @@ static void rtw_fwcd_dump(struct rtw_dev *rtwdev) * framework. Note that a new dump will be discarded if a previous one * hasn't been released yet. */ - dev_coredumpv(rtwdev->dev, desc->data, desc->size, GFP_KERNEL); + dev_coredumpv(rtwdev->dev, desc->data, desc->size); } static void rtw_fwcd_free(struct rtw_dev *rtwdev, bool free_self) diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index 9e95ed972710..d28fe01ad729 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -127,7 +127,7 @@ static void rtw89_ser_cd_send(struct rtw89_dev *rtwdev, * will be discarded if a previous one hasn't been released by * framework yet. */ - dev_coredumpv(rtwdev->dev, buf, sizeof(*buf), GFP_KERNEL); + dev_coredumpv(rtwdev->dev, buf, sizeof(*buf)); } static void rtw89_ser_cd_free(struct rtw89_dev *rtwdev, diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index af217de75e4d..813d87faef6c 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -597,7 +597,7 @@ static void q6v5_dump_mba_logs(struct q6v5 *qproc) data = vmalloc(MBA_LOG_SIZE); if (data) { memcpy(data, mba_region, MBA_LOG_SIZE); - dev_coredumpv(&rproc->dev, data, MBA_LOG_SIZE, GFP_KERNEL); + dev_coredumpv(&rproc->dev, data, MBA_LOG_SIZE); } memunmap(mba_region); } diff --git a/drivers/remoteproc/remoteproc_coredump.c b/drivers/remoteproc/remoteproc_coredump.c index 4b093420d98a..cd55c2abd227 100644 --- a/drivers/remoteproc/remoteproc_coredump.c +++ b/drivers/remoteproc/remoteproc_coredump.c @@ -309,7 +309,7 @@ void rproc_coredump(struct rproc *rproc) phdr += elf_size_of_phdr(class); } if (dump_conf == RPROC_COREDUMP_ENABLED) { - dev_coredumpv(&rproc->dev, data, data_size, GFP_KERNEL); + dev_coredumpv(&rproc->dev, data, data_size); return; } @@ -318,7 +318,7 @@ void rproc_coredump(struct rproc *rproc) dump_state.header = data; init_completion(&dump_state.dump_done); - dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, GFP_KERNEL, + dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, rproc_coredump_read, rproc_coredump_free); /* @@ -449,7 +449,7 @@ void rproc_coredump_using_sections(struct rproc *rproc) } if (dump_conf == RPROC_COREDUMP_ENABLED) { - dev_coredumpv(&rproc->dev, data, data_size, GFP_KERNEL); + dev_coredumpv(&rproc->dev, data, data_size); return; } @@ -458,7 +458,7 @@ void rproc_coredump_using_sections(struct rproc *rproc) dump_state.header = data; init_completion(&dump_state.dump_done); - dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, GFP_KERNEL, + dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, rproc_coredump_read, rproc_coredump_free); /* Wait until the dump is read and free is called. Data is freed diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h index 22fabdeed297..b41850366bcc 100644 --- a/include/drm/drm_print.h +++ b/include/drm/drm_print.h @@ -162,7 +162,7 @@ struct drm_print_iterator { * void makecoredump(...) * { * ... - * dev_coredumpm(dev, THIS_MODULE, data, 0, GFP_KERNEL, + * dev_coredumpm(dev, THIS_MODULE, data, 0, * coredump_read, ...) * } * diff --git a/include/linux/devcoredump.h b/include/linux/devcoredump.h index c008169ed2c6..c7d840d824c3 100644 --- a/include/linux/devcoredump.h +++ b/include/linux/devcoredump.h @@ -52,27 +52,26 @@ static inline void _devcd_free_sgtable(struct scatterlist *table) #ifdef CONFIG_DEV_COREDUMP -void dev_coredumpv(struct device *dev, void *data, size_t datalen, - gfp_t gfp); +void dev_coredumpv(struct device *dev, void *data, size_t datalen); void dev_coredumpm(struct device *dev, struct module *owner, - void *data, size_t datalen, gfp_t gfp, + void *data, size_t datalen, ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen), void (*free)(void *data)); void dev_coredumpsg(struct device *dev, struct scatterlist *table, - size_t datalen, gfp_t gfp); + size_t datalen); #else static inline void dev_coredumpv(struct device *dev, void *data, - size_t datalen, gfp_t gfp) + size_t datalen) { vfree(data); } static inline void dev_coredumpm(struct device *dev, struct module *owner, - void *data, size_t datalen, gfp_t gfp, + void *data, size_t datalen, ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen), void (*free)(void *data)) @@ -81,7 +80,7 @@ dev_coredumpm(struct device *dev, struct module *owner, } static inline void dev_coredumpsg(struct device *dev, struct scatterlist *table, - size_t datalen, gfp_t gfp) + size_t datalen) { _devcd_free_sgtable(table); } diff --git a/sound/soc/intel/avs/apl.c b/sound/soc/intel/avs/apl.c index b8e2b23c9f64..1ff57f1a483d 100644 --- a/sound/soc/intel/avs/apl.c +++ b/sound/soc/intel/avs/apl.c @@ -164,7 +164,7 @@ static int apl_coredump(struct avs_dev *adev, union avs_notify_msg *msg) } while (offset < msg->ext.coredump.stack_dump_size); exit: - dev_coredumpv(adev->dev, dump, dump_size, GFP_KERNEL); + dev_coredumpv(adev->dev, dump, dump_size); return 0; } diff --git a/sound/soc/intel/avs/skl.c b/sound/soc/intel/avs/skl.c index bda5ec7510fe..3413162768dc 100644 --- a/sound/soc/intel/avs/skl.c +++ b/sound/soc/intel/avs/skl.c @@ -88,7 +88,7 @@ static int skl_coredump(struct avs_dev *adev, union avs_notify_msg *msg) return -ENOMEM; memcpy_fromio(dump, avs_sram_addr(adev, AVS_FW_REGS_WINDOW), AVS_FW_REGS_SIZE); - dev_coredumpv(adev->dev, dump, AVS_FW_REGS_SIZE, GFP_KERNEL); + dev_coredumpv(adev->dev, dump, AVS_FW_REGS_SIZE); return 0; } diff --git a/sound/soc/intel/catpt/dsp.c b/sound/soc/intel/catpt/dsp.c index 346bec000306..d2afe9ff1e3a 100644 --- a/sound/soc/intel/catpt/dsp.c +++ b/sound/soc/intel/catpt/dsp.c @@ -539,7 +539,7 @@ int catpt_coredump(struct catpt_dev *cdev) pos += CATPT_DMA_REGS_SIZE; } - dev_coredumpv(cdev->dev, dump, dump_size, GFP_KERNEL); + dev_coredumpv(cdev->dev, dump, dump_size); return 0; } -- cgit v1.2.3 From e386b6725798eec07facedf4d4bb710c079fd25c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 2 Jun 2022 17:30:01 -0700 Subject: rcu-tasks: Eliminate RCU Tasks Trace IPIs to online CPUs Currently, the RCU Tasks Trace grace-period kthread IPIs each online CPU using smp_call_function_single() in order to track any tasks currently in RCU Tasks Trace read-side critical sections during which the corresponding task has neither blocked nor been preempted. These IPIs are annoying and are also not strictly necessary because any task that blocks or is preempted within its current RCU Tasks Trace read-side critical section will be tracked on one of the per-CPU rcu_tasks_percpu structure's ->rtp_blkd_tasks list. So the only time that this is a problem is if one of the CPUs runs through a long-duration RCU Tasks Trace read-side critical section without a context switch. Note that the task_call_func() function cannot help here because there is no safe way to identify the target task. Of course, the task_call_func() function will be very useful later, when processing the list of tasks, but it needs to know the task. This commit therefore creates a cpu_curr_snapshot() function that returns a pointer the task_struct structure of some task that happened to be running on the specified CPU more or less during the time that the cpu_curr_snapshot() function was executing. If there was no context switch during this time, this function will return a pointer to the task_struct structure of the task that was running throughout. If there was a context switch, then the outgoing task will be taken care of by RCU's context-switch hook, and the incoming task was either already taken care during some previous context switch, or it is not currently within an RCU Tasks Trace read-side critical section. And in this latter case, the grace period already started, so there is no need to wait on this task. This new cpu_curr_snapshot() function is invoked on each CPU early in the RCU Tasks Trace grace-period processing, and the resulting tasks are queued for later quiescent-state inspection. Signed-off-by: Paul E. McKenney Cc: Peter Zijlstra Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/sched.h | 1 + kernel/rcu/tasks.h | 24 +++++++----------------- kernel/sched/core.c | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index b88caf54e168..72242bc73d85 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2224,6 +2224,7 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) extern bool sched_task_on_rq(struct task_struct *p); extern unsigned long get_wchan(struct task_struct *p); +extern struct task_struct *cpu_curr_snapshot(int cpu); /* * In order to reduce various lock holder preemption latencies provide an diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 9d7d6fd4b8a7..c2aae2643a0b 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1479,21 +1479,6 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop trc_wait_for_one_reader(t, hop); } -/* - * Get the current CPU's current task on the holdout list. - * Calls to this function must be serialized. - */ -static void rcu_tasks_trace_pertask_handler(void *hop_in) -{ - struct list_head *hop = hop_in; - struct task_struct *t = current; - - // Pull in the currently running task, but only if it is currently - // in an RCU tasks trace read-side critical section. - if (rcu_tasks_trace_pertask_prep(t, false)) - trc_add_holdout(t, hop); -} - /* Initialize for a new RCU-tasks-trace grace period. */ static void rcu_tasks_trace_pregp_step(struct list_head *hop) { @@ -1513,8 +1498,13 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) // These smp_call_function_single() calls are serialized to // allow safe access to the hop list. - for_each_online_cpu(cpu) - smp_call_function_single(cpu, rcu_tasks_trace_pertask_handler, hop, 1); + for_each_online_cpu(cpu) { + rcu_read_lock(); + t = cpu_curr_snapshot(cpu); + if (rcu_tasks_trace_pertask_prep(t, true)) + trc_add_holdout(t, hop); + rcu_read_unlock(); + } // Only after all running tasks have been accounted for is it // safe to take care of the tasks that have blocked within their diff --git a/kernel/sched/core.c b/kernel/sched/core.c index da0bf6fe9ecd..9568019be124 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4263,6 +4263,38 @@ int task_call_func(struct task_struct *p, task_call_f func, void *arg) return ret; } +/** + * cpu_curr_snapshot - Return a snapshot of the currently running task + * @cpu: The CPU on which to snapshot the task. + * + * Returns the task_struct pointer of the task "currently" running on + * the specified CPU. If the same task is running on that CPU throughout, + * the return value will be a pointer to that task's task_struct structure. + * If the CPU did any context switches even vaguely concurrently with the + * execution of this function, the return value will be a pointer to the + * task_struct structure of a randomly chosen task that was running on + * that CPU somewhere around the time that this function was executing. + * + * If the specified CPU was offline, the return value is whatever it + * is, perhaps a pointer to the task_struct structure of that CPU's idle + * task, but there is no guarantee. Callers wishing a useful return + * value must take some action to ensure that the specified CPU remains + * online throughout. + * + * This function executes full memory barriers before and after fetching + * the pointer, which permits the caller to confine this function's fetch + * with respect to the caller's accesses to other shared variables. + */ +struct task_struct *cpu_curr_snapshot(int cpu) +{ + struct task_struct *t; + + smp_mb(); /* Pairing determined by caller's synchronization design. */ + t = rcu_dereference(cpu_curr(cpu)); + smp_mb(); /* Pairing determined by caller's synchronization design. */ + return t; +} + /** * wake_up_process - Wake up a specific process * @p: The process to be woken up. -- cgit v1.2.3 From 0ffc781a19ed3030c792ad1a0e44e6e047bb9adc Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:20 +0200 Subject: context_tracking: Rename __context_tracking_enter/exit() to __ct_user_enter/exit() The context tracking namespace is going to expand and some new functions will require even longer names. Start shrinking the context_tracking prefix to "ct" as is already the case for some existing macros, this will make the introduction of new functions easier. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 12 ++++++------ kernel/context_tracking.c | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 7a14807c9d1a..773035124bad 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -14,8 +14,8 @@ extern void context_tracking_cpu_set(int cpu); /* Called with interrupts disabled. */ -extern void __context_tracking_enter(enum ctx_state state); -extern void __context_tracking_exit(enum ctx_state state); +extern void __ct_user_enter(enum ctx_state state); +extern void __ct_user_exit(enum ctx_state state); extern void context_tracking_enter(enum ctx_state state); extern void context_tracking_exit(enum ctx_state state); @@ -38,13 +38,13 @@ static inline void user_exit(void) static __always_inline void user_enter_irqoff(void) { if (context_tracking_enabled()) - __context_tracking_enter(CONTEXT_USER); + __ct_user_enter(CONTEXT_USER); } static __always_inline void user_exit_irqoff(void) { if (context_tracking_enabled()) - __context_tracking_exit(CONTEXT_USER); + __ct_user_exit(CONTEXT_USER); } static inline enum ctx_state exception_enter(void) @@ -74,7 +74,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) static __always_inline bool context_tracking_guest_enter(void) { if (context_tracking_enabled()) - __context_tracking_enter(CONTEXT_GUEST); + __ct_user_enter(CONTEXT_GUEST); return context_tracking_enabled_this_cpu(); } @@ -82,7 +82,7 @@ static __always_inline bool context_tracking_guest_enter(void) static __always_inline void context_tracking_guest_exit(void) { if (context_tracking_enabled()) - __context_tracking_exit(CONTEXT_GUEST); + __ct_user_exit(CONTEXT_GUEST); } /** diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 3082332f6476..88c60ab39fbb 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -51,15 +51,15 @@ static __always_inline void context_tracking_recursion_exit(void) } /** - * context_tracking_enter - Inform the context tracking that the CPU is going - * enter user or guest space mode. + * __ct_user_enter - Inform the context tracking that the CPU is going + * to enter user or guest space mode. * * This function must be called right before we switch from the kernel * to user or guest space, when it's guaranteed the remaining kernel * instructions to execute won't use any RCU read side critical section * because this function sets RCU in extended quiescent state. */ -void noinstr __context_tracking_enter(enum ctx_state state) +void noinstr __ct_user_enter(enum ctx_state state) { /* Kernel threads aren't supposed to go to userspace */ WARN_ON_ONCE(!current->mm); @@ -101,7 +101,7 @@ void noinstr __context_tracking_enter(enum ctx_state state) } context_tracking_recursion_exit(); } -EXPORT_SYMBOL_GPL(__context_tracking_enter); +EXPORT_SYMBOL_GPL(__ct_user_enter); /* * OBSOLETE: @@ -128,7 +128,7 @@ void context_tracking_enter(enum ctx_state state) return; local_irq_save(flags); - __context_tracking_enter(state); + __ct_user_enter(state); local_irq_restore(flags); } NOKPROBE_SYMBOL(context_tracking_enter); @@ -149,8 +149,8 @@ void context_tracking_user_enter(void) NOKPROBE_SYMBOL(context_tracking_user_enter); /** - * context_tracking_exit - Inform the context tracking that the CPU is - * exiting user or guest mode and entering the kernel. + * __ct_user_exit - Inform the context tracking that the CPU is + * exiting user or guest mode and entering the kernel. * * This function must be called after we entered the kernel from user or * guest space before any use of RCU read side critical section. This @@ -160,7 +160,7 @@ NOKPROBE_SYMBOL(context_tracking_user_enter); * This call supports re-entrancy. This way it can be called from any exception * handler without needing to know if we came from userspace or not. */ -void noinstr __context_tracking_exit(enum ctx_state state) +void noinstr __ct_user_exit(enum ctx_state state) { if (!context_tracking_recursion_enter()) return; @@ -183,7 +183,7 @@ void noinstr __context_tracking_exit(enum ctx_state state) } context_tracking_recursion_exit(); } -EXPORT_SYMBOL_GPL(__context_tracking_exit); +EXPORT_SYMBOL_GPL(__ct_user_exit); /* * OBSOLETE: @@ -202,7 +202,7 @@ void context_tracking_exit(enum ctx_state state) return; local_irq_save(flags); - __context_tracking_exit(state); + __ct_user_exit(state); local_irq_restore(flags); } NOKPROBE_SYMBOL(context_tracking_exit); -- cgit v1.2.3 From e244a46a529a9a4c43ae3a2b4bf613e260ec8f81 Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Tue, 14 Jun 2022 21:41:17 +0200 Subject: platform/surface: aggregator: Reserve more event- and target-categories With the introduction of the Surface Laptop Studio, more event- and target categories have been added. Therefore, increase the number of reserved events and extend the enum of know target categories to accommodate this. Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20220614194117.4118897-1-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/surface/aggregator/trace.h | 80 +++++++++++++++------------ include/linux/surface_aggregator/serial_hub.h | 75 +++++++++++++------------ 2 files changed, 85 insertions(+), 70 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/surface/aggregator/trace.h b/drivers/platform/surface/aggregator/trace.h index de64cf169060..cc9e73fbc18e 100644 --- a/drivers/platform/surface/aggregator/trace.h +++ b/drivers/platform/surface/aggregator/trace.h @@ -76,7 +76,7 @@ TRACE_DEFINE_ENUM(SSAM_SSH_TC_HID); TRACE_DEFINE_ENUM(SSAM_SSH_TC_TCH); TRACE_DEFINE_ENUM(SSAM_SSH_TC_BKL); TRACE_DEFINE_ENUM(SSAM_SSH_TC_TAM); -TRACE_DEFINE_ENUM(SSAM_SSH_TC_ACC); +TRACE_DEFINE_ENUM(SSAM_SSH_TC_ACC0); TRACE_DEFINE_ENUM(SSAM_SSH_TC_UFI); TRACE_DEFINE_ENUM(SSAM_SSH_TC_USC); TRACE_DEFINE_ENUM(SSAM_SSH_TC_PEN); @@ -85,6 +85,11 @@ TRACE_DEFINE_ENUM(SSAM_SSH_TC_AUD); TRACE_DEFINE_ENUM(SSAM_SSH_TC_SMC); TRACE_DEFINE_ENUM(SSAM_SSH_TC_KPD); TRACE_DEFINE_ENUM(SSAM_SSH_TC_REG); +TRACE_DEFINE_ENUM(SSAM_SSH_TC_SPT); +TRACE_DEFINE_ENUM(SSAM_SSH_TC_SYS); +TRACE_DEFINE_ENUM(SSAM_SSH_TC_ACC1); +TRACE_DEFINE_ENUM(SSAM_SSH_TC_SHB); +TRACE_DEFINE_ENUM(SSAM_SSH_TC_POS); #define SSAM_PTR_UID_LEN 9 #define SSAM_U8_FIELD_NOT_APPLICABLE ((u16)-1) @@ -229,40 +234,45 @@ static inline u32 ssam_trace_get_request_tc(const struct ssh_packet *p) #define ssam_show_ssh_tc(rqid) \ __print_symbolic(rqid, \ - { SSAM_SSH_TC_NOT_APPLICABLE, "N/A" }, \ - { SSAM_SSH_TC_SAM, "SAM" }, \ - { SSAM_SSH_TC_BAT, "BAT" }, \ - { SSAM_SSH_TC_TMP, "TMP" }, \ - { SSAM_SSH_TC_PMC, "PMC" }, \ - { SSAM_SSH_TC_FAN, "FAN" }, \ - { SSAM_SSH_TC_PoM, "PoM" }, \ - { SSAM_SSH_TC_DBG, "DBG" }, \ - { SSAM_SSH_TC_KBD, "KBD" }, \ - { SSAM_SSH_TC_FWU, "FWU" }, \ - { SSAM_SSH_TC_UNI, "UNI" }, \ - { SSAM_SSH_TC_LPC, "LPC" }, \ - { SSAM_SSH_TC_TCL, "TCL" }, \ - { SSAM_SSH_TC_SFL, "SFL" }, \ - { SSAM_SSH_TC_KIP, "KIP" }, \ - { SSAM_SSH_TC_EXT, "EXT" }, \ - { SSAM_SSH_TC_BLD, "BLD" }, \ - { SSAM_SSH_TC_BAS, "BAS" }, \ - { SSAM_SSH_TC_SEN, "SEN" }, \ - { SSAM_SSH_TC_SRQ, "SRQ" }, \ - { SSAM_SSH_TC_MCU, "MCU" }, \ - { SSAM_SSH_TC_HID, "HID" }, \ - { SSAM_SSH_TC_TCH, "TCH" }, \ - { SSAM_SSH_TC_BKL, "BKL" }, \ - { SSAM_SSH_TC_TAM, "TAM" }, \ - { SSAM_SSH_TC_ACC, "ACC" }, \ - { SSAM_SSH_TC_UFI, "UFI" }, \ - { SSAM_SSH_TC_USC, "USC" }, \ - { SSAM_SSH_TC_PEN, "PEN" }, \ - { SSAM_SSH_TC_VID, "VID" }, \ - { SSAM_SSH_TC_AUD, "AUD" }, \ - { SSAM_SSH_TC_SMC, "SMC" }, \ - { SSAM_SSH_TC_KPD, "KPD" }, \ - { SSAM_SSH_TC_REG, "REG" } \ + { SSAM_SSH_TC_NOT_APPLICABLE, "N/A" }, \ + { SSAM_SSH_TC_SAM, "SAM" }, \ + { SSAM_SSH_TC_BAT, "BAT" }, \ + { SSAM_SSH_TC_TMP, "TMP" }, \ + { SSAM_SSH_TC_PMC, "PMC" }, \ + { SSAM_SSH_TC_FAN, "FAN" }, \ + { SSAM_SSH_TC_PoM, "PoM" }, \ + { SSAM_SSH_TC_DBG, "DBG" }, \ + { SSAM_SSH_TC_KBD, "KBD" }, \ + { SSAM_SSH_TC_FWU, "FWU" }, \ + { SSAM_SSH_TC_UNI, "UNI" }, \ + { SSAM_SSH_TC_LPC, "LPC" }, \ + { SSAM_SSH_TC_TCL, "TCL" }, \ + { SSAM_SSH_TC_SFL, "SFL" }, \ + { SSAM_SSH_TC_KIP, "KIP" }, \ + { SSAM_SSH_TC_EXT, "EXT" }, \ + { SSAM_SSH_TC_BLD, "BLD" }, \ + { SSAM_SSH_TC_BAS, "BAS" }, \ + { SSAM_SSH_TC_SEN, "SEN" }, \ + { SSAM_SSH_TC_SRQ, "SRQ" }, \ + { SSAM_SSH_TC_MCU, "MCU" }, \ + { SSAM_SSH_TC_HID, "HID" }, \ + { SSAM_SSH_TC_TCH, "TCH" }, \ + { SSAM_SSH_TC_BKL, "BKL" }, \ + { SSAM_SSH_TC_TAM, "TAM" }, \ + { SSAM_SSH_TC_ACC0, "ACC0" }, \ + { SSAM_SSH_TC_UFI, "UFI" }, \ + { SSAM_SSH_TC_USC, "USC" }, \ + { SSAM_SSH_TC_PEN, "PEN" }, \ + { SSAM_SSH_TC_VID, "VID" }, \ + { SSAM_SSH_TC_AUD, "AUD" }, \ + { SSAM_SSH_TC_SMC, "SMC" }, \ + { SSAM_SSH_TC_KPD, "KPD" }, \ + { SSAM_SSH_TC_REG, "REG" }, \ + { SSAM_SSH_TC_SPT, "SPT" }, \ + { SSAM_SSH_TC_SYS, "SYS" }, \ + { SSAM_SSH_TC_ACC1, "ACC1" }, \ + { SSAM_SSH_TC_SHB, "SMB" }, \ + { SSAM_SSH_TC_POS, "POS" } \ ) DECLARE_EVENT_CLASS(ssam_frame_class, diff --git a/include/linux/surface_aggregator/serial_hub.h b/include/linux/surface_aggregator/serial_hub.h index 26b95ec12733..45501b6e54e8 100644 --- a/include/linux/surface_aggregator/serial_hub.h +++ b/include/linux/surface_aggregator/serial_hub.h @@ -201,7 +201,7 @@ static inline u16 ssh_crc(const u8 *buf, size_t len) * exception of zero, which is not an event ID. Thus, this is also the * absolute maximum number of event handlers that can be registered. */ -#define SSH_NUM_EVENTS 34 +#define SSH_NUM_EVENTS 38 /* * SSH_NUM_TARGETS - The number of communication targets used in the protocol. @@ -292,40 +292,45 @@ struct ssam_span { * Windows driver. */ enum ssam_ssh_tc { - /* Category 0x00 is invalid for EC use. */ - SSAM_SSH_TC_SAM = 0x01, /* Generic system functionality, real-time clock. */ - SSAM_SSH_TC_BAT = 0x02, /* Battery/power subsystem. */ - SSAM_SSH_TC_TMP = 0x03, /* Thermal subsystem. */ - SSAM_SSH_TC_PMC = 0x04, - SSAM_SSH_TC_FAN = 0x05, - SSAM_SSH_TC_PoM = 0x06, - SSAM_SSH_TC_DBG = 0x07, - SSAM_SSH_TC_KBD = 0x08, /* Legacy keyboard (Laptop 1/2). */ - SSAM_SSH_TC_FWU = 0x09, - SSAM_SSH_TC_UNI = 0x0a, - SSAM_SSH_TC_LPC = 0x0b, - SSAM_SSH_TC_TCL = 0x0c, - SSAM_SSH_TC_SFL = 0x0d, - SSAM_SSH_TC_KIP = 0x0e, /* Manages detachable peripherals (Pro X/8 keyboard cover) */ - SSAM_SSH_TC_EXT = 0x0f, - SSAM_SSH_TC_BLD = 0x10, - SSAM_SSH_TC_BAS = 0x11, /* Detachment system (Surface Book 2/3). */ - SSAM_SSH_TC_SEN = 0x12, - SSAM_SSH_TC_SRQ = 0x13, - SSAM_SSH_TC_MCU = 0x14, - SSAM_SSH_TC_HID = 0x15, /* Generic HID input subsystem. */ - SSAM_SSH_TC_TCH = 0x16, - SSAM_SSH_TC_BKL = 0x17, - SSAM_SSH_TC_TAM = 0x18, - SSAM_SSH_TC_ACC = 0x19, - SSAM_SSH_TC_UFI = 0x1a, - SSAM_SSH_TC_USC = 0x1b, - SSAM_SSH_TC_PEN = 0x1c, - SSAM_SSH_TC_VID = 0x1d, - SSAM_SSH_TC_AUD = 0x1e, - SSAM_SSH_TC_SMC = 0x1f, - SSAM_SSH_TC_KPD = 0x20, - SSAM_SSH_TC_REG = 0x21, /* Extended event registry. */ + /* Category 0x00 is invalid for EC use. */ + SSAM_SSH_TC_SAM = 0x01, /* Generic system functionality, real-time clock. */ + SSAM_SSH_TC_BAT = 0x02, /* Battery/power subsystem. */ + SSAM_SSH_TC_TMP = 0x03, /* Thermal subsystem. */ + SSAM_SSH_TC_PMC = 0x04, + SSAM_SSH_TC_FAN = 0x05, + SSAM_SSH_TC_PoM = 0x06, + SSAM_SSH_TC_DBG = 0x07, + SSAM_SSH_TC_KBD = 0x08, /* Legacy keyboard (Laptop 1/2). */ + SSAM_SSH_TC_FWU = 0x09, + SSAM_SSH_TC_UNI = 0x0a, + SSAM_SSH_TC_LPC = 0x0b, + SSAM_SSH_TC_TCL = 0x0c, + SSAM_SSH_TC_SFL = 0x0d, + SSAM_SSH_TC_KIP = 0x0e, /* Manages detachable peripherals (Pro X/8 keyboard cover) */ + SSAM_SSH_TC_EXT = 0x0f, + SSAM_SSH_TC_BLD = 0x10, + SSAM_SSH_TC_BAS = 0x11, /* Detachment system (Surface Book 2/3). */ + SSAM_SSH_TC_SEN = 0x12, + SSAM_SSH_TC_SRQ = 0x13, + SSAM_SSH_TC_MCU = 0x14, + SSAM_SSH_TC_HID = 0x15, /* Generic HID input subsystem. */ + SSAM_SSH_TC_TCH = 0x16, + SSAM_SSH_TC_BKL = 0x17, + SSAM_SSH_TC_TAM = 0x18, + SSAM_SSH_TC_ACC0 = 0x19, + SSAM_SSH_TC_UFI = 0x1a, + SSAM_SSH_TC_USC = 0x1b, + SSAM_SSH_TC_PEN = 0x1c, + SSAM_SSH_TC_VID = 0x1d, + SSAM_SSH_TC_AUD = 0x1e, + SSAM_SSH_TC_SMC = 0x1f, + SSAM_SSH_TC_KPD = 0x20, + SSAM_SSH_TC_REG = 0x21, /* Extended event registry. */ + SSAM_SSH_TC_SPT = 0x22, + SSAM_SSH_TC_SYS = 0x23, + SSAM_SSH_TC_ACC1 = 0x24, + SSAM_SSH_TC_SHB = 0x25, + SSAM_SSH_TC_POS = 0x26, /* For obtaining Laptop Studio screen position. */ }; -- cgit v1.2.3 From 1a3c7d0841ae24d28a15bbf87ee7d08614cec957 Mon Sep 17 00:00:00 2001 From: Dongli Zhang Date: Sat, 11 Jun 2022 01:25:11 -0700 Subject: swiotlb: remove the unused swiotlb_force declaration The 'swiotlb_force' is removed since commit c6af2aa9ffc9 ("swiotlb: make the swiotlb_init interface more useful"). Signed-off-by: Dongli Zhang Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 7ed35dd3de6e..bdc58a0e20b1 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -60,7 +60,6 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs); #ifdef CONFIG_SWIOTLB -extern enum swiotlb_force swiotlb_force; /** * struct io_tlb_mem - IO TLB Memory Pool Descriptor -- cgit v1.2.3 From 0829c35dc5346e90f428de61896362b51ab58296 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sat, 21 May 2022 13:02:03 +0200 Subject: pwm: Drop support for legacy drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are no drivers left providing the legacy callbacks. So drop support for these. If this commit breaks your out-of-tree pwm driver, look at e.g. commit ec00cd5e63f0 ("pwm: renesas-tpu: Implement .apply() callback") for an example of the needed conversion for your driver. Signed-off-by: Uwe Kleine-König Signed-off-by: Thierry Reding --- drivers/pwm/core.c | 82 +---------------------------------------------------- include/linux/pwm.h | 12 -------- 2 files changed, 1 insertion(+), 93 deletions(-) (limited to 'include/linux') diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index c7552df32082..0e042410f6b9 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -235,18 +235,8 @@ EXPORT_SYMBOL_GPL(pwm_get_chip_data); static bool pwm_ops_check(const struct pwm_chip *chip) { - const struct pwm_ops *ops = chip->ops; - /* driver supports legacy, non-atomic operation */ - if (ops->config && ops->enable && ops->disable) { - if (IS_ENABLED(CONFIG_PWM_DEBUG)) - dev_warn(chip->dev, - "Driver needs updating to atomic API\n"); - - return true; - } - if (!ops->apply) return false; @@ -548,73 +538,6 @@ static void pwm_apply_state_debug(struct pwm_device *pwm, } } -static int pwm_apply_legacy(struct pwm_chip *chip, struct pwm_device *pwm, - const struct pwm_state *state) -{ - int err; - struct pwm_state initial_state = pwm->state; - - if (state->polarity != pwm->state.polarity) { - if (!chip->ops->set_polarity) - return -EINVAL; - - /* - * Changing the polarity of a running PWM is only allowed when - * the PWM driver implements ->apply(). - */ - if (pwm->state.enabled) { - chip->ops->disable(chip, pwm); - - /* - * Update pwm->state already here in case - * .set_polarity() or another callback depend on that. - */ - pwm->state.enabled = false; - } - - err = chip->ops->set_polarity(chip, pwm, state->polarity); - if (err) - goto rollback; - - pwm->state.polarity = state->polarity; - } - - if (!state->enabled) { - if (pwm->state.enabled) - chip->ops->disable(chip, pwm); - - return 0; - } - - /* - * We cannot skip calling ->config even if state->period == - * pwm->state.period && state->duty_cycle == pwm->state.duty_cycle - * because we might have exited early in the last call to - * pwm_apply_state because of !state->enabled and so the two values in - * pwm->state might not be configured in hardware. - */ - err = chip->ops->config(pwm->chip, pwm, - state->duty_cycle, - state->period); - if (err) - goto rollback; - - pwm->state.period = state->period; - pwm->state.duty_cycle = state->duty_cycle; - - if (!pwm->state.enabled) { - err = chip->ops->enable(chip, pwm); - if (err) - goto rollback; - } - - return 0; - -rollback: - pwm->state = initial_state; - return err; -} - /** * pwm_apply_state() - atomically apply a new state to a PWM device * @pwm: PWM device @@ -647,10 +570,7 @@ int pwm_apply_state(struct pwm_device *pwm, const struct pwm_state *state) state->usage_power == pwm->state.usage_power) return 0; - if (chip->ops->apply) - err = chip->ops->apply(chip, pwm, state); - else - err = pwm_apply_legacy(chip, pwm, state); + err = chip->ops->apply(chip, pwm, state); if (err) return err; diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 9771a0761a40..76463b71ad4e 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -261,10 +261,6 @@ pwm_set_relative_duty_cycle(struct pwm_state *state, unsigned int duty_cycle, * called once per PWM device when the PWM chip is * registered. * @owner: helps prevent removal of modules exporting active PWMs - * @config: configure duty cycles and period length for this PWM - * @set_polarity: configure the polarity of this PWM - * @enable: enable PWM output toggling - * @disable: disable PWM output toggling */ struct pwm_ops { int (*request)(struct pwm_chip *chip, struct pwm_device *pwm); @@ -276,14 +272,6 @@ struct pwm_ops { void (*get_state)(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state); struct module *owner; - - /* Only used by legacy drivers */ - int (*config)(struct pwm_chip *chip, struct pwm_device *pwm, - int duty_ns, int period_ns); - int (*set_polarity)(struct pwm_chip *chip, struct pwm_device *pwm, - enum pwm_polarity polarity); - int (*enable)(struct pwm_chip *chip, struct pwm_device *pwm); - void (*disable)(struct pwm_chip *chip, struct pwm_device *pwm); }; /** -- cgit v1.2.3 From ef2e35d90890932ee4546231c775326ffd7a3909 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Mon, 23 May 2022 19:45:00 +0200 Subject: pwm: Reorder header file to get rid of struct pwm_capture forward declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no cyclic dependency, so by reordering the forward declaration can be dropped. Signed-off-by: Uwe Kleine-König Signed-off-by: Thierry Reding --- include/linux/pwm.h | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 76463b71ad4e..6d6946dec771 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -6,7 +6,6 @@ #include #include -struct pwm_capture; struct seq_file; struct pwm_chip; @@ -251,6 +250,16 @@ pwm_set_relative_duty_cycle(struct pwm_state *state, unsigned int duty_cycle, return 0; } +/** + * struct pwm_capture - PWM capture data + * @period: period of the PWM signal (in nanoseconds) + * @duty_cycle: duty cycle of the PWM signal (in nanoseconds) + */ +struct pwm_capture { + unsigned int period; + unsigned int duty_cycle; +}; + /** * struct pwm_ops - PWM controller operations * @request: optional hook for requesting a PWM @@ -300,16 +309,6 @@ struct pwm_chip { struct pwm_device *pwms; }; -/** - * struct pwm_capture - PWM capture data - * @period: period of the PWM signal (in nanoseconds) - * @duty_cycle: duty cycle of the PWM signal (in nanoseconds) - */ -struct pwm_capture { - unsigned int period; - unsigned int duty_cycle; -}; - #if IS_ENABLED(CONFIG_PWM) /* PWM user APIs */ struct pwm_device *pwm_request(int pwm_id, const char *label); -- cgit v1.2.3 From 5c8dca97404bf23f6531456ae4d14c862f6871a3 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Mon, 23 May 2022 19:45:01 +0200 Subject: pwm: Drop unused forward declaration from pwm.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The declaration was necessary until commit cc2d22477779 ("pwm: Drop per-chip dbg_show callback"). Signed-off-by: Uwe Kleine-König Signed-off-by: Thierry Reding --- include/linux/pwm.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 6d6946dec771..9429930c5566 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -6,8 +6,6 @@ #include #include -struct seq_file; - struct pwm_chip; /** -- cgit v1.2.3 From 62c0aff64c8d6594357b6217db019552ea664b90 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 22 Jun 2022 20:11:47 +0300 Subject: clk: Remove never used devm_clk_*unregister() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For the entire history of the devm_clk_*unregister() existence they were used only once (*) in 2015. Remove them. *) The commit 264e3b75de4e ("clk: s2mps11: Simplify s2mps11_clk_probe unwind paths") exactly supports the point of the change proposed here. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20220622171147.85603-1-andriy.shevchenko@linux.intel.com Acked-by: Uwe Kleine-König Signed-off-by: Stephen Boyd --- drivers/clk/clk.c | 48 -------------------------------------------- include/linux/clk-provider.h | 2 -- 2 files changed, 50 deletions(-) (limited to 'include/linux') diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index f00d4c1158d7..7fc191c15507 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -4279,54 +4279,6 @@ int devm_clk_hw_register(struct device *dev, struct clk_hw *hw) } EXPORT_SYMBOL_GPL(devm_clk_hw_register); -static int devm_clk_match(struct device *dev, void *res, void *data) -{ - struct clk *c = res; - if (WARN_ON(!c)) - return 0; - return c == data; -} - -static int devm_clk_hw_match(struct device *dev, void *res, void *data) -{ - struct clk_hw *hw = res; - - if (WARN_ON(!hw)) - return 0; - return hw == data; -} - -/** - * devm_clk_unregister - resource managed clk_unregister() - * @dev: device that is unregistering the clock data - * @clk: clock to unregister - * - * Deallocate a clock allocated with devm_clk_register(). Normally - * this function will not need to be called and the resource management - * code will ensure that the resource is freed. - */ -void devm_clk_unregister(struct device *dev, struct clk *clk) -{ - WARN_ON(devres_release(dev, devm_clk_unregister_cb, devm_clk_match, clk)); -} -EXPORT_SYMBOL_GPL(devm_clk_unregister); - -/** - * devm_clk_hw_unregister - resource managed clk_hw_unregister() - * @dev: device that is unregistering the hardware-specific clock data - * @hw: link to hardware-specific clock data - * - * Unregister a clk_hw registered with devm_clk_hw_register(). Normally - * this function will not need to be called and the resource management - * code will ensure that the resource is freed. - */ -void devm_clk_hw_unregister(struct device *dev, struct clk_hw *hw) -{ - WARN_ON(devres_release(dev, devm_clk_hw_unregister_cb, devm_clk_hw_match, - hw)); -} -EXPORT_SYMBOL_GPL(devm_clk_hw_unregister); - static void devm_clk_release(struct device *dev, void *res) { clk_put(*(struct clk **)res); diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index c10dc4c659e2..72d937c03a3e 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -1176,10 +1176,8 @@ int __must_check devm_clk_hw_register(struct device *dev, struct clk_hw *hw); int __must_check of_clk_hw_register(struct device_node *node, struct clk_hw *hw); void clk_unregister(struct clk *clk); -void devm_clk_unregister(struct device *dev, struct clk *clk); void clk_hw_unregister(struct clk_hw *hw); -void devm_clk_hw_unregister(struct device *dev, struct clk_hw *hw); /* helper functions */ const char *__clk_get_name(const struct clk *clk); -- cgit v1.2.3 From 203184571388a988283543f0fd7da1a0da7c3f91 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 24 May 2022 10:21:53 -0500 Subject: dmaengine: dw-edma: Detach the private data and chip info structures "struct dw_edma_chip" contains an internal structure "struct dw_edma" that is used by the eDMA core internally and should not be touched by the eDMA controller drivers themselves. But currently, the eDMA controller drivers like "dw-edma-pci" allocate and populate this internal structure before passing it on to the eDMA core. The eDMA core further populates the structure and uses it. This is wrong! Hence, move all the "struct dw_edma" specifics from controller drivers to the eDMA core. Link: https://lore.kernel.org/r/20220524152159.2370739-3-Frank.Li@nxp.com Tested-by: Serge Semin Tested-by: Manivannan Sadhasivam Signed-off-by: Frank Li Signed-off-by: Bjorn Helgaas Reviewed-by: Serge Semin Reviewed-by: Manivannan Sadhasivam Acked-By: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 90 +++++++++++++++++--------------- drivers/dma/dw-edma/dw-edma-core.h | 31 ++--------- drivers/dma/dw-edma/dw-edma-pcie.c | 82 ++++++++++++----------------- drivers/dma/dw-edma/dw-edma-v0-core.c | 32 ++++++------ drivers/dma/dw-edma/dw-edma-v0-core.h | 4 +- drivers/dma/dw-edma/dw-edma-v0-debugfs.c | 18 +++---- drivers/dma/dw-edma/dw-edma-v0-debugfs.h | 8 +-- include/linux/dma/edma.h | 48 ++++++++++++++++- 8 files changed, 164 insertions(+), 149 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 468d1097a1ec..9a4c96f7d9d9 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -64,8 +64,8 @@ static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) { + struct dw_edma_chip *chip = desc->chan->dw->chip; struct dw_edma_chan *chan = desc->chan; - struct dw_edma *dw = chan->chip->dw; struct dw_edma_chunk *chunk; chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); @@ -82,11 +82,11 @@ static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) */ chunk->cb = !(desc->chunks_alloc % 2); if (chan->dir == EDMA_DIR_WRITE) { - chunk->ll_region.paddr = dw->ll_region_wr[chan->id].paddr; - chunk->ll_region.vaddr = dw->ll_region_wr[chan->id].vaddr; + chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; + chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; } else { - chunk->ll_region.paddr = dw->ll_region_rd[chan->id].paddr; - chunk->ll_region.vaddr = dw->ll_region_rd[chan->id].vaddr; + chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; + chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; } if (desc->chunk) { @@ -663,7 +663,7 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) if (chan->status != EDMA_ST_IDLE) return -EBUSY; - pm_runtime_get(chan->chip->dev); + pm_runtime_get(chan->dw->chip->dev); return 0; } @@ -685,15 +685,15 @@ static void dw_edma_free_chan_resources(struct dma_chan *dchan) cpu_relax(); } - pm_runtime_put(chan->chip->dev); + pm_runtime_put(chan->dw->chip->dev); } -static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, +static int dw_edma_channel_setup(struct dw_edma *dw, bool write, u32 wr_alloc, u32 rd_alloc) { + struct dw_edma_chip *chip = dw->chip; struct dw_edma_region *dt_region; struct device *dev = chip->dev; - struct dw_edma *dw = chip->dw; struct dw_edma_chan *chan; struct dw_edma_irq *irq; struct dma_device *dma; @@ -726,7 +726,7 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, chan->vc.chan.private = dt_region; - chan->chip = chip; + chan->dw = dw; chan->id = j; chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ; chan->configured = false; @@ -734,9 +734,9 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, chan->status = EDMA_ST_IDLE; if (write) - chan->ll_max = (dw->ll_region_wr[j].sz / EDMA_LL_SZ); + chan->ll_max = (chip->ll_region_wr[j].sz / EDMA_LL_SZ); else - chan->ll_max = (dw->ll_region_rd[j].sz / EDMA_LL_SZ); + chan->ll_max = (chip->ll_region_rd[j].sz / EDMA_LL_SZ); chan->ll_max -= 1; dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", @@ -766,13 +766,13 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, vchan_init(&chan->vc, dma); if (write) { - dt_region->paddr = dw->dt_region_wr[j].paddr; - dt_region->vaddr = dw->dt_region_wr[j].vaddr; - dt_region->sz = dw->dt_region_wr[j].sz; + dt_region->paddr = chip->dt_region_wr[j].paddr; + dt_region->vaddr = chip->dt_region_wr[j].vaddr; + dt_region->sz = chip->dt_region_wr[j].sz; } else { - dt_region->paddr = dw->dt_region_rd[j].paddr; - dt_region->vaddr = dw->dt_region_rd[j].vaddr; - dt_region->sz = dw->dt_region_rd[j].sz; + dt_region->paddr = chip->dt_region_rd[j].paddr; + dt_region->vaddr = chip->dt_region_rd[j].vaddr; + dt_region->sz = chip->dt_region_rd[j].sz; } dw_edma_v0_core_device_config(chan); @@ -826,11 +826,11 @@ static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) (*mask)++; } -static int dw_edma_irq_request(struct dw_edma_chip *chip, +static int dw_edma_irq_request(struct dw_edma *dw, u32 *wr_alloc, u32 *rd_alloc) { - struct device *dev = chip->dev; - struct dw_edma *dw = chip->dw; + struct dw_edma_chip *chip = dw->chip; + struct device *dev = dw->chip->dev; u32 wr_mask = 1; u32 rd_mask = 1; int i, err = 0; @@ -839,12 +839,16 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip, ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; - if (dw->nr_irqs < 1) + if (chip->nr_irqs < 1 || !chip->ops->irq_vector) return -EINVAL; - if (dw->nr_irqs == 1) { + dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL); + if (!dw->irq) + return -ENOMEM; + + if (chip->nr_irqs == 1) { /* Common IRQ shared among all channels */ - irq = dw->ops->irq_vector(dev, 0); + irq = chip->ops->irq_vector(dev, 0); err = request_irq(irq, dw_edma_interrupt_common, IRQF_SHARED, dw->name, &dw->irq[0]); if (err) { @@ -854,9 +858,11 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip, if (irq_get_msi_desc(irq)) get_cached_msi_msg(irq, &dw->irq[0].msi); + + dw->nr_irqs = 1; } else { /* Distribute IRQs equally among all channels */ - int tmp = dw->nr_irqs; + int tmp = chip->nr_irqs; while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); @@ -867,7 +873,7 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip, dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { - irq = dw->ops->irq_vector(dev, i); + irq = chip->ops->irq_vector(dev, i); err = request_irq(irq, i < *wr_alloc ? dw_edma_interrupt_write : @@ -901,20 +907,22 @@ int dw_edma_probe(struct dw_edma_chip *chip) return -EINVAL; dev = chip->dev; - if (!dev) + if (!dev || !chip->ops) return -EINVAL; - dw = chip->dw; - if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector) - return -EINVAL; + dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + dw->chip = chip; raw_spin_lock_init(&dw->lock); - dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, + dw->wr_ch_cnt = min_t(u16, chip->wr_ch_cnt, dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE)); dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); - dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, + dw->rd_ch_cnt = min_t(u16, chip->rd_ch_cnt, dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ)); dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); @@ -936,17 +944,17 @@ int dw_edma_probe(struct dw_edma_chip *chip) dw_edma_v0_core_off(dw); /* Request IRQs */ - err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc); + err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); if (err) return err; /* Setup write channels */ - err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc); + err = dw_edma_channel_setup(dw, true, wr_alloc, rd_alloc); if (err) goto err_irq_free; /* Setup read channels */ - err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc); + err = dw_edma_channel_setup(dw, false, wr_alloc, rd_alloc); if (err) goto err_irq_free; @@ -954,15 +962,15 @@ int dw_edma_probe(struct dw_edma_chip *chip) pm_runtime_enable(dev); /* Turn debugfs on */ - dw_edma_v0_core_debugfs_on(chip); + dw_edma_v0_core_debugfs_on(dw); + + chip->dw = dw; return 0; err_irq_free: for (i = (dw->nr_irqs - 1); i >= 0; i--) - free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); - - dw->nr_irqs = 0; + free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); return err; } @@ -980,7 +988,7 @@ int dw_edma_remove(struct dw_edma_chip *chip) /* Free irqs */ for (i = (dw->nr_irqs - 1); i >= 0; i--) - free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); + free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); /* Power management */ pm_runtime_disable(dev); @@ -1001,7 +1009,7 @@ int dw_edma_remove(struct dw_edma_chip *chip) } /* Turn debugfs off */ - dw_edma_v0_core_debugfs_off(chip); + dw_edma_v0_core_debugfs_off(dw); return 0; } diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 60316d408c3e..85df2d511907 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -15,20 +15,12 @@ #include "../virt-dma.h" #define EDMA_LL_SZ 24 -#define EDMA_MAX_WR_CH 8 -#define EDMA_MAX_RD_CH 8 enum dw_edma_dir { EDMA_DIR_WRITE = 0, EDMA_DIR_READ }; -enum dw_edma_map_format { - EDMA_MF_EDMA_LEGACY = 0x0, - EDMA_MF_EDMA_UNROLL = 0x1, - EDMA_MF_HDMA_COMPAT = 0x5 -}; - enum dw_edma_request { EDMA_REQ_NONE = 0, EDMA_REQ_STOP, @@ -57,12 +49,6 @@ struct dw_edma_burst { u32 sz; }; -struct dw_edma_region { - phys_addr_t paddr; - void __iomem *vaddr; - size_t sz; -}; - struct dw_edma_chunk { struct list_head list; struct dw_edma_chan *chan; @@ -87,7 +73,7 @@ struct dw_edma_desc { struct dw_edma_chan { struct virt_dma_chan vc; - struct dw_edma_chip *chip; + struct dw_edma *dw; int id; enum dw_edma_dir dir; @@ -109,10 +95,6 @@ struct dw_edma_irq { struct dw_edma *dw; }; -struct dw_edma_core_ops { - int (*irq_vector)(struct device *dev, unsigned int nr); -}; - struct dw_edma { char name[20]; @@ -122,21 +104,14 @@ struct dw_edma { struct dma_device rd_edma; u16 rd_ch_cnt; - struct dw_edma_region rg_region; /* Registers */ - struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH]; - struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH]; - struct dw_edma_irq *irq; int nr_irqs; - enum dw_edma_map_format mf; - struct dw_edma_chan *chan; - const struct dw_edma_core_ops *ops; raw_spinlock_t lock; /* Only for legacy */ + + struct dw_edma_chip *chip; #ifdef CONFIG_DEBUG_FS struct dentry *debugfs; #endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index bc07923c3bc0..965513698f3d 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -148,7 +148,6 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, struct dw_edma_pcie_data vsec_data; struct device *dev = &pdev->dev; struct dw_edma_chip *chip; - struct dw_edma *dw; int err, nr_irqs; int i, mask; @@ -197,10 +196,6 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, if (!chip) return -ENOMEM; - dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); - if (!dw) - return -ENOMEM; - /* IRQs allocation */ nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data.irqs, PCI_IRQ_MSI | PCI_IRQ_MSIX); @@ -211,28 +206,23 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, } /* Data structure initialization */ - chip->dw = dw; chip->dev = dev; chip->id = pdev->devfn; - dw->mf = vsec_data.mf; - dw->nr_irqs = nr_irqs; - dw->ops = &dw_edma_pcie_core_ops; - dw->wr_ch_cnt = vsec_data.wr_ch_cnt; - dw->rd_ch_cnt = vsec_data.rd_ch_cnt; + chip->mf = vsec_data.mf; + chip->nr_irqs = nr_irqs; + chip->ops = &dw_edma_pcie_core_ops; - dw->rg_region.vaddr = pcim_iomap_table(pdev)[vsec_data.rg.bar]; - if (!dw->rg_region.vaddr) - return -ENOMEM; + chip->wr_ch_cnt = vsec_data.wr_ch_cnt; + chip->rd_ch_cnt = vsec_data.rd_ch_cnt; - dw->rg_region.vaddr += vsec_data.rg.off; - dw->rg_region.paddr = pdev->resource[vsec_data.rg.bar].start; - dw->rg_region.paddr += vsec_data.rg.off; - dw->rg_region.sz = vsec_data.rg.sz; + chip->rg_region.vaddr = pcim_iomap_table(pdev)[vsec_data.rg.bar]; + if (!chip->rg_region.vaddr) + return -ENOMEM; - for (i = 0; i < dw->wr_ch_cnt; i++) { - struct dw_edma_region *ll_region = &dw->ll_region_wr[i]; - struct dw_edma_region *dt_region = &dw->dt_region_wr[i]; + for (i = 0; i < chip->wr_ch_cnt; i++) { + struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; + struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; struct dw_edma_block *ll_block = &vsec_data.ll_wr[i]; struct dw_edma_block *dt_block = &vsec_data.dt_wr[i]; @@ -255,9 +245,9 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, dt_region->sz = dt_block->sz; } - for (i = 0; i < dw->rd_ch_cnt; i++) { - struct dw_edma_region *ll_region = &dw->ll_region_rd[i]; - struct dw_edma_region *dt_region = &dw->dt_region_rd[i]; + for (i = 0; i < chip->rd_ch_cnt; i++) { + struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; + struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; struct dw_edma_block *ll_block = &vsec_data.ll_rd[i]; struct dw_edma_block *dt_block = &vsec_data.dt_rd[i]; @@ -281,45 +271,45 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, } /* Debug info */ - if (dw->mf == EDMA_MF_EDMA_LEGACY) - pci_dbg(pdev, "Version:\teDMA Port Logic (0x%x)\n", dw->mf); - else if (dw->mf == EDMA_MF_EDMA_UNROLL) - pci_dbg(pdev, "Version:\teDMA Unroll (0x%x)\n", dw->mf); - else if (dw->mf == EDMA_MF_HDMA_COMPAT) - pci_dbg(pdev, "Version:\tHDMA Compatible (0x%x)\n", dw->mf); + if (chip->mf == EDMA_MF_EDMA_LEGACY) + pci_dbg(pdev, "Version:\teDMA Port Logic (0x%x)\n", chip->mf); + else if (chip->mf == EDMA_MF_EDMA_UNROLL) + pci_dbg(pdev, "Version:\teDMA Unroll (0x%x)\n", chip->mf); + else if (chip->mf == EDMA_MF_HDMA_COMPAT) + pci_dbg(pdev, "Version:\tHDMA Compatible (0x%x)\n", chip->mf); else - pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", dw->mf); + pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", chip->mf); - pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", + pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n", vsec_data.rg.bar, vsec_data.rg.off, vsec_data.rg.sz, - dw->rg_region.vaddr, &dw->rg_region.paddr); + chip->rg_region.vaddr); - for (i = 0; i < dw->wr_ch_cnt; i++) { + for (i = 0; i < chip->wr_ch_cnt; i++) { pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, vsec_data.ll_wr[i].bar, - vsec_data.ll_wr[i].off, dw->ll_region_wr[i].sz, - dw->ll_region_wr[i].vaddr, &dw->ll_region_wr[i].paddr); + vsec_data.ll_wr[i].off, chip->ll_region_wr[i].sz, + chip->ll_region_wr[i].vaddr, &chip->ll_region_wr[i].paddr); pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, vsec_data.dt_wr[i].bar, - vsec_data.dt_wr[i].off, dw->dt_region_wr[i].sz, - dw->dt_region_wr[i].vaddr, &dw->dt_region_wr[i].paddr); + vsec_data.dt_wr[i].off, chip->dt_region_wr[i].sz, + chip->dt_region_wr[i].vaddr, &chip->dt_region_wr[i].paddr); } - for (i = 0; i < dw->rd_ch_cnt; i++) { + for (i = 0; i < chip->rd_ch_cnt; i++) { pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, vsec_data.ll_rd[i].bar, - vsec_data.ll_rd[i].off, dw->ll_region_rd[i].sz, - dw->ll_region_rd[i].vaddr, &dw->ll_region_rd[i].paddr); + vsec_data.ll_rd[i].off, chip->ll_region_rd[i].sz, + chip->ll_region_rd[i].vaddr, &chip->ll_region_rd[i].paddr); pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, vsec_data.dt_rd[i].bar, - vsec_data.dt_rd[i].off, dw->dt_region_rd[i].sz, - dw->dt_region_rd[i].vaddr, &dw->dt_region_rd[i].paddr); + vsec_data.dt_rd[i].off, chip->dt_region_rd[i].sz, + chip->dt_region_rd[i].vaddr, &chip->dt_region_rd[i].paddr); } - pci_dbg(pdev, "Nr. IRQs:\t%u\n", dw->nr_irqs); + pci_dbg(pdev, "Nr. IRQs:\t%u\n", chip->nr_irqs); /* Validating if PCI interrupts were enabled */ if (!pci_dev_msi_enabled(pdev)) { @@ -327,10 +317,6 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -EPERM; } - dw->irq = devm_kcalloc(dev, nr_irqs, sizeof(*dw->irq), GFP_KERNEL); - if (!dw->irq) - return -ENOMEM; - /* Starting eDMA driver */ err = dw_edma_probe(chip); if (err) { diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 33bc1e6c4cf2..999e03896186 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -25,7 +25,7 @@ enum dw_edma_control { static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) { - return dw->rg_region.vaddr; + return dw->chip->rg_region.vaddr; } #define SET_32(dw, name, value) \ @@ -96,7 +96,7 @@ static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) static inline struct dw_edma_v0_ch_regs __iomem * __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) { - if (dw->mf == EDMA_MF_EDMA_LEGACY) + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) return &(__dw_regs(dw)->type.legacy.ch); if (dir == EDMA_DIR_WRITE) @@ -108,7 +108,7 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, u32 value, void __iomem *addr) { - if (dw->mf == EDMA_MF_EDMA_LEGACY) { + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { u32 viewport_sel; unsigned long flags; @@ -133,7 +133,7 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, { u32 value; - if (dw->mf == EDMA_MF_EDMA_LEGACY) { + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { u32 viewport_sel; unsigned long flags; @@ -169,7 +169,7 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, static inline void writeq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, u64 value, void __iomem *addr) { - if (dw->mf == EDMA_MF_EDMA_LEGACY) { + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { u32 viewport_sel; unsigned long flags; @@ -194,7 +194,7 @@ static inline u64 readq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, { u32 value; - if (dw->mf == EDMA_MF_EDMA_LEGACY) { + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { u32 viewport_sel; unsigned long flags; @@ -256,7 +256,7 @@ u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; u32 tmp; tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK, @@ -272,7 +272,7 @@ enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; SET_RW_32(dw, chan->dir, int_clear, FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id))); @@ -280,7 +280,7 @@ void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan) void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; SET_RW_32(dw, chan->dir, int_clear, FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id))); @@ -357,7 +357,7 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; u32 tmp; dw_edma_v0_core_write_chunk(chunk); @@ -365,7 +365,7 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) if (first) { /* Enable engine */ SET_RW_32(dw, chan->dir, engine_en, BIT(0)); - if (dw->mf == EDMA_MF_HDMA_COMPAT) { + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { switch (chan->id) { case 0: SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, @@ -435,7 +435,7 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) int dw_edma_v0_core_device_config(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; u32 tmp = 0; /* MSI done addr - low, high */ @@ -505,12 +505,12 @@ int dw_edma_v0_core_device_config(struct dw_edma_chan *chan) } /* eDMA debugfs callbacks */ -void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip) +void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) { - dw_edma_v0_debugfs_on(chip); + dw_edma_v0_debugfs_on(dw); } -void dw_edma_v0_core_debugfs_off(struct dw_edma_chip *chip) +void dw_edma_v0_core_debugfs_off(struct dw_edma *dw) { - dw_edma_v0_debugfs_off(chip); + dw_edma_v0_debugfs_off(dw); } diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.h b/drivers/dma/dw-edma/dw-edma-v0-core.h index 2afa626b8300..75aec6d31b21 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.h +++ b/drivers/dma/dw-edma/dw-edma-v0-core.h @@ -22,7 +22,7 @@ u32 dw_edma_v0_core_status_abort_int(struct dw_edma *chan, enum dw_edma_dir dir) void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first); int dw_edma_v0_core_device_config(struct dw_edma_chan *chan); /* eDMA debug fs callbacks */ -void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip); -void dw_edma_v0_core_debugfs_off(struct dw_edma_chip *chip); +void dw_edma_v0_core_debugfs_on(struct dw_edma *dw); +void dw_edma_v0_core_debugfs_off(struct dw_edma *dw); #endif /* _DW_EDMA_V0_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c index 4b3bcffd15ef..b765adb96999 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c @@ -54,7 +54,7 @@ struct debugfs_entries { static int dw_edma_debugfs_u32_get(void *data, u64 *val) { void __iomem *reg = (void __force __iomem *)data; - if (dw->mf == EDMA_MF_EDMA_LEGACY && + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY && reg >= (void __iomem *)®s->type.legacy.ch) { void __iomem *ptr = ®s->type.legacy.ch; u32 viewport_sel = 0; @@ -173,7 +173,7 @@ static void dw_edma_debugfs_regs_wr(struct dentry *dir) nr_entries = ARRAY_SIZE(debugfs_regs); dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); - if (dw->mf == EDMA_MF_HDMA_COMPAT) { + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { nr_entries = ARRAY_SIZE(debugfs_unroll_regs); dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries, regs_dir); @@ -242,7 +242,7 @@ static void dw_edma_debugfs_regs_rd(struct dentry *dir) nr_entries = ARRAY_SIZE(debugfs_regs); dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); - if (dw->mf == EDMA_MF_HDMA_COMPAT) { + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { nr_entries = ARRAY_SIZE(debugfs_unroll_regs); dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries, regs_dir); @@ -282,13 +282,13 @@ static void dw_edma_debugfs_regs(void) dw_edma_debugfs_regs_rd(regs_dir); } -void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip) +void dw_edma_v0_debugfs_on(struct dw_edma *_dw) { - dw = chip->dw; + dw = _dw; if (!dw) return; - regs = dw->rg_region.vaddr; + regs = dw->chip->rg_region.vaddr; if (!regs) return; @@ -296,16 +296,16 @@ void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip) if (!dw->debugfs) return; - debugfs_create_u32("mf", 0444, dw->debugfs, &dw->mf); + debugfs_create_u32("mf", 0444, dw->debugfs, &dw->chip->mf); debugfs_create_u16("wr_ch_cnt", 0444, dw->debugfs, &dw->wr_ch_cnt); debugfs_create_u16("rd_ch_cnt", 0444, dw->debugfs, &dw->rd_ch_cnt); dw_edma_debugfs_regs(); } -void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip) +void dw_edma_v0_debugfs_off(struct dw_edma *_dw) { - dw = chip->dw; + dw = _dw; if (!dw) return; diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.h b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h index d0ff25a9ea5c..3391b86edf5a 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-debugfs.h +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h @@ -12,14 +12,14 @@ #include #ifdef CONFIG_DEBUG_FS -void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip); -void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip); +void dw_edma_v0_debugfs_on(struct dw_edma *dw); +void dw_edma_v0_debugfs_off(struct dw_edma *dw); #else -static inline void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip) +static inline void dw_edma_v0_debugfs_on(struct dw_edma *dw) { } -static inline void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip) +static inline void dw_edma_v0_debugfs_off(struct dw_edma *dw) { } #endif /* CONFIG_DEBUG_FS */ diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index d4333e721588..6f64e90d5c38 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -12,17 +12,63 @@ #include #include +#define EDMA_MAX_WR_CH 8 +#define EDMA_MAX_RD_CH 8 + struct dw_edma; +struct dw_edma_region { + phys_addr_t paddr; + void __iomem *vaddr; + size_t sz; +}; + +struct dw_edma_core_ops { + int (*irq_vector)(struct device *dev, unsigned int nr); +}; + +enum dw_edma_map_format { + EDMA_MF_EDMA_LEGACY = 0x0, + EDMA_MF_EDMA_UNROLL = 0x1, + EDMA_MF_HDMA_COMPAT = 0x5 +}; + /** * struct dw_edma_chip - representation of DesignWare eDMA controller hardware * @dev: struct device of the eDMA controller * @id: instance ID - * @dw: struct dw_edma that is filed by dw_edma_probe() + * @nr_irqs: total number of DMA IRQs + * @ops DMA channel to IRQ number mapping + * @wr_ch_cnt DMA write channel number + * @rd_ch_cnt DMA read channel number + * @rg_region DMA register region + * @ll_region_wr DMA descriptor link list memory for write channel + * @ll_region_rd DMA descriptor link list memory for read channel + * @dt_region_wr DMA data memory for write channel + * @dt_region_rd DMA data memory for read channel + * @mf DMA register map format + * @dw: struct dw_edma that is filled by dw_edma_probe() */ struct dw_edma_chip { struct device *dev; int id; + int nr_irqs; + const struct dw_edma_core_ops *ops; + + struct dw_edma_region rg_region; + + u16 wr_ch_cnt; + u16 rd_ch_cnt; + /* link list address */ + struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH]; + struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH]; + + /* data region */ + struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH]; + struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH]; + + enum dw_edma_map_format mf; + struct dw_edma *dw; }; -- cgit v1.2.3 From e51b3048116a6e10b96bd5298cbcb209b6d729cd Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 24 May 2022 10:21:54 -0500 Subject: dmaengine: dw-edma: Change rg_region to reg_base in struct dw_edma_chip struct dw_edma_region rg_region included virtual address, physical address and size information. But only the virtual address is used by EDMA driver. Change it to void __iomem *reg_base to clean up code. Link: https://lore.kernel.org/r/20220524152159.2370739-4-Frank.Li@nxp.com Tested-by: Serge Semin Tested-by: Manivannan Sadhasivam Signed-off-by: Frank Li Signed-off-by: Bjorn Helgaas Reviewed-by: Serge Semin Reviewed-by: Manivannan Sadhasivam Acked-By: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 6 +++--- drivers/dma/dw-edma/dw-edma-v0-core.c | 2 +- drivers/dma/dw-edma/dw-edma-v0-debugfs.c | 2 +- include/linux/dma/edma.h | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 965513698f3d..9553fb75091d 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -216,8 +216,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->wr_ch_cnt = vsec_data.wr_ch_cnt; chip->rd_ch_cnt = vsec_data.rd_ch_cnt; - chip->rg_region.vaddr = pcim_iomap_table(pdev)[vsec_data.rg.bar]; - if (!chip->rg_region.vaddr) + chip->reg_base = pcim_iomap_table(pdev)[vsec_data.rg.bar]; + if (!chip->reg_base) return -ENOMEM; for (i = 0; i < chip->wr_ch_cnt; i++) { @@ -282,7 +282,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n", vsec_data.rg.bar, vsec_data.rg.off, vsec_data.rg.sz, - chip->rg_region.vaddr); + chip->reg_base); for (i = 0; i < chip->wr_ch_cnt; i++) { diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 999e03896186..403ade40c1b1 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -25,7 +25,7 @@ enum dw_edma_control { static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) { - return dw->chip->rg_region.vaddr; + return dw->chip->reg_base; } #define SET_32(dw, name, value) \ diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c index b765adb96999..5226c9014703 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c @@ -288,7 +288,7 @@ void dw_edma_v0_debugfs_on(struct dw_edma *_dw) if (!dw) return; - regs = dw->chip->rg_region.vaddr; + regs = dw->chip->reg_base; if (!regs) return; diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index 6f64e90d5c38..df9ba3ecb437 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -39,6 +39,7 @@ enum dw_edma_map_format { * @id: instance ID * @nr_irqs: total number of DMA IRQs * @ops DMA channel to IRQ number mapping + * @reg_base DMA register base address * @wr_ch_cnt DMA write channel number * @rd_ch_cnt DMA read channel number * @rg_region DMA register region @@ -55,7 +56,7 @@ struct dw_edma_chip { int nr_irqs; const struct dw_edma_core_ops *ops; - struct dw_edma_region rg_region; + void __iomem *reg_base; u16 wr_ch_cnt; u16 rd_ch_cnt; -- cgit v1.2.3 From 6951ee96c649f6e963b98c11b2b1a92697d3c45c Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 24 May 2022 10:21:55 -0500 Subject: dmaengine: dw-edma: Rename wr(rd)_ch_cnt to ll_wr(rd)_cnt in struct dw_edma_chip The struct dw_edma contains wr(rd)_ch_cnt fields. The EDMA driver gets write(read) channel number from register, then saves these into dw_edma. The wr(rd)_ch_cnt in dw_edma_chip actually means how many link list memory are available in ll_region_wr(rd)[EDMA_MAX_WR_CH]. Rename it to ll_wr(rd)_cnt to indicate actual usage. Link: https://lore.kernel.org/r/20220524152159.2370739-5-Frank.Li@nxp.com Tested-by: Serge Semin Tested-by: Manivannan Sadhasivam Signed-off-by: Frank Li Signed-off-by: Bjorn Helgaas Reviewed-by: Serge Semin Reviewed-by: Manivannan Sadhasivam Acked-By: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 4 ++-- drivers/dma/dw-edma/dw-edma-pcie.c | 12 ++++++------ include/linux/dma/edma.h | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 9a4c96f7d9d9..af037ec61a86 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -918,11 +918,11 @@ int dw_edma_probe(struct dw_edma_chip *chip) raw_spin_lock_init(&dw->lock); - dw->wr_ch_cnt = min_t(u16, chip->wr_ch_cnt, + dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE)); dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); - dw->rd_ch_cnt = min_t(u16, chip->rd_ch_cnt, + dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ)); dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 9553fb75091d..d6b5e2463884 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -213,14 +213,14 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->nr_irqs = nr_irqs; chip->ops = &dw_edma_pcie_core_ops; - chip->wr_ch_cnt = vsec_data.wr_ch_cnt; - chip->rd_ch_cnt = vsec_data.rd_ch_cnt; + chip->ll_wr_cnt = vsec_data.wr_ch_cnt; + chip->ll_rd_cnt = vsec_data.rd_ch_cnt; chip->reg_base = pcim_iomap_table(pdev)[vsec_data.rg.bar]; if (!chip->reg_base) return -ENOMEM; - for (i = 0; i < chip->wr_ch_cnt; i++) { + for (i = 0; i < chip->ll_wr_cnt; i++) { struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; struct dw_edma_block *ll_block = &vsec_data.ll_wr[i]; @@ -245,7 +245,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, dt_region->sz = dt_block->sz; } - for (i = 0; i < chip->rd_ch_cnt; i++) { + for (i = 0; i < chip->ll_rd_cnt; i++) { struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; struct dw_edma_block *ll_block = &vsec_data.ll_rd[i]; @@ -285,7 +285,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->reg_base); - for (i = 0; i < chip->wr_ch_cnt; i++) { + for (i = 0; i < chip->ll_wr_cnt; i++) { pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, vsec_data.ll_wr[i].bar, vsec_data.ll_wr[i].off, chip->ll_region_wr[i].sz, @@ -297,7 +297,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->dt_region_wr[i].vaddr, &chip->dt_region_wr[i].paddr); } - for (i = 0; i < chip->rd_ch_cnt; i++) { + for (i = 0; i < chip->ll_rd_cnt; i++) { pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, vsec_data.ll_rd[i].bar, vsec_data.ll_rd[i].off, chip->ll_region_rd[i].sz, diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index df9ba3ecb437..fdbbeda170a9 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -40,8 +40,8 @@ enum dw_edma_map_format { * @nr_irqs: total number of DMA IRQs * @ops DMA channel to IRQ number mapping * @reg_base DMA register base address - * @wr_ch_cnt DMA write channel number - * @rd_ch_cnt DMA read channel number + * @ll_wr_cnt DMA write link list count + * @ll_rd_cnt DMA read link list count * @rg_region DMA register region * @ll_region_wr DMA descriptor link list memory for write channel * @ll_region_rd DMA descriptor link list memory for read channel @@ -58,8 +58,8 @@ struct dw_edma_chip { void __iomem *reg_base; - u16 wr_ch_cnt; - u16 rd_ch_cnt; + u16 ll_wr_cnt; + u16 ll_rd_cnt; /* link list address */ struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH]; struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH]; -- cgit v1.2.3 From d6b03171f9fc8127b3a7adfd4e74ee5d4dae5d14 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 24 May 2022 10:21:58 -0500 Subject: dmaengine: dw-edma: Add support for chip-specific flags Add a "flags" field to the "struct dw_edma_chip" so that the controller drivers can pass flags that are relevant to the platform. DW_EDMA_CHIP_LOCAL - Used by the controller drivers accessing eDMA locally. Local eDMA access doesn't require generating MSIs to the remote. Link: https://lore.kernel.org/r/20220524152159.2370739-8-Frank.Li@nxp.com Tested-by: Serge Semin Tested-by: Manivannan Sadhasivam Signed-off-by: Frank Li Signed-off-by: Bjorn Helgaas Reviewed-by: Serge Semin Reviewed-by: Manivannan Sadhasivam Acked-By: Vinod Koul --- drivers/dma/dw-edma/dw-edma-v0-core.c | 9 ++++++--- include/linux/dma/edma.h | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 403ade40c1b1..607647dacc29 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -301,6 +301,7 @@ u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) { struct dw_edma_burst *child; + struct dw_edma_chan *chan = chunk->chan; struct dw_edma_v0_lli __iomem *lli; struct dw_edma_v0_llp __iomem *llp; u32 control = 0, i = 0; @@ -314,9 +315,11 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) j = chunk->bursts_alloc; list_for_each_entry(child, &chunk->burst->list, list) { j--; - if (!j) - control |= (DW_EDMA_V0_LIE | DW_EDMA_V0_RIE); - + if (!j) { + control |= DW_EDMA_V0_LIE; + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + control |= DW_EDMA_V0_RIE; + } /* Channel control */ SET_LL_32(&lli[i].control, control); /* Transfer size */ diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index fdbbeda170a9..7d8062e9c544 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -33,12 +33,21 @@ enum dw_edma_map_format { EDMA_MF_HDMA_COMPAT = 0x5 }; +/** + * enum dw_edma_chip_flags - Flags specific to an eDMA chip + * @DW_EDMA_CHIP_LOCAL: eDMA is used locally by an endpoint + */ +enum dw_edma_chip_flags { + DW_EDMA_CHIP_LOCAL = BIT(0), +}; + /** * struct dw_edma_chip - representation of DesignWare eDMA controller hardware * @dev: struct device of the eDMA controller * @id: instance ID * @nr_irqs: total number of DMA IRQs * @ops DMA channel to IRQ number mapping + * @flags dw_edma_chip_flags * @reg_base DMA register base address * @ll_wr_cnt DMA write link list count * @ll_rd_cnt DMA read link list count @@ -55,6 +64,7 @@ struct dw_edma_chip { int id; int nr_irqs; const struct dw_edma_core_ops *ops; + u32 flags; void __iomem *reg_base; -- cgit v1.2.3 From 3cc624beba6310a8a534fb00841f22445a200d54 Mon Sep 17 00:00:00 2001 From: Ivan Bornyakov Date: Thu, 23 Jun 2022 19:32:44 +0300 Subject: fpga: fpga-mgr: support bitstream offset in image buffer At the moment FPGA manager core loads to the device entire image provided to fpga_mgr_load(). But it is not always whole FPGA image buffer meant to be written to the device. In particular, .dat formatted image for Microchip MPF contains meta info in the header that is not meant to be written to the device. This is issue for those low level drivers that loads data to the device with write() fpga_manager_ops callback, since write() can be called in iterator over scatter-gather table, not only linear image buffer. On the other hand, write_sg() callback is provided with whole image in scatter-gather form and can decide itself which part should be sent to the device. Add header_size and data_size to the fpga_image_info struct, add skip_header to the fpga_manager_ops struct and adjust fpga_mgr_write() callers with respect to them. * info->header_size indicates part at the beginning of image buffer that contains some meta info. It is optional and can be 0, initialized with mops->initial_header_size. * mops->skip_header tells fpga-mgr core whether write should start from the beginning of image buffer or at the offset of header_size. * info->data_size is the size of bitstream data that is meant to be written to the device. It is also optional and can be 0, which means bitstream data is up to the end of image buffer. Also add parse_header() callback to fpga_manager_ops, which purpose is to set info->header_size and info->data_size. At least initial_header_size bytes of image buffer will be passed into parse_header() first time. If it is not enough, parse_header() should set desired size into info->header_size and return -EAGAIN, then it will be called again with greater part of image buffer on the input. Suggested-by: Xu Yilun Signed-off-by: Ivan Bornyakov Acked-by: Xu Yilun Link: https://lore.kernel.org/r/20220623163248.3672-2-i.bornyakov@metrotek.ru Signed-off-by: Xu Yilun --- drivers/fpga/fpga-mgr.c | 223 +++++++++++++++++++++++++++++++++++++----- include/linux/fpga/fpga-mgr.h | 24 ++++- 2 files changed, 220 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c index 08dc85fcd511..a0fa0a2cb8af 100644 --- a/drivers/fpga/fpga-mgr.c +++ b/drivers/fpga/fpga-mgr.c @@ -74,6 +74,15 @@ static inline int fpga_mgr_write_complete(struct fpga_manager *mgr, return 0; } +static inline int fpga_mgr_parse_header(struct fpga_manager *mgr, + struct fpga_image_info *info, + const char *buf, size_t count) +{ + if (mgr->mops->parse_header) + return mgr->mops->parse_header(mgr, info, buf, count); + return 0; +} + static inline int fpga_mgr_write_init(struct fpga_manager *mgr, struct fpga_image_info *info, const char *buf, size_t count) @@ -136,24 +145,141 @@ void fpga_image_info_free(struct fpga_image_info *info) EXPORT_SYMBOL_GPL(fpga_image_info_free); /* - * Call the low level driver's write_init function. This will do the + * Call the low level driver's parse_header function with entire FPGA image + * buffer on the input. This will set info->header_size and info->data_size. + */ +static int fpga_mgr_parse_header_mapped(struct fpga_manager *mgr, + struct fpga_image_info *info, + const char *buf, size_t count) +{ + int ret; + + mgr->state = FPGA_MGR_STATE_PARSE_HEADER; + ret = fpga_mgr_parse_header(mgr, info, buf, count); + + if (info->header_size + info->data_size > count) { + dev_err(&mgr->dev, "Bitsream data outruns FPGA image\n"); + ret = -EINVAL; + } + + if (ret) { + dev_err(&mgr->dev, "Error while parsing FPGA image header\n"); + mgr->state = FPGA_MGR_STATE_PARSE_HEADER_ERR; + } + + return ret; +} + +/* + * Call the low level driver's parse_header function with first fragment of + * scattered FPGA image on the input. If header fits first fragment, + * parse_header will set info->header_size and info->data_size. If it is not, + * parse_header will set desired size to info->header_size and -EAGAIN will be + * returned. + */ +static int fpga_mgr_parse_header_sg_first(struct fpga_manager *mgr, + struct fpga_image_info *info, + struct sg_table *sgt) +{ + struct sg_mapping_iter miter; + int ret; + + mgr->state = FPGA_MGR_STATE_PARSE_HEADER; + + sg_miter_start(&miter, sgt->sgl, sgt->nents, SG_MITER_FROM_SG); + if (sg_miter_next(&miter) && + miter.length >= info->header_size) + ret = fpga_mgr_parse_header(mgr, info, miter.addr, miter.length); + else + ret = -EAGAIN; + sg_miter_stop(&miter); + + if (ret && ret != -EAGAIN) { + dev_err(&mgr->dev, "Error while parsing FPGA image header\n"); + mgr->state = FPGA_MGR_STATE_PARSE_HEADER_ERR; + } + + return ret; +} + +/* + * Copy scattered FPGA image fragments to temporary buffer and call the + * low level driver's parse_header function. This should be called after + * fpga_mgr_parse_header_sg_first() returned -EAGAIN. In case of success, + * pointer to the newly allocated image header copy will be returned and + * its size will be set into *ret_size. Returned buffer needs to be freed. + */ +static void *fpga_mgr_parse_header_sg(struct fpga_manager *mgr, + struct fpga_image_info *info, + struct sg_table *sgt, size_t *ret_size) +{ + size_t len, new_header_size, header_size = 0; + char *new_buf, *buf = NULL; + int ret; + + do { + new_header_size = info->header_size; + if (new_header_size <= header_size) { + dev_err(&mgr->dev, "Requested invalid header size\n"); + ret = -EFAULT; + break; + } + + new_buf = krealloc(buf, new_header_size, GFP_KERNEL); + if (!new_buf) { + ret = -ENOMEM; + break; + } + + buf = new_buf; + + len = sg_pcopy_to_buffer(sgt->sgl, sgt->nents, + buf + header_size, + new_header_size - header_size, + header_size); + if (len != new_header_size - header_size) { + ret = -EFAULT; + break; + } + + header_size = new_header_size; + ret = fpga_mgr_parse_header(mgr, info, buf, header_size); + } while (ret == -EAGAIN); + + if (ret) { + dev_err(&mgr->dev, "Error while parsing FPGA image header\n"); + mgr->state = FPGA_MGR_STATE_PARSE_HEADER_ERR; + kfree(buf); + buf = ERR_PTR(ret); + } + + *ret_size = header_size; + + return buf; +} + +/* + * Call the low level driver's write_init function. This will do the * device-specific things to get the FPGA into the state where it is ready to - * receive an FPGA image. The low level driver only gets to see the first - * initial_header_size bytes in the buffer. + * receive an FPGA image. The low level driver gets to see at least first + * info->header_size bytes in the buffer. If info->header_size is 0, + * write_init will not get any bytes of image buffer. */ static int fpga_mgr_write_init_buf(struct fpga_manager *mgr, struct fpga_image_info *info, const char *buf, size_t count) { + size_t header_size = info->header_size; int ret; mgr->state = FPGA_MGR_STATE_WRITE_INIT; - if (!mgr->mops->initial_header_size) { + + if (header_size > count) + ret = -EINVAL; + else if (!header_size) ret = fpga_mgr_write_init(mgr, info, NULL, 0); - } else { - count = min(mgr->mops->initial_header_size, count); + else ret = fpga_mgr_write_init(mgr, info, buf, count); - } if (ret) { dev_err(&mgr->dev, "Error preparing FPGA for writing\n"); @@ -164,39 +290,50 @@ static int fpga_mgr_write_init_buf(struct fpga_manager *mgr, return 0; } -static int fpga_mgr_write_init_sg(struct fpga_manager *mgr, - struct fpga_image_info *info, - struct sg_table *sgt) +static int fpga_mgr_prepare_sg(struct fpga_manager *mgr, + struct fpga_image_info *info, + struct sg_table *sgt) { struct sg_mapping_iter miter; size_t len; char *buf; int ret; - if (!mgr->mops->initial_header_size) + /* Short path. Low level driver don't care about image header. */ + if (!mgr->mops->initial_header_size && !mgr->mops->parse_header) return fpga_mgr_write_init_buf(mgr, info, NULL, 0); /* * First try to use miter to map the first fragment to access the * header, this is the typical path. */ - sg_miter_start(&miter, sgt->sgl, sgt->nents, SG_MITER_FROM_SG); - if (sg_miter_next(&miter) && - miter.length >= mgr->mops->initial_header_size) { - ret = fpga_mgr_write_init_buf(mgr, info, miter.addr, - miter.length); + ret = fpga_mgr_parse_header_sg_first(mgr, info, sgt); + /* If 0, header fits first fragment, call write_init on it */ + if (!ret) { + sg_miter_start(&miter, sgt->sgl, sgt->nents, SG_MITER_FROM_SG); + if (sg_miter_next(&miter)) { + ret = fpga_mgr_write_init_buf(mgr, info, miter.addr, + miter.length); + sg_miter_stop(&miter); + return ret; + } sg_miter_stop(&miter); + /* + * If -EAGAIN, more sg buffer is needed, + * otherwise an error has occurred. + */ + } else if (ret != -EAGAIN) { return ret; } - sg_miter_stop(&miter); - /* Otherwise copy the fragments into temporary memory. */ - buf = kmalloc(mgr->mops->initial_header_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; + /* + * Copy the fragments into temporary memory. + * Copying is done inside fpga_mgr_parse_header_sg(). + */ + buf = fpga_mgr_parse_header_sg(mgr, info, sgt, &len); + if (IS_ERR(buf)) + return PTR_ERR(buf); - len = sg_copy_to_buffer(sgt->sgl, sgt->nents, buf, - mgr->mops->initial_header_size); ret = fpga_mgr_write_init_buf(mgr, info, buf, len); kfree(buf); @@ -227,7 +364,7 @@ static int fpga_mgr_buf_load_sg(struct fpga_manager *mgr, { int ret; - ret = fpga_mgr_write_init_sg(mgr, info, sgt); + ret = fpga_mgr_prepare_sg(mgr, info, sgt); if (ret) return ret; @@ -236,17 +373,35 @@ static int fpga_mgr_buf_load_sg(struct fpga_manager *mgr, if (mgr->mops->write_sg) { ret = fpga_mgr_write_sg(mgr, sgt); } else { + size_t length, count = 0, data_size = info->data_size; struct sg_mapping_iter miter; sg_miter_start(&miter, sgt->sgl, sgt->nents, SG_MITER_FROM_SG); + + if (mgr->mops->skip_header && + !sg_miter_skip(&miter, info->header_size)) { + ret = -EINVAL; + goto out; + } + while (sg_miter_next(&miter)) { - ret = fpga_mgr_write(mgr, miter.addr, miter.length); + if (data_size) + length = min(miter.length, data_size - count); + else + length = miter.length; + + ret = fpga_mgr_write(mgr, miter.addr, length); if (ret) break; + + count += length; + if (data_size && count >= data_size) + break; } sg_miter_stop(&miter); } +out: if (ret) { dev_err(&mgr->dev, "Error while writing image data to FPGA\n"); mgr->state = FPGA_MGR_STATE_WRITE_ERR; @@ -262,10 +417,22 @@ static int fpga_mgr_buf_load_mapped(struct fpga_manager *mgr, { int ret; + ret = fpga_mgr_parse_header_mapped(mgr, info, buf, count); + if (ret) + return ret; + ret = fpga_mgr_write_init_buf(mgr, info, buf, count); if (ret) return ret; + if (mgr->mops->skip_header) { + buf += info->header_size; + count -= info->header_size; + } + + if (info->data_size) + count = info->data_size; + /* * Write the FPGA image to the FPGA. */ @@ -404,6 +571,8 @@ static int fpga_mgr_firmware_load(struct fpga_manager *mgr, */ int fpga_mgr_load(struct fpga_manager *mgr, struct fpga_image_info *info) { + info->header_size = mgr->mops->initial_header_size; + if (info->sgt) return fpga_mgr_buf_load_sg(mgr, info, info->sgt); if (info->buf && info->count) @@ -424,6 +593,10 @@ static const char * const state_str[] = { [FPGA_MGR_STATE_FIRMWARE_REQ] = "firmware request", [FPGA_MGR_STATE_FIRMWARE_REQ_ERR] = "firmware request error", + /* Parse FPGA image header */ + [FPGA_MGR_STATE_PARSE_HEADER] = "parse header", + [FPGA_MGR_STATE_PARSE_HEADER_ERR] = "parse header error", + /* Preparing FPGA to receive image */ [FPGA_MGR_STATE_WRITE_INIT] = "write init", [FPGA_MGR_STATE_WRITE_INIT_ERR] = "write init error", diff --git a/include/linux/fpga/fpga-mgr.h b/include/linux/fpga/fpga-mgr.h index 0f9468771bb9..54f63459efd6 100644 --- a/include/linux/fpga/fpga-mgr.h +++ b/include/linux/fpga/fpga-mgr.h @@ -22,6 +22,8 @@ struct sg_table; * @FPGA_MGR_STATE_RESET: FPGA in reset state * @FPGA_MGR_STATE_FIRMWARE_REQ: firmware request in progress * @FPGA_MGR_STATE_FIRMWARE_REQ_ERR: firmware request failed + * @FPGA_MGR_STATE_PARSE_HEADER: parse FPGA image header + * @FPGA_MGR_STATE_PARSE_HEADER_ERR: Error during PARSE_HEADER stage * @FPGA_MGR_STATE_WRITE_INIT: preparing FPGA for programming * @FPGA_MGR_STATE_WRITE_INIT_ERR: Error during WRITE_INIT stage * @FPGA_MGR_STATE_WRITE: writing image to FPGA @@ -41,7 +43,9 @@ enum fpga_mgr_states { FPGA_MGR_STATE_FIRMWARE_REQ, FPGA_MGR_STATE_FIRMWARE_REQ_ERR, - /* write sequence: init, write, complete */ + /* write sequence: parse header, init, write, complete */ + FPGA_MGR_STATE_PARSE_HEADER, + FPGA_MGR_STATE_PARSE_HEADER_ERR, FPGA_MGR_STATE_WRITE_INIT, FPGA_MGR_STATE_WRITE_INIT_ERR, FPGA_MGR_STATE_WRITE, @@ -85,6 +89,9 @@ enum fpga_mgr_states { * @sgt: scatter/gather table containing FPGA image * @buf: contiguous buffer containing FPGA image * @count: size of buf + * @header_size: size of image header. + * @data_size: size of image data to be sent to the device. If not specified, + * whole image will be used. Header may be skipped in either case. * @region_id: id of target region * @dev: device that owns this * @overlay: Device Tree overlay @@ -98,6 +105,8 @@ struct fpga_image_info { struct sg_table *sgt; const char *buf; size_t count; + size_t header_size; + size_t data_size; int region_id; struct device *dev; #ifdef CONFIG_OF @@ -137,9 +146,16 @@ struct fpga_manager_info { /** * struct fpga_manager_ops - ops for low level fpga manager drivers - * @initial_header_size: Maximum number of bytes that should be passed into write_init + * @initial_header_size: minimum number of bytes that should be passed into + * parse_header and write_init. + * @skip_header: bool flag to tell fpga-mgr core whether it should skip + * info->header_size part at the beginning of the image when invoking + * write callback. * @state: returns an enum value of the FPGA's state * @status: returns status of the FPGA, including reconfiguration error code + * @parse_header: parse FPGA image header to set info->header_size and + * info->data_size. In case the input buffer is not large enough, set + * required size to info->header_size and return -EAGAIN. * @write_init: prepare the FPGA to receive configuration data * @write: write count bytes of configuration data to the FPGA * @write_sg: write the scatter list of configuration data to the FPGA @@ -153,8 +169,12 @@ struct fpga_manager_info { */ struct fpga_manager_ops { size_t initial_header_size; + bool skip_header; enum fpga_mgr_states (*state)(struct fpga_manager *mgr); u64 (*status)(struct fpga_manager *mgr); + int (*parse_header)(struct fpga_manager *mgr, + struct fpga_image_info *info, + const char *buf, size_t count); int (*write_init)(struct fpga_manager *mgr, struct fpga_image_info *info, const char *buf, size_t count); -- cgit v1.2.3 From 837f66c71207542283831d0762c5dca3db5b397a Mon Sep 17 00:00:00 2001 From: David Matlack Date: Wed, 22 Jun 2022 15:27:08 -0400 Subject: KVM: Allow for different capacities in kvm_mmu_memory_cache structs Allow the capacity of the kvm_mmu_memory_cache struct to be chosen at declaration time rather than being fixed for all declarations. This will be used in a follow-up commit to declare an cache in x86 with a capacity of 512+ objects without having to increase the capacity of all caches in KVM. This change requires each cache now specify its capacity at runtime, since the cache struct itself no longer has a fixed capacity known at compile time. To protect against someone accidentally defining a kvm_mmu_memory_cache struct directly (without the extra storage), this commit includes a WARN_ON() in kvm_mmu_topup_memory_cache(). In order to support different capacities, this commit changes the objects pointer array to be dynamically allocated the first time the cache is topped-up. While here, opportunistically clean up the stack-allocated kvm_mmu_memory_cache structs in riscv and arm64 to use designated initializers. No functional change intended. Reviewed-by: Marc Zyngier Signed-off-by: David Matlack Message-Id: <20220516232138.1783324-22-dmatlack@google.com> Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/mmu.c | 2 +- arch/riscv/kvm/mmu.c | 5 +---- include/linux/kvm_host.h | 1 + include/linux/kvm_types.h | 6 +++++- virt/kvm/kvm_main.c | 33 ++++++++++++++++++++++++++++++--- 5 files changed, 38 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index f5651a05b6a8..87f1cd0df36e 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -786,7 +786,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, { phys_addr_t addr; int ret = 0; - struct kvm_mmu_memory_cache cache = { 0, __GFP_ZERO, NULL, }; + struct kvm_mmu_memory_cache cache = { .gfp_zero = __GFP_ZERO }; struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_DEVICE | KVM_PGTABLE_PROT_R | diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 1c00695ebee7..081f8d2b9cf3 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -350,10 +350,7 @@ static int gstage_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, int ret = 0; unsigned long pfn; phys_addr_t addr, end; - struct kvm_mmu_memory_cache pcache; - - memset(&pcache, 0, sizeof(pcache)); - pcache.gfp_zero = __GFP_ZERO; + struct kvm_mmu_memory_cache pcache = { .gfp_zero = __GFP_ZERO }; end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK; pfn = __phys_to_pfn(hpa); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index a2bbdf3ab086..3554e48406e4 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1356,6 +1356,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm); #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min); +int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min); int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc); void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc); void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index f328a01db4fe..4d933518060f 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -85,12 +85,16 @@ struct gfn_to_pfn_cache { * MMU flows is problematic, as is triggering reclaim, I/O, etc... while * holding MMU locks. Note, these caches act more like prefetch buffers than * classical caches, i.e. objects are not returned to the cache on being freed. + * + * The @capacity field and @objects array are lazily initialized when the cache + * is topped up (__kvm_mmu_topup_memory_cache()). */ struct kvm_mmu_memory_cache { int nobjs; gfp_t gfp_zero; struct kmem_cache *kmem_cache; - void *objects[KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE]; + int capacity; + void **objects; }; #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 5b8ae83e09d7..45188d11812c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -396,14 +396,31 @@ static inline void *mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache *mc, return (void *)__get_free_page(gfp_flags); } -int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min) +int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min) { + gfp_t gfp = GFP_KERNEL_ACCOUNT; void *obj; if (mc->nobjs >= min) return 0; - while (mc->nobjs < ARRAY_SIZE(mc->objects)) { - obj = mmu_memory_cache_alloc_obj(mc, GFP_KERNEL_ACCOUNT); + + if (unlikely(!mc->objects)) { + if (WARN_ON_ONCE(!capacity)) + return -EIO; + + mc->objects = kvmalloc_array(sizeof(void *), capacity, gfp); + if (!mc->objects) + return -ENOMEM; + + mc->capacity = capacity; + } + + /* It is illegal to request a different capacity across topups. */ + if (WARN_ON_ONCE(mc->capacity != capacity)) + return -EIO; + + while (mc->nobjs < mc->capacity) { + obj = mmu_memory_cache_alloc_obj(mc, gfp); if (!obj) return mc->nobjs >= min ? 0 : -ENOMEM; mc->objects[mc->nobjs++] = obj; @@ -411,6 +428,11 @@ int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min) return 0; } +int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min) +{ + return __kvm_mmu_topup_memory_cache(mc, KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE, min); +} + int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc) { return mc->nobjs; @@ -424,6 +446,11 @@ void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) else free_page((unsigned long)mc->objects[--mc->nobjs]); } + + kvfree(mc->objects); + + mc->objects = NULL; + mc->capacity = 0; } void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) -- cgit v1.2.3 From ebc3197963fc42841b183d0280bd3f9f85f13c30 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 23 Jun 2022 04:34:32 +0000 Subject: ipmr: add rcu protection over (struct vif_device)->dev We will soon use RCU instead of rwlock in ipmr & ip6mr This preliminary patch adds proper rcu verbs to read/write (struct vif_device)->dev Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/mroute_base.h | 11 +++++--- net/ipv4/ipmr.c | 65 ++++++++++++++++++++++++++------------------- net/ipv4/ipmr_base.c | 49 +++++++++++++++++++++------------- net/ipv6/ip6mr.c | 63 ++++++++++++++++++++++++------------------- 4 files changed, 111 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h index e05ee9f001ff..10d1e4fb4e9f 100644 --- a/include/linux/mroute_base.h +++ b/include/linux/mroute_base.h @@ -26,7 +26,7 @@ * @remote: Remote address for tunnels */ struct vif_device { - struct net_device *dev; + struct net_device __rcu *dev; netdevice_tracker dev_tracker; unsigned long bytes_in, bytes_out; unsigned long pkt_in, pkt_out; @@ -52,6 +52,7 @@ static inline int mr_call_vif_notifier(struct notifier_block *nb, unsigned short family, enum fib_event_type event_type, struct vif_device *vif, + struct net_device *vif_dev, unsigned short vif_index, u32 tb_id, struct netlink_ext_ack *extack) { @@ -60,7 +61,7 @@ static inline int mr_call_vif_notifier(struct notifier_block *nb, .family = family, .extack = extack, }, - .dev = vif->dev, + .dev = vif_dev, .vif_index = vif_index, .vif_flags = vif->flags, .tb_id = tb_id, @@ -73,6 +74,7 @@ static inline int mr_call_vif_notifiers(struct net *net, unsigned short family, enum fib_event_type event_type, struct vif_device *vif, + struct net_device *vif_dev, unsigned short vif_index, u32 tb_id, unsigned int *ipmr_seq) { @@ -80,7 +82,7 @@ static inline int mr_call_vif_notifiers(struct net *net, .info = { .family = family, }, - .dev = vif->dev, + .dev = vif_dev, .vif_index = vif_index, .vif_flags = vif->flags, .tb_id = tb_id, @@ -98,7 +100,8 @@ static inline int mr_call_vif_notifiers(struct net *net, #define MAXVIFS 32 #endif -#define VIF_EXISTS(_mrt, _idx) (!!((_mrt)->vif_table[_idx].dev)) +/* Note: This helper is deprecated. */ +#define VIF_EXISTS(_mrt, _idx) (!!rcu_access_pointer((_mrt)->vif_table[_idx].dev)) /* mfc_flags: * MFC_STATIC - the entry was added statically (not by a routing daemon) diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 8324e541d193..10371a9e78fc 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -79,6 +79,12 @@ struct ipmr_result { static DEFINE_RWLOCK(mrt_lock); +static struct net_device *vif_dev_read(const struct vif_device *vif) +{ + return rcu_dereference_check(vif->dev, + lockdep_is_held(&mrt_lock)); +} + /* Multicast router control variables */ /* Special spinlock for queue of unresolved entries */ @@ -586,7 +592,7 @@ static int __pim_rcv(struct mr_table *mrt, struct sk_buff *skb, read_lock(&mrt_lock); if (mrt->mroute_reg_vif_num >= 0) - reg_dev = mrt->vif_table[mrt->mroute_reg_vif_num].dev; + reg_dev = vif_dev_read(&mrt->vif_table[mrt->mroute_reg_vif_num]); read_unlock(&mrt_lock); if (!reg_dev) @@ -614,10 +620,11 @@ static struct net_device *ipmr_reg_vif(struct net *net, struct mr_table *mrt) static int call_ipmr_vif_entry_notifiers(struct net *net, enum fib_event_type event_type, struct vif_device *vif, + struct net_device *vif_dev, vifi_t vif_index, u32 tb_id) { return mr_call_vif_notifiers(net, RTNL_FAMILY_IPMR, event_type, - vif, vif_index, tb_id, + vif, vif_dev, vif_index, tb_id, &net->ipv4.ipmr_seq); } @@ -649,18 +656,14 @@ static int vif_delete(struct mr_table *mrt, int vifi, int notify, v = &mrt->vif_table[vifi]; - if (VIF_EXISTS(mrt, vifi)) - call_ipmr_vif_entry_notifiers(net, FIB_EVENT_VIF_DEL, v, vifi, - mrt->id); + dev = rtnl_dereference(v->dev); + if (!dev) + return -EADDRNOTAVAIL; write_lock_bh(&mrt_lock); - dev = v->dev; - v->dev = NULL; - - if (!dev) { - write_unlock_bh(&mrt_lock); - return -EADDRNOTAVAIL; - } + call_ipmr_vif_entry_notifiers(net, FIB_EVENT_VIF_DEL, v, dev, + vifi, mrt->id); + RCU_INIT_POINTER(v->dev, NULL); if (vifi == mrt->mroute_reg_vif_num) mrt->mroute_reg_vif_num = -1; @@ -890,14 +893,15 @@ static int vif_add(struct net *net, struct mr_table *mrt, /* And finish update writing critical data */ write_lock_bh(&mrt_lock); - v->dev = dev; + rcu_assign_pointer(v->dev, dev); netdev_tracker_alloc(dev, &v->dev_tracker, GFP_ATOMIC); if (v->flags & VIFF_REGISTER) mrt->mroute_reg_vif_num = vifi; if (vifi+1 > mrt->maxvif) mrt->maxvif = vifi+1; write_unlock_bh(&mrt_lock); - call_ipmr_vif_entry_notifiers(net, FIB_EVENT_VIF_ADD, v, vifi, mrt->id); + call_ipmr_vif_entry_notifiers(net, FIB_EVENT_VIF_ADD, v, dev, + vifi, mrt->id); return 0; } @@ -1726,7 +1730,7 @@ static int ipmr_device_event(struct notifier_block *this, unsigned long event, v ipmr_for_each_table(mrt, net) { v = &mrt->vif_table[0]; for (ct = 0; ct < mrt->maxvif; ct++, v++) { - if (v->dev == dev) + if (rcu_access_pointer(v->dev) == dev) vif_delete(mrt, ct, 1, NULL); } } @@ -1811,19 +1815,21 @@ static void ipmr_queue_xmit(struct net *net, struct mr_table *mrt, { const struct iphdr *iph = ip_hdr(skb); struct vif_device *vif = &mrt->vif_table[vifi]; + struct net_device *vif_dev; struct net_device *dev; struct rtable *rt; struct flowi4 fl4; int encap = 0; - if (!vif->dev) + vif_dev = vif_dev_read(vif); + if (!vif_dev) goto out_free; if (vif->flags & VIFF_REGISTER) { vif->pkt_out++; vif->bytes_out += skb->len; - vif->dev->stats.tx_bytes += skb->len; - vif->dev->stats.tx_packets++; + vif_dev->stats.tx_bytes += skb->len; + vif_dev->stats.tx_packets++; ipmr_cache_report(mrt, skb, vifi, IGMPMSG_WHOLEPKT); goto out_free; } @@ -1881,8 +1887,8 @@ static void ipmr_queue_xmit(struct net *net, struct mr_table *mrt, if (vif->flags & VIFF_TUNNEL) { ip_encap(net, skb, vif->local, vif->remote); /* FIXME: extra output firewall step used to be here. --RR */ - vif->dev->stats.tx_packets++; - vif->dev->stats.tx_bytes += skb->len; + vif_dev->stats.tx_packets++; + vif_dev->stats.tx_bytes += skb->len; } IPCB(skb)->flags |= IPSKB_FORWARDED; @@ -1911,7 +1917,7 @@ static int ipmr_find_vif(struct mr_table *mrt, struct net_device *dev) int ct; for (ct = mrt->maxvif-1; ct >= 0; ct--) { - if (mrt->vif_table[ct].dev == dev) + if (rcu_access_pointer(mrt->vif_table[ct].dev) == dev) break; } return ct; @@ -1944,7 +1950,7 @@ static void ip_mr_forward(struct net *net, struct mr_table *mrt, } /* Wrong interface: drop packet and (maybe) send PIM assert. */ - if (mrt->vif_table[vif].dev != dev) { + if (rcu_access_pointer(mrt->vif_table[vif].dev) != dev) { if (rt_is_output_route(skb_rtable(skb))) { /* It is our own packet, looped back. * Very complicated situation... @@ -2744,18 +2750,21 @@ static bool ipmr_fill_table(struct mr_table *mrt, struct sk_buff *skb) static bool ipmr_fill_vif(struct mr_table *mrt, u32 vifid, struct sk_buff *skb) { + struct net_device *vif_dev; struct nlattr *vif_nest; struct vif_device *vif; + vif = &mrt->vif_table[vifid]; + vif_dev = vif_dev_read(vif); /* if the VIF doesn't exist just continue */ - if (!VIF_EXISTS(mrt, vifid)) + if (!vif_dev) return true; - vif = &mrt->vif_table[vifid]; vif_nest = nla_nest_start_noflag(skb, IPMRA_VIF); if (!vif_nest) return false; - if (nla_put_u32(skb, IPMRA_VIFA_IFINDEX, vif->dev->ifindex) || + + if (nla_put_u32(skb, IPMRA_VIFA_IFINDEX, vif_dev->ifindex) || nla_put_u32(skb, IPMRA_VIFA_VIF_ID, vifid) || nla_put_u16(skb, IPMRA_VIFA_FLAGS, vif->flags) || nla_put_u64_64bit(skb, IPMRA_VIFA_BYTES_IN, vif->bytes_in, @@ -2919,9 +2928,11 @@ static int ipmr_vif_seq_show(struct seq_file *seq, void *v) "Interface BytesIn PktsIn BytesOut PktsOut Flags Local Remote\n"); } else { const struct vif_device *vif = v; - const char *name = vif->dev ? - vif->dev->name : "none"; + const struct net_device *vif_dev; + const char *name; + vif_dev = vif_dev_read(vif); + name = vif_dev ? vif_dev->name : "none"; seq_printf(seq, "%2td %-10s %8ld %7ld %8ld %7ld %05X %08X %08X\n", vif - mrt->vif_table, diff --git a/net/ipv4/ipmr_base.c b/net/ipv4/ipmr_base.c index aa8738a91210..59f62b938472 100644 --- a/net/ipv4/ipmr_base.c +++ b/net/ipv4/ipmr_base.c @@ -13,7 +13,7 @@ void vif_device_init(struct vif_device *v, unsigned short flags, unsigned short get_iflink_mask) { - v->dev = NULL; + RCU_INIT_POINTER(v->dev, NULL); v->bytes_in = 0; v->bytes_out = 0; v->pkt_in = 0; @@ -208,6 +208,7 @@ EXPORT_SYMBOL(mr_mfc_seq_next); int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb, struct mr_mfc *c, struct rtmsg *rtm) { + struct net_device *vif_dev; struct rta_mfc_stats mfcs; struct nlattr *mp_attr; struct rtnexthop *nhp; @@ -220,10 +221,13 @@ int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb, return -ENOENT; } - if (VIF_EXISTS(mrt, c->mfc_parent) && - nla_put_u32(skb, RTA_IIF, - mrt->vif_table[c->mfc_parent].dev->ifindex) < 0) + rcu_read_lock(); + vif_dev = rcu_dereference(mrt->vif_table[c->mfc_parent].dev); + if (vif_dev && nla_put_u32(skb, RTA_IIF, vif_dev->ifindex) < 0) { + rcu_read_unlock(); return -EMSGSIZE; + } + rcu_read_unlock(); if (c->mfc_flags & MFC_OFFLOAD) rtm->rtm_flags |= RTNH_F_OFFLOAD; @@ -232,23 +236,27 @@ int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb, if (!mp_attr) return -EMSGSIZE; + rcu_read_lock(); for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) { - if (VIF_EXISTS(mrt, ct) && c->mfc_un.res.ttls[ct] < 255) { - struct vif_device *vif; + struct vif_device *vif = &mrt->vif_table[ct]; + + vif_dev = rcu_dereference(vif->dev); + if (vif_dev && c->mfc_un.res.ttls[ct] < 255) { nhp = nla_reserve_nohdr(skb, sizeof(*nhp)); if (!nhp) { + rcu_read_unlock(); nla_nest_cancel(skb, mp_attr); return -EMSGSIZE; } nhp->rtnh_flags = 0; nhp->rtnh_hops = c->mfc_un.res.ttls[ct]; - vif = &mrt->vif_table[ct]; - nhp->rtnh_ifindex = vif->dev->ifindex; + nhp->rtnh_ifindex = vif_dev->ifindex; nhp->rtnh_len = sizeof(*nhp); } } + rcu_read_unlock(); nla_nest_end(skb, mp_attr); @@ -275,13 +283,14 @@ static bool mr_mfc_uses_dev(const struct mr_table *mrt, int ct; for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) { - if (VIF_EXISTS(mrt, ct) && c->mfc_un.res.ttls[ct] < 255) { - const struct vif_device *vif; - - vif = &mrt->vif_table[ct]; - if (vif->dev == dev) - return true; - } + const struct net_device *vif_dev; + const struct vif_device *vif; + + vif = &mrt->vif_table[ct]; + vif_dev = rcu_access_pointer(vif->dev); + if (vif_dev && c->mfc_un.res.ttls[ct] < 255 && + vif_dev == dev) + return true; } return false; } @@ -402,18 +411,22 @@ int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family, for (mrt = mr_iter(net, NULL); mrt; mrt = mr_iter(net, mrt)) { struct vif_device *v = &mrt->vif_table[0]; + struct net_device *vif_dev; struct mr_mfc *mfc; int vifi; /* Notifiy on table VIF entries */ read_lock(mrt_lock); for (vifi = 0; vifi < mrt->maxvif; vifi++, v++) { - if (!v->dev) + vif_dev = rcu_dereference_check(v->dev, + lockdep_is_held(mrt_lock)); + if (!vif_dev) continue; err = mr_call_vif_notifier(nb, family, - FIB_EVENT_VIF_ADD, - v, vifi, mrt->id, extack); + FIB_EVENT_VIF_ADD, v, + vif_dev, vifi, + mrt->id, extack); if (err) break; } diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index aa66c032ba97..44cb3d88bbd6 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -64,6 +64,12 @@ struct ip6mr_result { static DEFINE_RWLOCK(mrt_lock); +static struct net_device *vif_dev_read(const struct vif_device *vif) +{ + return rcu_dereference_check(vif->dev, + lockdep_is_held(&mrt_lock)); +} + /* Multicast router control variables */ /* Special spinlock for queue of unresolved entries */ @@ -430,7 +436,11 @@ static int ip6mr_vif_seq_show(struct seq_file *seq, void *v) "Interface BytesIn PktsIn BytesOut PktsOut Flags\n"); } else { const struct vif_device *vif = v; - const char *name = vif->dev ? vif->dev->name : "none"; + const struct net_device *vif_dev; + const char *name; + + vif_dev = vif_dev_read(vif); + name = vif_dev ? vif_dev->name : "none"; seq_printf(seq, "%2td %-10s %8ld %7ld %8ld %7ld %05X\n", @@ -553,7 +563,7 @@ static int pim6_rcv(struct sk_buff *skb) read_lock(&mrt_lock); if (reg_vif_num >= 0) - reg_dev = mrt->vif_table[reg_vif_num].dev; + reg_dev = vif_dev_read(&mrt->vif_table[reg_vif_num]); read_unlock(&mrt_lock); if (!reg_dev) @@ -668,10 +678,11 @@ failure: static int call_ip6mr_vif_entry_notifiers(struct net *net, enum fib_event_type event_type, struct vif_device *vif, + struct net_device *vif_dev, mifi_t vif_index, u32 tb_id) { return mr_call_vif_notifiers(net, RTNL_FAMILY_IP6MR, event_type, - vif, vif_index, tb_id, + vif, vif_dev, vif_index, tb_id, &net->ipv6.ipmr_seq); } @@ -696,19 +707,15 @@ static int mif6_delete(struct mr_table *mrt, int vifi, int notify, v = &mrt->vif_table[vifi]; - if (VIF_EXISTS(mrt, vifi)) - call_ip6mr_vif_entry_notifiers(read_pnet(&mrt->net), - FIB_EVENT_VIF_DEL, v, vifi, - mrt->id); + dev = rtnl_dereference(v->dev); + if (!dev) + return -EADDRNOTAVAIL; + call_ip6mr_vif_entry_notifiers(read_pnet(&mrt->net), + FIB_EVENT_VIF_DEL, v, dev, + vifi, mrt->id); write_lock_bh(&mrt_lock); - dev = v->dev; - v->dev = NULL; - - if (!dev) { - write_unlock_bh(&mrt_lock); - return -EADDRNOTAVAIL; - } + RCU_INIT_POINTER(v->dev, NULL); #ifdef CONFIG_IPV6_PIMSM_V2 if (vifi == mrt->mroute_reg_vif_num) @@ -911,7 +918,7 @@ static int mif6_add(struct net *net, struct mr_table *mrt, /* And finish update writing critical data */ write_lock_bh(&mrt_lock); - v->dev = dev; + rcu_assign_pointer(v->dev, dev); netdev_tracker_alloc(dev, &v->dev_tracker, GFP_ATOMIC); #ifdef CONFIG_IPV6_PIMSM_V2 if (v->flags & MIFF_REGISTER) @@ -921,7 +928,7 @@ static int mif6_add(struct net *net, struct mr_table *mrt, mrt->maxvif = vifi + 1; write_unlock_bh(&mrt_lock); call_ip6mr_vif_entry_notifiers(net, FIB_EVENT_VIF_ADD, - v, vifi, mrt->id); + v, dev, vifi, mrt->id); return 0; } @@ -1241,7 +1248,7 @@ static int ip6mr_device_event(struct notifier_block *this, ip6mr_for_each_table(mrt, net) { v = &mrt->vif_table[0]; for (ct = 0; ct < mrt->maxvif; ct++, v++) { - if (v->dev == dev) + if (rcu_access_pointer(v->dev) == dev) mif6_delete(mrt, ct, 1, NULL); } } @@ -2019,21 +2026,22 @@ static inline int ip6mr_forward2_finish(struct net *net, struct sock *sk, struct static int ip6mr_forward2(struct net *net, struct mr_table *mrt, struct sk_buff *skb, int vifi) { - struct ipv6hdr *ipv6h; struct vif_device *vif = &mrt->vif_table[vifi]; - struct net_device *dev; + struct net_device *vif_dev; + struct ipv6hdr *ipv6h; struct dst_entry *dst; struct flowi6 fl6; - if (!vif->dev) + vif_dev = vif_dev_read(vif); + if (!vif_dev) goto out_free; #ifdef CONFIG_IPV6_PIMSM_V2 if (vif->flags & MIFF_REGISTER) { vif->pkt_out++; vif->bytes_out += skb->len; - vif->dev->stats.tx_bytes += skb->len; - vif->dev->stats.tx_packets++; + vif_dev->stats.tx_bytes += skb->len; + vif_dev->stats.tx_packets++; ip6mr_cache_report(mrt, skb, vifi, MRT6MSG_WHOLEPKT); goto out_free; } @@ -2066,14 +2074,13 @@ static int ip6mr_forward2(struct net *net, struct mr_table *mrt, * not mrouter) cannot join to more than one interface - it will * result in receiving multiple packets. */ - dev = vif->dev; - skb->dev = dev; + skb->dev = vif_dev; vif->pkt_out++; vif->bytes_out += skb->len; /* We are about to write */ /* XXX: extension headers? */ - if (skb_cow(skb, sizeof(*ipv6h) + LL_RESERVED_SPACE(dev))) + if (skb_cow(skb, sizeof(*ipv6h) + LL_RESERVED_SPACE(vif_dev))) goto out_free; ipv6h = ipv6_hdr(skb); @@ -2082,7 +2089,7 @@ static int ip6mr_forward2(struct net *net, struct mr_table *mrt, IP6CB(skb)->flags |= IP6SKB_FORWARDED; return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, - net, NULL, skb, skb->dev, dev, + net, NULL, skb, skb->dev, vif_dev, ip6mr_forward2_finish); out_free: @@ -2095,7 +2102,7 @@ static int ip6mr_find_vif(struct mr_table *mrt, struct net_device *dev) int ct; for (ct = mrt->maxvif - 1; ct >= 0; ct--) { - if (mrt->vif_table[ct].dev == dev) + if (rcu_access_pointer(mrt->vif_table[ct].dev) == dev) break; } return ct; @@ -2133,7 +2140,7 @@ static void ip6_mr_forward(struct net *net, struct mr_table *mrt, /* * Wrong interface: drop packet and (maybe) send PIM assert. */ - if (mrt->vif_table[vif].dev != dev) { + if (rcu_access_pointer(mrt->vif_table[vif].dev) != dev) { c->_c.mfc_un.res.wrong_if++; if (true_vifi >= 0 && mrt->mroute_do_assert && -- cgit v1.2.3 From 194366b28b8306b7a24596c57c09635ab2891252 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 23 Jun 2022 04:34:46 +0000 Subject: ipmr: adopt rcu_read_lock() in mr_dump() We no longer need to acquire mrt_lock() in mr_dump, using rcu_read_lock() is enough. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/mroute_base.h | 4 ++-- net/ipv4/ipmr.c | 2 +- net/ipv4/ipmr_base.c | 8 +++----- net/ipv6/ip6mr.c | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h index 10d1e4fb4e9f..9dd4bf157255 100644 --- a/include/linux/mroute_base.h +++ b/include/linux/mroute_base.h @@ -308,7 +308,7 @@ int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family, struct netlink_ext_ack *extack), struct mr_table *(*mr_iter)(struct net *net, struct mr_table *mrt), - rwlock_t *mrt_lock, struct netlink_ext_ack *extack); + struct netlink_ext_ack *extack); #else static inline void vif_device_init(struct vif_device *v, struct net_device *dev, @@ -363,7 +363,7 @@ static inline int mr_dump(struct net *net, struct notifier_block *nb, struct netlink_ext_ack *extack), struct mr_table *(*mr_iter)(struct net *net, struct mr_table *mrt), - rwlock_t *mrt_lock, struct netlink_ext_ack *extack) + struct netlink_ext_ack *extack) { return -EINVAL; } diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 69ccd3d7c655..38963b8de7af 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -3027,7 +3027,7 @@ static int ipmr_dump(struct net *net, struct notifier_block *nb, struct netlink_ext_ack *extack) { return mr_dump(net, nb, RTNL_FAMILY_IPMR, ipmr_rules_dump, - ipmr_mr_table_iter, &mrt_lock, extack); + ipmr_mr_table_iter, extack); } static const struct fib_notifier_ops ipmr_notifier_ops_template = { diff --git a/net/ipv4/ipmr_base.c b/net/ipv4/ipmr_base.c index 59f62b938472..271dc03fc6db 100644 --- a/net/ipv4/ipmr_base.c +++ b/net/ipv4/ipmr_base.c @@ -399,7 +399,6 @@ int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family, struct netlink_ext_ack *extack), struct mr_table *(*mr_iter)(struct net *net, struct mr_table *mrt), - rwlock_t *mrt_lock, struct netlink_ext_ack *extack) { struct mr_table *mrt; @@ -416,10 +415,9 @@ int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family, int vifi; /* Notifiy on table VIF entries */ - read_lock(mrt_lock); + rcu_read_lock(); for (vifi = 0; vifi < mrt->maxvif; vifi++, v++) { - vif_dev = rcu_dereference_check(v->dev, - lockdep_is_held(mrt_lock)); + vif_dev = rcu_dereference(v->dev); if (!vif_dev) continue; @@ -430,7 +428,7 @@ int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family, if (err) break; } - read_unlock(mrt_lock); + rcu_read_unlock(); if (err) return err; diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 08ac177fe30c..f0a9bceb8e3c 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1267,7 +1267,7 @@ static int ip6mr_dump(struct net *net, struct notifier_block *nb, struct netlink_ext_ack *extack) { return mr_dump(net, nb, RTNL_FAMILY_IP6MR, ip6mr_rules_dump, - ip6mr_mr_table_iter, &mrt_lock, extack); + ip6mr_mr_table_iter, extack); } static struct notifier_block ip6_mr_notifier = { -- cgit v1.2.3 From ba1afa676d0babf99e99f5415db43fdd7ecef104 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 24 Jun 2022 17:31:47 +0800 Subject: lib: bitmap: fix the duplicated comments on bitmap_to_arr64() Thanks to the recent commit 0a97953fd221 ("lib: add bitmap_{from,to}_arr64") now we can directly convert a U64 value into a bitmap and vice verse. However when checking the header there is duplicated helper for bitmap_to_arr64(), but no bitmap_from_arr64(). Just fix the copy-n-paste error. Signed-off-by: Qu Wenruo Signed-off-by: Yury Norov --- include/linux/bitmap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 2e6cd5681040..f091a1664bf1 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -71,9 +71,9 @@ struct device; * bitmap_release_region(bitmap, pos, order) Free specified bit region * bitmap_allocate_region(bitmap, pos, order) Allocate specified bit region * bitmap_from_arr32(dst, buf, nbits) Copy nbits from u32[] buf to dst + * bitmap_from_arr64(dst, buf, nbits) Copy nbits from u64[] buf to dst * bitmap_to_arr32(buf, src, nbits) Copy nbits from buf to u32[] dst * bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst - * bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst * bitmap_get_value8(map, start) Get 8bit value from map at start * bitmap_set_value8(map, value, start) Set 8bit value to map at start * -- cgit v1.2.3 From e61c451476e61450f6771ce03bbc01210a09be16 Mon Sep 17 00:00:00 2001 From: Mark-PK Tsai Date: Fri, 22 Apr 2022 14:24:35 +0800 Subject: dma-mapping: Add dma_release_coherent_memory to DMA API Add dma_release_coherent_memory to DMA API to allow dma user call it to release dev->dma_mem when the device is removed. Signed-off-by: Mark-PK Tsai Acked-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220422062436.14384-2-mark-pk.tsai@mediatek.com Signed-off-by: Mathieu Poirier --- include/linux/dma-map-ops.h | 3 +++ kernel/dma/coherent.c | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 0d5b06b3a4a6..53db9655efe9 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -166,6 +166,7 @@ static inline void dma_pernuma_cma_reserve(void) { } #ifdef CONFIG_DMA_DECLARE_COHERENT int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, dma_addr_t device_addr, size_t size); +void dma_release_coherent_memory(struct device *dev); int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size, dma_addr_t *dma_handle, void **ret); int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr); @@ -177,6 +178,8 @@ static inline int dma_declare_coherent_memory(struct device *dev, { return -ENOSYS; } + +#define dma_release_coherent_memory(dev) (0) #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0) #define dma_release_from_dev_coherent(dev, order, vaddr) (0) #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0) diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c index 375fb3c9538d..c21abc77c53e 100644 --- a/kernel/dma/coherent.c +++ b/kernel/dma/coherent.c @@ -74,7 +74,7 @@ out_unmap_membase: return ERR_PTR(-ENOMEM); } -static void dma_release_coherent_memory(struct dma_coherent_mem *mem) +static void _dma_release_coherent_memory(struct dma_coherent_mem *mem) { if (!mem) return; @@ -126,10 +126,16 @@ int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, ret = dma_assign_coherent_memory(dev, mem); if (ret) - dma_release_coherent_memory(mem); + _dma_release_coherent_memory(mem); return ret; } +void dma_release_coherent_memory(struct device *dev) +{ + if (dev) + _dma_release_coherent_memory(dev->dma_mem); +} + static void *__dma_alloc_from_coherent(struct device *dev, struct dma_coherent_mem *mem, ssize_t size, dma_addr_t *dma_handle) -- cgit v1.2.3 From e36de87d34a7f2f26b2e2129c6dc18a0024663eb Mon Sep 17 00:00:00 2001 From: Vineeth Pillai Date: Mon, 23 May 2022 15:03:27 -0400 Subject: KVM: debugfs: expose pid of vcpu threads Add a new debugfs file to expose the pid of each vcpu threads. This is very helpful for userland tools to get the vcpu pids without worrying about thread naming conventions of the VMM. Signed-off-by: Vineeth Pillai (Google) Message-Id: <20220523190327.2658-1-vineeth@bitbyteword.org> Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 2 ++ virt/kvm/kvm_main.c | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 3554e48406e4..3b40f8d68fbb 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1435,6 +1435,8 @@ int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state); #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry); +#else +static inline void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) {} #endif int kvm_arch_hardware_enable(void); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 45188d11812c..da263c370d00 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -3822,9 +3822,18 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu) return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); } +#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS +static int vcpu_get_pid(void *data, u64 *val) +{ + struct kvm_vcpu *vcpu = (struct kvm_vcpu *) data; + *val = pid_nr(rcu_access_pointer(vcpu->pid)); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(vcpu_get_pid_fops, vcpu_get_pid, NULL, "%llu\n"); + static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) { -#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS struct dentry *debugfs_dentry; char dir_name[ITOA_MAX_LEN * 2]; @@ -3834,10 +3843,12 @@ static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); debugfs_dentry = debugfs_create_dir(dir_name, vcpu->kvm->debugfs_dentry); + debugfs_create_file("pid", 0444, debugfs_dentry, vcpu, + &vcpu_get_pid_fops); kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry); -#endif } +#endif /* * Creates some virtual cpus. Good luck creating more than one. -- cgit v1.2.3 From 8ca869b24538a7b5501af368e87e4a59b0c04117 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 22 Jun 2022 09:31:31 +0200 Subject: pstore: Add priv field to pstore_record for backend specific use The EFI pstore backend will need to store per-record variable name data when we switch away from the efivars layer. Add a priv field to struct pstore_record, and document it as holding a backend specific pointer that is assumed to be a kmalloc()d buffer, and will be kfree()d when the entire record is freed. Acked-by: Kees Cook Signed-off-by: Ard Biesheuvel --- fs/pstore/inode.c | 1 + fs/pstore/platform.c | 1 + include/linux/pstore.h | 4 ++++ 3 files changed, 6 insertions(+) (limited to 'include/linux') diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c index 14658b009f1b..ffbadb8b3032 100644 --- a/fs/pstore/inode.c +++ b/fs/pstore/inode.c @@ -55,6 +55,7 @@ static void free_pstore_private(struct pstore_private *private) return; if (private->record) { kfree(private->record->buf); + kfree(private->record->priv); kfree(private->record); } kfree(private); diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c index e26162f102ff..0c034ea39954 100644 --- a/fs/pstore/platform.c +++ b/fs/pstore/platform.c @@ -769,6 +769,7 @@ void pstore_get_backend_records(struct pstore_info *psi, if (rc) { /* pstore_mkfile() did not take record, so free it. */ kfree(record->buf); + kfree(record->priv); kfree(record); if (rc != -EEXIST || !quiet) failed++; diff --git a/include/linux/pstore.h b/include/linux/pstore.h index e97a8188f0fd..638507a3c8ff 100644 --- a/include/linux/pstore.h +++ b/include/linux/pstore.h @@ -57,6 +57,9 @@ struct pstore_info; * @size: size of @buf * @ecc_notice_size: * ECC information for @buf + * @priv: pointer for backend specific use, will be + * kfree()d by the pstore core if non-NULL + * when the record is freed. * * Valid for PSTORE_TYPE_DMESG @type: * @@ -74,6 +77,7 @@ struct pstore_record { char *buf; ssize_t size; ssize_t ecc_notice_size; + void *priv; int count; enum kmsg_dump_reason reason; -- cgit v1.2.3 From ec3507b2ca51286de6ecd85fdac8e722219cdef8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Jun 2022 17:04:32 +0200 Subject: efi: vars: Don't drop lock in the middle of efivar_init() Even though the efivars_lock lock is documented as protecting the efivars->ops pointer (among other things), efivar_init() happily releases and reacquires the lock for every EFI variable that it enumerates. This used to be needed because the lock was originally a spinlock, which prevented the callback that is invoked for every variable from being able to sleep. However, releasing the lock could potentially invalidate the ops pointer, but more importantly, it might allow a SetVariable() runtime service call to take place concurrently, and the UEFI spec does not define how this affects an enumeration that is running in parallel using the GetNextVariable() runtime service, which is what efivar_init() uses. In the meantime, the lock has been converted into a semaphore, and the only reason we need to drop the lock is because the efivarfs pseudo filesystem driver will otherwise deadlock when it invokes the efivars API from the callback to create the efivar_entry items and insert them into the linked list. (EFI pstore is affected in a similar way) So let's switch to helpers that can be used while the lock is already taken. This way, we can hold on to the lock throughout the enumeration. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/efi-pstore.c | 7 ++----- drivers/firmware/efi/efivars.c | 5 +---- drivers/firmware/efi/vars.c | 22 +++++++++++----------- fs/efivarfs/super.c | 6 ++---- include/linux/efi.h | 1 + 5 files changed, 17 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/efi-pstore.c b/drivers/firmware/efi/efi-pstore.c index 7e771c56c13c..0d80cc7ff6ca 100644 --- a/drivers/firmware/efi/efi-pstore.c +++ b/drivers/firmware/efi/efi-pstore.c @@ -364,7 +364,6 @@ static int efi_pstore_callback(efi_char16_t *name, efi_guid_t vendor, unsigned long name_size, void *data) { struct efivar_entry *entry; - int ret; entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (!entry) @@ -373,11 +372,9 @@ static int efi_pstore_callback(efi_char16_t *name, efi_guid_t vendor, memcpy(entry->var.VariableName, name, name_size); entry->var.VendorGuid = vendor; - ret = efivar_entry_add(entry, &efi_pstore_list); - if (ret) - kfree(entry); + __efivar_entry_add(entry, &efi_pstore_list); - return ret; + return 0; } static int efi_pstore_update_entry(efi_char16_t *name, efi_guid_t vendor, diff --git a/drivers/firmware/efi/efivars.c b/drivers/firmware/efi/efivars.c index ea0bc39dc965..c19db0b35c0d 100644 --- a/drivers/firmware/efi/efivars.c +++ b/drivers/firmware/efi/efivars.c @@ -527,10 +527,7 @@ efivar_create_sysfs_entry(struct efivar_entry *new_var) } kobject_uevent(&new_var->kobj, KOBJ_ADD); - if (efivar_entry_add(new_var, &efivar_sysfs_list)) { - efivar_unregister(new_var); - return -EINTR; - } + __efivar_entry_add(new_var, &efivar_sysfs_list); return 0; } diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index cae590bd08f2..146360e2f1cb 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -450,9 +450,6 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), &vendor_guid); switch (status) { case EFI_SUCCESS: - if (duplicates) - up(&efivars_lock); - variable_name_size = var_name_strnsize(variable_name, variable_name_size); @@ -476,14 +473,6 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), if (err) status = EFI_NOT_FOUND; } - - if (duplicates) { - if (down_interruptible(&efivars_lock)) { - err = -EINTR; - goto free; - } - } - break; case EFI_UNSUPPORTED: err = -EOPNOTSUPP; @@ -526,6 +515,17 @@ int efivar_entry_add(struct efivar_entry *entry, struct list_head *head) } EXPORT_SYMBOL_GPL(efivar_entry_add); +/** + * __efivar_entry_add - add entry to variable list + * @entry: entry to add to list + * @head: list head + */ +void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head) +{ + list_add(&entry->list, head); +} +EXPORT_SYMBOL_GPL(__efivar_entry_add); + /** * efivar_entry_remove - remove entry from variable list * @entry: entry to remove from list diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c index 15880a68faad..09dfa8362f50 100644 --- a/fs/efivarfs/super.c +++ b/fs/efivarfs/super.c @@ -155,10 +155,8 @@ static int efivarfs_callback(efi_char16_t *name16, efi_guid_t vendor, goto fail_inode; } - efivar_entry_size(entry, &size); - err = efivar_entry_add(entry, &efivarfs_list); - if (err) - goto fail_inode; + __efivar_entry_get(entry, NULL, &size, NULL); + __efivar_entry_add(entry, &efivarfs_list); /* copied by the above to local storage in the dentry. */ kfree(name); diff --git a/include/linux/efi.h b/include/linux/efi.h index 53f64c14a525..56f04b6daeb0 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1064,6 +1064,7 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), void *data, bool duplicates, struct list_head *head); int efivar_entry_add(struct efivar_entry *entry, struct list_head *head); +void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head); int efivar_entry_remove(struct efivar_entry *entry); int __efivar_entry_delete(struct efivar_entry *entry); -- cgit v1.2.3 From 472831d4c4b2d8eac783b256e5c829487d5310df Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Jun 2022 13:17:20 +0200 Subject: efi: vars: Add thin wrapper around EFI get/set variable interface The current efivars layer is a jumble of list iterators, shadow data structures and safe variable manipulation helpers that really belong in the efivarfs pseudo file system once the obsolete sysfs access method to EFI variables is removed. So split off a minimal efivar get/set variable API that reuses the existing efivars_lock semaphore to mediate access to the various runtime services, primarily to ensure that performing a SetVariable() on one CPU while another is calling GetNextVariable() in a loop to enumerate the contents of the EFI variable store does not result in surprises. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/vars.c | 154 +++++++++++++++++++++++++++++++++++++++++--- include/linux/efi.h | 20 ++++++ 2 files changed, 164 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index 146360e2f1cb..41c82614a4b2 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -298,14 +298,10 @@ efivar_variable_is_removable(efi_guid_t vendor, const char *var_name, } EXPORT_SYMBOL_GPL(efivar_variable_is_removable); -static efi_status_t -check_var_size(u32 attributes, unsigned long size) +efi_status_t check_var_size(u32 attributes, unsigned long size) { const struct efivar_operations *fops; - if (!__efivars) - return EFI_UNSUPPORTED; - fops = __efivars->ops; if (!fops->query_variable_store) @@ -313,15 +309,12 @@ check_var_size(u32 attributes, unsigned long size) return fops->query_variable_store(attributes, size, false); } +EXPORT_SYMBOL_NS_GPL(check_var_size, EFIVAR); -static efi_status_t -check_var_size_nonblocking(u32 attributes, unsigned long size) +efi_status_t check_var_size_nonblocking(u32 attributes, unsigned long size) { const struct efivar_operations *fops; - if (!__efivars) - return EFI_UNSUPPORTED; - fops = __efivars->ops; if (!fops->query_variable_store) @@ -329,6 +322,7 @@ check_var_size_nonblocking(u32 attributes, unsigned long size) return fops->query_variable_store(attributes, size, true); } +EXPORT_SYMBOL_NS_GPL(check_var_size_nonblocking, EFIVAR); static bool variable_is_present(efi_char16_t *variable_name, efi_guid_t *vendor, struct list_head *head) @@ -1220,3 +1214,143 @@ int efivar_supports_writes(void) return __efivars && __efivars->ops->set_variable; } EXPORT_SYMBOL_GPL(efivar_supports_writes); + +/* + * efivar_lock() - obtain the efivar lock, wait for it if needed + * @return 0 on success, error code on failure + */ +int efivar_lock(void) +{ + if (down_interruptible(&efivars_lock)) + return -EINTR; + if (!__efivars->ops) { + up(&efivars_lock); + return -ENODEV; + } + return 0; +} +EXPORT_SYMBOL_NS_GPL(efivar_lock, EFIVAR); + +/* + * efivar_lock() - obtain the efivar lock if it is free + * @return 0 on success, error code on failure + */ +int efivar_trylock(void) +{ + if (down_trylock(&efivars_lock)) + return -EBUSY; + if (!__efivars->ops) { + up(&efivars_lock); + return -ENODEV; + } + return 0; +} +EXPORT_SYMBOL_NS_GPL(efivar_trylock, EFIVAR); + +/* + * efivar_unlock() - release the efivar lock + */ +void efivar_unlock(void) +{ + up(&efivars_lock); +} +EXPORT_SYMBOL_NS_GPL(efivar_unlock, EFIVAR); + +/* + * efivar_get_variable() - retrieve a variable identified by name/vendor + * + * Must be called with efivars_lock held. + */ +efi_status_t efivar_get_variable(efi_char16_t *name, efi_guid_t *vendor, + u32 *attr, unsigned long *size, void *data) +{ + return __efivars->ops->get_variable(name, vendor, attr, size, data); +} +EXPORT_SYMBOL_NS_GPL(efivar_get_variable, EFIVAR); + +/* + * efivar_get_next_variable() - enumerate the next name/vendor pair + * + * Must be called with efivars_lock held. + */ +efi_status_t efivar_get_next_variable(unsigned long *name_size, + efi_char16_t *name, efi_guid_t *vendor) +{ + return __efivars->ops->get_next_variable(name_size, name, vendor); +} +EXPORT_SYMBOL_NS_GPL(efivar_get_next_variable, EFIVAR); + +/* + * efivar_set_variable_blocking() - local helper function for set_variable + * + * Must be called with efivars_lock held. + */ +static efi_status_t +efivar_set_variable_blocking(efi_char16_t *name, efi_guid_t *vendor, + u32 attr, unsigned long data_size, void *data) +{ + efi_status_t status; + + if (data_size > 0) { + status = check_var_size(attr, data_size + + ucs2_strsize(name, 1024)); + if (status != EFI_SUCCESS) + return status; + } + return __efivars->ops->set_variable(name, vendor, attr, data_size, data); +} + +/* + * efivar_set_variable_locked() - set a variable identified by name/vendor + * + * Must be called with efivars_lock held. If @nonblocking is set, it will use + * non-blocking primitives so it is guaranteed not to sleep. + */ +efi_status_t efivar_set_variable_locked(efi_char16_t *name, efi_guid_t *vendor, + u32 attr, unsigned long data_size, + void *data, bool nonblocking) +{ + efi_set_variable_t *setvar; + efi_status_t status; + + if (!nonblocking) + return efivar_set_variable_blocking(name, vendor, attr, + data_size, data); + + /* + * If no _nonblocking variant exists, the ordinary one + * is assumed to be non-blocking. + */ + setvar = __efivars->ops->set_variable_nonblocking ?: + __efivars->ops->set_variable; + + if (data_size > 0) { + status = check_var_size_nonblocking(attr, data_size + + ucs2_strsize(name, 1024)); + if (status != EFI_SUCCESS) + return status; + } + return setvar(name, vendor, attr, data_size, data); +} +EXPORT_SYMBOL_NS_GPL(efivar_set_variable_locked, EFIVAR); + +/* + * efivar_set_variable() - set a variable identified by name/vendor + * + * Can be called without holding the efivars_lock. Will sleep on obtaining the + * lock, or on obtaining other locks that are needed in order to complete the + * call. + */ +efi_status_t efivar_set_variable(efi_char16_t *name, efi_guid_t *vendor, + u32 attr, unsigned long data_size, void *data) +{ + efi_status_t status; + + if (efivar_lock()) + return EFI_ABORTED; + + status = efivar_set_variable_blocking(name, vendor, attr, data_size, data); + efivar_unlock(); + return status; +} +EXPORT_SYMBOL_NS_GPL(efivar_set_variable, EFIVAR); diff --git a/include/linux/efi.h b/include/linux/efi.h index 56f04b6daeb0..c828ab6f0e2a 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1099,6 +1099,26 @@ bool efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, bool efivar_variable_is_removable(efi_guid_t vendor, const char *name, size_t len); +int efivar_lock(void); +int efivar_trylock(void); +void efivar_unlock(void); + +efi_status_t efivar_get_variable(efi_char16_t *name, efi_guid_t *vendor, + u32 *attr, unsigned long *size, void *data); + +efi_status_t efivar_get_next_variable(unsigned long *name_size, + efi_char16_t *name, efi_guid_t *vendor); + +efi_status_t efivar_set_variable_locked(efi_char16_t *name, efi_guid_t *vendor, + u32 attr, unsigned long data_size, + void *data, bool nonblocking); + +efi_status_t efivar_set_variable(efi_char16_t *name, efi_guid_t *vendor, + u32 attr, unsigned long data_size, void *data); + +efi_status_t check_var_size(u32 attributes, unsigned long size); +efi_status_t check_var_size_nonblocking(u32 attributes, unsigned long size); + #if IS_ENABLED(CONFIG_EFI_CAPSULE_LOADER) extern bool efi_capsule_pending(int *reset_type); -- cgit v1.2.3 From 859748255b43460685e93a1f8a40b8cdc3be02f2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Jun 2022 13:21:26 +0200 Subject: efi: pstore: Omit efivars caching EFI varstore access layer Avoid the efivars layer and simply call the newly introduced EFI varstore helpers instead. This simplifies the code substantially, and also allows us to remove some hacks in the shared efivars layer that were added for efi-pstore specifically. In order to be able to delete the EFI variable associated with a record, store the UTF-16 name of the variable in the pstore record's priv field. That way, we don't have to make guesses regarding which variable the record may have been loaded from. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/Kconfig | 1 + drivers/firmware/efi/efi-pstore.c | 374 +++++++++----------------------------- drivers/firmware/efi/efivars.c | 12 +- drivers/firmware/efi/vars.c | 12 +- include/linux/efi.h | 2 - 5 files changed, 91 insertions(+), 310 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index 6fd4414c4836..7fe8b5c686d2 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -22,6 +22,7 @@ config EFI_ESRT config EFI_VARS_PSTORE tristate "Register efivars backend for pstore" depends on PSTORE + select UCS2_STRING default y help Say Y here to enable use efivars as a backend to pstore. This diff --git a/drivers/firmware/efi/efi-pstore.c b/drivers/firmware/efi/efi-pstore.c index 0d80cc7ff6ca..3bddc152fcd4 100644 --- a/drivers/firmware/efi/efi-pstore.c +++ b/drivers/firmware/efi/efi-pstore.c @@ -6,6 +6,8 @@ #include #include +MODULE_IMPORT_NS(EFIVAR); + #define DUMP_NAME_LEN 66 #define EFIVARS_DATA_SIZE_MAX 1024 @@ -20,18 +22,25 @@ module_param_named(pstore_disable, efivars_pstore_disable, bool, 0644); EFI_VARIABLE_BOOTSERVICE_ACCESS | \ EFI_VARIABLE_RUNTIME_ACCESS) -static LIST_HEAD(efi_pstore_list); -static DECLARE_WORK(efivar_work, NULL); - static int efi_pstore_open(struct pstore_info *psi) { - psi->data = NULL; + int err; + + err = efivar_lock(); + if (err) + return err; + + psi->data = kzalloc(EFIVARS_DATA_SIZE_MAX, GFP_KERNEL); + if (!psi->data) + return -ENOMEM; + return 0; } static int efi_pstore_close(struct pstore_info *psi) { - psi->data = NULL; + efivar_unlock(); + kfree(psi->data); return 0; } @@ -40,22 +49,17 @@ static inline u64 generic_id(u64 timestamp, unsigned int part, int count) return (timestamp * 100 + part) * 1000 + count; } -static int efi_pstore_read_func(struct efivar_entry *entry, - struct pstore_record *record) +static int efi_pstore_read_func(struct pstore_record *record, + efi_char16_t *varname) { - efi_guid_t vendor = LINUX_EFI_CRASH_GUID; + unsigned long wlen, size = EFIVARS_DATA_SIZE_MAX; char name[DUMP_NAME_LEN], data_type; - int i; + efi_status_t status; int cnt; unsigned int part; - unsigned long size; u64 time; - if (efi_guidcmp(entry->var.VendorGuid, vendor)) - return 0; - - for (i = 0; i < DUMP_NAME_LEN; i++) - name[i] = entry->var.VariableName[i]; + ucs2_as_utf8(name, varname, DUMP_NAME_LEN); if (sscanf(name, "dump-type%u-%u-%d-%llu-%c", &record->type, &part, &cnt, &time, &data_type) == 5) { @@ -95,161 +99,75 @@ static int efi_pstore_read_func(struct efivar_entry *entry, } else return 0; - entry->var.DataSize = 1024; - __efivar_entry_get(entry, &entry->var.Attributes, - &entry->var.DataSize, entry->var.Data); - size = entry->var.DataSize; - memcpy(record->buf, entry->var.Data, - (size_t)min_t(unsigned long, EFIVARS_DATA_SIZE_MAX, size)); - - return size; -} - -/** - * efi_pstore_scan_sysfs_enter - * @pos: scanning entry - * @next: next entry - * @head: list head - */ -static void efi_pstore_scan_sysfs_enter(struct efivar_entry *pos, - struct efivar_entry *next, - struct list_head *head) -{ - pos->scanning = true; - if (&next->list != head) - next->scanning = true; -} - -/** - * __efi_pstore_scan_sysfs_exit - * @entry: deleting entry - * @turn_off_scanning: Check if a scanning flag should be turned off - */ -static inline int __efi_pstore_scan_sysfs_exit(struct efivar_entry *entry, - bool turn_off_scanning) -{ - if (entry->deleting) { - list_del(&entry->list); - efivar_entry_iter_end(); - kfree(entry); - if (efivar_entry_iter_begin()) - return -EINTR; - } else if (turn_off_scanning) - entry->scanning = false; - - return 0; -} - -/** - * efi_pstore_scan_sysfs_exit - * @pos: scanning entry - * @next: next entry - * @head: list head - * @stop: a flag checking if scanning will stop - */ -static int efi_pstore_scan_sysfs_exit(struct efivar_entry *pos, - struct efivar_entry *next, - struct list_head *head, bool stop) -{ - int ret = __efi_pstore_scan_sysfs_exit(pos, true); - - if (ret) - return ret; - - if (stop) - ret = __efi_pstore_scan_sysfs_exit(next, &next->list != head); - return ret; -} + record->buf = kmalloc(size, GFP_KERNEL); + if (!record->buf) + return -ENOMEM; -/** - * efi_pstore_sysfs_entry_iter - * - * @record: pstore record to pass to callback - * - * You MUST call efivar_entry_iter_begin() before this function, and - * efivar_entry_iter_end() afterwards. - * - */ -static int efi_pstore_sysfs_entry_iter(struct pstore_record *record) -{ - struct efivar_entry **pos = (struct efivar_entry **)&record->psi->data; - struct efivar_entry *entry, *n; - struct list_head *head = &efi_pstore_list; - int size = 0; - int ret; - - if (!*pos) { - list_for_each_entry_safe(entry, n, head, list) { - efi_pstore_scan_sysfs_enter(entry, n, head); - - size = efi_pstore_read_func(entry, record); - ret = efi_pstore_scan_sysfs_exit(entry, n, head, - size < 0); - if (ret) - return ret; - if (size) - break; - } - *pos = n; - return size; + status = efivar_get_variable(varname, &LINUX_EFI_CRASH_GUID, NULL, + &size, record->buf); + if (status != EFI_SUCCESS) { + kfree(record->buf); + return -EIO; } - list_for_each_entry_safe_from((*pos), n, head, list) { - efi_pstore_scan_sysfs_enter((*pos), n, head); - - size = efi_pstore_read_func((*pos), record); - ret = efi_pstore_scan_sysfs_exit((*pos), n, head, size < 0); - if (ret) - return ret; - if (size) - break; + /* + * Store the name of the variable in the pstore_record priv field, so + * we can reuse it later if we need to delete the EFI variable from the + * variable store. + */ + wlen = (ucs2_strnlen(varname, DUMP_NAME_LEN) + 1) * sizeof(efi_char16_t); + record->priv = kmemdup(varname, wlen, GFP_KERNEL); + if (!record->priv) { + kfree(record->buf); + return -ENOMEM; } - *pos = n; + return size; } -/** - * efi_pstore_read - * - * This function returns a size of NVRAM entry logged via efi_pstore_write(). - * The meaning and behavior of efi_pstore/pstore are as below. - * - * size > 0: Got data of an entry logged via efi_pstore_write() successfully, - * and pstore filesystem will continue reading subsequent entries. - * size == 0: Entry was not logged via efi_pstore_write(), - * and efi_pstore driver will continue reading subsequent entries. - * size < 0: Failed to get data of entry logging via efi_pstore_write(), - * and pstore will stop reading entry. - */ static ssize_t efi_pstore_read(struct pstore_record *record) { - ssize_t size; + efi_char16_t *varname = record->psi->data; + efi_guid_t guid = LINUX_EFI_CRASH_GUID; + unsigned long varname_size; + efi_status_t status; - record->buf = kzalloc(EFIVARS_DATA_SIZE_MAX, GFP_KERNEL); - if (!record->buf) - return -ENOMEM; + for (;;) { + varname_size = EFIVARS_DATA_SIZE_MAX; - if (efivar_entry_iter_begin()) { - size = -EINTR; - goto out; - } - size = efi_pstore_sysfs_entry_iter(record); - efivar_entry_iter_end(); + /* + * If this is the first read() call in the pstore enumeration, + * varname will be the empty string, and the GetNextVariable() + * runtime service call will return the first EFI variable in + * its own enumeration order, ignoring the guid argument. + * + * Subsequent calls to GetNextVariable() must pass the name and + * guid values returned by the previous call, which is why we + * store varname in record->psi->data. Given that we only + * enumerate variables with the efi-pstore GUID, there is no + * need to record the guid return value. + */ + status = efivar_get_next_variable(&varname_size, varname, &guid); + if (status == EFI_NOT_FOUND) + return 0; -out: - if (size <= 0) { - kfree(record->buf); - record->buf = NULL; + if (status != EFI_SUCCESS) + return -EIO; + + /* skip variables that don't concern us */ + if (efi_guidcmp(guid, LINUX_EFI_CRASH_GUID)) + continue; + + return efi_pstore_read_func(record, varname); } - return size; } static int efi_pstore_write(struct pstore_record *record) { char name[DUMP_NAME_LEN]; efi_char16_t efi_name[DUMP_NAME_LEN]; - efi_guid_t vendor = LINUX_EFI_CRASH_GUID; - int i, ret = 0; + efi_status_t status; + int i; record->id = generic_id(record->time.tv_sec, record->part, record->count); @@ -265,88 +183,26 @@ static int efi_pstore_write(struct pstore_record *record) for (i = 0; i < DUMP_NAME_LEN; i++) efi_name[i] = name[i]; - ret = efivar_entry_set_safe(efi_name, vendor, PSTORE_EFI_ATTRIBUTES, - false, record->size, record->psi->buf); - - if (record->reason == KMSG_DUMP_OOPS && try_module_get(THIS_MODULE)) - if (!schedule_work(&efivar_work)) - module_put(THIS_MODULE); - - return ret; + if (efivar_trylock()) + return -EBUSY; + status = efivar_set_variable_locked(efi_name, &LINUX_EFI_CRASH_GUID, + PSTORE_EFI_ATTRIBUTES, + record->size, record->psi->buf, + true); + efivar_unlock(); + return status == EFI_SUCCESS ? 0 : -EIO; }; -/* - * Clean up an entry with the same name - */ -static int efi_pstore_erase_func(struct efivar_entry *entry, void *data) -{ - efi_char16_t *efi_name = data; - efi_guid_t vendor = LINUX_EFI_CRASH_GUID; - unsigned long ucs2_len = ucs2_strlen(efi_name); - - if (efi_guidcmp(entry->var.VendorGuid, vendor)) - return 0; - - if (ucs2_strncmp(entry->var.VariableName, efi_name, (size_t)ucs2_len)) - return 0; - - if (entry->scanning) { - /* - * Skip deletion because this entry will be deleted - * after scanning is completed. - */ - entry->deleting = true; - } else - list_del(&entry->list); - - /* found */ - __efivar_entry_delete(entry); - - return 1; -} - -static int efi_pstore_erase_name(const char *name) -{ - struct efivar_entry *entry = NULL; - efi_char16_t efi_name[DUMP_NAME_LEN]; - int found, i; - - for (i = 0; i < DUMP_NAME_LEN; i++) { - efi_name[i] = name[i]; - if (name[i] == '\0') - break; - } - - if (efivar_entry_iter_begin()) - return -EINTR; - - found = __efivar_entry_iter(efi_pstore_erase_func, &efi_pstore_list, - efi_name, &entry); - efivar_entry_iter_end(); - - if (found && !entry->scanning) - kfree(entry); - - return found ? 0 : -ENOENT; -} - static int efi_pstore_erase(struct pstore_record *record) { - char name[DUMP_NAME_LEN]; - int ret; + efi_status_t status; - snprintf(name, sizeof(name), "dump-type%u-%u-%d-%lld", - record->type, record->part, record->count, - (long long)record->time.tv_sec); - ret = efi_pstore_erase_name(name); - if (ret != -ENOENT) - return ret; - - snprintf(name, sizeof(name), "dump-type%u-%u-%lld", - record->type, record->part, (long long)record->time.tv_sec); - ret = efi_pstore_erase_name(name); + status = efivar_set_variable(record->priv, &LINUX_EFI_CRASH_GUID, + PSTORE_EFI_ATTRIBUTES, 0, NULL); - return ret; + if (status != EFI_SUCCESS && status != EFI_NOT_FOUND) + return -EIO; + return 0; } static struct pstore_info efi_pstore_info = { @@ -360,74 +216,14 @@ static struct pstore_info efi_pstore_info = { .erase = efi_pstore_erase, }; -static int efi_pstore_callback(efi_char16_t *name, efi_guid_t vendor, - unsigned long name_size, void *data) -{ - struct efivar_entry *entry; - - entry = kzalloc(sizeof(*entry), GFP_KERNEL); - if (!entry) - return -ENOMEM; - - memcpy(entry->var.VariableName, name, name_size); - entry->var.VendorGuid = vendor; - - __efivar_entry_add(entry, &efi_pstore_list); - - return 0; -} - -static int efi_pstore_update_entry(efi_char16_t *name, efi_guid_t vendor, - unsigned long name_size, void *data) -{ - struct efivar_entry *entry = data; - - if (efivar_entry_find(name, vendor, &efi_pstore_list, false)) - return 0; - - memcpy(entry->var.VariableName, name, name_size); - memcpy(&(entry->var.VendorGuid), &vendor, sizeof(efi_guid_t)); - - return 1; -} - -static void efi_pstore_update_entries(struct work_struct *work) -{ - struct efivar_entry *entry; - int err; - - /* Add new sysfs entries */ - while (1) { - entry = kzalloc(sizeof(*entry), GFP_KERNEL); - if (!entry) - return; - - err = efivar_init(efi_pstore_update_entry, entry, - false, &efi_pstore_list); - if (!err) - break; - - efivar_entry_add(entry, &efi_pstore_list); - } - - kfree(entry); - module_put(THIS_MODULE); -} - static __init int efivars_pstore_init(void) { - int ret; - - if (!efivars_kobject() || !efivar_supports_writes()) + if (!efivar_supports_writes()) return 0; if (efivars_pstore_disable) return 0; - ret = efivar_init(efi_pstore_callback, NULL, true, &efi_pstore_list); - if (ret) - return ret; - efi_pstore_info.buf = kmalloc(4096, GFP_KERNEL); if (!efi_pstore_info.buf) return -ENOMEM; @@ -440,8 +236,6 @@ static __init int efivars_pstore_init(void) efi_pstore_info.bufsize = 0; } - INIT_WORK(&efivar_work, efi_pstore_update_entries); - return 0; } diff --git a/drivers/firmware/efi/efivars.c b/drivers/firmware/efi/efivars.c index c19db0b35c0d..8341fb15f62e 100644 --- a/drivers/firmware/efi/efivars.c +++ b/drivers/firmware/efi/efivars.c @@ -467,16 +467,12 @@ static ssize_t efivar_delete(struct file *filp, struct kobject *kobj, else if (__efivar_entry_delete(entry)) err = -EIO; - if (err) { - efivar_entry_iter_end(); + efivar_entry_iter_end(); + + if (err) return err; - } - if (!entry->scanning) { - efivar_entry_iter_end(); - efivar_unregister(entry); - } else - efivar_entry_iter_end(); + efivar_unregister(entry); /* It's dead Jim.... */ return count; diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index 41c82614a4b2..5640ffa81544 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -821,16 +821,8 @@ struct efivar_entry *efivar_entry_find(efi_char16_t *name, efi_guid_t guid, if (!found) return NULL; - if (remove) { - if (entry->scanning) { - /* - * The entry will be deleted - * after scanning is completed. - */ - entry->deleting = true; - } else - list_del(&entry->list); - } + if (remove) + list_del(&entry->list); return entry; } diff --git a/include/linux/efi.h b/include/linux/efi.h index c828ab6f0e2a..08bc6215e3b4 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1043,8 +1043,6 @@ struct efivar_entry { struct efi_variable var; struct list_head list; struct kobject kobj; - bool scanning; - bool deleting; }; static inline void -- cgit v1.2.3 From 0f5b2c69a4cbe4166ca24b76d5ada98ed2867741 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Jun 2022 13:34:03 +0200 Subject: efi: vars: Remove deprecated 'efivars' sysfs interface Commit 5d9db883761a ("efi: Add support for a UEFI variable filesystem") dated Oct 5, 2012, introduced a new efivarfs pseudo-filesystem to replace the efivars sysfs interface that was used up to that point to expose EFI variables to user space. The main problem with the sysfs interface was that it only supported up to 1024 bytes of payload per file, whereas the underlying variables themselves are only bounded by a platform specific per-variable and global limit that is typically much higher than 1024 bytes. The deprecated sysfs interface is only enabled on x86 and Itanium, other EFI enabled architectures only support the efivarfs pseudo-filesystem. So let's finally rip off the band aid, and drop the old interface entirely. This will make it easier to refactor and clean up the underlying infrastructure that is shared between efivars, efivarfs and efi-pstore, and is long overdue for a makeover. Signed-off-by: Ard Biesheuvel --- Documentation/x86/x86_64/uefi.rst | 2 +- arch/arm/configs/milbeaut_m10v_defconfig | 1 - arch/ia64/configs/bigsur_defconfig | 1 - arch/ia64/configs/generic_defconfig | 1 - arch/ia64/configs/gensparse_defconfig | 1 - arch/ia64/configs/tiger_defconfig | 1 - arch/ia64/configs/zx1_defconfig | 1 - arch/x86/configs/i386_defconfig | 1 - arch/x86/configs/x86_64_defconfig | 1 - drivers/firmware/efi/Kconfig | 12 - drivers/firmware/efi/Makefile | 1 - drivers/firmware/efi/efivars.c | 660 ------------------------------- drivers/firmware/efi/vars.c | 265 +------------ include/linux/efi.h | 18 - 14 files changed, 3 insertions(+), 963 deletions(-) delete mode 100644 drivers/firmware/efi/efivars.c (limited to 'include/linux') diff --git a/Documentation/x86/x86_64/uefi.rst b/Documentation/x86/x86_64/uefi.rst index 3b894103a734..fbc30c9a071d 100644 --- a/Documentation/x86/x86_64/uefi.rst +++ b/Documentation/x86/x86_64/uefi.rst @@ -29,7 +29,7 @@ Mechanics be selected:: CONFIG_EFI=y - CONFIG_EFI_VARS=y or m # optional + CONFIG_EFIVAR_FS=y or m # optional - Create a VFAT partition on the disk - Copy the following to the VFAT partition: diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 7c07f9893a0f..9b4789af0201 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -44,7 +44,6 @@ CONFIG_ARM_CPUIDLE=y CONFIG_VFP=y CONFIG_NEON=y CONFIG_KERNEL_MODE_NEON=y -CONFIG_EFI_VARS=m CONFIG_EFI_CAPSULE_LOADER=m CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m diff --git a/arch/ia64/configs/bigsur_defconfig b/arch/ia64/configs/bigsur_defconfig index 0341a67cc1bf..a3724882295c 100644 --- a/arch/ia64/configs/bigsur_defconfig +++ b/arch/ia64/configs/bigsur_defconfig @@ -10,7 +10,6 @@ CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_PREEMPT=y CONFIG_IA64_PALINFO=y -CONFIG_EFI_VARS=y CONFIG_BINFMT_MISC=m CONFIG_ACPI_BUTTON=m CONFIG_ACPI_FAN=m diff --git a/arch/ia64/configs/generic_defconfig b/arch/ia64/configs/generic_defconfig index 8916a2850c48..a3dff482a3d7 100644 --- a/arch/ia64/configs/generic_defconfig +++ b/arch/ia64/configs/generic_defconfig @@ -21,7 +21,6 @@ CONFIG_IA64_MCA_RECOVERY=y CONFIG_IA64_PALINFO=y CONFIG_KEXEC=y CONFIG_CRASH_DUMP=y -CONFIG_EFI_VARS=y CONFIG_BINFMT_MISC=m CONFIG_ACPI_BUTTON=m CONFIG_ACPI_FAN=m diff --git a/arch/ia64/configs/gensparse_defconfig b/arch/ia64/configs/gensparse_defconfig index 281eb9c544f9..4cd46105b020 100644 --- a/arch/ia64/configs/gensparse_defconfig +++ b/arch/ia64/configs/gensparse_defconfig @@ -18,7 +18,6 @@ CONFIG_HOTPLUG_CPU=y CONFIG_SPARSEMEM_MANUAL=y CONFIG_IA64_MCA_RECOVERY=y CONFIG_IA64_PALINFO=y -CONFIG_EFI_VARS=y CONFIG_BINFMT_MISC=m CONFIG_ACPI_BUTTON=m CONFIG_ACPI_FAN=m diff --git a/arch/ia64/configs/tiger_defconfig b/arch/ia64/configs/tiger_defconfig index b4f9819a1a45..a2045d73adfa 100644 --- a/arch/ia64/configs/tiger_defconfig +++ b/arch/ia64/configs/tiger_defconfig @@ -23,7 +23,6 @@ CONFIG_FORCE_CPEI_RETARGET=y CONFIG_IA64_MCA_RECOVERY=y CONFIG_IA64_PALINFO=y CONFIG_KEXEC=y -CONFIG_EFI_VARS=y CONFIG_BINFMT_MISC=m CONFIG_ACPI_BUTTON=m CONFIG_ACPI_FAN=m diff --git a/arch/ia64/configs/zx1_defconfig b/arch/ia64/configs/zx1_defconfig index 851d8594cdb8..99f8b2a0332b 100644 --- a/arch/ia64/configs/zx1_defconfig +++ b/arch/ia64/configs/zx1_defconfig @@ -12,7 +12,6 @@ CONFIG_FLATMEM_MANUAL=y CONFIG_IA64_MCA_RECOVERY=y CONFIG_IA64_PALINFO=y CONFIG_CRASH_DUMP=y -CONFIG_EFI_VARS=y CONFIG_BINFMT_MISC=y CONFIG_HOTPLUG_PCI=y CONFIG_HOTPLUG_PCI_ACPI=y diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index 98a4852ed6a0..7207219509f6 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -135,7 +135,6 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_DEBUG_DEVRES=y CONFIG_CONNECTOR=y -CONFIG_EFI_VARS=y CONFIG_EFI_CAPSULE_LOADER=y CONFIG_BLK_DEV_LOOP=y CONFIG_VIRTIO_BLK=y diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 69784505a7a8..5ce67b73e218 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -134,7 +134,6 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_DEBUG_DEVRES=y CONFIG_CONNECTOR=y -CONFIG_EFI_VARS=y CONFIG_BLK_DEV_LOOP=y CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_SD=y diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index 7fe8b5c686d2..6cb7384ad2ac 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -2,18 +2,6 @@ menu "EFI (Extensible Firmware Interface) Support" depends on EFI -config EFI_VARS - tristate "EFI Variable Support via sysfs" - depends on EFI && (X86 || IA64) - default n - help - If you say Y here, you are able to get EFI (Extensible Firmware - Interface) variable information via sysfs. You may read, - write, create, and destroy EFI variables through this interface. - Note that this driver is only retained for compatibility with - legacy users: new users should use the efivarfs filesystem - instead. - config EFI_ESRT bool depends on EFI && !IA64 diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile index c02ff25dd477..8d151e332584 100644 --- a/drivers/firmware/efi/Makefile +++ b/drivers/firmware/efi/Makefile @@ -17,7 +17,6 @@ ifneq ($(CONFIG_EFI_CAPSULE_LOADER),) obj-$(CONFIG_EFI) += capsule.o endif obj-$(CONFIG_EFI_PARAMS_FROM_FDT) += fdtparams.o -obj-$(CONFIG_EFI_VARS) += efivars.o obj-$(CONFIG_EFI_ESRT) += esrt.o obj-$(CONFIG_EFI_VARS_PSTORE) += efi-pstore.o obj-$(CONFIG_UEFI_CPER) += cper.o diff --git a/drivers/firmware/efi/efivars.c b/drivers/firmware/efi/efivars.c deleted file mode 100644 index 801a65582172..000000000000 --- a/drivers/firmware/efi/efivars.c +++ /dev/null @@ -1,660 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Originally from efivars.c, - * - * Copyright (C) 2001,2003,2004 Dell - * Copyright (C) 2004 Intel Corporation - * - * This code takes all variables accessible from EFI runtime and - * exports them via sysfs - */ - -#include -#include -#include -#include -#include - -#define EFIVARS_VERSION "0.08" -#define EFIVARS_DATE "2004-May-17" - -MODULE_AUTHOR("Matt Domsch "); -MODULE_DESCRIPTION("sysfs interface to EFI Variables"); -MODULE_LICENSE("GPL"); -MODULE_VERSION(EFIVARS_VERSION); - -static LIST_HEAD(efivar_sysfs_list); - -static struct kset *efivars_kset; - -static struct bin_attribute *efivars_new_var; -static struct bin_attribute *efivars_del_var; - -struct compat_efi_variable { - efi_char16_t VariableName[EFI_VAR_NAME_LEN/sizeof(efi_char16_t)]; - efi_guid_t VendorGuid; - __u32 DataSize; - __u8 Data[1024]; - __u32 Status; - __u32 Attributes; -} __packed; - -struct efivar_attribute { - struct attribute attr; - ssize_t (*show) (struct efivar_entry *entry, char *buf); - ssize_t (*store)(struct efivar_entry *entry, const char *buf, size_t count); -}; - -#define EFIVAR_ATTR(_name, _mode, _show, _store) \ -struct efivar_attribute efivar_attr_##_name = { \ - .attr = {.name = __stringify(_name), .mode = _mode}, \ - .show = _show, \ - .store = _store, \ -}; - -#define to_efivar_attr(_attr) container_of(_attr, struct efivar_attribute, attr) -#define to_efivar_entry(obj) container_of(obj, struct efivar_entry, kobj) - -/* - * Prototype for sysfs creation function - */ -static int -efivar_create_sysfs_entry(struct efivar_entry *new_var); - -static ssize_t -efivar_guid_read(struct efivar_entry *entry, char *buf) -{ - struct efi_variable *var = &entry->var; - char *str = buf; - - if (!entry || !buf) - return 0; - - efi_guid_to_str(&var->VendorGuid, str); - str += strlen(str); - str += sprintf(str, "\n"); - - return str - buf; -} - -static ssize_t -efivar_attr_read(struct efivar_entry *entry, char *buf) -{ - struct efi_variable *var = &entry->var; - unsigned long size = sizeof(var->Data); - char *str = buf; - int ret; - - if (!entry || !buf) - return -EINVAL; - - ret = efivar_entry_get(entry, &var->Attributes, &size, var->Data); - var->DataSize = size; - if (ret) - return -EIO; - - if (var->Attributes & EFI_VARIABLE_NON_VOLATILE) - str += sprintf(str, "EFI_VARIABLE_NON_VOLATILE\n"); - if (var->Attributes & EFI_VARIABLE_BOOTSERVICE_ACCESS) - str += sprintf(str, "EFI_VARIABLE_BOOTSERVICE_ACCESS\n"); - if (var->Attributes & EFI_VARIABLE_RUNTIME_ACCESS) - str += sprintf(str, "EFI_VARIABLE_RUNTIME_ACCESS\n"); - if (var->Attributes & EFI_VARIABLE_HARDWARE_ERROR_RECORD) - str += sprintf(str, "EFI_VARIABLE_HARDWARE_ERROR_RECORD\n"); - if (var->Attributes & EFI_VARIABLE_AUTHENTICATED_WRITE_ACCESS) - str += sprintf(str, - "EFI_VARIABLE_AUTHENTICATED_WRITE_ACCESS\n"); - if (var->Attributes & - EFI_VARIABLE_TIME_BASED_AUTHENTICATED_WRITE_ACCESS) - str += sprintf(str, - "EFI_VARIABLE_TIME_BASED_AUTHENTICATED_WRITE_ACCESS\n"); - if (var->Attributes & EFI_VARIABLE_APPEND_WRITE) - str += sprintf(str, "EFI_VARIABLE_APPEND_WRITE\n"); - return str - buf; -} - -static ssize_t -efivar_size_read(struct efivar_entry *entry, char *buf) -{ - struct efi_variable *var = &entry->var; - unsigned long size = sizeof(var->Data); - char *str = buf; - int ret; - - if (!entry || !buf) - return -EINVAL; - - ret = efivar_entry_get(entry, &var->Attributes, &size, var->Data); - var->DataSize = size; - if (ret) - return -EIO; - - str += sprintf(str, "0x%lx\n", var->DataSize); - return str - buf; -} - -static ssize_t -efivar_data_read(struct efivar_entry *entry, char *buf) -{ - struct efi_variable *var = &entry->var; - unsigned long size = sizeof(var->Data); - int ret; - - if (!entry || !buf) - return -EINVAL; - - ret = efivar_entry_get(entry, &var->Attributes, &size, var->Data); - var->DataSize = size; - if (ret) - return -EIO; - - memcpy(buf, var->Data, var->DataSize); - return var->DataSize; -} - -static inline int -sanity_check(struct efi_variable *var, efi_char16_t *name, efi_guid_t vendor, - unsigned long size, u32 attributes, u8 *data) -{ - /* - * If only updating the variable data, then the name - * and guid should remain the same - */ - if (memcmp(name, var->VariableName, sizeof(var->VariableName)) || - efi_guidcmp(vendor, var->VendorGuid)) { - printk(KERN_ERR "efivars: Cannot edit the wrong variable!\n"); - return -EINVAL; - } - - if ((size <= 0) || (attributes == 0)){ - printk(KERN_ERR "efivars: DataSize & Attributes must be valid!\n"); - return -EINVAL; - } - - if ((attributes & ~EFI_VARIABLE_MASK) != 0 || - efivar_validate(vendor, name, data, size) == false) { - printk(KERN_ERR "efivars: Malformed variable content\n"); - return -EINVAL; - } - - return 0; -} - -static void -copy_out_compat(struct efi_variable *dst, struct compat_efi_variable *src) -{ - memcpy(dst->VariableName, src->VariableName, EFI_VAR_NAME_LEN); - memcpy(dst->Data, src->Data, sizeof(src->Data)); - - dst->VendorGuid = src->VendorGuid; - dst->DataSize = src->DataSize; - dst->Attributes = src->Attributes; -} - -/* - * We allow each variable to be edited via rewriting the - * entire efi variable structure. - */ -static ssize_t -efivar_store_raw(struct efivar_entry *entry, const char *buf, size_t count) -{ - struct efi_variable *new_var, *var = &entry->var; - efi_char16_t *name; - unsigned long size; - efi_guid_t vendor; - u32 attributes; - u8 *data; - int err; - - if (!entry || !buf) - return -EINVAL; - - if (in_compat_syscall()) { - struct compat_efi_variable *compat; - - if (count != sizeof(*compat)) - return -EINVAL; - - compat = (struct compat_efi_variable *)buf; - attributes = compat->Attributes; - vendor = compat->VendorGuid; - name = compat->VariableName; - size = compat->DataSize; - data = compat->Data; - - err = sanity_check(var, name, vendor, size, attributes, data); - if (err) - return err; - - copy_out_compat(&entry->var, compat); - } else { - if (count != sizeof(struct efi_variable)) - return -EINVAL; - - new_var = (struct efi_variable *)buf; - - attributes = new_var->Attributes; - vendor = new_var->VendorGuid; - name = new_var->VariableName; - size = new_var->DataSize; - data = new_var->Data; - - err = sanity_check(var, name, vendor, size, attributes, data); - if (err) - return err; - - memcpy(&entry->var, new_var, count); - } - - err = efivar_entry_set(entry, attributes, size, data, NULL); - if (err) { - printk(KERN_WARNING "efivars: set_variable() failed: status=%d\n", err); - return -EIO; - } - - return count; -} - -static ssize_t -efivar_show_raw(struct efivar_entry *entry, char *buf) -{ - struct efi_variable *var = &entry->var; - struct compat_efi_variable *compat; - unsigned long datasize = sizeof(var->Data); - size_t size; - int ret; - - if (!entry || !buf) - return 0; - - ret = efivar_entry_get(entry, &var->Attributes, &datasize, var->Data); - var->DataSize = datasize; - if (ret) - return -EIO; - - if (in_compat_syscall()) { - compat = (struct compat_efi_variable *)buf; - - size = sizeof(*compat); - memcpy(compat->VariableName, var->VariableName, - EFI_VAR_NAME_LEN); - memcpy(compat->Data, var->Data, sizeof(compat->Data)); - - compat->VendorGuid = var->VendorGuid; - compat->DataSize = var->DataSize; - compat->Attributes = var->Attributes; - } else { - size = sizeof(*var); - memcpy(buf, var, size); - } - - return size; -} - -/* - * Generic read/write functions that call the specific functions of - * the attributes... - */ -static ssize_t efivar_attr_show(struct kobject *kobj, struct attribute *attr, - char *buf) -{ - struct efivar_entry *var = to_efivar_entry(kobj); - struct efivar_attribute *efivar_attr = to_efivar_attr(attr); - ssize_t ret = -EIO; - - if (!capable(CAP_SYS_ADMIN)) - return -EACCES; - - if (efivar_attr->show) { - ret = efivar_attr->show(var, buf); - } - return ret; -} - -static ssize_t efivar_attr_store(struct kobject *kobj, struct attribute *attr, - const char *buf, size_t count) -{ - struct efivar_entry *var = to_efivar_entry(kobj); - struct efivar_attribute *efivar_attr = to_efivar_attr(attr); - ssize_t ret = -EIO; - - if (!capable(CAP_SYS_ADMIN)) - return -EACCES; - - if (efivar_attr->store) - ret = efivar_attr->store(var, buf, count); - - return ret; -} - -static const struct sysfs_ops efivar_attr_ops = { - .show = efivar_attr_show, - .store = efivar_attr_store, -}; - -static void efivar_release(struct kobject *kobj) -{ - struct efivar_entry *var = to_efivar_entry(kobj); - kfree(var); -} - -static EFIVAR_ATTR(guid, 0400, efivar_guid_read, NULL); -static EFIVAR_ATTR(attributes, 0400, efivar_attr_read, NULL); -static EFIVAR_ATTR(size, 0400, efivar_size_read, NULL); -static EFIVAR_ATTR(data, 0400, efivar_data_read, NULL); -static EFIVAR_ATTR(raw_var, 0600, efivar_show_raw, efivar_store_raw); - -static struct attribute *def_attrs[] = { - &efivar_attr_guid.attr, - &efivar_attr_size.attr, - &efivar_attr_attributes.attr, - &efivar_attr_data.attr, - &efivar_attr_raw_var.attr, - NULL, -}; -ATTRIBUTE_GROUPS(def); - -static struct kobj_type efivar_ktype = { - .release = efivar_release, - .sysfs_ops = &efivar_attr_ops, - .default_groups = def_groups, -}; - -static ssize_t efivar_create(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, - char *buf, loff_t pos, size_t count) -{ - struct compat_efi_variable *compat = (struct compat_efi_variable *)buf; - struct efi_variable *new_var = (struct efi_variable *)buf; - struct efivar_entry *new_entry; - bool need_compat = in_compat_syscall(); - efi_char16_t *name; - unsigned long size; - u32 attributes; - u8 *data; - int err; - - if (!capable(CAP_SYS_ADMIN)) - return -EACCES; - - if (need_compat) { - if (count != sizeof(*compat)) - return -EINVAL; - - attributes = compat->Attributes; - name = compat->VariableName; - size = compat->DataSize; - data = compat->Data; - } else { - if (count != sizeof(*new_var)) - return -EINVAL; - - attributes = new_var->Attributes; - name = new_var->VariableName; - size = new_var->DataSize; - data = new_var->Data; - } - - if ((attributes & ~EFI_VARIABLE_MASK) != 0 || - efivar_validate(new_var->VendorGuid, name, data, - size) == false) { - printk(KERN_ERR "efivars: Malformed variable content\n"); - return -EINVAL; - } - - new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL); - if (!new_entry) - return -ENOMEM; - - if (need_compat) - copy_out_compat(&new_entry->var, compat); - else - memcpy(&new_entry->var, new_var, sizeof(*new_var)); - - err = efivar_entry_set(new_entry, attributes, size, - data, &efivar_sysfs_list); - if (err) { - if (err == -EEXIST) - err = -EINVAL; - goto out; - } - - if (efivar_create_sysfs_entry(new_entry)) { - printk(KERN_WARNING "efivars: failed to create sysfs entry.\n"); - kfree(new_entry); - } - return count; - -out: - kfree(new_entry); - return err; -} - -static ssize_t efivar_delete(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, - char *buf, loff_t pos, size_t count) -{ - struct efi_variable *del_var = (struct efi_variable *)buf; - struct compat_efi_variable *compat; - struct efivar_entry *entry; - efi_char16_t *name; - efi_guid_t vendor; - int err = 0; - - if (!capable(CAP_SYS_ADMIN)) - return -EACCES; - - if (in_compat_syscall()) { - if (count != sizeof(*compat)) - return -EINVAL; - - compat = (struct compat_efi_variable *)buf; - name = compat->VariableName; - vendor = compat->VendorGuid; - } else { - if (count != sizeof(*del_var)) - return -EINVAL; - - name = del_var->VariableName; - vendor = del_var->VendorGuid; - } - - if (efivar_entry_iter_begin()) - return -EINTR; - entry = efivar_entry_find(name, vendor, &efivar_sysfs_list, true); - if (!entry) - err = -EINVAL; - else if (__efivar_entry_delete(entry)) - err = -EIO; - - efivar_entry_iter_end(); - - if (err) - return err; - - efivar_unregister(entry); - - /* It's dead Jim.... */ - return count; -} - -/** - * efivar_create_sysfs_entry - create a new entry in sysfs - * @new_var: efivar entry to create - * - * Returns 0 on success, negative error code on failure - */ -static int -efivar_create_sysfs_entry(struct efivar_entry *new_var) -{ - int short_name_size; - char *short_name; - unsigned long utf8_name_size; - efi_char16_t *variable_name = new_var->var.VariableName; - int ret; - - /* - * Length of the variable bytes in UTF8, plus the '-' separator, - * plus the GUID, plus trailing NUL - */ - utf8_name_size = ucs2_utf8size(variable_name); - short_name_size = utf8_name_size + 1 + EFI_VARIABLE_GUID_LEN + 1; - - short_name = kmalloc(short_name_size, GFP_KERNEL); - if (!short_name) - return -ENOMEM; - - ucs2_as_utf8(short_name, variable_name, short_name_size); - - /* This is ugly, but necessary to separate one vendor's - private variables from another's. */ - short_name[utf8_name_size] = '-'; - efi_guid_to_str(&new_var->var.VendorGuid, - short_name + utf8_name_size + 1); - - new_var->kobj.kset = efivars_kset; - - ret = kobject_init_and_add(&new_var->kobj, &efivar_ktype, - NULL, "%s", short_name); - kfree(short_name); - if (ret) { - kobject_put(&new_var->kobj); - return ret; - } - - kobject_uevent(&new_var->kobj, KOBJ_ADD); - __efivar_entry_add(new_var, &efivar_sysfs_list); - - return 0; -} - -static int -create_efivars_bin_attributes(void) -{ - struct bin_attribute *attr; - int error; - - /* new_var */ - attr = kzalloc(sizeof(*attr), GFP_KERNEL); - if (!attr) - return -ENOMEM; - - attr->attr.name = "new_var"; - attr->attr.mode = 0200; - attr->write = efivar_create; - efivars_new_var = attr; - - /* del_var */ - attr = kzalloc(sizeof(*attr), GFP_KERNEL); - if (!attr) { - error = -ENOMEM; - goto out_free; - } - attr->attr.name = "del_var"; - attr->attr.mode = 0200; - attr->write = efivar_delete; - efivars_del_var = attr; - - sysfs_bin_attr_init(efivars_new_var); - sysfs_bin_attr_init(efivars_del_var); - - /* Register */ - error = sysfs_create_bin_file(&efivars_kset->kobj, efivars_new_var); - if (error) { - printk(KERN_ERR "efivars: unable to create new_var sysfs file" - " due to error %d\n", error); - goto out_free; - } - - error = sysfs_create_bin_file(&efivars_kset->kobj, efivars_del_var); - if (error) { - printk(KERN_ERR "efivars: unable to create del_var sysfs file" - " due to error %d\n", error); - sysfs_remove_bin_file(&efivars_kset->kobj, efivars_new_var); - goto out_free; - } - - return 0; -out_free: - kfree(efivars_del_var); - efivars_del_var = NULL; - kfree(efivars_new_var); - efivars_new_var = NULL; - return error; -} - -static int efivars_sysfs_callback(efi_char16_t *name, efi_guid_t vendor, - unsigned long name_size, void *data) -{ - struct efivar_entry *entry; - - entry = kzalloc(sizeof(*entry), GFP_KERNEL); - if (!entry) - return -ENOMEM; - - memcpy(entry->var.VariableName, name, name_size); - memcpy(&(entry->var.VendorGuid), &vendor, sizeof(efi_guid_t)); - - efivar_create_sysfs_entry(entry); - - return 0; -} - -static int efivar_sysfs_destroy(struct efivar_entry *entry, void *data) -{ - efivar_entry_remove(entry); - efivar_unregister(entry); - return 0; -} - -static void efivars_sysfs_exit(void) -{ - /* Remove all entries and destroy */ - int err; - - err = efivar_entry_iter(efivar_sysfs_destroy, &efivar_sysfs_list, NULL); - if (err) { - pr_err("efivars: Failed to destroy sysfs entries\n"); - return; - } - - if (efivars_new_var) - sysfs_remove_bin_file(&efivars_kset->kobj, efivars_new_var); - if (efivars_del_var) - sysfs_remove_bin_file(&efivars_kset->kobj, efivars_del_var); - kfree(efivars_new_var); - kfree(efivars_del_var); - kset_unregister(efivars_kset); -} - -static int efivars_sysfs_init(void) -{ - struct kobject *parent_kobj = efivars_kobject(); - int error = 0; - - /* No efivars has been registered yet */ - if (!parent_kobj || !efivar_supports_writes()) - return 0; - - printk(KERN_INFO "EFI Variables Facility v%s %s\n", EFIVARS_VERSION, - EFIVARS_DATE); - - efivars_kset = kset_create_and_add("vars", NULL, parent_kobj); - if (!efivars_kset) { - printk(KERN_ERR "efivars: Subsystem registration failed.\n"); - return -ENOMEM; - } - - efivar_init(efivars_sysfs_callback, NULL, true, &efivar_sysfs_list); - - error = create_efivars_bin_attributes(); - if (error) { - efivars_sysfs_exit(); - return error; - } - - return 0; -} - -module_init(efivars_sysfs_init); -module_exit(efivars_sysfs_exit); diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index 932435945c85..94de1d0cb4e7 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -547,36 +547,6 @@ static void efivar_entry_list_del_unlock(struct efivar_entry *entry) up(&efivars_lock); } -/** - * __efivar_entry_delete - delete an EFI variable - * @entry: entry containing EFI variable to delete - * - * Delete the variable from the firmware but leave @entry on the - * variable list. - * - * This function differs from efivar_entry_delete() because it does - * not remove @entry from the variable list. Also, it is safe to be - * called from within a efivar_entry_iter_begin() and - * efivar_entry_iter_end() region, unlike efivar_entry_delete(). - * - * Returns 0 on success, or a converted EFI status code if - * set_variable() fails. - */ -int __efivar_entry_delete(struct efivar_entry *entry) -{ - efi_status_t status; - - if (!__efivars) - return -EINVAL; - - status = __efivars->ops->set_variable(entry->var.VariableName, - &entry->var.VendorGuid, - 0, 0, NULL); - - return efi_status_to_err(status); -} -EXPORT_SYMBOL_GPL(__efivar_entry_delete); - /** * efivar_entry_delete - delete variable and remove entry from list * @entry: entry containing variable to delete @@ -614,213 +584,6 @@ int efivar_entry_delete(struct efivar_entry *entry) } EXPORT_SYMBOL_GPL(efivar_entry_delete); -/** - * efivar_entry_set - call set_variable() - * @entry: entry containing the EFI variable to write - * @attributes: variable attributes - * @size: size of @data buffer - * @data: buffer containing variable data - * @head: head of variable list - * - * Calls set_variable() for an EFI variable. If creating a new EFI - * variable, this function is usually followed by efivar_entry_add(). - * - * Before writing the variable, the remaining EFI variable storage - * space is checked to ensure there is enough room available. - * - * If @head is not NULL a lookup is performed to determine whether - * the entry is already on the list. - * - * Returns 0 on success, -EINTR if we can't grab the semaphore, - * -EEXIST if a lookup is performed and the entry already exists on - * the list, or a converted EFI status code if set_variable() fails. - */ -int efivar_entry_set(struct efivar_entry *entry, u32 attributes, - unsigned long size, void *data, struct list_head *head) -{ - const struct efivar_operations *ops; - efi_status_t status; - efi_char16_t *name = entry->var.VariableName; - efi_guid_t vendor = entry->var.VendorGuid; - - if (down_interruptible(&efivars_lock)) - return -EINTR; - - if (!__efivars) { - up(&efivars_lock); - return -EINVAL; - } - ops = __efivars->ops; - if (head && efivar_entry_find(name, vendor, head, false)) { - up(&efivars_lock); - return -EEXIST; - } - - status = check_var_size(attributes, size + ucs2_strsize(name, 1024)); - if (status == EFI_SUCCESS || status == EFI_UNSUPPORTED) - status = ops->set_variable(name, &vendor, - attributes, size, data); - - up(&efivars_lock); - - return efi_status_to_err(status); - -} -EXPORT_SYMBOL_GPL(efivar_entry_set); - -/* - * efivar_entry_set_nonblocking - call set_variable_nonblocking() - * - * This function is guaranteed to not block and is suitable for calling - * from crash/panic handlers. - * - * Crucially, this function will not block if it cannot acquire - * efivars_lock. Instead, it returns -EBUSY. - */ -static int -efivar_entry_set_nonblocking(efi_char16_t *name, efi_guid_t vendor, - u32 attributes, unsigned long size, void *data) -{ - const struct efivar_operations *ops; - efi_status_t status; - - if (down_trylock(&efivars_lock)) - return -EBUSY; - - if (!__efivars) { - up(&efivars_lock); - return -EINVAL; - } - - status = check_var_size_nonblocking(attributes, - size + ucs2_strsize(name, 1024)); - if (status != EFI_SUCCESS) { - up(&efivars_lock); - return -ENOSPC; - } - - ops = __efivars->ops; - status = ops->set_variable_nonblocking(name, &vendor, attributes, - size, data); - - up(&efivars_lock); - return efi_status_to_err(status); -} - -/** - * efivar_entry_set_safe - call set_variable() if enough space in firmware - * @name: buffer containing the variable name - * @vendor: variable vendor guid - * @attributes: variable attributes - * @block: can we block in this context? - * @size: size of @data buffer - * @data: buffer containing variable data - * - * Ensures there is enough free storage in the firmware for this variable, and - * if so, calls set_variable(). If creating a new EFI variable, this function - * is usually followed by efivar_entry_add(). - * - * Returns 0 on success, -ENOSPC if the firmware does not have enough - * space for set_variable() to succeed, or a converted EFI status code - * if set_variable() fails. - */ -int efivar_entry_set_safe(efi_char16_t *name, efi_guid_t vendor, u32 attributes, - bool block, unsigned long size, void *data) -{ - const struct efivar_operations *ops; - efi_status_t status; - unsigned long varsize; - - if (!__efivars) - return -EINVAL; - - ops = __efivars->ops; - if (!ops->query_variable_store) - return -ENOSYS; - - /* - * If the EFI variable backend provides a non-blocking - * ->set_variable() operation and we're in a context where we - * cannot block, then we need to use it to avoid live-locks, - * since the implication is that the regular ->set_variable() - * will block. - * - * If no ->set_variable_nonblocking() is provided then - * ->set_variable() is assumed to be non-blocking. - */ - if (!block && ops->set_variable_nonblocking) - return efivar_entry_set_nonblocking(name, vendor, attributes, - size, data); - - varsize = size + ucs2_strsize(name, 1024); - if (!block) { - if (down_trylock(&efivars_lock)) - return -EBUSY; - status = check_var_size_nonblocking(attributes, varsize); - } else { - if (down_interruptible(&efivars_lock)) - return -EINTR; - status = check_var_size(attributes, varsize); - } - - if (status != EFI_SUCCESS) { - up(&efivars_lock); - return -ENOSPC; - } - - status = ops->set_variable(name, &vendor, attributes, size, data); - - up(&efivars_lock); - - return efi_status_to_err(status); -} -EXPORT_SYMBOL_GPL(efivar_entry_set_safe); - -/** - * efivar_entry_find - search for an entry - * @name: the EFI variable name - * @guid: the EFI variable vendor's guid - * @head: head of the variable list - * @remove: should we remove the entry from the list? - * - * Search for an entry on the variable list that has the EFI variable - * name @name and vendor guid @guid. If an entry is found on the list - * and @remove is true, the entry is removed from the list. - * - * The caller MUST call efivar_entry_iter_begin() and - * efivar_entry_iter_end() before and after the invocation of this - * function, respectively. - * - * Returns the entry if found on the list, %NULL otherwise. - */ -struct efivar_entry *efivar_entry_find(efi_char16_t *name, efi_guid_t guid, - struct list_head *head, bool remove) -{ - struct efivar_entry *entry, *n; - int strsize1, strsize2; - bool found = false; - - list_for_each_entry_safe(entry, n, head, list) { - strsize1 = ucs2_strsize(name, 1024); - strsize2 = ucs2_strsize(entry->var.VariableName, 1024); - if (strsize1 == strsize2 && - !memcmp(name, &(entry->var.VariableName), strsize1) && - !efi_guidcmp(guid, entry->var.VendorGuid)) { - found = true; - break; - } - } - - if (!found) - return NULL; - - if (remove) - list_del(&entry->list); - - return entry; -} -EXPORT_SYMBOL_GPL(efivar_entry_find); - /** * efivar_entry_size - obtain the size of a variable * @entry: entry for this variable @@ -1010,30 +773,6 @@ out: } EXPORT_SYMBOL_GPL(efivar_entry_set_get_size); -/** - * efivar_entry_iter_begin - begin iterating the variable list - * - * Lock the variable list to prevent entry insertion and removal until - * efivar_entry_iter_end() is called. This function is usually used in - * conjunction with __efivar_entry_iter() or efivar_entry_iter(). - */ -int efivar_entry_iter_begin(void) -{ - return down_interruptible(&efivars_lock); -} -EXPORT_SYMBOL_GPL(efivar_entry_iter_begin); - -/** - * efivar_entry_iter_end - finish iterating the variable list - * - * Unlock the variable list and allow modifications to the list again. - */ -void efivar_entry_iter_end(void) -{ - up(&efivars_lock); -} -EXPORT_SYMBOL_GPL(efivar_entry_iter_end); - /** * efivar_entry_iter - iterate over variable list * @func: callback function @@ -1054,7 +793,7 @@ int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), struct efivar_entry *entry, *n; int err = 0; - err = efivar_entry_iter_begin(); + err = down_interruptible(&efivars_lock); if (err) return err; @@ -1063,7 +802,7 @@ int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), if (err) break; } - efivar_entry_iter_end(); + up(&efivars_lock); return err; } diff --git a/include/linux/efi.h b/include/linux/efi.h index 93ce85a14a46..10ef0a0d5e9a 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1045,12 +1045,6 @@ struct efivar_entry { struct kobject kobj; }; -static inline void -efivar_unregister(struct efivar_entry *var) -{ - kobject_put(&var->kobj); -} - int efivars_register(struct efivars *efivars, const struct efivar_operations *ops, struct kobject *kobject); @@ -1064,8 +1058,6 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), int efivar_entry_add(struct efivar_entry *entry, struct list_head *head); void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head); void efivar_entry_remove(struct efivar_entry *entry); - -int __efivar_entry_delete(struct efivar_entry *entry); int efivar_entry_delete(struct efivar_entry *entry); int efivar_entry_size(struct efivar_entry *entry, unsigned long *size); @@ -1073,22 +1065,12 @@ int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes, unsigned long *size, void *data); int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, unsigned long *size, void *data); -int efivar_entry_set(struct efivar_entry *entry, u32 attributes, - unsigned long size, void *data, struct list_head *head); int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes, unsigned long *size, void *data, bool *set); -int efivar_entry_set_safe(efi_char16_t *name, efi_guid_t vendor, u32 attributes, - bool block, unsigned long size, void *data); - -int efivar_entry_iter_begin(void); -void efivar_entry_iter_end(void); int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), struct list_head *head, void *data); -struct efivar_entry *efivar_entry_find(efi_char16_t *name, efi_guid_t guid, - struct list_head *head, bool remove); - bool efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, unsigned long data_size); bool efivar_variable_is_removable(efi_guid_t vendor, const char *name, -- cgit v1.2.3 From 3a75f9f2f9ad19bb9a0f566373ae91d8f09db85e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 21 Jun 2022 15:48:29 +0200 Subject: efi: vars: Use locking version to iterate over efivars linked lists Both efivars and efivarfs uses __efivar_entry_iter() to go over the linked list that shadows the list of EFI variables held by the firmware, but fail to call the begin/end helpers that are documented as a prerequisite. So switch to the proper version, which is efivar_entry_iter(). Given that in both cases, efivar_entry_remove() is invoked with the lock held already, don't take the lock there anymore. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/efivars.c | 8 ++------ drivers/firmware/efi/vars.c | 9 +-------- fs/efivarfs/super.c | 9 +++------ include/linux/efi.h | 2 +- 4 files changed, 7 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/efivars.c b/drivers/firmware/efi/efivars.c index 8341fb15f62e..801a65582172 100644 --- a/drivers/firmware/efi/efivars.c +++ b/drivers/firmware/efi/efivars.c @@ -602,10 +602,7 @@ static int efivars_sysfs_callback(efi_char16_t *name, efi_guid_t vendor, static int efivar_sysfs_destroy(struct efivar_entry *entry, void *data) { - int err = efivar_entry_remove(entry); - - if (err) - return err; + efivar_entry_remove(entry); efivar_unregister(entry); return 0; } @@ -615,8 +612,7 @@ static void efivars_sysfs_exit(void) /* Remove all entries and destroy */ int err; - err = __efivar_entry_iter(efivar_sysfs_destroy, &efivar_sysfs_list, - NULL, NULL); + err = efivar_entry_iter(efivar_sysfs_destroy, &efivar_sysfs_list, NULL); if (err) { pr_err("efivars: Failed to destroy sysfs entries\n"); return; diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index 5640ffa81544..29540013b358 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -523,17 +523,10 @@ EXPORT_SYMBOL_GPL(__efivar_entry_add); /** * efivar_entry_remove - remove entry from variable list * @entry: entry to remove from list - * - * Returns 0 on success, or a kernel error code on failure. */ -int efivar_entry_remove(struct efivar_entry *entry) +void efivar_entry_remove(struct efivar_entry *entry) { - if (down_interruptible(&efivars_lock)) - return -EINTR; list_del(&entry->list); - up(&efivars_lock); - - return 0; } EXPORT_SYMBOL_GPL(efivar_entry_remove); diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c index 09dfa8362f50..6780fc81cc11 100644 --- a/fs/efivarfs/super.c +++ b/fs/efivarfs/super.c @@ -180,10 +180,7 @@ fail: static int efivarfs_destroy(struct efivar_entry *entry, void *data) { - int err = efivar_entry_remove(entry); - - if (err) - return err; + efivar_entry_remove(entry); kfree(entry); return 0; } @@ -219,7 +216,7 @@ static int efivarfs_fill_super(struct super_block *sb, struct fs_context *fc) err = efivar_init(efivarfs_callback, (void *)sb, true, &efivarfs_list); if (err) - __efivar_entry_iter(efivarfs_destroy, &efivarfs_list, NULL, NULL); + efivar_entry_iter(efivarfs_destroy, &efivarfs_list, NULL); return err; } @@ -244,7 +241,7 @@ static void efivarfs_kill_sb(struct super_block *sb) kill_litter_super(sb); /* Remove all entries and destroy */ - __efivar_entry_iter(efivarfs_destroy, &efivarfs_list, NULL, NULL); + efivar_entry_iter(efivarfs_destroy, &efivarfs_list, NULL); } static struct file_system_type efivarfs_type = { diff --git a/include/linux/efi.h b/include/linux/efi.h index 08bc6215e3b4..54ca2d6b6c78 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1063,7 +1063,7 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), int efivar_entry_add(struct efivar_entry *entry, struct list_head *head); void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head); -int efivar_entry_remove(struct efivar_entry *entry); +void efivar_entry_remove(struct efivar_entry *entry); int __efivar_entry_delete(struct efivar_entry *entry); int efivar_entry_delete(struct efivar_entry *entry); -- cgit v1.2.3 From 5ac941367a6f85777ef34ec15d60e17ea8e446d4 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 21 Jun 2022 15:54:53 +0200 Subject: efi: vars: Drop __efivar_entry_iter() helper which is no longer used __efivar_entry_iter() uses a list iterator in a dubious way, i.e., it assumes that the iteration variable always points to an object of the appropriate type, even if the list traversal exhausts the list completely, in which case it will point somewhere in the vicinity of the list's anchor instead. Fortunately, we no longer use this function so we can just get rid of it entirely. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/vars.c | 61 ++++++--------------------------------------- include/linux/efi.h | 3 --- 2 files changed, 7 insertions(+), 57 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index 29540013b358..932435945c85 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -1034,59 +1034,6 @@ void efivar_entry_iter_end(void) } EXPORT_SYMBOL_GPL(efivar_entry_iter_end); -/** - * __efivar_entry_iter - iterate over variable list - * @func: callback function - * @head: head of the variable list - * @data: function-specific data to pass to callback - * @prev: entry to begin iterating from - * - * Iterate over the list of EFI variables and call @func with every - * entry on the list. It is safe for @func to remove entries in the - * list via efivar_entry_delete(). - * - * You MUST call efivar_entry_iter_begin() before this function, and - * efivar_entry_iter_end() afterwards. - * - * It is possible to begin iteration from an arbitrary entry within - * the list by passing @prev. @prev is updated on return to point to - * the last entry passed to @func. To begin iterating from the - * beginning of the list @prev must be %NULL. - * - * The restrictions for @func are the same as documented for - * efivar_entry_iter(). - */ -int __efivar_entry_iter(int (*func)(struct efivar_entry *, void *), - struct list_head *head, void *data, - struct efivar_entry **prev) -{ - struct efivar_entry *entry, *n; - int err = 0; - - if (!prev || !*prev) { - list_for_each_entry_safe(entry, n, head, list) { - err = func(entry, data); - if (err) - break; - } - - if (prev) - *prev = entry; - - return err; - } - - - list_for_each_entry_safe_continue((*prev), n, head, list) { - err = func(*prev, data); - if (err) - break; - } - - return err; -} -EXPORT_SYMBOL_GPL(__efivar_entry_iter); - /** * efivar_entry_iter - iterate over variable list * @func: callback function @@ -1104,12 +1051,18 @@ EXPORT_SYMBOL_GPL(__efivar_entry_iter); int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), struct list_head *head, void *data) { + struct efivar_entry *entry, *n; int err = 0; err = efivar_entry_iter_begin(); if (err) return err; - err = __efivar_entry_iter(func, head, data, NULL); + + list_for_each_entry_safe(entry, n, head, list) { + err = func(entry, data); + if (err) + break; + } efivar_entry_iter_end(); return err; diff --git a/include/linux/efi.h b/include/linux/efi.h index 54ca2d6b6c78..93ce85a14a46 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1083,9 +1083,6 @@ int efivar_entry_set_safe(efi_char16_t *name, efi_guid_t vendor, u32 attributes, int efivar_entry_iter_begin(void); void efivar_entry_iter_end(void); -int __efivar_entry_iter(int (*func)(struct efivar_entry *, void *), - struct list_head *head, void *data, - struct efivar_entry **prev); int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), struct list_head *head, void *data); -- cgit v1.2.3 From 2d82e6227ea189c0589e7383a36616ac2a2d248c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Jun 2022 18:19:43 +0200 Subject: efi: vars: Move efivar caching layer into efivarfs Move the fiddly bits of the efivar layer into its only remaining user, efivarfs, and confine its use to that particular module. All other uses of the EFI variable store have no need for this additional layer of complexity, given that they either only read variables, or read and write variables into a separate GUIDed namespace, and cannot be used to manipulate EFI variables that are covered by the EFI spec and/or affect the boot flow. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/efi.c | 1 + drivers/firmware/efi/vars.c | 733 ------------------------------------------- fs/efivarfs/Makefile | 2 +- fs/efivarfs/internal.h | 40 +++ fs/efivarfs/vars.c | 738 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/efi.h | 38 --- 6 files changed, 780 insertions(+), 772 deletions(-) create mode 100644 fs/efivarfs/vars.c (limited to 'include/linux') diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 7f06065d3eb0..e4080ad96089 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -899,6 +899,7 @@ int efi_status_to_err(efi_status_t status) return err; } +EXPORT_SYMBOL_GPL(efi_status_to_err); static DEFINE_SPINLOCK(efi_mem_reserve_persistent_lock); static struct linux_efi_memreserve *efi_memreserve_root __ro_after_init; diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index cafc128a5774..dd74d2ad3184 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -6,298 +6,20 @@ * Copyright (C) 2004 Intel Corporation */ -#include #include #include #include -#include #include #include #include #include -#include -#include -#include -#include #include /* Private pointer to registered efivars */ static struct efivars *__efivars; -/* - * efivars_lock protects three things: - * 1) efivarfs_list and efivars_sysfs_list - * 2) ->ops calls - * 3) (un)registration of __efivars - */ static DEFINE_SEMAPHORE(efivars_lock); -static bool -validate_device_path(efi_char16_t *var_name, int match, u8 *buffer, - unsigned long len) -{ - struct efi_generic_dev_path *node; - int offset = 0; - - node = (struct efi_generic_dev_path *)buffer; - - if (len < sizeof(*node)) - return false; - - while (offset <= len - sizeof(*node) && - node->length >= sizeof(*node) && - node->length <= len - offset) { - offset += node->length; - - if ((node->type == EFI_DEV_END_PATH || - node->type == EFI_DEV_END_PATH2) && - node->sub_type == EFI_DEV_END_ENTIRE) - return true; - - node = (struct efi_generic_dev_path *)(buffer + offset); - } - - /* - * If we're here then either node->length pointed past the end - * of the buffer or we reached the end of the buffer without - * finding a device path end node. - */ - return false; -} - -static bool -validate_boot_order(efi_char16_t *var_name, int match, u8 *buffer, - unsigned long len) -{ - /* An array of 16-bit integers */ - if ((len % 2) != 0) - return false; - - return true; -} - -static bool -validate_load_option(efi_char16_t *var_name, int match, u8 *buffer, - unsigned long len) -{ - u16 filepathlength; - int i, desclength = 0, namelen; - - namelen = ucs2_strnlen(var_name, EFI_VAR_NAME_LEN); - - /* Either "Boot" or "Driver" followed by four digits of hex */ - for (i = match; i < match+4; i++) { - if (var_name[i] > 127 || - hex_to_bin(var_name[i] & 0xff) < 0) - return true; - } - - /* Reject it if there's 4 digits of hex and then further content */ - if (namelen > match + 4) - return false; - - /* A valid entry must be at least 8 bytes */ - if (len < 8) - return false; - - filepathlength = buffer[4] | buffer[5] << 8; - - /* - * There's no stored length for the description, so it has to be - * found by hand - */ - desclength = ucs2_strsize((efi_char16_t *)(buffer + 6), len - 6) + 2; - - /* Each boot entry must have a descriptor */ - if (!desclength) - return false; - - /* - * If the sum of the length of the description, the claimed filepath - * length and the original header are greater than the length of the - * variable, it's malformed - */ - if ((desclength + filepathlength + 6) > len) - return false; - - /* - * And, finally, check the filepath - */ - return validate_device_path(var_name, match, buffer + desclength + 6, - filepathlength); -} - -static bool -validate_uint16(efi_char16_t *var_name, int match, u8 *buffer, - unsigned long len) -{ - /* A single 16-bit integer */ - if (len != 2) - return false; - - return true; -} - -static bool -validate_ascii_string(efi_char16_t *var_name, int match, u8 *buffer, - unsigned long len) -{ - int i; - - for (i = 0; i < len; i++) { - if (buffer[i] > 127) - return false; - - if (buffer[i] == 0) - return true; - } - - return false; -} - -struct variable_validate { - efi_guid_t vendor; - char *name; - bool (*validate)(efi_char16_t *var_name, int match, u8 *data, - unsigned long len); -}; - -/* - * This is the list of variables we need to validate, as well as the - * whitelist for what we think is safe not to default to immutable. - * - * If it has a validate() method that's not NULL, it'll go into the - * validation routine. If not, it is assumed valid, but still used for - * whitelisting. - * - * Note that it's sorted by {vendor,name}, but globbed names must come after - * any other name with the same prefix. - */ -static const struct variable_validate variable_validate[] = { - { EFI_GLOBAL_VARIABLE_GUID, "BootNext", validate_uint16 }, - { EFI_GLOBAL_VARIABLE_GUID, "BootOrder", validate_boot_order }, - { EFI_GLOBAL_VARIABLE_GUID, "Boot*", validate_load_option }, - { EFI_GLOBAL_VARIABLE_GUID, "DriverOrder", validate_boot_order }, - { EFI_GLOBAL_VARIABLE_GUID, "Driver*", validate_load_option }, - { EFI_GLOBAL_VARIABLE_GUID, "ConIn", validate_device_path }, - { EFI_GLOBAL_VARIABLE_GUID, "ConInDev", validate_device_path }, - { EFI_GLOBAL_VARIABLE_GUID, "ConOut", validate_device_path }, - { EFI_GLOBAL_VARIABLE_GUID, "ConOutDev", validate_device_path }, - { EFI_GLOBAL_VARIABLE_GUID, "ErrOut", validate_device_path }, - { EFI_GLOBAL_VARIABLE_GUID, "ErrOutDev", validate_device_path }, - { EFI_GLOBAL_VARIABLE_GUID, "Lang", validate_ascii_string }, - { EFI_GLOBAL_VARIABLE_GUID, "OsIndications", NULL }, - { EFI_GLOBAL_VARIABLE_GUID, "PlatformLang", validate_ascii_string }, - { EFI_GLOBAL_VARIABLE_GUID, "Timeout", validate_uint16 }, - { LINUX_EFI_CRASH_GUID, "*", NULL }, - { NULL_GUID, "", NULL }, -}; - -/* - * Check if @var_name matches the pattern given in @match_name. - * - * @var_name: an array of @len non-NUL characters. - * @match_name: a NUL-terminated pattern string, optionally ending in "*". A - * final "*" character matches any trailing characters @var_name, - * including the case when there are none left in @var_name. - * @match: on output, the number of non-wildcard characters in @match_name - * that @var_name matches, regardless of the return value. - * @return: whether @var_name fully matches @match_name. - */ -static bool -variable_matches(const char *var_name, size_t len, const char *match_name, - int *match) -{ - for (*match = 0; ; (*match)++) { - char c = match_name[*match]; - - switch (c) { - case '*': - /* Wildcard in @match_name means we've matched. */ - return true; - - case '\0': - /* @match_name has ended. Has @var_name too? */ - return (*match == len); - - default: - /* - * We've reached a non-wildcard char in @match_name. - * Continue only if there's an identical character in - * @var_name. - */ - if (*match < len && c == var_name[*match]) - continue; - return false; - } - } -} - -bool -efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, - unsigned long data_size) -{ - int i; - unsigned long utf8_size; - u8 *utf8_name; - - utf8_size = ucs2_utf8size(var_name); - utf8_name = kmalloc(utf8_size + 1, GFP_KERNEL); - if (!utf8_name) - return false; - - ucs2_as_utf8(utf8_name, var_name, utf8_size); - utf8_name[utf8_size] = '\0'; - - for (i = 0; variable_validate[i].name[0] != '\0'; i++) { - const char *name = variable_validate[i].name; - int match = 0; - - if (efi_guidcmp(vendor, variable_validate[i].vendor)) - continue; - - if (variable_matches(utf8_name, utf8_size+1, name, &match)) { - if (variable_validate[i].validate == NULL) - break; - kfree(utf8_name); - return variable_validate[i].validate(var_name, match, - data, data_size); - } - } - kfree(utf8_name); - return true; -} -EXPORT_SYMBOL_GPL(efivar_validate); - -bool -efivar_variable_is_removable(efi_guid_t vendor, const char *var_name, - size_t len) -{ - int i; - bool found = false; - int match = 0; - - /* - * Check if our variable is in the validated variables list - */ - for (i = 0; variable_validate[i].name[0] != '\0'; i++) { - if (efi_guidcmp(variable_validate[i].vendor, vendor)) - continue; - - if (variable_matches(var_name, len, - variable_validate[i].name, &match)) { - found = true; - break; - } - } - - /* - * If it's in our list, it is removable. - */ - return found; -} -EXPORT_SYMBOL_GPL(efivar_variable_is_removable); - efi_status_t check_var_size(u32 attributes, unsigned long size) { const struct efivar_operations *fops; @@ -324,461 +46,6 @@ efi_status_t check_var_size_nonblocking(u32 attributes, unsigned long size) } EXPORT_SYMBOL_NS_GPL(check_var_size_nonblocking, EFIVAR); -static bool variable_is_present(efi_char16_t *variable_name, efi_guid_t *vendor, - struct list_head *head) -{ - struct efivar_entry *entry, *n; - unsigned long strsize1, strsize2; - bool found = false; - - strsize1 = ucs2_strsize(variable_name, 1024); - list_for_each_entry_safe(entry, n, head, list) { - strsize2 = ucs2_strsize(entry->var.VariableName, 1024); - if (strsize1 == strsize2 && - !memcmp(variable_name, &(entry->var.VariableName), - strsize2) && - !efi_guidcmp(entry->var.VendorGuid, - *vendor)) { - found = true; - break; - } - } - return found; -} - -/* - * Returns the size of variable_name, in bytes, including the - * terminating NULL character, or variable_name_size if no NULL - * character is found among the first variable_name_size bytes. - */ -static unsigned long var_name_strnsize(efi_char16_t *variable_name, - unsigned long variable_name_size) -{ - unsigned long len; - efi_char16_t c; - - /* - * The variable name is, by definition, a NULL-terminated - * string, so make absolutely sure that variable_name_size is - * the value we expect it to be. If not, return the real size. - */ - for (len = 2; len <= variable_name_size; len += sizeof(c)) { - c = variable_name[(len / sizeof(c)) - 1]; - if (!c) - break; - } - - return min(len, variable_name_size); -} - -/* - * Print a warning when duplicate EFI variables are encountered and - * disable the sysfs workqueue since the firmware is buggy. - */ -static void dup_variable_bug(efi_char16_t *str16, efi_guid_t *vendor_guid, - unsigned long len16) -{ - size_t i, len8 = len16 / sizeof(efi_char16_t); - char *str8; - - str8 = kzalloc(len8, GFP_KERNEL); - if (!str8) - return; - - for (i = 0; i < len8; i++) - str8[i] = str16[i]; - - printk(KERN_WARNING "efivars: duplicate variable: %s-%pUl\n", - str8, vendor_guid); - kfree(str8); -} - -/** - * efivar_init - build the initial list of EFI variables - * @func: callback function to invoke for every variable - * @data: function-specific data to pass to @func - * @duplicates: error if we encounter duplicates on @head? - * @head: initialised head of variable list - * - * Get every EFI variable from the firmware and invoke @func. @func - * should call efivar_entry_add() to build the list of variables. - * - * Returns 0 on success, or a kernel error code on failure. - */ -int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), - void *data, bool duplicates, struct list_head *head) -{ - unsigned long variable_name_size = 1024; - efi_char16_t *variable_name; - efi_status_t status; - efi_guid_t vendor_guid; - int err = 0; - - variable_name = kzalloc(variable_name_size, GFP_KERNEL); - if (!variable_name) { - printk(KERN_ERR "efivars: Memory allocation failed.\n"); - return -ENOMEM; - } - - err = efivar_lock(); - if (err) - goto free; - - /* - * Per EFI spec, the maximum storage allocated for both - * the variable name and variable data is 1024 bytes. - */ - - do { - variable_name_size = 1024; - - status = efivar_get_next_variable(&variable_name_size, - variable_name, - &vendor_guid); - switch (status) { - case EFI_SUCCESS: - variable_name_size = var_name_strnsize(variable_name, - variable_name_size); - - /* - * Some firmware implementations return the - * same variable name on multiple calls to - * get_next_variable(). Terminate the loop - * immediately as there is no guarantee that - * we'll ever see a different variable name, - * and may end up looping here forever. - */ - if (duplicates && - variable_is_present(variable_name, &vendor_guid, - head)) { - dup_variable_bug(variable_name, &vendor_guid, - variable_name_size); - status = EFI_NOT_FOUND; - } else { - err = func(variable_name, vendor_guid, - variable_name_size, data); - if (err) - status = EFI_NOT_FOUND; - } - break; - case EFI_UNSUPPORTED: - err = -EOPNOTSUPP; - status = EFI_NOT_FOUND; - break; - case EFI_NOT_FOUND: - break; - default: - printk(KERN_WARNING "efivars: get_next_variable: status=%lx\n", - status); - status = EFI_NOT_FOUND; - break; - } - - } while (status != EFI_NOT_FOUND); - - efivar_unlock(); -free: - kfree(variable_name); - - return err; -} -EXPORT_SYMBOL_GPL(efivar_init); - -/** - * efivar_entry_add - add entry to variable list - * @entry: entry to add to list - * @head: list head - * - * Returns 0 on success, or a kernel error code on failure. - */ -int efivar_entry_add(struct efivar_entry *entry, struct list_head *head) -{ - int err; - - err = efivar_lock(); - if (err) - return err; - list_add(&entry->list, head); - efivar_unlock(); - - return 0; -} -EXPORT_SYMBOL_GPL(efivar_entry_add); - -/** - * __efivar_entry_add - add entry to variable list - * @entry: entry to add to list - * @head: list head - */ -void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head) -{ - list_add(&entry->list, head); -} -EXPORT_SYMBOL_GPL(__efivar_entry_add); - -/** - * efivar_entry_remove - remove entry from variable list - * @entry: entry to remove from list - */ -void efivar_entry_remove(struct efivar_entry *entry) -{ - list_del(&entry->list); -} -EXPORT_SYMBOL_GPL(efivar_entry_remove); - -/* - * efivar_entry_list_del_unlock - remove entry from variable list - * @entry: entry to remove - * - * Remove @entry from the variable list and release the list lock. - * - * NOTE: slightly weird locking semantics here - we expect to be - * called with the efivars lock already held, and we release it before - * returning. This is because this function is usually called after - * set_variable() while the lock is still held. - */ -static void efivar_entry_list_del_unlock(struct efivar_entry *entry) -{ - list_del(&entry->list); - efivar_unlock(); -} - -/** - * efivar_entry_delete - delete variable and remove entry from list - * @entry: entry containing variable to delete - * - * Delete the variable from the firmware and remove @entry from the - * variable list. It is the caller's responsibility to free @entry - * once we return. - * - * Returns 0 on success, -EINTR if we can't grab the semaphore, - * converted EFI status code if set_variable() fails. - */ -int efivar_entry_delete(struct efivar_entry *entry) -{ - efi_status_t status; - int err; - - err = efivar_lock(); - if (err) - return err; - - status = efivar_set_variable_locked(entry->var.VariableName, - &entry->var.VendorGuid, - 0, 0, NULL, false); - if (!(status == EFI_SUCCESS || status == EFI_NOT_FOUND)) { - efivar_unlock(); - return efi_status_to_err(status); - } - - efivar_entry_list_del_unlock(entry); - return 0; -} -EXPORT_SYMBOL_GPL(efivar_entry_delete); - -/** - * efivar_entry_size - obtain the size of a variable - * @entry: entry for this variable - * @size: location to store the variable's size - */ -int efivar_entry_size(struct efivar_entry *entry, unsigned long *size) -{ - efi_status_t status; - int err; - - *size = 0; - - err = efivar_lock(); - if (err) - return err; - - status = efivar_get_variable(entry->var.VariableName, - &entry->var.VendorGuid, NULL, size, NULL); - efivar_unlock(); - - if (status != EFI_BUFFER_TOO_SMALL) - return efi_status_to_err(status); - - return 0; -} -EXPORT_SYMBOL_GPL(efivar_entry_size); - -/** - * __efivar_entry_get - call get_variable() - * @entry: read data for this variable - * @attributes: variable attributes - * @size: size of @data buffer - * @data: buffer to store variable data - * - * The caller MUST hold the efivar lock when calling this function. - */ -int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes, - unsigned long *size, void *data) -{ - efi_status_t status; - - status = efivar_get_variable(entry->var.VariableName, - &entry->var.VendorGuid, - attributes, size, data); - - return efi_status_to_err(status); -} -EXPORT_SYMBOL_GPL(__efivar_entry_get); - -/** - * efivar_entry_get - call get_variable() - * @entry: read data for this variable - * @attributes: variable attributes - * @size: size of @data buffer - * @data: buffer to store variable data - */ -int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, - unsigned long *size, void *data) -{ - int err; - - err = efivar_lock(); - if (err) - return err; - err = __efivar_entry_get(entry, attributes, size, data); - efivar_unlock(); - - return err; -} -EXPORT_SYMBOL_GPL(efivar_entry_get); - -/** - * efivar_entry_set_get_size - call set_variable() and get new size (atomic) - * @entry: entry containing variable to set and get - * @attributes: attributes of variable to be written - * @size: size of data buffer - * @data: buffer containing data to write - * @set: did the set_variable() call succeed? - * - * This is a pretty special (complex) function. See efivarfs_file_write(). - * - * Atomically call set_variable() for @entry and if the call is - * successful, return the new size of the variable from get_variable() - * in @size. The success of set_variable() is indicated by @set. - * - * Returns 0 on success, -EINVAL if the variable data is invalid, - * -ENOSPC if the firmware does not have enough available space, or a - * converted EFI status code if either of set_variable() or - * get_variable() fail. - * - * If the EFI variable does not exist when calling set_variable() - * (EFI_NOT_FOUND), @entry is removed from the variable list. - */ -int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes, - unsigned long *size, void *data, bool *set) -{ - efi_char16_t *name = entry->var.VariableName; - efi_guid_t *vendor = &entry->var.VendorGuid; - efi_status_t status; - int err; - - *set = false; - - if (efivar_validate(*vendor, name, data, *size) == false) - return -EINVAL; - - /* - * The lock here protects the get_variable call, the conditional - * set_variable call, and removal of the variable from the efivars - * list (in the case of an authenticated delete). - */ - err = efivar_lock(); - if (err) - return err; - - /* - * Ensure that the available space hasn't shrunk below the safe level - */ - status = check_var_size(attributes, *size + ucs2_strsize(name, 1024)); - if (status != EFI_SUCCESS) { - if (status != EFI_UNSUPPORTED) { - err = efi_status_to_err(status); - goto out; - } - - if (*size > 65536) { - err = -ENOSPC; - goto out; - } - } - - status = efivar_set_variable_locked(name, vendor, attributes, *size, - data, false); - if (status != EFI_SUCCESS) { - err = efi_status_to_err(status); - goto out; - } - - *set = true; - - /* - * Writing to the variable may have caused a change in size (which - * could either be an append or an overwrite), or the variable to be - * deleted. Perform a GetVariable() so we can tell what actually - * happened. - */ - *size = 0; - status = efivar_get_variable(entry->var.VariableName, - &entry->var.VendorGuid, - NULL, size, NULL); - - if (status == EFI_NOT_FOUND) - efivar_entry_list_del_unlock(entry); - else - efivar_unlock(); - - if (status && status != EFI_BUFFER_TOO_SMALL) - return efi_status_to_err(status); - - return 0; - -out: - efivar_unlock(); - return err; - -} -EXPORT_SYMBOL_GPL(efivar_entry_set_get_size); - -/** - * efivar_entry_iter - iterate over variable list - * @func: callback function - * @head: head of variable list - * @data: function-specific data to pass to callback - * - * Iterate over the list of EFI variables and call @func with every - * entry on the list. It is safe for @func to remove entries in the - * list via efivar_entry_delete() while iterating. - * - * Some notes for the callback function: - * - a non-zero return value indicates an error and terminates the loop - * - @func is called from atomic context - */ -int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), - struct list_head *head, void *data) -{ - struct efivar_entry *entry, *n; - int err = 0; - - err = efivar_lock(); - if (err) - return err; - - list_for_each_entry_safe(entry, n, head, list) { - err = func(entry, data); - if (err) - break; - } - efivar_unlock(); - - return err; -} -EXPORT_SYMBOL_GPL(efivar_entry_iter); - /** * efivars_kobject - get the kobject for the registered efivars * diff --git a/fs/efivarfs/Makefile b/fs/efivarfs/Makefile index 0b1c5e63eb71..7bfc2f9754a8 100644 --- a/fs/efivarfs/Makefile +++ b/fs/efivarfs/Makefile @@ -5,4 +5,4 @@ obj-$(CONFIG_EFIVAR_FS) += efivarfs.o -efivarfs-objs := inode.o file.o super.o +efivarfs-objs := inode.o file.o super.o vars.o diff --git a/fs/efivarfs/internal.h b/fs/efivarfs/internal.h index 30ae44cb7453..8ebf3a6a8aa2 100644 --- a/fs/efivarfs/internal.h +++ b/fs/efivarfs/internal.h @@ -7,6 +7,46 @@ #define EFIVAR_FS_INTERNAL_H #include +#include + +struct efi_variable { + efi_char16_t VariableName[EFI_VAR_NAME_LEN/sizeof(efi_char16_t)]; + efi_guid_t VendorGuid; + unsigned long DataSize; + __u8 Data[1024]; + efi_status_t Status; + __u32 Attributes; +} __attribute__((packed)); + +struct efivar_entry { + struct efi_variable var; + struct list_head list; + struct kobject kobj; +}; + +int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), + void *data, bool duplicates, struct list_head *head); + +int efivar_entry_add(struct efivar_entry *entry, struct list_head *head); +void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head); +void efivar_entry_remove(struct efivar_entry *entry); +int efivar_entry_delete(struct efivar_entry *entry); + +int efivar_entry_size(struct efivar_entry *entry, unsigned long *size); +int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data); +int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data); +int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes, + unsigned long *size, void *data, bool *set); + +int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), + struct list_head *head, void *data); + +bool efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, + unsigned long data_size); +bool efivar_variable_is_removable(efi_guid_t vendor, const char *name, + size_t len); extern const struct file_operations efivarfs_file_operations; extern const struct inode_operations efivarfs_dir_inode_operations; diff --git a/fs/efivarfs/vars.c b/fs/efivarfs/vars.c new file mode 100644 index 000000000000..a0ef63cfcecb --- /dev/null +++ b/fs/efivarfs/vars.c @@ -0,0 +1,738 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Originally from efivars.c + * + * Copyright (C) 2001,2003,2004 Dell + * Copyright (C) 2004 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +MODULE_IMPORT_NS(EFIVAR); + +static bool +validate_device_path(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + struct efi_generic_dev_path *node; + int offset = 0; + + node = (struct efi_generic_dev_path *)buffer; + + if (len < sizeof(*node)) + return false; + + while (offset <= len - sizeof(*node) && + node->length >= sizeof(*node) && + node->length <= len - offset) { + offset += node->length; + + if ((node->type == EFI_DEV_END_PATH || + node->type == EFI_DEV_END_PATH2) && + node->sub_type == EFI_DEV_END_ENTIRE) + return true; + + node = (struct efi_generic_dev_path *)(buffer + offset); + } + + /* + * If we're here then either node->length pointed past the end + * of the buffer or we reached the end of the buffer without + * finding a device path end node. + */ + return false; +} + +static bool +validate_boot_order(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + /* An array of 16-bit integers */ + if ((len % 2) != 0) + return false; + + return true; +} + +static bool +validate_load_option(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + u16 filepathlength; + int i, desclength = 0, namelen; + + namelen = ucs2_strnlen(var_name, EFI_VAR_NAME_LEN); + + /* Either "Boot" or "Driver" followed by four digits of hex */ + for (i = match; i < match+4; i++) { + if (var_name[i] > 127 || + hex_to_bin(var_name[i] & 0xff) < 0) + return true; + } + + /* Reject it if there's 4 digits of hex and then further content */ + if (namelen > match + 4) + return false; + + /* A valid entry must be at least 8 bytes */ + if (len < 8) + return false; + + filepathlength = buffer[4] | buffer[5] << 8; + + /* + * There's no stored length for the description, so it has to be + * found by hand + */ + desclength = ucs2_strsize((efi_char16_t *)(buffer + 6), len - 6) + 2; + + /* Each boot entry must have a descriptor */ + if (!desclength) + return false; + + /* + * If the sum of the length of the description, the claimed filepath + * length and the original header are greater than the length of the + * variable, it's malformed + */ + if ((desclength + filepathlength + 6) > len) + return false; + + /* + * And, finally, check the filepath + */ + return validate_device_path(var_name, match, buffer + desclength + 6, + filepathlength); +} + +static bool +validate_uint16(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + /* A single 16-bit integer */ + if (len != 2) + return false; + + return true; +} + +static bool +validate_ascii_string(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + int i; + + for (i = 0; i < len; i++) { + if (buffer[i] > 127) + return false; + + if (buffer[i] == 0) + return true; + } + + return false; +} + +struct variable_validate { + efi_guid_t vendor; + char *name; + bool (*validate)(efi_char16_t *var_name, int match, u8 *data, + unsigned long len); +}; + +/* + * This is the list of variables we need to validate, as well as the + * whitelist for what we think is safe not to default to immutable. + * + * If it has a validate() method that's not NULL, it'll go into the + * validation routine. If not, it is assumed valid, but still used for + * whitelisting. + * + * Note that it's sorted by {vendor,name}, but globbed names must come after + * any other name with the same prefix. + */ +static const struct variable_validate variable_validate[] = { + { EFI_GLOBAL_VARIABLE_GUID, "BootNext", validate_uint16 }, + { EFI_GLOBAL_VARIABLE_GUID, "BootOrder", validate_boot_order }, + { EFI_GLOBAL_VARIABLE_GUID, "Boot*", validate_load_option }, + { EFI_GLOBAL_VARIABLE_GUID, "DriverOrder", validate_boot_order }, + { EFI_GLOBAL_VARIABLE_GUID, "Driver*", validate_load_option }, + { EFI_GLOBAL_VARIABLE_GUID, "ConIn", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ConInDev", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ConOut", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ConOutDev", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ErrOut", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ErrOutDev", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "Lang", validate_ascii_string }, + { EFI_GLOBAL_VARIABLE_GUID, "OsIndications", NULL }, + { EFI_GLOBAL_VARIABLE_GUID, "PlatformLang", validate_ascii_string }, + { EFI_GLOBAL_VARIABLE_GUID, "Timeout", validate_uint16 }, + { LINUX_EFI_CRASH_GUID, "*", NULL }, + { NULL_GUID, "", NULL }, +}; + +/* + * Check if @var_name matches the pattern given in @match_name. + * + * @var_name: an array of @len non-NUL characters. + * @match_name: a NUL-terminated pattern string, optionally ending in "*". A + * final "*" character matches any trailing characters @var_name, + * including the case when there are none left in @var_name. + * @match: on output, the number of non-wildcard characters in @match_name + * that @var_name matches, regardless of the return value. + * @return: whether @var_name fully matches @match_name. + */ +static bool +variable_matches(const char *var_name, size_t len, const char *match_name, + int *match) +{ + for (*match = 0; ; (*match)++) { + char c = match_name[*match]; + + switch (c) { + case '*': + /* Wildcard in @match_name means we've matched. */ + return true; + + case '\0': + /* @match_name has ended. Has @var_name too? */ + return (*match == len); + + default: + /* + * We've reached a non-wildcard char in @match_name. + * Continue only if there's an identical character in + * @var_name. + */ + if (*match < len && c == var_name[*match]) + continue; + return false; + } + } +} + +bool +efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, + unsigned long data_size) +{ + int i; + unsigned long utf8_size; + u8 *utf8_name; + + utf8_size = ucs2_utf8size(var_name); + utf8_name = kmalloc(utf8_size + 1, GFP_KERNEL); + if (!utf8_name) + return false; + + ucs2_as_utf8(utf8_name, var_name, utf8_size); + utf8_name[utf8_size] = '\0'; + + for (i = 0; variable_validate[i].name[0] != '\0'; i++) { + const char *name = variable_validate[i].name; + int match = 0; + + if (efi_guidcmp(vendor, variable_validate[i].vendor)) + continue; + + if (variable_matches(utf8_name, utf8_size+1, name, &match)) { + if (variable_validate[i].validate == NULL) + break; + kfree(utf8_name); + return variable_validate[i].validate(var_name, match, + data, data_size); + } + } + kfree(utf8_name); + return true; +} + +bool +efivar_variable_is_removable(efi_guid_t vendor, const char *var_name, + size_t len) +{ + int i; + bool found = false; + int match = 0; + + /* + * Check if our variable is in the validated variables list + */ + for (i = 0; variable_validate[i].name[0] != '\0'; i++) { + if (efi_guidcmp(variable_validate[i].vendor, vendor)) + continue; + + if (variable_matches(var_name, len, + variable_validate[i].name, &match)) { + found = true; + break; + } + } + + /* + * If it's in our list, it is removable. + */ + return found; +} + +static bool variable_is_present(efi_char16_t *variable_name, efi_guid_t *vendor, + struct list_head *head) +{ + struct efivar_entry *entry, *n; + unsigned long strsize1, strsize2; + bool found = false; + + strsize1 = ucs2_strsize(variable_name, 1024); + list_for_each_entry_safe(entry, n, head, list) { + strsize2 = ucs2_strsize(entry->var.VariableName, 1024); + if (strsize1 == strsize2 && + !memcmp(variable_name, &(entry->var.VariableName), + strsize2) && + !efi_guidcmp(entry->var.VendorGuid, + *vendor)) { + found = true; + break; + } + } + return found; +} + +/* + * Returns the size of variable_name, in bytes, including the + * terminating NULL character, or variable_name_size if no NULL + * character is found among the first variable_name_size bytes. + */ +static unsigned long var_name_strnsize(efi_char16_t *variable_name, + unsigned long variable_name_size) +{ + unsigned long len; + efi_char16_t c; + + /* + * The variable name is, by definition, a NULL-terminated + * string, so make absolutely sure that variable_name_size is + * the value we expect it to be. If not, return the real size. + */ + for (len = 2; len <= variable_name_size; len += sizeof(c)) { + c = variable_name[(len / sizeof(c)) - 1]; + if (!c) + break; + } + + return min(len, variable_name_size); +} + +/* + * Print a warning when duplicate EFI variables are encountered and + * disable the sysfs workqueue since the firmware is buggy. + */ +static void dup_variable_bug(efi_char16_t *str16, efi_guid_t *vendor_guid, + unsigned long len16) +{ + size_t i, len8 = len16 / sizeof(efi_char16_t); + char *str8; + + str8 = kzalloc(len8, GFP_KERNEL); + if (!str8) + return; + + for (i = 0; i < len8; i++) + str8[i] = str16[i]; + + printk(KERN_WARNING "efivars: duplicate variable: %s-%pUl\n", + str8, vendor_guid); + kfree(str8); +} + +/** + * efivar_init - build the initial list of EFI variables + * @func: callback function to invoke for every variable + * @data: function-specific data to pass to @func + * @duplicates: error if we encounter duplicates on @head? + * @head: initialised head of variable list + * + * Get every EFI variable from the firmware and invoke @func. @func + * should call efivar_entry_add() to build the list of variables. + * + * Returns 0 on success, or a kernel error code on failure. + */ +int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), + void *data, bool duplicates, struct list_head *head) +{ + unsigned long variable_name_size = 1024; + efi_char16_t *variable_name; + efi_status_t status; + efi_guid_t vendor_guid; + int err = 0; + + variable_name = kzalloc(variable_name_size, GFP_KERNEL); + if (!variable_name) { + printk(KERN_ERR "efivars: Memory allocation failed.\n"); + return -ENOMEM; + } + + err = efivar_lock(); + if (err) + goto free; + + /* + * Per EFI spec, the maximum storage allocated for both + * the variable name and variable data is 1024 bytes. + */ + + do { + variable_name_size = 1024; + + status = efivar_get_next_variable(&variable_name_size, + variable_name, + &vendor_guid); + switch (status) { + case EFI_SUCCESS: + variable_name_size = var_name_strnsize(variable_name, + variable_name_size); + + /* + * Some firmware implementations return the + * same variable name on multiple calls to + * get_next_variable(). Terminate the loop + * immediately as there is no guarantee that + * we'll ever see a different variable name, + * and may end up looping here forever. + */ + if (duplicates && + variable_is_present(variable_name, &vendor_guid, + head)) { + dup_variable_bug(variable_name, &vendor_guid, + variable_name_size); + status = EFI_NOT_FOUND; + } else { + err = func(variable_name, vendor_guid, + variable_name_size, data); + if (err) + status = EFI_NOT_FOUND; + } + break; + case EFI_UNSUPPORTED: + err = -EOPNOTSUPP; + status = EFI_NOT_FOUND; + break; + case EFI_NOT_FOUND: + break; + default: + printk(KERN_WARNING "efivars: get_next_variable: status=%lx\n", + status); + status = EFI_NOT_FOUND; + break; + } + + } while (status != EFI_NOT_FOUND); + + efivar_unlock(); +free: + kfree(variable_name); + + return err; +} + +/** + * efivar_entry_add - add entry to variable list + * @entry: entry to add to list + * @head: list head + * + * Returns 0 on success, or a kernel error code on failure. + */ +int efivar_entry_add(struct efivar_entry *entry, struct list_head *head) +{ + int err; + + err = efivar_lock(); + if (err) + return err; + list_add(&entry->list, head); + efivar_unlock(); + + return 0; +} + +/** + * __efivar_entry_add - add entry to variable list + * @entry: entry to add to list + * @head: list head + */ +void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head) +{ + list_add(&entry->list, head); +} + +/** + * efivar_entry_remove - remove entry from variable list + * @entry: entry to remove from list + * + * Returns 0 on success, or a kernel error code on failure. + */ +void efivar_entry_remove(struct efivar_entry *entry) +{ + list_del(&entry->list); +} + +/* + * efivar_entry_list_del_unlock - remove entry from variable list + * @entry: entry to remove + * + * Remove @entry from the variable list and release the list lock. + * + * NOTE: slightly weird locking semantics here - we expect to be + * called with the efivars lock already held, and we release it before + * returning. This is because this function is usually called after + * set_variable() while the lock is still held. + */ +static void efivar_entry_list_del_unlock(struct efivar_entry *entry) +{ + list_del(&entry->list); + efivar_unlock(); +} + +/** + * efivar_entry_delete - delete variable and remove entry from list + * @entry: entry containing variable to delete + * + * Delete the variable from the firmware and remove @entry from the + * variable list. It is the caller's responsibility to free @entry + * once we return. + * + * Returns 0 on success, -EINTR if we can't grab the semaphore, + * converted EFI status code if set_variable() fails. + */ +int efivar_entry_delete(struct efivar_entry *entry) +{ + efi_status_t status; + int err; + + err = efivar_lock(); + if (err) + return err; + + status = efivar_set_variable_locked(entry->var.VariableName, + &entry->var.VendorGuid, + 0, 0, NULL, false); + if (!(status == EFI_SUCCESS || status == EFI_NOT_FOUND)) { + efivar_unlock(); + return efi_status_to_err(status); + } + + efivar_entry_list_del_unlock(entry); + return 0; +} + +/** + * efivar_entry_size - obtain the size of a variable + * @entry: entry for this variable + * @size: location to store the variable's size + */ +int efivar_entry_size(struct efivar_entry *entry, unsigned long *size) +{ + efi_status_t status; + int err; + + *size = 0; + + err = efivar_lock(); + if (err) + return err; + + status = efivar_get_variable(entry->var.VariableName, + &entry->var.VendorGuid, NULL, size, NULL); + efivar_unlock(); + + if (status != EFI_BUFFER_TOO_SMALL) + return efi_status_to_err(status); + + return 0; +} + +/** + * __efivar_entry_get - call get_variable() + * @entry: read data for this variable + * @attributes: variable attributes + * @size: size of @data buffer + * @data: buffer to store variable data + * + * The caller MUST call efivar_entry_iter_begin() and + * efivar_entry_iter_end() before and after the invocation of this + * function, respectively. + */ +int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data) +{ + efi_status_t status; + + status = efivar_get_variable(entry->var.VariableName, + &entry->var.VendorGuid, + attributes, size, data); + + return efi_status_to_err(status); +} + +/** + * efivar_entry_get - call get_variable() + * @entry: read data for this variable + * @attributes: variable attributes + * @size: size of @data buffer + * @data: buffer to store variable data + */ +int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data) +{ + int err; + + err = efivar_lock(); + if (err) + return err; + err = __efivar_entry_get(entry, attributes, size, data); + efivar_unlock(); + + return 0; +} + +/** + * efivar_entry_set_get_size - call set_variable() and get new size (atomic) + * @entry: entry containing variable to set and get + * @attributes: attributes of variable to be written + * @size: size of data buffer + * @data: buffer containing data to write + * @set: did the set_variable() call succeed? + * + * This is a pretty special (complex) function. See efivarfs_file_write(). + * + * Atomically call set_variable() for @entry and if the call is + * successful, return the new size of the variable from get_variable() + * in @size. The success of set_variable() is indicated by @set. + * + * Returns 0 on success, -EINVAL if the variable data is invalid, + * -ENOSPC if the firmware does not have enough available space, or a + * converted EFI status code if either of set_variable() or + * get_variable() fail. + * + * If the EFI variable does not exist when calling set_variable() + * (EFI_NOT_FOUND), @entry is removed from the variable list. + */ +int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes, + unsigned long *size, void *data, bool *set) +{ + efi_char16_t *name = entry->var.VariableName; + efi_guid_t *vendor = &entry->var.VendorGuid; + efi_status_t status; + int err; + + *set = false; + + if (efivar_validate(*vendor, name, data, *size) == false) + return -EINVAL; + + /* + * The lock here protects the get_variable call, the conditional + * set_variable call, and removal of the variable from the efivars + * list (in the case of an authenticated delete). + */ + err = efivar_lock(); + if (err) + return err; + + /* + * Ensure that the available space hasn't shrunk below the safe level + */ + status = check_var_size(attributes, *size + ucs2_strsize(name, 1024)); + if (status != EFI_SUCCESS) { + if (status != EFI_UNSUPPORTED) { + err = efi_status_to_err(status); + goto out; + } + + if (*size > 65536) { + err = -ENOSPC; + goto out; + } + } + + status = efivar_set_variable_locked(name, vendor, attributes, *size, + data, false); + if (status != EFI_SUCCESS) { + err = efi_status_to_err(status); + goto out; + } + + *set = true; + + /* + * Writing to the variable may have caused a change in size (which + * could either be an append or an overwrite), or the variable to be + * deleted. Perform a GetVariable() so we can tell what actually + * happened. + */ + *size = 0; + status = efivar_get_variable(entry->var.VariableName, + &entry->var.VendorGuid, + NULL, size, NULL); + + if (status == EFI_NOT_FOUND) + efivar_entry_list_del_unlock(entry); + else + efivar_unlock(); + + if (status && status != EFI_BUFFER_TOO_SMALL) + return efi_status_to_err(status); + + return 0; + +out: + efivar_unlock(); + return err; + +} + +/** + * efivar_entry_iter - iterate over variable list + * @func: callback function + * @head: head of variable list + * @data: function-specific data to pass to callback + * + * Iterate over the list of EFI variables and call @func with every + * entry on the list. It is safe for @func to remove entries in the + * list via efivar_entry_delete() while iterating. + * + * Some notes for the callback function: + * - a non-zero return value indicates an error and terminates the loop + * - @func is called from atomic context + */ +int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), + struct list_head *head, void *data) +{ + struct efivar_entry *entry, *n; + int err = 0; + + err = efivar_lock(); + if (err) + return err; + + list_for_each_entry_safe(entry, n, head, list) { + err = func(entry, data); + if (err) + break; + } + efivar_unlock(); + + return err; +} diff --git a/include/linux/efi.h b/include/linux/efi.h index 10ef0a0d5e9a..8122c2ed505c 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1030,21 +1030,6 @@ struct efivars { #define EFI_VAR_NAME_LEN 1024 -struct efi_variable { - efi_char16_t VariableName[EFI_VAR_NAME_LEN/sizeof(efi_char16_t)]; - efi_guid_t VendorGuid; - unsigned long DataSize; - __u8 Data[1024]; - efi_status_t Status; - __u32 Attributes; -} __attribute__((packed)); - -struct efivar_entry { - struct efi_variable var; - struct list_head list; - struct kobject kobj; -}; - int efivars_register(struct efivars *efivars, const struct efivar_operations *ops, struct kobject *kobject); @@ -1052,29 +1037,6 @@ int efivars_unregister(struct efivars *efivars); struct kobject *efivars_kobject(void); int efivar_supports_writes(void); -int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), - void *data, bool duplicates, struct list_head *head); - -int efivar_entry_add(struct efivar_entry *entry, struct list_head *head); -void __efivar_entry_add(struct efivar_entry *entry, struct list_head *head); -void efivar_entry_remove(struct efivar_entry *entry); -int efivar_entry_delete(struct efivar_entry *entry); - -int efivar_entry_size(struct efivar_entry *entry, unsigned long *size); -int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes, - unsigned long *size, void *data); -int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, - unsigned long *size, void *data); -int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes, - unsigned long *size, void *data, bool *set); - -int efivar_entry_iter(int (*func)(struct efivar_entry *, void *), - struct list_head *head, void *data); - -bool efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, - unsigned long data_size); -bool efivar_variable_is_removable(efi_guid_t vendor, const char *name, - size_t len); int efivar_lock(void); int efivar_trylock(void); -- cgit v1.2.3 From ede57d58e6f38d5bc66137368e4a1e68a157af6e Mon Sep 17 00:00:00 2001 From: Richard Gobert Date: Wed, 22 Jun 2022 18:09:03 +0200 Subject: net: helper function skb_len_add Move the len fields manipulation in the skbs to a helper function. There is a comment specifically requesting this and there are several other areas in the code displaying the same pattern which can be refactored. This improves code readability. Signed-off-by: Richard Gobert Link: https://lore.kernel.org/r/20220622160853.GA6478@debian Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 12 ++++++++++++ include/net/sock.h | 4 +--- net/core/skbuff.c | 13 +++---------- net/ipv4/esp4.c | 4 +--- net/ipv4/ip_output.c | 8 ++------ 5 files changed, 19 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index cd4a8268894a..f6a27ab19202 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2351,6 +2351,18 @@ static inline unsigned int skb_pagelen(const struct sk_buff *skb) return skb_headlen(skb) + __skb_pagelen(skb); } +/** + * skb_len_add - adds a number to len fields of skb + * @skb: buffer to add len to + * @delta: number of bytes to add + */ +static inline void skb_len_add(struct sk_buff *skb, int delta) +{ + skb->len += delta; + skb->data_len += delta; + skb->truesize += delta; +} + /** * __skb_fill_page_desc - initialise a paged fragment in an skb * @skb: buffer containing fragment to be initialised diff --git a/include/net/sock.h b/include/net/sock.h index 5bed1ea7a722..40bbd0e8925b 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2219,9 +2219,7 @@ static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *fro if (err) return err; - skb->len += copy; - skb->data_len += copy; - skb->truesize += copy; + skb_len_add(skb, copy); sk_wmem_queued_add(sk, copy); sk_mem_charge(sk, copy); return 0; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 00bf35ee8205..c62e42d0c531 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3195,9 +3195,7 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) } } - to->truesize += len + plen; - to->len += len + plen; - to->data_len += len + plen; + skb_len_add(to, len + plen); if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { skb_tx_error(from); @@ -3634,13 +3632,8 @@ onlymerged: tgt->ip_summed = CHECKSUM_PARTIAL; skb->ip_summed = CHECKSUM_PARTIAL; - /* Yak, is it really working this way? Some helper please? */ - skb->len -= shiftlen; - skb->data_len -= shiftlen; - skb->truesize -= shiftlen; - tgt->len += shiftlen; - tgt->data_len += shiftlen; - tgt->truesize += shiftlen; + skb_len_add(skb, -shiftlen); + skb_len_add(tgt, shiftlen); return shiftlen; } diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index b21238df3301..7eae8d686e20 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -502,9 +502,7 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * nfrags++; - skb->len += tailen; - skb->data_len += tailen; - skb->truesize += tailen; + skb_len_add(skb, tailen); if (sk && sk_fullsock(sk)) refcount_add(tailen, &sk->sk_wmem_alloc); diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 00b4bf26fd93..5e32a2f86fbd 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1214,9 +1214,7 @@ alloc_new_skb: pfrag->offset += copy; skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); - skb->len += copy; - skb->data_len += copy; - skb->truesize += copy; + skb_len_add(skb, copy); wmem_alloc_delta += copy; } else { err = skb_zerocopy_iter_dgram(skb, from, copy); @@ -1443,9 +1441,7 @@ ssize_t ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page, skb->csum = csum_block_add(skb->csum, csum, skb->len); } - skb->len += len; - skb->data_len += len; - skb->truesize += len; + skb_len_add(skb, len); refcount_add(len, &sk->sk_wmem_alloc); offset += len; size -= len; -- cgit v1.2.3 From 0cae04373b77a117830e5f7d7aaa7eaf01f950d5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 6 Jun 2022 09:47:33 +0200 Subject: dmaengine: remove DMA_MEMCPY_SG once again This was removed before due to the complete lack of users, but 3218910fd585 ("dmaengine: Add core function and capability check for DMA_MEMCPY_SG") and 29cf37fa6dd9 ("dmaengine: Add consumer for the new DMA_MEMCPY_SG API function.") added it back despite still not having any users whatsoever. Fixes: 3218910fd585 ("dmaengine: Add core function and capability check for DMA_MEMCPY_SG") Fixes: 29cf37fa6dd9 ("dmaengine: Add consumer for the new DMA_MEMCPY_SG API function.") Signed-off-by: Christoph Hellwig Acked-by: Michal Simek Link: https://lore.kernel.org/r/20220606074733.622616-1-hch@lst.de Signed-off-by: Vinod Koul --- Documentation/driver-api/dmaengine/provider.rst | 10 -- drivers/dma/dmaengine.c | 7 -- drivers/dma/xilinx/xilinx_dma.c | 122 ------------------------ include/linux/dmaengine.h | 20 ---- 4 files changed, 159 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/dmaengine/provider.rst b/Documentation/driver-api/dmaengine/provider.rst index 1e0f1f85d10e..ceac2a300e32 100644 --- a/Documentation/driver-api/dmaengine/provider.rst +++ b/Documentation/driver-api/dmaengine/provider.rst @@ -162,16 +162,6 @@ Currently, the types available are: - The device is able to do memory to memory copies -- - DMA_MEMCPY_SG - - - The device supports memory to memory scatter-gather transfers. - - - Even though a plain memcpy can look like a particular case of a - scatter-gather transfer, with a single chunk to copy, it's a distinct - transaction type in the mem2mem transfer case. This is because some very - simple devices might be able to do contiguous single-chunk memory copies, - but have no support for more complex SG transfers. - - No matter what the overall size of the combined chunks for source and destination is, only as many bytes as the smallest of the two will be transmitted. That means the number and size of the scatter-gather buffers in diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index e80feeea0e01..c741b6431958 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -1153,13 +1153,6 @@ int dma_async_device_register(struct dma_device *device) return -EIO; } - if (dma_has_cap(DMA_MEMCPY_SG, device->cap_mask) && !device->device_prep_dma_memcpy_sg) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_MEMCPY_SG"); - return -EIO; - } - if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) { dev_err(device->dev, "Device claims capability %s, but op is not defined\n", diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index cd62bbb50e8b..6276934d4d2b 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -2127,126 +2127,6 @@ error: return NULL; } -/** - * xilinx_cdma_prep_memcpy_sg - prepare descriptors for a memcpy_sg transaction - * @dchan: DMA channel - * @dst_sg: Destination scatter list - * @dst_sg_len: Number of entries in destination scatter list - * @src_sg: Source scatter list - * @src_sg_len: Number of entries in source scatter list - * @flags: transfer ack flags - * - * Return: Async transaction descriptor on success and NULL on failure - */ -static struct dma_async_tx_descriptor *xilinx_cdma_prep_memcpy_sg( - struct dma_chan *dchan, struct scatterlist *dst_sg, - unsigned int dst_sg_len, struct scatterlist *src_sg, - unsigned int src_sg_len, unsigned long flags) -{ - struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); - struct xilinx_dma_tx_descriptor *desc; - struct xilinx_cdma_tx_segment *segment, *prev = NULL; - struct xilinx_cdma_desc_hw *hw; - size_t len, dst_avail, src_avail; - dma_addr_t dma_dst, dma_src; - - if (unlikely(dst_sg_len == 0 || src_sg_len == 0)) - return NULL; - - if (unlikely(!dst_sg || !src_sg)) - return NULL; - - desc = xilinx_dma_alloc_tx_descriptor(chan); - if (!desc) - return NULL; - - dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); - desc->async_tx.tx_submit = xilinx_dma_tx_submit; - - dst_avail = sg_dma_len(dst_sg); - src_avail = sg_dma_len(src_sg); - /* - * loop until there is either no more source or no more destination - * scatterlist entry - */ - while (true) { - len = min_t(size_t, src_avail, dst_avail); - len = min_t(size_t, len, chan->xdev->max_buffer_len); - if (len == 0) - goto fetch; - - /* Allocate the link descriptor from DMA pool */ - segment = xilinx_cdma_alloc_tx_segment(chan); - if (!segment) - goto error; - - dma_dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - - dst_avail; - dma_src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - - src_avail; - hw = &segment->hw; - hw->control = len; - hw->src_addr = dma_src; - hw->dest_addr = dma_dst; - if (chan->ext_addr) { - hw->src_addr_msb = upper_32_bits(dma_src); - hw->dest_addr_msb = upper_32_bits(dma_dst); - } - - if (prev) { - prev->hw.next_desc = segment->phys; - if (chan->ext_addr) - prev->hw.next_desc_msb = - upper_32_bits(segment->phys); - } - - prev = segment; - dst_avail -= len; - src_avail -= len; - list_add_tail(&segment->node, &desc->segments); - -fetch: - /* Fetch the next dst scatterlist entry */ - if (dst_avail == 0) { - if (dst_sg_len == 0) - break; - dst_sg = sg_next(dst_sg); - if (dst_sg == NULL) - break; - dst_sg_len--; - dst_avail = sg_dma_len(dst_sg); - } - /* Fetch the next src scatterlist entry */ - if (src_avail == 0) { - if (src_sg_len == 0) - break; - src_sg = sg_next(src_sg); - if (src_sg == NULL) - break; - src_sg_len--; - src_avail = sg_dma_len(src_sg); - } - } - - if (list_empty(&desc->segments)) { - dev_err(chan->xdev->dev, - "%s: Zero-size SG transfer requested\n", __func__); - goto error; - } - - /* Link the last hardware descriptor with the first. */ - segment = list_first_entry(&desc->segments, - struct xilinx_cdma_tx_segment, node); - desc->async_tx.phys = segment->phys; - prev->hw.next_desc = segment->phys; - - return &desc->async_tx; - -error: - xilinx_dma_free_tx_descriptor(chan, desc); - return NULL; -} - /** * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction * @dchan: DMA channel @@ -3240,9 +3120,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) DMA_RESIDUE_GRANULARITY_SEGMENT; } else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask); - dma_cap_set(DMA_MEMCPY_SG, xdev->common.cap_mask); xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy; - xdev->common.device_prep_dma_memcpy_sg = xilinx_cdma_prep_memcpy_sg; /* Residue calculation is supported by only AXI DMA and CDMA */ xdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index b46b88e6aa0d..c923f4e60f24 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -50,7 +50,6 @@ enum dma_status { */ enum dma_transaction_type { DMA_MEMCPY, - DMA_MEMCPY_SG, DMA_XOR, DMA_PQ, DMA_XOR_VAL, @@ -887,11 +886,6 @@ struct dma_device { struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)( struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, size_t len, unsigned long flags); - struct dma_async_tx_descriptor *(*device_prep_dma_memcpy_sg)( - struct dma_chan *chan, - struct scatterlist *dst_sg, unsigned int dst_nents, - struct scatterlist *src_sg, unsigned int src_nents, - unsigned long flags); struct dma_async_tx_descriptor *(*device_prep_dma_xor)( struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, unsigned int src_cnt, size_t len, unsigned long flags); @@ -1060,20 +1054,6 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy( len, flags); } -static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy_sg( - struct dma_chan *chan, - struct scatterlist *dst_sg, unsigned int dst_nents, - struct scatterlist *src_sg, unsigned int src_nents, - unsigned long flags) -{ - if (!chan || !chan->device || !chan->device->device_prep_dma_memcpy_sg) - return NULL; - - return chan->device->device_prep_dma_memcpy_sg(chan, dst_sg, dst_nents, - src_sg, src_nents, - flags); -} - static inline bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan, enum dma_desc_metadata_mode mode) { -- cgit v1.2.3 From 7283f862bd991c8657e9bf1c02db772fcf018f13 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Wed, 22 Jun 2022 16:01:33 +0200 Subject: drm: Implement DRM aperture helpers under video/ Implement DRM's aperture helpers under video/ for sharing with other sub-systems. Remove DRM-isms from the interface. The helpers track the ownership of framebuffer apertures and provide hand-over from firmware, such as EFI and VESA, to native graphics drivers. Other subsystems, such as fbdev and vfio, also have to maintain ownership of framebuffer apertures. Moving DRM's aperture helpers to a more public location allows all subsystems to interact with each other and share a common implementation. The aperture helpers are selected by the various firmware drivers within DRM and fbdev, and the VGA text-console driver. The original DRM interface is kept in place for use by DRM drivers. v3: * prefix all interfaces with aperture_ (Javier) * rework and simplify documentation (Javier) * rename struct dev_aperture to struct aperture_range * rebase onto latest DRM * update MAINTAINERS entry Signed-off-by: Thomas Zimmermann Reviewed-by: Javier Martinez Canillas Tested-by: Laszlo Ersek Link: https://patchwork.freedesktop.org/patch/msgid/20220622140134.12763-3-tzimmermann@suse.de --- Documentation/driver-api/aperture.rst | 13 ++ Documentation/driver-api/index.rst | 1 + MAINTAINERS | 2 + drivers/gpu/drm/drm_aperture.c | 178 +---------------- drivers/gpu/drm/tiny/Kconfig | 1 + drivers/video/Kconfig | 6 + drivers/video/Makefile | 2 + drivers/video/aperture.c | 351 ++++++++++++++++++++++++++++++++++ drivers/video/console/Kconfig | 1 + drivers/video/fbdev/Kconfig | 7 +- include/linux/aperture.h | 56 ++++++ 11 files changed, 448 insertions(+), 170 deletions(-) create mode 100644 Documentation/driver-api/aperture.rst create mode 100644 drivers/video/aperture.c create mode 100644 include/linux/aperture.h (limited to 'include/linux') diff --git a/Documentation/driver-api/aperture.rst b/Documentation/driver-api/aperture.rst new file mode 100644 index 000000000000..d173f4e7a7d9 --- /dev/null +++ b/Documentation/driver-api/aperture.rst @@ -0,0 +1,13 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Managing Ownership of the Framebuffer Aperture +============================================== + +.. kernel-doc:: drivers/video/aperture.c + :doc: overview + +.. kernel-doc:: include/linux/aperture.h + :internal: + +.. kernel-doc:: drivers/video/aperture.c + :export: diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index a6d525cd9fc4..d3a58f77328e 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -27,6 +27,7 @@ available subsections can be seen below. component message-based infiniband + aperture frame-buffer regulator reset diff --git a/MAINTAINERS b/MAINTAINERS index 7428a59437b3..6a66d91e02a2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6481,7 +6481,9 @@ S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: drivers/gpu/drm/drm_aperture.c F: drivers/gpu/drm/tiny/simpledrm.c +F: drivers/video/aperture.c F: include/drm/drm_aperture.h +F: include/linux/aperture.h DRM DRIVER FOR SIS VIDEO CARDS S: Orphan / Obsolete diff --git a/drivers/gpu/drm/drm_aperture.c b/drivers/gpu/drm/drm_aperture.c index 059fd71424f6..fdb7d5c17ba1 100644 --- a/drivers/gpu/drm/drm_aperture.c +++ b/drivers/gpu/drm/drm_aperture.c @@ -1,14 +1,7 @@ // SPDX-License-Identifier: MIT -#include -#include -#include -#include -#include -#include /* for firmware helpers */ -#include -#include -#include +#include +#include #include #include @@ -126,92 +119,6 @@ * afterwards. */ -struct drm_aperture { - struct drm_device *dev; - resource_size_t base; - resource_size_t size; - struct list_head lh; - void (*detach)(struct drm_device *dev); -}; - -static LIST_HEAD(drm_apertures); -static DEFINE_MUTEX(drm_apertures_lock); - -static bool overlap(resource_size_t base1, resource_size_t end1, - resource_size_t base2, resource_size_t end2) -{ - return (base1 < end2) && (end1 > base2); -} - -static void devm_aperture_acquire_release(void *data) -{ - struct drm_aperture *ap = data; - bool detached = !ap->dev; - - if (detached) - return; - - mutex_lock(&drm_apertures_lock); - list_del(&ap->lh); - mutex_unlock(&drm_apertures_lock); -} - -static int devm_aperture_acquire(struct drm_device *dev, - resource_size_t base, resource_size_t size, - void (*detach)(struct drm_device *)) -{ - size_t end = base + size; - struct list_head *pos; - struct drm_aperture *ap; - - mutex_lock(&drm_apertures_lock); - - list_for_each(pos, &drm_apertures) { - ap = container_of(pos, struct drm_aperture, lh); - if (overlap(base, end, ap->base, ap->base + ap->size)) { - mutex_unlock(&drm_apertures_lock); - return -EBUSY; - } - } - - ap = devm_kzalloc(dev->dev, sizeof(*ap), GFP_KERNEL); - if (!ap) { - mutex_unlock(&drm_apertures_lock); - return -ENOMEM; - } - - ap->dev = dev; - ap->base = base; - ap->size = size; - ap->detach = detach; - INIT_LIST_HEAD(&ap->lh); - - list_add(&ap->lh, &drm_apertures); - - mutex_unlock(&drm_apertures_lock); - - return devm_add_action_or_reset(dev->dev, devm_aperture_acquire_release, ap); -} - -static void drm_aperture_detach_firmware(struct drm_device *dev) -{ - struct platform_device *pdev = to_platform_device(dev->dev); - - /* - * Remove the device from the device hierarchy. This is the right thing - * to do for firmware-based DRM drivers, such as EFI, VESA or VGA. After - * the new driver takes over the hardware, the firmware device's state - * will be lost. - * - * For non-platform devices, a new callback would be required. - * - * If the aperture helpers ever need to handle native drivers, this call - * would only have to unplug the DRM device, so that the hardware device - * stays around after detachment. - */ - platform_device_unregister(pdev); -} - /** * devm_aperture_acquire_from_firmware - Acquires ownership of a firmware framebuffer * on behalf of a DRM driver. @@ -239,39 +146,16 @@ static void drm_aperture_detach_firmware(struct drm_device *dev) int devm_aperture_acquire_from_firmware(struct drm_device *dev, resource_size_t base, resource_size_t size) { + struct platform_device *pdev; + if (drm_WARN_ON(dev, !dev_is_platform(dev->dev))) return -EINVAL; - return devm_aperture_acquire(dev, base, size, drm_aperture_detach_firmware); -} -EXPORT_SYMBOL(devm_aperture_acquire_from_firmware); - -static void drm_aperture_detach_drivers(resource_size_t base, resource_size_t size) -{ - resource_size_t end = base + size; - struct list_head *pos, *n; - - mutex_lock(&drm_apertures_lock); - - list_for_each_safe(pos, n, &drm_apertures) { - struct drm_aperture *ap = - container_of(pos, struct drm_aperture, lh); - struct drm_device *dev = ap->dev; - - if (WARN_ON_ONCE(!dev)) - continue; - - if (!overlap(base, end, ap->base, ap->base + ap->size)) - continue; - - ap->dev = NULL; /* detach from device */ - list_del(&ap->lh); + pdev = to_platform_device(dev->dev); - ap->detach(dev); - } - - mutex_unlock(&drm_apertures_lock); + return devm_aperture_acquire_for_platform_device(pdev, base, size); } +EXPORT_SYMBOL(devm_aperture_acquire_from_firmware); /** * drm_aperture_remove_conflicting_framebuffers - remove existing framebuffers in the given range @@ -289,27 +173,7 @@ static void drm_aperture_detach_drivers(resource_size_t base, resource_size_t si int drm_aperture_remove_conflicting_framebuffers(resource_size_t base, resource_size_t size, bool primary, const struct drm_driver *req_driver) { -#if IS_REACHABLE(CONFIG_FB) - struct apertures_struct *a; - int ret; - - a = alloc_apertures(1); - if (!a) - return -ENOMEM; - - a->ranges[0].base = base; - a->ranges[0].size = size; - - ret = remove_conflicting_framebuffers(a, req_driver->name, primary); - kfree(a); - - if (ret) - return ret; -#endif - - drm_aperture_detach_drivers(base, size); - - return 0; + return aperture_remove_conflicting_devices(base, size, primary, req_driver->name); } EXPORT_SYMBOL(drm_aperture_remove_conflicting_framebuffers); @@ -328,30 +192,6 @@ EXPORT_SYMBOL(drm_aperture_remove_conflicting_framebuffers); int drm_aperture_remove_conflicting_pci_framebuffers(struct pci_dev *pdev, const struct drm_driver *req_driver) { - resource_size_t base, size; - int bar, ret; - - /* - * WARNING: Apparently we must kick fbdev drivers before vgacon, - * otherwise the vga fbdev driver falls over. - */ -#if IS_REACHABLE(CONFIG_FB) - ret = remove_conflicting_pci_framebuffers(pdev, req_driver->name); - if (ret) - return ret; -#endif - ret = vga_remove_vgacon(pdev); - if (ret) - return ret; - - for (bar = 0; bar < PCI_STD_NUM_BARS; ++bar) { - if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) - continue; - base = pci_resource_start(pdev, bar); - size = pci_resource_len(pdev, bar); - drm_aperture_detach_drivers(base, size); - } - - return 0; + return aperture_remove_conflicting_pci_devices(pdev, req_driver->name); } EXPORT_SYMBOL(drm_aperture_remove_conflicting_pci_framebuffers); diff --git a/drivers/gpu/drm/tiny/Kconfig b/drivers/gpu/drm/tiny/Kconfig index 627d637a1e7e..027cd87c3d0d 100644 --- a/drivers/gpu/drm/tiny/Kconfig +++ b/drivers/gpu/drm/tiny/Kconfig @@ -69,6 +69,7 @@ config DRM_PANEL_MIPI_DBI config DRM_SIMPLEDRM tristate "Simple framebuffer driver" depends on DRM && MMU + select APERTURE_HELPERS select DRM_GEM_SHMEM_HELPER select DRM_KMS_HELPER help diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 427a993c7f57..0587e21abad9 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -5,6 +5,12 @@ menu "Graphics support" +config APERTURE_HELPERS + bool + help + Support tracking and hand-over of aperture ownership. Required + by graphics drivers for firmware-provided framebuffers. + if HAS_IOMEM config HAVE_FB_ATMEL diff --git a/drivers/video/Makefile b/drivers/video/Makefile index df7650adede9..5bb6b452cc83 100644 --- a/drivers/video/Makefile +++ b/drivers/video/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_APERTURE_HELPERS) += aperture.o obj-$(CONFIG_VGASTATE) += vgastate.o obj-$(CONFIG_HDMI) += hdmi.o diff --git a/drivers/video/aperture.c b/drivers/video/aperture.c new file mode 100644 index 000000000000..538f2d40acda --- /dev/null +++ b/drivers/video/aperture.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: MIT + +#include +#include +#include /* for old fbdev helpers */ +#include +#include +#include +#include +#include +#include +#include + +/** + * DOC: overview + * + * A graphics device might be supported by different drivers, but only one + * driver can be active at any given time. Many systems load a generic + * graphics drivers, such as EFI-GOP or VESA, early during the boot process. + * During later boot stages, they replace the generic driver with a dedicated, + * hardware-specific driver. To take over the device the dedicated driver + * first has to remove the generic driver. Aperture functions manage + * ownership of framebuffer memory and hand-over between drivers. + * + * Graphics drivers should call aperture_remove_conflicting_devices() + * at the top of their probe function. The function removes any generic + * driver that is currently associated with the given framebuffer memory. + * An example for a graphics device on the platform bus is shown below. + * + * .. code-block:: c + * + * static int example_probe(struct platform_device *pdev) + * { + * struct resource *mem; + * resource_size_t base, size; + * int ret; + * + * mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + * if (!mem) + * return -ENODEV; + * base = mem->start; + * size = resource_size(mem); + * + * ret = aperture_remove_conflicting_devices(base, size, false, "example"); + * if (ret) + * return ret; + * + * // Initialize the hardware + * ... + * + * return 0; + * } + * + * static const struct platform_driver example_driver = { + * .probe = example_probe, + * ... + * }; + * + * The given example reads the platform device's I/O-memory range from the + * device instance. An active framebuffer will be located within this range. + * The call to aperture_remove_conflicting_devices() releases drivers that + * have previously claimed ownership of the range and are currently driving + * output on the framebuffer. If successful, the new driver can take over + * the device. + * + * While the given example uses a platform device, the aperture helpers work + * with every bus that has an addressable framebuffer. In the case of PCI, + * device drivers can also call aperture_remove_conflicting_pci_devices() and + * let the function detect the apertures automatically. Device drivers without + * knowledge of the framebuffer's location can call + * aperture_remove_all_conflicting_devices(), which removes all known devices. + * + * Drivers that are susceptible to being removed by other drivers, such as + * generic EFI or VESA drivers, have to register themselves as owners of their + * framebuffer apertures. Ownership of the framebuffer memory is achieved + * by calling devm_aperture_acquire_for_platform_device(). If successful, the + * driveris the owner of the framebuffer range. The function fails if the + * framebuffer is already owned by another driver. See below for an example. + * + * .. code-block:: c + * + * static int generic_probe(struct platform_device *pdev) + * { + * struct resource *mem; + * resource_size_t base, size; + * + * mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + * if (!mem) + * return -ENODEV; + * base = mem->start; + * size = resource_size(mem); + * + * ret = devm_aperture_acquire_for_platform_device(pdev, base, size); + * if (ret) + * return ret; + * + * // Initialize the hardware + * ... + * + * return 0; + * } + * + * static int generic_remove(struct platform_device *) + * { + * // Hot-unplug the device + * ... + * + * return 0; + * } + * + * static const struct platform_driver generic_driver = { + * .probe = generic_probe, + * .remove = generic_remove, + * ... + * }; + * + * The similar to the previous example, the generic driver claims ownership + * of the framebuffer memory from its probe function. This will fail if the + * memory range, or parts of it, is already owned by another driver. + * + * If successful, the generic driver is now subject to forced removal by + * another driver. This only works for platform drivers that support hot + * unplugging. When a driver calls aperture_remove_conflicting_devices() + * et al for the registered framebuffer range, the aperture helpers call + * platform_device_unregister() and the generic driver unloads itself. The + * generic driver also has to provide a remove function to make this work. + * Once hot unplugged fro mhardware, it may not access the device's + * registers, framebuffer memory, ROM, etc afterwards. + */ + +struct aperture_range { + struct device *dev; + resource_size_t base; + resource_size_t size; + struct list_head lh; + void (*detach)(struct device *dev); +}; + +static LIST_HEAD(apertures); +static DEFINE_MUTEX(apertures_lock); + +static bool overlap(resource_size_t base1, resource_size_t end1, + resource_size_t base2, resource_size_t end2) +{ + return (base1 < end2) && (end1 > base2); +} + +static void devm_aperture_acquire_release(void *data) +{ + struct aperture_range *ap = data; + bool detached = !ap->dev; + + if (detached) + return; + + mutex_lock(&apertures_lock); + list_del(&ap->lh); + mutex_unlock(&apertures_lock); +} + +static int devm_aperture_acquire(struct device *dev, + resource_size_t base, resource_size_t size, + void (*detach)(struct device *)) +{ + size_t end = base + size; + struct list_head *pos; + struct aperture_range *ap; + + mutex_lock(&apertures_lock); + + list_for_each(pos, &apertures) { + ap = container_of(pos, struct aperture_range, lh); + if (overlap(base, end, ap->base, ap->base + ap->size)) { + mutex_unlock(&apertures_lock); + return -EBUSY; + } + } + + ap = devm_kzalloc(dev, sizeof(*ap), GFP_KERNEL); + if (!ap) { + mutex_unlock(&apertures_lock); + return -ENOMEM; + } + + ap->dev = dev; + ap->base = base; + ap->size = size; + ap->detach = detach; + INIT_LIST_HEAD(&ap->lh); + + list_add(&ap->lh, &apertures); + + mutex_unlock(&apertures_lock); + + return devm_add_action_or_reset(dev, devm_aperture_acquire_release, ap); +} + +static void aperture_detach_platform_device(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + + /* + * Remove the device from the device hierarchy. This is the right thing + * to do for firmware-based DRM drivers, such as EFI, VESA or VGA. After + * the new driver takes over the hardware, the firmware device's state + * will be lost. + * + * For non-platform devices, a new callback would be required. + * + * If the aperture helpers ever need to handle native drivers, this call + * would only have to unplug the DRM device, so that the hardware device + * stays around after detachment. + */ + platform_device_unregister(pdev); +} + +/** + * devm_aperture_acquire_for_platform_device - Acquires ownership of an aperture + * on behalf of a platform device. + * @pdev: the platform device to own the aperture + * @base: the aperture's byte offset in physical memory + * @size: the aperture size in bytes + * + * Installs the given device as the new owner of the aperture. The function + * expects the aperture to be provided by a platform device. If another + * driver takes over ownership of the aperture, aperture helpers will then + * unregister the platform device automatically. All acquired apertures are + * released automatically when the underlying device goes away. + * + * The function fails if the aperture, or parts of it, is currently + * owned by another device. To evict current owners, callers should use + * remove_conflicting_devices() et al. before calling this function. + * + * Returns: + * 0 on success, or a negative errno value otherwise. + */ +int devm_aperture_acquire_for_platform_device(struct platform_device *pdev, + resource_size_t base, + resource_size_t size) +{ + return devm_aperture_acquire(&pdev->dev, base, size, aperture_detach_platform_device); +} +EXPORT_SYMBOL(devm_aperture_acquire_for_platform_device); + +static void aperture_detach_devices(resource_size_t base, resource_size_t size) +{ + resource_size_t end = base + size; + struct list_head *pos, *n; + + mutex_lock(&apertures_lock); + + list_for_each_safe(pos, n, &apertures) { + struct aperture_range *ap = container_of(pos, struct aperture_range, lh); + struct device *dev = ap->dev; + + if (WARN_ON_ONCE(!dev)) + continue; + + if (!overlap(base, end, ap->base, ap->base + ap->size)) + continue; + + ap->dev = NULL; /* detach from device */ + list_del(&ap->lh); + + ap->detach(dev); + } + + mutex_unlock(&apertures_lock); +} + +/** + * aperture_remove_conflicting_devices - remove devices in the given range + * @base: the aperture's base address in physical memory + * @size: aperture size in bytes + * @primary: also kick vga16fb if present; only relevant for VGA devices + * @name: a descriptive name of the requesting driver + * + * This function removes devices that own apertures within @base and @size. + * + * Returns: + * 0 on success, or a negative errno code otherwise + */ +int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size, + bool primary, const char *name) +{ +#if IS_REACHABLE(CONFIG_FB) + struct apertures_struct *a; + int ret; + + a = alloc_apertures(1); + if (!a) + return -ENOMEM; + + a->ranges[0].base = base; + a->ranges[0].size = size; + + ret = remove_conflicting_framebuffers(a, name, primary); + kfree(a); + + if (ret) + return ret; +#endif + + aperture_detach_devices(base, size); + + return 0; +} +EXPORT_SYMBOL(aperture_remove_conflicting_devices); + +/** + * aperture_remove_conflicting_pci_devices - remove existing framebuffers for PCI devices + * @pdev: PCI device + * @name: a descriptive name of the requesting driver + * + * This function removes devices that own apertures within any of @pdev's + * memory bars. The function assumes that PCI device with shadowed ROM + * drives a primary display and therefore kicks out vga16fb as well. + * + * Returns: + * 0 on success, or a negative errno code otherwise + */ +int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name) +{ + resource_size_t base, size; + int bar, ret; + + /* + * WARNING: Apparently we must kick fbdev drivers before vgacon, + * otherwise the vga fbdev driver falls over. + */ +#if IS_REACHABLE(CONFIG_FB) + ret = remove_conflicting_pci_framebuffers(pdev, name); + if (ret) + return ret; +#endif + ret = vga_remove_vgacon(pdev); + if (ret) + return ret; + + for (bar = 0; bar < PCI_STD_NUM_BARS; ++bar) { + if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) + continue; + base = pci_resource_start(pdev, bar); + size = pci_resource_len(pdev, bar); + aperture_detach_devices(base, size); + } + + return 0; + +} +EXPORT_SYMBOL(aperture_remove_conflicting_pci_devices); diff --git a/drivers/video/console/Kconfig b/drivers/video/console/Kconfig index 40c50fa2dd70..22cea5082ac4 100644 --- a/drivers/video/console/Kconfig +++ b/drivers/video/console/Kconfig @@ -10,6 +10,7 @@ config VGA_CONSOLE depends on !4xx && !PPC_8xx && !SPARC && !M68K && !PARISC && !SUPERH && \ (!ARM || ARCH_FOOTBRIDGE || ARCH_INTEGRATOR || ARCH_NETWINDER) && \ !ARM64 && !ARC && !MICROBLAZE && !OPENRISC && !S390 && !UML + select APERTURE_HELPERS if (DRM || FB || VFIO_PCI_CORE) default y help Saying Y here will allow you to use Linux in text mode through a diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig index f2a6b81e45c4..cfc55273dc5d 100644 --- a/drivers/video/fbdev/Kconfig +++ b/drivers/video/fbdev/Kconfig @@ -455,6 +455,7 @@ config FB_ATARI config FB_OF bool "Open Firmware frame buffer device support" depends on (FB = y) && PPC && (!PPC_PSERIES || PCI) + select APERTURE_HELPERS select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT @@ -527,6 +528,7 @@ config FB_IMSTT config FB_VGA16 tristate "VGA 16-color graphics support" depends on FB && (X86 || PPC) + select APERTURE_HELPERS select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT @@ -551,7 +553,7 @@ config FB_STI BIOS routines contained in a ROM chip in HP PA-RISC based machines. Enabling this option will implement the linux framebuffer device using calls to the STI BIOS routines for initialisation. - + If you enable this option, you will get a planar framebuffer device /dev/fb which will work on the most common HP graphic cards of the NGLE family, including the artist chips (in the 7xx and Bxxx series), @@ -617,6 +619,7 @@ config FB_UVESA config FB_VESA bool "VESA VGA graphics support" depends on (FB = y) && X86 + select APERTURE_HELPERS select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT @@ -630,6 +633,7 @@ config FB_VESA config FB_EFI bool "EFI-based Framebuffer Support" depends on (FB = y) && !IA64 && EFI + select APERTURE_HELPERS select DRM_PANEL_ORIENTATION_QUIRKS select FB_CFB_FILLRECT select FB_CFB_COPYAREA @@ -2190,6 +2194,7 @@ config FB_SIMPLE tristate "Simple framebuffer support" depends on FB depends on !DRM_SIMPLEDRM + select APERTURE_HELPERS select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT diff --git a/include/linux/aperture.h b/include/linux/aperture.h new file mode 100644 index 000000000000..442f15a57cad --- /dev/null +++ b/include/linux/aperture.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT */ + +#ifndef _LINUX_APERTURE_H_ +#define _LINUX_APERTURE_H_ + +#include + +struct pci_dev; +struct platform_device; + +#if defined(CONFIG_APERTURE_HELPERS) +int devm_aperture_acquire_for_platform_device(struct platform_device *pdev, + resource_size_t base, + resource_size_t size); + +int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size, + bool primary, const char *name); + +int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name); +#else +static inline int devm_aperture_acquire_for_platform_device(struct platform_device *pdev, + resource_size_t base, + resource_size_t size) +{ + return 0; +} + +static inline int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size, + bool primary, const char *name) +{ + return 0; +} + +static inline int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name) +{ + return 0; +} +#endif + +/** + * aperture_remove_all_conflicting_devices - remove all existing framebuffers + * @primary: also kick vga16fb if present; only relevant for VGA devices + * @name: a descriptive name of the requesting driver + * + * This function removes all graphics device drivers. Use this function on systems + * that can have their framebuffer located anywhere in memory. + * + * Returns: + * 0 on success, or a negative errno code otherwise + */ +static inline int aperture_remove_all_conflicting_devices(bool primary, const char *name) +{ + return aperture_remove_conflicting_devices(0, (resource_size_t)-1, primary, name); +} + +#endif -- cgit v1.2.3 From 7dc54d3b8d9100c65b0860f76342fc9f3b4694d9 Mon Sep 17 00:00:00 2001 From: Clément Léger Date: Fri, 24 Jun 2022 16:39:50 +0200 Subject: net: pcs: add Renesas MII converter driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a PCS driver for the MII converter that is present on the Renesas RZ/N1 SoC. This MII converter is reponsible for converting MII to RMII/RGMII or act as a MII pass-trough. Exposing it as a PCS allows to reuse it in both the switch driver and the stmmac driver. Currently, this driver only allows the PCS to be used by the dual Cortex-A7 subsystem since the register locking system is not used. Signed-off-by: Clément Léger Reviewed-by: Vladimir Oltean Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/pcs/Kconfig | 8 + drivers/net/pcs/Makefile | 1 + drivers/net/pcs/pcs-rzn1-miic.c | 520 ++++++++++++++++++++++++++++++++++++++++ include/linux/pcs-rzn1-miic.h | 18 ++ 4 files changed, 547 insertions(+) create mode 100644 drivers/net/pcs/pcs-rzn1-miic.c create mode 100644 include/linux/pcs-rzn1-miic.h (limited to 'include/linux') diff --git a/drivers/net/pcs/Kconfig b/drivers/net/pcs/Kconfig index f778e5155fae..6289b7c765f1 100644 --- a/drivers/net/pcs/Kconfig +++ b/drivers/net/pcs/Kconfig @@ -18,4 +18,12 @@ config PCS_LYNX This module provides helpers to phylink for managing the Lynx PCS which is part of the Layerscape and QorIQ Ethernet SERDES. +config PCS_RZN1_MIIC + tristate "Renesas RZ/N1 MII converter" + depends on OF && (ARCH_RZN1 || COMPILE_TEST) + help + This module provides a driver for the MII converter that is available + on RZ/N1 SoCs. This PCS converts MII to RMII/RGMII or can be set in + pass-through mode for MII. + endmenu diff --git a/drivers/net/pcs/Makefile b/drivers/net/pcs/Makefile index 0603d469bd57..0ff5388fcdea 100644 --- a/drivers/net/pcs/Makefile +++ b/drivers/net/pcs/Makefile @@ -5,3 +5,4 @@ pcs_xpcs-$(CONFIG_PCS_XPCS) := pcs-xpcs.o pcs-xpcs-nxp.o obj-$(CONFIG_PCS_XPCS) += pcs_xpcs.o obj-$(CONFIG_PCS_LYNX) += pcs-lynx.o +obj-$(CONFIG_PCS_RZN1_MIIC) += pcs-rzn1-miic.o diff --git a/drivers/net/pcs/pcs-rzn1-miic.c b/drivers/net/pcs/pcs-rzn1-miic.c new file mode 100644 index 000000000000..8f5e910f443d --- /dev/null +++ b/drivers/net/pcs/pcs-rzn1-miic.c @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Schneider Electric + * + * Clément Léger + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MIIC_PRCMD 0x0 +#define MIIC_ESID_CODE 0x4 + +#define MIIC_MODCTRL 0x20 +#define MIIC_MODCTRL_SW_MODE GENMASK(4, 0) + +#define MIIC_CONVCTRL(port) (0x100 + (port) * 4) + +#define MIIC_CONVCTRL_CONV_SPEED GENMASK(1, 0) +#define CONV_MODE_10MBPS 0 +#define CONV_MODE_100MBPS 1 +#define CONV_MODE_1000MBPS 2 + +#define MIIC_CONVCTRL_CONV_MODE GENMASK(3, 2) +#define CONV_MODE_MII 0 +#define CONV_MODE_RMII 1 +#define CONV_MODE_RGMII 2 + +#define MIIC_CONVCTRL_FULLD BIT(8) +#define MIIC_CONVCTRL_RGMII_LINK BIT(12) +#define MIIC_CONVCTRL_RGMII_DUPLEX BIT(13) +#define MIIC_CONVCTRL_RGMII_SPEED GENMASK(15, 14) + +#define MIIC_CONVRST 0x114 +#define MIIC_CONVRST_PHYIF_RST(port) BIT(port) +#define MIIC_CONVRST_PHYIF_RST_MASK GENMASK(4, 0) + +#define MIIC_SWCTRL 0x304 +#define MIIC_SWDUPC 0x308 + +#define MIIC_MAX_NR_PORTS 5 + +#define MIIC_MODCTRL_CONF_CONV_NUM 6 +#define MIIC_MODCTRL_CONF_NONE -1 + +/** + * struct modctrl_match - Matching table entry for convctrl configuration + * See section 8.2.1 of manual. + * @mode_cfg: Configuration value for convctrl + * @conv: Configuration of ethernet port muxes. First index is SWITCH_PORTIN, + * then index 1 - 5 are CONV1 - CONV5. + */ +struct modctrl_match { + u32 mode_cfg; + u8 conv[MIIC_MODCTRL_CONF_CONV_NUM]; +}; + +static struct modctrl_match modctrl_match_table[] = { + {0x0, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_SERCOS_PORTB, MIIC_SERCOS_PORTA}}, + {0x1, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_ETHERCAT_PORTB, MIIC_ETHERCAT_PORTA}}, + {0x2, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_ETHERCAT_PORTC, MIIC_ETHERCAT_PORTB, MIIC_ETHERCAT_PORTA}}, + {0x3, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_SWITCH_PORTB, MIIC_SWITCH_PORTA}}, + + {0x8, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_SERCOS_PORTB, MIIC_SERCOS_PORTA}}, + {0x9, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_ETHERCAT_PORTB, MIIC_ETHERCAT_PORTA}}, + {0xA, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_ETHERCAT_PORTC, MIIC_ETHERCAT_PORTB, MIIC_ETHERCAT_PORTA}}, + {0xB, {MIIC_RTOS_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_SWITCH_PORTB, MIIC_SWITCH_PORTA}}, + + {0x10, {MIIC_GMAC2_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_SERCOS_PORTB, MIIC_SERCOS_PORTA}}, + {0x11, {MIIC_GMAC2_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_ETHERCAT_PORTB, MIIC_ETHERCAT_PORTA}}, + {0x12, {MIIC_GMAC2_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_ETHERCAT_PORTC, MIIC_ETHERCAT_PORTB, MIIC_ETHERCAT_PORTA}}, + {0x13, {MIIC_GMAC2_PORT, MIIC_GMAC1_PORT, MIIC_SWITCH_PORTD, + MIIC_SWITCH_PORTC, MIIC_SWITCH_PORTB, MIIC_SWITCH_PORTA}} +}; + +static const char * const conf_to_string[] = { + [MIIC_GMAC1_PORT] = "GMAC1_PORT", + [MIIC_GMAC2_PORT] = "GMAC2_PORT", + [MIIC_RTOS_PORT] = "RTOS_PORT", + [MIIC_SERCOS_PORTA] = "SERCOS_PORTA", + [MIIC_SERCOS_PORTB] = "SERCOS_PORTB", + [MIIC_ETHERCAT_PORTA] = "ETHERCAT_PORTA", + [MIIC_ETHERCAT_PORTB] = "ETHERCAT_PORTB", + [MIIC_ETHERCAT_PORTC] = "ETHERCAT_PORTC", + [MIIC_SWITCH_PORTA] = "SWITCH_PORTA", + [MIIC_SWITCH_PORTB] = "SWITCH_PORTB", + [MIIC_SWITCH_PORTC] = "SWITCH_PORTC", + [MIIC_SWITCH_PORTD] = "SWITCH_PORTD", + [MIIC_HSR_PORTA] = "HSR_PORTA", + [MIIC_HSR_PORTB] = "HSR_PORTB", +}; + +static const char *index_to_string[MIIC_MODCTRL_CONF_CONV_NUM] = { + "SWITCH_PORTIN", + "CONV1", + "CONV2", + "CONV3", + "CONV4", + "CONV5", +}; + +/** + * struct miic - MII converter structure + * @base: base address of the MII converter + * @dev: Device associated to the MII converter + * @clks: Clocks used for this device + * @nclk: Number of clocks + * @lock: Lock used for read-modify-write access + */ +struct miic { + void __iomem *base; + struct device *dev; + struct clk_bulk_data *clks; + int nclk; + spinlock_t lock; +}; + +/** + * struct miic_port - Per port MII converter struct + * @miic: backiling to MII converter structure + * @pcs: PCS structure associated to the port + * @port: port number + */ +struct miic_port { + struct miic *miic; + struct phylink_pcs pcs; + int port; +}; + +static struct miic_port *phylink_pcs_to_miic_port(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct miic_port, pcs); +} + +static void miic_reg_writel(struct miic *miic, int offset, u32 value) +{ + writel(value, miic->base + offset); +} + +static u32 miic_reg_readl(struct miic *miic, int offset) +{ + return readl(miic->base + offset); +} + +static void miic_reg_rmw(struct miic *miic, int offset, u32 mask, u32 val) +{ + u32 reg; + + spin_lock(&miic->lock); + + reg = miic_reg_readl(miic, offset); + reg &= ~mask; + reg |= val; + miic_reg_writel(miic, offset, reg); + + spin_unlock(&miic->lock); +} + +static void miic_converter_enable(struct miic *miic, int port, int enable) +{ + u32 val = 0; + + if (enable) + val = MIIC_CONVRST_PHYIF_RST(port); + + miic_reg_rmw(miic, MIIC_CONVRST, MIIC_CONVRST_PHYIF_RST(port), val); +} + +static int miic_config(struct phylink_pcs *pcs, unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising, bool permit) +{ + struct miic_port *miic_port = phylink_pcs_to_miic_port(pcs); + struct miic *miic = miic_port->miic; + int port = miic_port->port; + u32 speed, conv_mode, val; + + switch (interface) { + case PHY_INTERFACE_MODE_RMII: + conv_mode = CONV_MODE_RMII; + speed = CONV_MODE_100MBPS; + break; + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + conv_mode = CONV_MODE_RGMII; + speed = CONV_MODE_1000MBPS; + break; + case PHY_INTERFACE_MODE_MII: + conv_mode = CONV_MODE_MII; + /* When in MII mode, speed should be set to 0 (which is actually + * CONV_MODE_10MBPS) + */ + speed = CONV_MODE_10MBPS; + break; + default: + return -EOPNOTSUPP; + } + + val = FIELD_PREP(MIIC_CONVCTRL_CONV_MODE, conv_mode) | + FIELD_PREP(MIIC_CONVCTRL_CONV_SPEED, speed); + + miic_reg_rmw(miic, MIIC_CONVCTRL(port), + MIIC_CONVCTRL_CONV_MODE | MIIC_CONVCTRL_CONV_SPEED, val); + miic_converter_enable(miic_port->miic, miic_port->port, 1); + + return 0; +} + +static void miic_link_up(struct phylink_pcs *pcs, unsigned int mode, + phy_interface_t interface, int speed, int duplex) +{ + struct miic_port *miic_port = phylink_pcs_to_miic_port(pcs); + struct miic *miic = miic_port->miic; + u32 conv_speed = 0, val = 0; + int port = miic_port->port; + + if (duplex == DUPLEX_FULL) + val |= MIIC_CONVCTRL_FULLD; + + /* No speed in MII through-mode */ + if (interface != PHY_INTERFACE_MODE_MII) { + switch (speed) { + case SPEED_1000: + conv_speed = CONV_MODE_1000MBPS; + break; + case SPEED_100: + conv_speed = CONV_MODE_100MBPS; + break; + case SPEED_10: + conv_speed = CONV_MODE_10MBPS; + break; + default: + return; + } + } + + val |= FIELD_PREP(MIIC_CONVCTRL_CONV_SPEED, conv_speed); + + miic_reg_rmw(miic, MIIC_CONVCTRL(port), + (MIIC_CONVCTRL_CONV_SPEED | MIIC_CONVCTRL_FULLD), val); +} + +static int miic_validate(struct phylink_pcs *pcs, unsigned long *supported, + const struct phylink_link_state *state) +{ + if (phy_interface_mode_is_rgmii(state->interface) || + state->interface == PHY_INTERFACE_MODE_RMII || + state->interface == PHY_INTERFACE_MODE_MII) + return 1; + + return -EINVAL; +} + +static const struct phylink_pcs_ops miic_phylink_ops = { + .pcs_validate = miic_validate, + .pcs_config = miic_config, + .pcs_link_up = miic_link_up, +}; + +struct phylink_pcs *miic_create(struct device *dev, struct device_node *np) +{ + struct platform_device *pdev; + struct miic_port *miic_port; + struct device_node *pcs_np; + struct miic *miic; + u32 port; + + if (!of_device_is_available(np)) + return ERR_PTR(-ENODEV); + + if (of_property_read_u32(np, "reg", &port)) + return ERR_PTR(-EINVAL); + + if (port > MIIC_MAX_NR_PORTS || port < 1) + return ERR_PTR(-EINVAL); + + /* The PCS pdev is attached to the parent node */ + pcs_np = of_get_parent(np); + if (!pcs_np) + return ERR_PTR(-ENODEV); + + if (!of_device_is_available(pcs_np)) { + of_node_put(pcs_np); + return ERR_PTR(-ENODEV); + } + + pdev = of_find_device_by_node(pcs_np); + of_node_put(pcs_np); + if (!pdev || !platform_get_drvdata(pdev)) + return ERR_PTR(-EPROBE_DEFER); + + miic_port = kzalloc(sizeof(*miic_port), GFP_KERNEL); + if (!miic_port) + return ERR_PTR(-ENOMEM); + + miic = platform_get_drvdata(pdev); + device_link_add(dev, miic->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + + miic_port->miic = miic; + miic_port->port = port - 1; + miic_port->pcs.ops = &miic_phylink_ops; + + return &miic_port->pcs; +} +EXPORT_SYMBOL(miic_create); + +void miic_destroy(struct phylink_pcs *pcs) +{ + struct miic_port *miic_port = phylink_pcs_to_miic_port(pcs); + + miic_converter_enable(miic_port->miic, miic_port->port, 0); + kfree(miic_port); +} +EXPORT_SYMBOL(miic_destroy); + +static int miic_init_hw(struct miic *miic, u32 cfg_mode) +{ + int port; + + /* Unlock write access to accessory registers (cf datasheet). If this + * is going to be used in conjunction with the Cortex-M3, this sequence + * will have to be moved in register write + */ + miic_reg_writel(miic, MIIC_PRCMD, 0x00A5); + miic_reg_writel(miic, MIIC_PRCMD, 0x0001); + miic_reg_writel(miic, MIIC_PRCMD, 0xFFFE); + miic_reg_writel(miic, MIIC_PRCMD, 0x0001); + + miic_reg_writel(miic, MIIC_MODCTRL, + FIELD_PREP(MIIC_MODCTRL_SW_MODE, cfg_mode)); + + for (port = 0; port < MIIC_MAX_NR_PORTS; port++) { + miic_converter_enable(miic, port, 0); + /* Disable speed/duplex control from these registers, datasheet + * says switch registers should be used to setup switch port + * speed and duplex. + */ + miic_reg_writel(miic, MIIC_SWCTRL, 0x0); + miic_reg_writel(miic, MIIC_SWDUPC, 0x0); + } + + return 0; +} + +static bool miic_modctrl_match(s8 table_val[MIIC_MODCTRL_CONF_CONV_NUM], + s8 dt_val[MIIC_MODCTRL_CONF_CONV_NUM]) +{ + int i; + + for (i = 0; i < MIIC_MODCTRL_CONF_CONV_NUM; i++) { + if (dt_val[i] == MIIC_MODCTRL_CONF_NONE) + continue; + + if (dt_val[i] != table_val[i]) + return false; + } + + return true; +} + +static void miic_dump_conf(struct device *dev, + s8 conf[MIIC_MODCTRL_CONF_CONV_NUM]) +{ + const char *conf_name; + int i; + + for (i = 0; i < MIIC_MODCTRL_CONF_CONV_NUM; i++) { + if (conf[i] != MIIC_MODCTRL_CONF_NONE) + conf_name = conf_to_string[conf[i]]; + else + conf_name = "NONE"; + + dev_err(dev, "%s: %s\n", index_to_string[i], conf_name); + } +} + +static int miic_match_dt_conf(struct device *dev, + s8 dt_val[MIIC_MODCTRL_CONF_CONV_NUM], + u32 *mode_cfg) +{ + struct modctrl_match *table_entry; + int i; + + for (i = 0; i < ARRAY_SIZE(modctrl_match_table); i++) { + table_entry = &modctrl_match_table[i]; + + if (miic_modctrl_match(table_entry->conv, dt_val)) { + *mode_cfg = table_entry->mode_cfg; + return 0; + } + } + + dev_err(dev, "Failed to apply requested configuration\n"); + miic_dump_conf(dev, dt_val); + + return -EINVAL; +} + +static int miic_parse_dt(struct device *dev, u32 *mode_cfg) +{ + s8 dt_val[MIIC_MODCTRL_CONF_CONV_NUM]; + struct device_node *np = dev->of_node; + struct device_node *conv; + u32 conf; + int port; + + memset(dt_val, MIIC_MODCTRL_CONF_NONE, sizeof(dt_val)); + + if (of_property_read_u32(np, "renesas,miic-switch-portin", &conf) == 0) + dt_val[0] = conf; + + for_each_child_of_node(np, conv) { + if (of_property_read_u32(conv, "reg", &port)) + continue; + + if (!of_device_is_available(conv)) + continue; + + if (of_property_read_u32(conv, "renesas,miic-input", &conf) == 0) + dt_val[port] = conf; + } + + return miic_match_dt_conf(dev, dt_val, mode_cfg); +} + +static int miic_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct miic *miic; + u32 mode_cfg; + int ret; + + ret = miic_parse_dt(dev, &mode_cfg); + if (ret < 0) + return ret; + + miic = devm_kzalloc(dev, sizeof(*miic), GFP_KERNEL); + if (!miic) + return -ENOMEM; + + spin_lock_init(&miic->lock); + miic->dev = dev; + miic->base = devm_platform_ioremap_resource(pdev, 0); + if (!miic->base) + return -EINVAL; + + ret = devm_pm_runtime_enable(dev); + if (ret < 0) + return ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; + + ret = miic_init_hw(miic, mode_cfg); + if (ret) + goto disable_runtime_pm; + + /* miic_create() relies on that fact that data are attached to the + * platform device to determine if the driver is ready so this needs to + * be the last thing to be done after everything is initialized + * properly. + */ + platform_set_drvdata(pdev, miic); + + return 0; + +disable_runtime_pm: + pm_runtime_put(dev); + + return ret; +} + +static int miic_remove(struct platform_device *pdev) +{ + pm_runtime_put(&pdev->dev); + + return 0; +} + +static const struct of_device_id miic_of_mtable[] = { + { .compatible = "renesas,rzn1-miic" }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, miic_of_mtable); + +static struct platform_driver miic_driver = { + .driver = { + .name = "rzn1_miic", + .suppress_bind_attrs = true, + .of_match_table = miic_of_mtable, + }, + .probe = miic_probe, + .remove = miic_remove, +}; +module_platform_driver(miic_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Renesas MII converter PCS driver"); +MODULE_AUTHOR("Clément Léger "); diff --git a/include/linux/pcs-rzn1-miic.h b/include/linux/pcs-rzn1-miic.h new file mode 100644 index 000000000000..56d12b21365d --- /dev/null +++ b/include/linux/pcs-rzn1-miic.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 Schneider Electric + * + * Clément Léger + */ + +#ifndef __LINUX_PCS_MIIC_H +#define __LINUX_PCS_MIIC_H + +struct phylink; +struct device_node; + +struct phylink_pcs *miic_create(struct device *dev, struct device_node *np); + +void miic_destroy(struct phylink_pcs *pcs); + +#endif /* __LINUX_PCS_MIIC_H */ -- cgit v1.2.3 From 8da443b1a4036b5863fd2ec7e0251164b704c726 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 14 Jun 2022 11:05:34 +0200 Subject: tty/vt: consolemap: rename struct vc_data::vc_uni_pagedir* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As a follow-up to the commit 4173f018aae1 (tty/vt: consolemap: rename and document struct uni_pagedir), rename also the members of struct vc_data. I.e. pagedir -> pagedict. And while touching all the places, remove also the unnecessary vc_ prefix. Suggested-by: Ilpo Järvinen Reviewed-by: Ilpo Järvinen Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20220614090537.15557-5-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/consolemap.c | 46 ++++++++++++++++----------------- drivers/tty/vt/vt.c | 8 +++--- drivers/usb/misc/sisusbvga/sisusb_con.c | 2 +- drivers/video/console/vgacon.c | 8 +++--- drivers/video/fbdev/core/fbcon.c | 8 +++--- include/linux/console_struct.h | 4 +-- 6 files changed, 38 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c index 3d0e10dac6d9..16d0d8f04f0e 100644 --- a/drivers/tty/vt/consolemap.c +++ b/drivers/tty/vt/consolemap.c @@ -296,7 +296,7 @@ u16 inverse_translate(const struct vc_data *conp, u16 glyph, bool use_unicode) if (glyph >= MAX_GLYPH) return 0; - p = *conp->vc_uni_pagedir_loc; + p = *conp->uni_pagedict_loc; if (!p) return glyph; @@ -323,7 +323,7 @@ static void update_user_maps(void) for (i = 0; i < MAX_NR_CONSOLES; i++) { if (!vc_cons_allocated(i)) continue; - p = *vc_cons[i].d->vc_uni_pagedir_loc; + p = *vc_cons[i].d->uni_pagedict_loc; if (p && p != q) { set_inverse_transl(vc_cons[i].d, p, USER_MAP); set_inverse_trans_unicode(p); @@ -445,10 +445,10 @@ void con_free_unimap(struct vc_data *vc) { struct uni_pagedict *p; - p = *vc->vc_uni_pagedir_loc; + p = *vc->uni_pagedict_loc; if (!p) return; - *vc->vc_uni_pagedir_loc = NULL; + *vc->uni_pagedict_loc = NULL; if (--p->refcount) return; con_release_unimap(p); @@ -463,7 +463,7 @@ static int con_unify_unimap(struct vc_data *conp, struct uni_pagedict *dict1) for (cons = 0; cons < MAX_NR_CONSOLES; cons++) { if (!vc_cons_allocated(cons)) continue; - dict2 = *vc_cons[cons].d->vc_uni_pagedir_loc; + dict2 = *vc_cons[cons].d->uni_pagedict_loc; if (!dict2 || dict2 == dict1 || dict2->sum != dict1->sum) continue; for (d = 0; d < UNI_DIRS; d++) { @@ -487,7 +487,7 @@ static int con_unify_unimap(struct vc_data *conp, struct uni_pagedict *dict1) } if (d == UNI_DIRS) { dict2->refcount++; - *conp->vc_uni_pagedir_loc = dict2; + *conp->uni_pagedict_loc = dict2; con_release_unimap(dict1); kfree(dict1); return 1; @@ -531,14 +531,14 @@ con_insert_unipair(struct uni_pagedict *p, u_short unicode, u_short fontpos) static int con_allocate_new(struct vc_data *vc) { - struct uni_pagedict *new, *old = *vc->vc_uni_pagedir_loc; + struct uni_pagedict *new, *old = *vc->uni_pagedict_loc; new = kzalloc(sizeof(*new), GFP_KERNEL); if (!new) return -ENOMEM; new->refcount = 1; - *vc->vc_uni_pagedir_loc = new; + *vc->uni_pagedict_loc = new; if (old) old->refcount--; @@ -549,7 +549,7 @@ static int con_allocate_new(struct vc_data *vc) /* Caller must hold the lock */ static int con_do_clear_unimap(struct vc_data *vc) { - struct uni_pagedict *old = *vc->vc_uni_pagedir_loc; + struct uni_pagedict *old = *vc->uni_pagedict_loc; if (!old || old->refcount > 1) return con_allocate_new(vc); @@ -583,7 +583,7 @@ static struct uni_pagedict *con_unshare_unimap(struct vc_data *vc, if (ret) return ERR_PTR(ret); - new = *vc->vc_uni_pagedir_loc; + new = *vc->uni_pagedict_loc; /* * uni_pgdir is a 32*32*64 table with rows allocated when its first @@ -616,7 +616,7 @@ static struct uni_pagedict *con_unshare_unimap(struct vc_data *vc, ret = con_insert_unipair(new, uni, row[g]); if (ret) { old->refcount++; - *vc->vc_uni_pagedir_loc = old; + *vc->uni_pagedict_loc = old; con_release_unimap(new); kfree(new); return ERR_PTR(ret); @@ -644,7 +644,7 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list) console_lock(); /* Save original vc_unipagdir_loc in case we allocate a new one */ - dict = *vc->vc_uni_pagedir_loc; + dict = *vc->uni_pagedict_loc; if (!dict) { err = -EINVAL; goto out_unlock; @@ -704,12 +704,12 @@ int con_set_default_unimap(struct vc_data *vc) u16 *dfont; if (dflt) { - dict = *vc->vc_uni_pagedir_loc; + dict = *vc->uni_pagedict_loc; if (dict == dflt) return 0; dflt->refcount++; - *vc->vc_uni_pagedir_loc = dflt; + *vc->uni_pagedict_loc = dflt; if (dict && !--dict->refcount) { con_release_unimap(dict); kfree(dict); @@ -723,7 +723,7 @@ int con_set_default_unimap(struct vc_data *vc) if (err) return err; - dict = *vc->vc_uni_pagedir_loc; + dict = *vc->uni_pagedict_loc; dfont = dfont_unitable; for (fontpos = 0; fontpos < 256U; fontpos++) @@ -734,7 +734,7 @@ int con_set_default_unimap(struct vc_data *vc) } if (con_unify_unimap(vc, dict)) { - dflt = *vc->vc_uni_pagedir_loc; + dflt = *vc->uni_pagedict_loc; return err; } @@ -757,14 +757,14 @@ int con_copy_unimap(struct vc_data *dst_vc, struct vc_data *src_vc) { struct uni_pagedict *src; - if (!*src_vc->vc_uni_pagedir_loc) + if (!*src_vc->uni_pagedict_loc) return -EINVAL; - if (*dst_vc->vc_uni_pagedir_loc == *src_vc->vc_uni_pagedir_loc) + if (*dst_vc->uni_pagedict_loc == *src_vc->uni_pagedict_loc) return 0; con_free_unimap(dst_vc); - src = *src_vc->vc_uni_pagedir_loc; + src = *src_vc->uni_pagedict_loc; src->refcount++; - *dst_vc->vc_uni_pagedir_loc = src; + *dst_vc->uni_pagedict_loc = src; return 0; } EXPORT_SYMBOL(con_copy_unimap); @@ -791,7 +791,7 @@ int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct, console_lock(); ect = 0; - dict = *vc->vc_uni_pagedir_loc; + dict = *vc->uni_pagedict_loc; if (!dict) goto unlock; @@ -873,7 +873,7 @@ int conv_uni_to_pc(struct vc_data *conp, long ucs) else if ((ucs & ~UNI_DIRECT_MASK) == UNI_DIRECT_BASE) return ucs & UNI_DIRECT_MASK; - dict = *conp->vc_uni_pagedir_loc; + dict = *conp->uni_pagedict_loc; if (!dict) return -3; @@ -903,7 +903,7 @@ console_map_init(void) int i; for (i = 0; i < MAX_NR_CONSOLES; i++) - if (vc_cons_allocated(i) && !*vc_cons[i].d->vc_uni_pagedir_loc) + if (vc_cons_allocated(i) && !*vc_cons[i].d->uni_pagedict_loc) con_set_default_unimap(vc_cons[i].d); } diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index c718b0d01e3d..1899b8a5d73e 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1063,10 +1063,10 @@ static void visual_init(struct vc_data *vc, int num, int init) __module_get(vc->vc_sw->owner); vc->vc_num = num; vc->vc_display_fg = &master_display_fg; - if (vc->vc_uni_pagedir_loc) + if (vc->uni_pagedict_loc) con_free_unimap(vc); - vc->vc_uni_pagedir_loc = &vc->vc_uni_pagedir; - vc->vc_uni_pagedir = NULL; + vc->uni_pagedict_loc = &vc->uni_pagedict; + vc->uni_pagedict = NULL; vc->vc_hi_font_mask = 0; vc->vc_complement_mask = 0; vc->vc_can_do_color = 0; @@ -1136,7 +1136,7 @@ int vc_allocate(unsigned int currcons) /* return 0 on success */ visual_init(vc, currcons, 1); - if (!*vc->vc_uni_pagedir_loc) + if (!*vc->uni_pagedict_loc) con_set_default_unimap(vc); err = -EINVAL; diff --git a/drivers/usb/misc/sisusbvga/sisusb_con.c b/drivers/usb/misc/sisusbvga/sisusb_con.c index dfa0d5ce6012..fcb95fb639e0 100644 --- a/drivers/usb/misc/sisusbvga/sisusb_con.c +++ b/drivers/usb/misc/sisusbvga/sisusb_con.c @@ -248,7 +248,7 @@ sisusbcon_init(struct vc_data *c, int init) */ kref_get(&sisusb->kref); - if (!*c->vc_uni_pagedir_loc) + if (!*c->uni_pagedict_loc) con_set_default_unimap(c); mutex_unlock(&sisusb->lock); diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 058a78b8dbcf..fcdf017e2665 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -367,10 +367,10 @@ static void vgacon_init(struct vc_data *c, int init) c->vc_complement_mask = 0x7700; if (vga_512_chars) c->vc_hi_font_mask = 0x0800; - p = *c->vc_uni_pagedir_loc; - if (c->vc_uni_pagedir_loc != &vgacon_uni_pagedir) { + p = *c->uni_pagedict_loc; + if (c->uni_pagedict_loc != &vgacon_uni_pagedir) { con_free_unimap(c); - c->vc_uni_pagedir_loc = &vgacon_uni_pagedir; + c->uni_pagedict_loc = &vgacon_uni_pagedir; vgacon_refcount++; } if (!vgacon_uni_pagedir && p) @@ -392,7 +392,7 @@ static void vgacon_deinit(struct vc_data *c) if (!--vgacon_refcount) con_free_unimap(c); - c->vc_uni_pagedir_loc = &c->vc_uni_pagedir; + c->uni_pagedict_loc = &c->uni_pagedict; con_set_default_unimap(c); } diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index c4e91715ef00..b10236fe5886 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1058,9 +1058,9 @@ static void fbcon_init(struct vc_data *vc, int init) vc->vc_complement_mask <<= 1; } - if (!*svc->vc_uni_pagedir_loc) + if (!*svc->uni_pagedict_loc) con_set_default_unimap(svc); - if (!*vc->vc_uni_pagedir_loc) + if (!*vc->uni_pagedict_loc) con_copy_unimap(vc, svc); ops = info->fbcon_par; @@ -1382,9 +1382,9 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var, vc->vc_complement_mask <<= 1; } - if (!*svc->vc_uni_pagedir_loc) + if (!*svc->uni_pagedict_loc) con_set_default_unimap(svc); - if (!*vc->vc_uni_pagedir_loc) + if (!*vc->uni_pagedict_loc) con_copy_unimap(vc, svc); cols = FBCON_SWAP(ops->rotate, info->var.xres, info->var.yres); diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h index f75033f0277f..1518568aaf0f 100644 --- a/include/linux/console_struct.h +++ b/include/linux/console_struct.h @@ -157,8 +157,8 @@ struct vc_data { unsigned int vc_bell_duration; /* Console bell duration */ unsigned short vc_cur_blink_ms; /* Cursor blink duration */ struct vc_data **vc_display_fg; /* [!] Ptr to var holding fg console for this display */ - struct uni_pagedict *vc_uni_pagedir; - struct uni_pagedict **vc_uni_pagedir_loc; /* [!] Location of uni_pagedict variable for this console */ + struct uni_pagedict *uni_pagedict; + struct uni_pagedict **uni_pagedict_loc; /* [!] Location of uni_pagedict variable for this console */ struct uni_screen *vc_uni_screen; /* unicode screen content */ /* additional information is in vt_kern.h */ }; -- cgit v1.2.3 From 1714582a3a087eda8786d5a1b32b2ec86ca8a303 Mon Sep 17 00:00:00 2001 From: David Jander Date: Tue, 21 Jun 2022 08:12:24 +0200 Subject: spi: Move ctlr->cur_msg_prepared to struct spi_message This enables the possibility to transfer a message that is not at the current tip of the async message queue. This is in preparation of the next patch(es) which enable spi_sync messages to skip the queue altogether. Signed-off-by: David Jander Link: https://lore.kernel.org/r/20220621061234.3626638-2-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 7 ++++--- include/linux/spi/spi.h | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index c78d1ceeaa42..eb6360153fa1 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1684,7 +1684,7 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) spi_finalize_current_message(ctlr); goto out; } - ctlr->cur_msg_prepared = true; + msg->prepared = true; } ret = spi_map_msg(ctlr, msg); @@ -1926,7 +1926,7 @@ void spi_finalize_current_message(struct spi_controller *ctlr) */ spi_res_release(ctlr, mesg); - if (ctlr->cur_msg_prepared && ctlr->unprepare_message) { + if (mesg->prepared && ctlr->unprepare_message) { ret = ctlr->unprepare_message(ctlr, mesg); if (ret) { dev_err(&ctlr->dev, "failed to unprepare message: %d\n", @@ -1934,9 +1934,10 @@ void spi_finalize_current_message(struct spi_controller *ctlr) } } + mesg->prepared = false; + spin_lock_irqsave(&ctlr->queue_lock, flags); ctlr->cur_msg = NULL; - ctlr->cur_msg_prepared = false; ctlr->fallback = false; kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); spin_unlock_irqrestore(&ctlr->queue_lock, flags); diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index c96f526d9a20..1a75c26742f2 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -385,8 +385,6 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @queue: message queue * @idling: the device is entering idle state * @cur_msg: the currently in-flight message - * @cur_msg_prepared: spi_prepare_message was called for the currently - * in-flight message * @cur_msg_mapped: message has been mapped for DMA * @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip * selected @@ -621,7 +619,6 @@ struct spi_controller { bool running; bool rt; bool auto_runtime_pm; - bool cur_msg_prepared; bool cur_msg_mapped; char last_cs; bool last_cs_mode_high; @@ -988,6 +985,7 @@ struct spi_transfer { * @queue: for use by whichever driver currently owns the message * @state: for use by whichever driver currently owns the message * @resources: for resource management when the spi message is processed + * @prepared: spi_prepare_message was called for the this message * * A @spi_message is used to execute an atomic sequence of data transfers, * each represented by a struct spi_transfer. The sequence is "atomic" @@ -1037,6 +1035,9 @@ struct spi_message { /* list of spi_res reources when the spi message is processed */ struct list_head resources; + + /* spi_prepare_message was called for this message */ + bool prepared; }; static inline void spi_message_init_no_memset(struct spi_message *m) -- cgit v1.2.3 From ae7d2346dc89ae89a6e0aabe6037591a11e593c0 Mon Sep 17 00:00:00 2001 From: David Jander Date: Tue, 21 Jun 2022 08:12:25 +0200 Subject: spi: Don't use the message queue if possible in spi_sync The interaction with the controller message queue and its corresponding auxiliary flags and variables requires the use of the queue_lock which is costly. Since spi_sync will transfer the complete message anyway, and not return until it is finished, there is no need to put the message into the queue if the queue is empty. This can save a lot of overhead. As an example of how significant this is, when using the MCP2518FD SPI CAN controller on a i.MX8MM SoC, the time during which the interrupt line stays active (during 3 relatively short spi_sync messages), is reduced from 98us to 72us by this patch. Signed-off-by: David Jander Link: https://lore.kernel.org/r/20220621061234.3626638-3-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 246 +++++++++++++++++++++++++++++------------------- include/linux/spi/spi.h | 11 ++- 2 files changed, 159 insertions(+), 98 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index eb6360153fa1..2d057d03c4f7 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1549,6 +1549,80 @@ static void spi_idle_runtime_pm(struct spi_controller *ctlr) } } +static int __spi_pump_transfer_message(struct spi_controller *ctlr, + struct spi_message *msg, bool was_busy) +{ + struct spi_transfer *xfer; + int ret; + + if (!was_busy && ctlr->auto_runtime_pm) { + ret = pm_runtime_get_sync(ctlr->dev.parent); + if (ret < 0) { + pm_runtime_put_noidle(ctlr->dev.parent); + dev_err(&ctlr->dev, "Failed to power device: %d\n", + ret); + return ret; + } + } + + if (!was_busy) + trace_spi_controller_busy(ctlr); + + if (!was_busy && ctlr->prepare_transfer_hardware) { + ret = ctlr->prepare_transfer_hardware(ctlr); + if (ret) { + dev_err(&ctlr->dev, + "failed to prepare transfer hardware: %d\n", + ret); + + if (ctlr->auto_runtime_pm) + pm_runtime_put(ctlr->dev.parent); + + msg->status = ret; + spi_finalize_current_message(ctlr); + + return ret; + } + } + + trace_spi_message_start(msg); + + if (ctlr->prepare_message) { + ret = ctlr->prepare_message(ctlr, msg); + if (ret) { + dev_err(&ctlr->dev, "failed to prepare message: %d\n", + ret); + msg->status = ret; + spi_finalize_current_message(ctlr); + return ret; + } + msg->prepared = true; + } + + ret = spi_map_msg(ctlr, msg); + if (ret) { + msg->status = ret; + spi_finalize_current_message(ctlr); + return ret; + } + + if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) { + list_for_each_entry(xfer, &msg->transfers, transfer_list) { + xfer->ptp_sts_word_pre = 0; + ptp_read_system_prets(xfer->ptp_sts); + } + } + + ret = ctlr->transfer_one_message(ctlr, msg); + if (ret) { + dev_err(&ctlr->dev, + "failed to transfer one message from queue\n"); + return ret; + } + + return 0; +} + /** * __spi_pump_messages - function which processes spi message queue * @ctlr: controller to process queue for @@ -1564,7 +1638,6 @@ static void spi_idle_runtime_pm(struct spi_controller *ctlr) */ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) { - struct spi_transfer *xfer; struct spi_message *msg; bool was_busy = false; unsigned long flags; @@ -1599,6 +1672,7 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) !ctlr->unprepare_transfer_hardware) { spi_idle_runtime_pm(ctlr); ctlr->busy = false; + ctlr->queue_empty = true; trace_spi_controller_idle(ctlr); } else { kthread_queue_work(ctlr->kworker, @@ -1625,6 +1699,7 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) spin_lock_irqsave(&ctlr->queue_lock, flags); ctlr->idling = false; + ctlr->queue_empty = true; spin_unlock_irqrestore(&ctlr->queue_lock, flags); return; } @@ -1641,75 +1716,7 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) spin_unlock_irqrestore(&ctlr->queue_lock, flags); mutex_lock(&ctlr->io_mutex); - - if (!was_busy && ctlr->auto_runtime_pm) { - ret = pm_runtime_resume_and_get(ctlr->dev.parent); - if (ret < 0) { - dev_err(&ctlr->dev, "Failed to power device: %d\n", - ret); - mutex_unlock(&ctlr->io_mutex); - return; - } - } - - if (!was_busy) - trace_spi_controller_busy(ctlr); - - if (!was_busy && ctlr->prepare_transfer_hardware) { - ret = ctlr->prepare_transfer_hardware(ctlr); - if (ret) { - dev_err(&ctlr->dev, - "failed to prepare transfer hardware: %d\n", - ret); - - if (ctlr->auto_runtime_pm) - pm_runtime_put(ctlr->dev.parent); - - msg->status = ret; - spi_finalize_current_message(ctlr); - - mutex_unlock(&ctlr->io_mutex); - return; - } - } - - trace_spi_message_start(msg); - - if (ctlr->prepare_message) { - ret = ctlr->prepare_message(ctlr, msg); - if (ret) { - dev_err(&ctlr->dev, "failed to prepare message: %d\n", - ret); - msg->status = ret; - spi_finalize_current_message(ctlr); - goto out; - } - msg->prepared = true; - } - - ret = spi_map_msg(ctlr, msg); - if (ret) { - msg->status = ret; - spi_finalize_current_message(ctlr); - goto out; - } - - if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) { - list_for_each_entry(xfer, &msg->transfers, transfer_list) { - xfer->ptp_sts_word_pre = 0; - ptp_read_system_prets(xfer->ptp_sts); - } - } - - ret = ctlr->transfer_one_message(ctlr, msg); - if (ret) { - dev_err(&ctlr->dev, - "failed to transfer one message from queue: %d\n", - ret); - goto out; - } - -out: + ret = __spi_pump_transfer_message(ctlr, msg, was_busy); mutex_unlock(&ctlr->io_mutex); /* Prod the scheduler in case transfer_one() was busy waiting */ @@ -1839,6 +1846,7 @@ static int spi_init_queue(struct spi_controller *ctlr) { ctlr->running = false; ctlr->busy = false; + ctlr->queue_empty = true; ctlr->kworker = kthread_create_worker(0, dev_name(&ctlr->dev)); if (IS_ERR(ctlr->kworker)) { @@ -1936,11 +1944,20 @@ void spi_finalize_current_message(struct spi_controller *ctlr) mesg->prepared = false; - spin_lock_irqsave(&ctlr->queue_lock, flags); - ctlr->cur_msg = NULL; - ctlr->fallback = false; - kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); - spin_unlock_irqrestore(&ctlr->queue_lock, flags); + if (!mesg->sync) { + /* + * This message was sent via the async message queue. Handle + * the queue and kick the worker thread to do the + * idling/shutdown or send the next message if needed. + */ + spin_lock_irqsave(&ctlr->queue_lock, flags); + WARN(ctlr->cur_msg != mesg, + "Finalizing queued message that is not the current head of queue!"); + ctlr->cur_msg = NULL; + ctlr->fallback = false; + kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); + spin_unlock_irqrestore(&ctlr->queue_lock, flags); + } trace_spi_message_done(mesg); @@ -2043,6 +2060,7 @@ static int __spi_queued_transfer(struct spi_device *spi, msg->status = -EINPROGRESS; list_add_tail(&msg->queue, &ctlr->queue); + ctlr->queue_empty = false; if (!ctlr->busy && need_pump) kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); @@ -3938,6 +3956,39 @@ static int spi_async_locked(struct spi_device *spi, struct spi_message *message) } +static void __spi_transfer_message_noqueue(struct spi_controller *ctlr, struct spi_message *msg) +{ + bool was_busy; + int ret; + + mutex_lock(&ctlr->io_mutex); + + /* If another context is idling the device then wait */ + while (ctlr->idling) + usleep_range(10000, 11000); + + was_busy = READ_ONCE(ctlr->busy); + + ret = __spi_pump_transfer_message(ctlr, msg, was_busy); + if (ret) + goto out; + + if (!was_busy) { + kfree(ctlr->dummy_rx); + ctlr->dummy_rx = NULL; + kfree(ctlr->dummy_tx); + ctlr->dummy_tx = NULL; + if (ctlr->unprepare_transfer_hardware && + ctlr->unprepare_transfer_hardware(ctlr)) + dev_err(&ctlr->dev, + "failed to unprepare transfer hardware\n"); + spi_idle_runtime_pm(ctlr); + } + +out: + mutex_unlock(&ctlr->io_mutex); +} + /*-------------------------------------------------------------------------*/ /* @@ -3956,51 +4007,52 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) DECLARE_COMPLETION_ONSTACK(done); int status; struct spi_controller *ctlr = spi->controller; - unsigned long flags; status = __spi_validate(spi, message); if (status != 0) return status; - message->complete = spi_complete; - message->context = &done; message->spi = spi; SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync); SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync); /* - * If we're not using the legacy transfer method then we will - * try to transfer in the calling context so special case. - * This code would be less tricky if we could remove the - * support for driver implemented message queues. + * Checking queue_empty here only guarantees async/sync message + * ordering when coming from the same context. It does not need to + * guard against reentrancy from a different context. The io_mutex + * will catch those cases. */ - if (ctlr->transfer == spi_queued_transfer) { - spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); + if (READ_ONCE(ctlr->queue_empty)) { + message->sync = true; + message->actual_length = 0; + message->status = -EINPROGRESS; trace_spi_message_submit(message); - status = __spi_queued_transfer(spi, message, false); + SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync_immediate); + SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync_immediate); - spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); - } else { - status = spi_async_locked(spi, message); + __spi_transfer_message_noqueue(ctlr, message); + + return message->status; } + /* + * There are messages in the async queue that could have originated + * from the same context, so we need to preserve ordering. + * Therefor we send the message to the async queue and wait until they + * are completed. + */ + message->complete = spi_complete; + message->context = &done; + status = spi_async_locked(spi, message); if (status == 0) { - /* Push out the messages in the calling context if we can */ - if (ctlr->transfer == spi_queued_transfer) { - SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, - spi_sync_immediate); - SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, - spi_sync_immediate); - __spi_pump_messages(ctlr, false); - } - wait_for_completion(&done); status = message->status; } message->context = NULL; + return status; } diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 1a75c26742f2..74261a83b5fa 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -461,6 +461,8 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @irq_flags: Interrupt enable state during PTP system timestamping * @fallback: fallback to pio if dma transfer return failure with * SPI_TRANS_FAIL_NO_START. + * @queue_empty: signal green light for opportunistically skipping the queue + * for spi_sync transfers. * * Each SPI controller can communicate with one or more @spi_device * children. These make a small bus, sharing MOSI, MISO and SCK signals @@ -677,6 +679,9 @@ struct spi_controller { /* Interrupt enable state during PTP system timestamping */ unsigned long irq_flags; + + /* Flag for enabling opportunistic skipping of the queue in spi_sync */ + bool queue_empty; }; static inline void *spi_controller_get_devdata(struct spi_controller *ctlr) @@ -986,6 +991,7 @@ struct spi_transfer { * @state: for use by whichever driver currently owns the message * @resources: for resource management when the spi message is processed * @prepared: spi_prepare_message was called for the this message + * @sync: this message took the direct sync path skipping the async queue * * A @spi_message is used to execute an atomic sequence of data transfers, * each represented by a struct spi_transfer. The sequence is "atomic" @@ -1037,7 +1043,10 @@ struct spi_message { struct list_head resources; /* spi_prepare_message was called for this message */ - bool prepared; + bool prepared; + + /* this message is skipping the async queue */ + bool sync; }; static inline void spi_message_init_no_memset(struct spi_message *m) -- cgit v1.2.3 From 66a221593cb26dd6aabba63bcd18173f4e69c7ab Mon Sep 17 00:00:00 2001 From: David Jander Date: Tue, 21 Jun 2022 08:12:30 +0200 Subject: spi: Remove the now unused ctlr->idling flag The ctlr->idling flag is never checked now, so we don't need to set it either. Signed-off-by: David Jander Link: https://lore.kernel.org/r/20220621061234.3626638-8-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 2 -- include/linux/spi/spi.h | 2 -- 2 files changed, 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 71b767a9ad77..52736e339645 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1674,7 +1674,6 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) } ctlr->busy = false; - ctlr->idling = true; spin_unlock_irqrestore(&ctlr->queue_lock, flags); kfree(ctlr->dummy_rx); @@ -1689,7 +1688,6 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) trace_spi_controller_idle(ctlr); spin_lock_irqsave(&ctlr->queue_lock, flags); - ctlr->idling = false; ctlr->queue_empty = true; goto out_unlock; } diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 74261a83b5fa..c58f46be762f 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -383,7 +383,6 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @pump_messages: work struct for scheduling work to the message pump * @queue_lock: spinlock to syncronise access to message queue * @queue: message queue - * @idling: the device is entering idle state * @cur_msg: the currently in-flight message * @cur_msg_mapped: message has been mapped for DMA * @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip @@ -616,7 +615,6 @@ struct spi_controller { spinlock_t queue_lock; struct list_head queue; struct spi_message *cur_msg; - bool idling; bool busy; bool running; bool rt; -- cgit v1.2.3 From 69fa95905d40846756d22402690ddf5361a9d13b Mon Sep 17 00:00:00 2001 From: David Jander Date: Tue, 21 Jun 2022 08:12:33 +0200 Subject: spi: Ensure the io_mutex is held until spi_finalize_current_message() This patch introduces a completion that is completed in spi_finalize_current_message() and waited for in __spi_pump_transfer_message(). This way all manipulation of ctlr->cur_msg is done with the io_mutex held and strictly ordered: __spi_pump_transfer_message() will not return until spi_finalize_current_message() is done using ctlr->cur_msg, and its calling context is only touching ctlr->cur_msg after returning. Due to this, we can safely drop the spin-locks around ctlr->cur_msg. Signed-off-by: David Jander Link: https://lore.kernel.org/r/20220621061234.3626638-11-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 32 ++++++++++++++------------------ include/linux/spi/spi.h | 6 ++---- 2 files changed, 16 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 3df84f43918c..db08cb868652 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1613,11 +1613,14 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, } } + reinit_completion(&ctlr->cur_msg_completion); ret = ctlr->transfer_one_message(ctlr, msg); if (ret) { dev_err(&ctlr->dev, "failed to transfer one message from queue\n"); return ret; + } else { + wait_for_completion(&ctlr->cur_msg_completion); } return 0; @@ -1704,6 +1707,12 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) spin_unlock_irqrestore(&ctlr->queue_lock, flags); ret = __spi_pump_transfer_message(ctlr, msg, was_busy); + + if (!ret) + kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); + ctlr->cur_msg = NULL; + ctlr->fallback = false; + mutex_unlock(&ctlr->io_mutex); /* Prod the scheduler in case transfer_one() was busy waiting */ @@ -1897,12 +1906,9 @@ void spi_finalize_current_message(struct spi_controller *ctlr) { struct spi_transfer *xfer; struct spi_message *mesg; - unsigned long flags; int ret; - spin_lock_irqsave(&ctlr->queue_lock, flags); mesg = ctlr->cur_msg; - spin_unlock_irqrestore(&ctlr->queue_lock, flags); if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) { list_for_each_entry(xfer, &mesg->transfers, transfer_list) { @@ -1936,20 +1942,7 @@ void spi_finalize_current_message(struct spi_controller *ctlr) mesg->prepared = false; - if (!mesg->sync) { - /* - * This message was sent via the async message queue. Handle - * the queue and kick the worker thread to do the - * idling/shutdown or send the next message if needed. - */ - spin_lock_irqsave(&ctlr->queue_lock, flags); - WARN(ctlr->cur_msg != mesg, - "Finalizing queued message that is not the current head of queue!"); - ctlr->cur_msg = NULL; - ctlr->fallback = false; - kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); - spin_unlock_irqrestore(&ctlr->queue_lock, flags); - } + complete(&ctlr->cur_msg_completion); trace_spi_message_done(mesg); @@ -3036,6 +3029,7 @@ int spi_register_controller(struct spi_controller *ctlr) } ctlr->bus_lock_flag = 0; init_completion(&ctlr->xfer_completion); + init_completion(&ctlr->cur_msg_completion); if (!ctlr->max_dma_len) ctlr->max_dma_len = INT_MAX; @@ -3962,6 +3956,9 @@ static void __spi_transfer_message_noqueue(struct spi_controller *ctlr, struct s if (ret) goto out; + ctlr->cur_msg = NULL; + ctlr->fallback = false; + if (!was_busy) { kfree(ctlr->dummy_rx); ctlr->dummy_rx = NULL; @@ -4013,7 +4010,6 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) * will catch those cases. */ if (READ_ONCE(ctlr->queue_empty)) { - message->sync = true; message->actual_length = 0; message->status = -EINPROGRESS; diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index c58f46be762f..c56e0d240a58 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -384,6 +384,7 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @queue_lock: spinlock to syncronise access to message queue * @queue: message queue * @cur_msg: the currently in-flight message + * @cur_msg_completion: a completion for the current in-flight message * @cur_msg_mapped: message has been mapped for DMA * @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip * selected @@ -615,6 +616,7 @@ struct spi_controller { spinlock_t queue_lock; struct list_head queue; struct spi_message *cur_msg; + struct completion cur_msg_completion; bool busy; bool running; bool rt; @@ -989,7 +991,6 @@ struct spi_transfer { * @state: for use by whichever driver currently owns the message * @resources: for resource management when the spi message is processed * @prepared: spi_prepare_message was called for the this message - * @sync: this message took the direct sync path skipping the async queue * * A @spi_message is used to execute an atomic sequence of data transfers, * each represented by a struct spi_transfer. The sequence is "atomic" @@ -1042,9 +1043,6 @@ struct spi_message { /* spi_prepare_message was called for this message */ bool prepared; - - /* this message is skipping the async queue */ - bool sync; }; static inline void spi_message_init_no_memset(struct spi_message *m) -- cgit v1.2.3 From dc3029056b02414c29b6627e3dd7b16624725ae9 Mon Sep 17 00:00:00 2001 From: David Jander Date: Tue, 21 Jun 2022 08:12:34 +0200 Subject: spi: opportunistically skip ctlr->cur_msg_completion There are only a few drivers that do not call spi_finalize_current_message() in the context of transfer_one_message(), and even for those cases the completion ctlr->cur_msg_completion is not needed always. The calls to complete() and wait_for_completion() each take a spin-lock, which is costly. This patch makes it possible to avoid those calls in the big majority of cases, by introducing two flags that with the help of ordering via barriers can avoid using the completion safely. In case of a race with the context calling spi_finalize_current_message(), the scheme errs on the safe side and takes the completion. The impact of this patch is worth the effort: On a i.MX8MM SoC, the time the SPI bus is idle between two consecutive calls to spi_sync(), is reduced from 19.6us to 16.8us... roughly 15%. Signed-off-by: David Jander Link: https://lore.kernel.org/r/20220621061234.3626638-12-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 27 +++++++++++++++++++++++++-- include/linux/spi/spi.h | 8 ++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index db08cb868652..ef37f043fd17 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1613,14 +1613,34 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, } } + /* + * Drivers implementation of transfer_one_message() must arrange for + * spi_finalize_current_message() to get called. Most drivers will do + * this in the calling context, but some don't. For those cases, a + * completion is used to guarantee that this function does not return + * until spi_finalize_current_message() is done accessing + * ctlr->cur_msg. + * Use of the following two flags enable to opportunistically skip the + * use of the completion since its use involves expensive spin locks. + * In case of a race with the context that calls + * spi_finalize_current_message() the completion will always be used, + * due to strict ordering of these flags using barriers. + */ + WRITE_ONCE(ctlr->cur_msg_incomplete, true); + WRITE_ONCE(ctlr->cur_msg_need_completion, false); reinit_completion(&ctlr->cur_msg_completion); + smp_wmb(); /* make these available to spi_finalize_current_message */ + ret = ctlr->transfer_one_message(ctlr, msg); if (ret) { dev_err(&ctlr->dev, "failed to transfer one message from queue\n"); return ret; } else { - wait_for_completion(&ctlr->cur_msg_completion); + WRITE_ONCE(ctlr->cur_msg_need_completion, true); + smp_mb(); /* see spi_finalize_current_message()... */ + if (READ_ONCE(ctlr->cur_msg_incomplete)) + wait_for_completion(&ctlr->cur_msg_completion); } return 0; @@ -1942,7 +1962,10 @@ void spi_finalize_current_message(struct spi_controller *ctlr) mesg->prepared = false; - complete(&ctlr->cur_msg_completion); + WRITE_ONCE(ctlr->cur_msg_incomplete, false); + smp_mb(); /* See __spi_pump_transfer_message()... */ + if (READ_ONCE(ctlr->cur_msg_need_completion)) + complete(&ctlr->cur_msg_completion); trace_spi_message_done(mesg); diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index c56e0d240a58..eb0d316e3c36 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -385,6 +385,12 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @queue: message queue * @cur_msg: the currently in-flight message * @cur_msg_completion: a completion for the current in-flight message + * @cur_msg_incomplete: Flag used internally to opportunistically skip + * the @cur_msg_completion. This flag is used to check if the driver has + * already called spi_finalize_current_message(). + * @cur_msg_need_completion: Flag used internally to opportunistically skip + * the @cur_msg_completion. This flag is used to signal the context that + * is running spi_finalize_current_message() that it needs to complete() * @cur_msg_mapped: message has been mapped for DMA * @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip * selected @@ -617,6 +623,8 @@ struct spi_controller { struct list_head queue; struct spi_message *cur_msg; struct completion cur_msg_completion; + bool cur_msg_incomplete; + bool cur_msg_need_completion; bool busy; bool running; bool rt; -- cgit v1.2.3 From 4a2dcc35911324d6fcde09b1760cf4f2962699ef Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 10 Jun 2022 12:58:23 -0700 Subject: block: introduce bdev_dma_alignment helper Preparing for upcoming dma_alignment users that have a block_device, but don't need the request_queue. Signed-off-by: Keith Busch Reviewed-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220610195830.3574005-5-kbusch@fb.com Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2f7b43444c5f..2556fcdb645b 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1365,6 +1365,11 @@ static inline int queue_dma_alignment(const struct request_queue *q) return q ? q->dma_alignment : 511; } +static inline unsigned int bdev_dma_alignment(struct block_device *bdev) +{ + return queue_dma_alignment(bdev_get_queue(bdev)); +} + static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr, unsigned int len) { -- cgit v1.2.3 From cfa320f72882f0e944e2237287db84b0f7df877d Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 10 Jun 2022 12:58:27 -0700 Subject: iov: introduce iov_iter_aligned The existing iov_iter_alignment() function returns the logical OR of address and length. For cases where address and length need to be considered separately, introduce a helper function that a caller can specificy length and address masks that indicate if the iov is unaligned. Cc: Alexander Viro Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220610195830.3574005-9-kbusch@fb.com Signed-off-by: Jens Axboe --- include/linux/uio.h | 2 ++ lib/iov_iter.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) (limited to 'include/linux') diff --git a/include/linux/uio.h b/include/linux/uio.h index 739285fe5a2f..34ba4a731179 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -219,6 +219,8 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i); #endif size_t iov_iter_zero(size_t bytes, struct iov_iter *); +bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask, + unsigned len_mask); unsigned long iov_iter_alignment(const struct iov_iter *i); unsigned long iov_iter_gap_alignment(const struct iov_iter *i); void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov, diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 0b64695ab632..507e732ef7cf 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1268,6 +1268,98 @@ void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) } EXPORT_SYMBOL(iov_iter_discard); +static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask, + unsigned len_mask) +{ + size_t size = i->count; + size_t skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->iov[k].iov_len - skip; + + if (len > size) + len = size; + if (len & len_mask) + return false; + if ((unsigned long)(i->iov[k].iov_base + skip) & addr_mask) + return false; + + size -= len; + if (!size) + break; + } + return true; +} + +static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask, + unsigned len_mask) +{ + size_t size = i->count; + unsigned skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->bvec[k].bv_len - skip; + + if (len > size) + len = size; + if (len & len_mask) + return false; + if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask) + return false; + + size -= len; + if (!size) + break; + } + return true; +} + +/** + * iov_iter_is_aligned() - Check if the addresses and lengths of each segments + * are aligned to the parameters. + * + * @i: &struct iov_iter to restore + * @addr_mask: bit mask to check against the iov element's addresses + * @len_mask: bit mask to check against the iov element's lengths + * + * Return: false if any addresses or lengths intersect with the provided masks + */ +bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask, + unsigned len_mask) +{ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_iter_aligned_iovec(i, addr_mask, len_mask); + + if (iov_iter_is_bvec(i)) + return iov_iter_aligned_bvec(i, addr_mask, len_mask); + + if (iov_iter_is_pipe(i)) { + unsigned int p_mask = i->pipe->ring_size - 1; + size_t size = i->count; + + if (size & len_mask) + return false; + if (size && allocated(&i->pipe->bufs[i->head & p_mask])) { + if (i->iov_offset & addr_mask) + return false; + } + + return true; + } + + if (iov_iter_is_xarray(i)) { + if (i->count & len_mask) + return false; + if ((i->xarray_start + i->iov_offset) & addr_mask) + return false; + } + + return true; +} +EXPORT_SYMBOL_GPL(iov_iter_is_aligned); + static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) { unsigned long res = 0; -- cgit v1.2.3 From 5debd9691c3ac64c3acd6867c264ad38bbe48cdc Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 10 Jun 2022 12:58:28 -0700 Subject: block: introduce bdev_iter_is_aligned helper Provide a convenient function for this repeatable coding pattern. Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220610195830.3574005-10-kbusch@fb.com Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2556fcdb645b..0b8bc1fe0b2c 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1370,6 +1370,13 @@ static inline unsigned int bdev_dma_alignment(struct block_device *bdev) return queue_dma_alignment(bdev_get_queue(bdev)); } +static inline bool bdev_iter_is_aligned(struct block_device *bdev, + struct iov_iter *iter) +{ + return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev), + bdev_logical_block_size(bdev) - 1); +} + static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr, unsigned int len) { -- cgit v1.2.3 From b1a000d3b8ec582da64bb644be633e5a0beffcbf Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 10 Jun 2022 12:58:29 -0700 Subject: block: relax direct io memory alignment Use the address alignment requirements from the block_device for direct io instead of requiring addresses be aligned to the block size. User space can discover the alignment requirements from the dma_alignment queue attribute. User space can specify any hardware compatible DMA offset for each segment, but every segment length is still required to be a multiple of the block size. Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220610195830.3574005-11-kbusch@fb.com Signed-off-by: Jens Axboe --- block/bio.c | 9 +++++++++ block/fops.c | 4 ++-- include/linux/blkdev.h | 5 +++++ 3 files changed, 16 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index ee5fe1bb015e..933ea3210954 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1220,7 +1220,16 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); + /* + * Each segment in the iov is required to be a block size multiple. + * However, we may not be able to get the entire segment if it spans + * more pages than bi_max_vecs allows, so we have to ALIGN_DOWN the + * result to ensure the bio's total size is correct. The remainder of + * the iov data will be picked up in the next bio iteration. + */ size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); + if (size > 0) + size = ALIGN_DOWN(size, bdev_logical_block_size(bio->bi_bdev)); if (unlikely(size <= 0)) return size ? size : -EFAULT; diff --git a/block/fops.c b/block/fops.c index 9d32df6fc315..86d3cab9bf93 100644 --- a/block/fops.c +++ b/block/fops.c @@ -45,8 +45,8 @@ static unsigned int dio_bio_write_op(struct kiocb *iocb) static bool blkdev_dio_unaligned(struct block_device *bdev, loff_t pos, struct iov_iter *iter) { - return ((pos | iov_iter_alignment(iter)) & - (bdev_logical_block_size(bdev) - 1)); + return pos & (bdev_logical_block_size(bdev) - 1) || + !bdev_iter_is_aligned(bdev, iter); } #define DIO_INLINE_BIO_VECS 4 diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 0b8bc1fe0b2c..886c44e97434 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -424,6 +424,11 @@ struct request_queue { unsigned long nr_requests; /* Max # of requests */ unsigned int dma_pad_mask; + /* + * Drivers that set dma_alignment to less than 511 must be prepared to + * handle individual bvec's that are not a multiple of a SECTOR_SIZE + * due to possible offsets. + */ unsigned int dma_alignment; #ifdef CONFIG_BLK_INLINE_ENCRYPTION -- cgit v1.2.3 From 8689461be3f1ce6686bc26f1f379790bb0fc7a8c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 14 Jun 2022 11:09:29 +0200 Subject: block: factor out a chunk_size_left helper Factor out a helper from blk_max_size_offset so that it can be reused independently. Signed-off-by: Christoph Hellwig Reviewed-by: Bart Van Assche Reviewed-by: Pankaj Raghav Link: https://lore.kernel.org/r/20220614090934.570632-2-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 886c44e97434..283961257cc9 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -934,6 +934,17 @@ static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q, return q->limits.max_sectors; } +/* + * Return how much of the chunk is left to be used for I/O at a given offset. + */ +static inline unsigned int blk_chunk_sectors_left(sector_t offset, + unsigned int chunk_sectors) +{ + if (unlikely(!is_power_of_2(chunk_sectors))) + return chunk_sectors - sector_div(offset, chunk_sectors); + return chunk_sectors - (offset & (chunk_sectors - 1)); +} + /* * Return maximum size of a request at given offset. Only valid for * file system requests. @@ -949,12 +960,8 @@ static inline unsigned int blk_max_size_offset(struct request_queue *q, return q->limits.max_sectors; } - if (likely(is_power_of_2(chunk_sectors))) - chunk_sectors -= offset & (chunk_sectors - 1); - else - chunk_sectors -= sector_div(offset, chunk_sectors); - - return min(q->limits.max_sectors, chunk_sectors); + return min(q->limits.max_sectors, + blk_chunk_sectors_left(offset, chunk_sectors)); } /* -- cgit v1.2.3 From efef739d5f37dc998b113fb965aea68d42a9eddc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 14 Jun 2022 11:09:33 +0200 Subject: block: fold blk_max_size_offset into get_max_io_size Now that blk_max_size_offset has a single caller left, fold it into that and clean up the naming convention for the local variables there. Signed-off-by: Christoph Hellwig Reviewed-by: Pankaj Raghav Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/20220614090934.570632-6-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-merge.c | 9 +++++++-- include/linux/blkdev.h | 19 ------------------- 2 files changed, 7 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/block/blk-merge.c b/block/blk-merge.c index 4da981efddee..0f5f42ebd0bb 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -166,9 +166,14 @@ static inline unsigned get_max_io_size(struct request_queue *q, { unsigned pbs = queue_physical_block_size(q) >> SECTOR_SHIFT; unsigned lbs = queue_logical_block_size(q) >> SECTOR_SHIFT; - unsigned max_sectors, start, end; + unsigned max_sectors = queue_max_sectors(q), start, end; + + if (q->limits.chunk_sectors) { + max_sectors = min(max_sectors, + blk_chunk_sectors_left(bio->bi_iter.bi_sector, + q->limits.chunk_sectors)); + } - max_sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector, 0); start = bio->bi_iter.bi_sector & (pbs - 1); end = (start + max_sectors) & ~(pbs - 1); if (end > start) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 283961257cc9..652c357dafb9 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -945,25 +945,6 @@ static inline unsigned int blk_chunk_sectors_left(sector_t offset, return chunk_sectors - (offset & (chunk_sectors - 1)); } -/* - * Return maximum size of a request at given offset. Only valid for - * file system requests. - */ -static inline unsigned int blk_max_size_offset(struct request_queue *q, - sector_t offset, - unsigned int chunk_sectors) -{ - if (!chunk_sectors) { - if (q->limits.chunk_sectors) - chunk_sectors = q->limits.chunk_sectors; - else - return q->limits.max_sectors; - } - - return min(q->limits.max_sectors, - blk_chunk_sectors_left(offset, chunk_sectors)); -} - /* * Access functions for manipulating queue properties */ -- cgit v1.2.3 From 2a9336c42a6abdcef564d522648a684a474a3483 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 14 Jun 2022 11:09:34 +0200 Subject: block: move blk_queue_get_max_sectors to blk.h blk_queue_get_max_sectors is private to the block layer, so move it out of blkdev.h. Signed-off-by: Christoph Hellwig Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/20220614090934.570632-7-hch@lst.de Signed-off-by: Jens Axboe --- block/blk.h | 13 +++++++++++++ include/linux/blkdev.h | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/block/blk.h b/block/blk.h index 434017701403..8e79296ee97a 100644 --- a/block/blk.h +++ b/block/blk.h @@ -159,6 +159,19 @@ static inline bool blk_discard_mergable(struct request *req) return false; } +static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q, + int op) +{ + if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) + return min(q->limits.max_discard_sectors, + UINT_MAX >> SECTOR_SHIFT); + + if (unlikely(op == REQ_OP_WRITE_ZEROES)) + return q->limits.max_write_zeroes_sectors; + + return q->limits.max_sectors; +} + #ifdef CONFIG_BLK_DEV_INTEGRITY void blk_flush_integrity(void); bool __bio_integrity_endio(struct bio *); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 652c357dafb9..b2d42201bd5d 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -921,19 +921,6 @@ static inline unsigned int bio_zone_is_seq(struct bio *bio) } #endif /* CONFIG_BLK_DEV_ZONED */ -static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q, - int op) -{ - if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) - return min(q->limits.max_discard_sectors, - UINT_MAX >> SECTOR_SHIFT); - - if (unlikely(op == REQ_OP_WRITE_ZEROES)) - return q->limits.max_write_zeroes_sectors; - - return q->limits.max_sectors; -} - /* * Return how much of the chunk is left to be used for I/O at a given offset. */ -- cgit v1.2.3 From e589f46445960c274cc813a1cc8e2fc73b2a1849 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 23 Jun 2022 09:48:26 +0200 Subject: block: fix default IO priority handling again Commit e70344c05995 ("block: fix default IO priority handling") introduced an inconsistency in get_current_ioprio() that tasks without IO context return IOPRIO_DEFAULT priority while tasks with freshly allocated IO context will return 0 (IOPRIO_CLASS_NONE/0) IO priority. Tasks without IO context used to be rare before 5a9d041ba2f6 ("block: move io_context creation into where it's needed") but after this commit they became common because now only BFQ IO scheduler setups task's IO context. Similar inconsistency is there for get_task_ioprio() so this inconsistency is now exposed to userspace and userspace will see different IO priority for tasks operating on devices with BFQ compared to devices without BFQ. Furthemore the changes done by commit e70344c05995 change the behavior when no IO priority is set for BFQ IO scheduler which is also documented in ioprio_set(2) manpage: "If no I/O scheduler has been set for a thread, then by default the I/O priority will follow the CPU nice value (setpriority(2)). In Linux kernels before version 2.6.24, once an I/O priority had been set using ioprio_set(), there was no way to reset the I/O scheduling behavior to the default. Since Linux 2.6.24, specifying ioprio as 0 can be used to reset to the default I/O scheduling behavior." So make sure we default to IOPRIO_CLASS_NONE as used to be the case before commit e70344c05995. Also cleanup alloc_io_context() to explicitely set this IO priority for the allocated IO context to avoid future surprises. Note that we tweak ioprio_best() to maintain ioprio_get(2) behavior and make this commit easily backportable. CC: stable@vger.kernel.org Fixes: e70344c05995 ("block: fix default IO priority handling") Reviewed-by: Damien Le Moal Tested-by: Damien Le Moal Signed-off-by: Jan Kara Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220623074840.5960-1-jack@suse.cz Signed-off-by: Jens Axboe --- block/blk-ioc.c | 2 ++ block/ioprio.c | 4 ++-- include/linux/ioprio.h | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/block/blk-ioc.c b/block/blk-ioc.c index df9cfe4ca532..63fc02042408 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -247,6 +247,8 @@ static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) INIT_HLIST_HEAD(&ioc->icq_list); INIT_WORK(&ioc->release_work, ioc_release_fn); #endif + ioc->ioprio = IOPRIO_DEFAULT; + return ioc; } diff --git a/block/ioprio.c b/block/ioprio.c index 2fe068fcaad5..2a34cbca18ae 100644 --- a/block/ioprio.c +++ b/block/ioprio.c @@ -157,9 +157,9 @@ out: int ioprio_best(unsigned short aprio, unsigned short bprio) { if (!ioprio_valid(aprio)) - aprio = IOPRIO_DEFAULT; + aprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_BE_NORM); if (!ioprio_valid(bprio)) - bprio = IOPRIO_DEFAULT; + bprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_BE_NORM); return min(aprio, bprio); } diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 3f53bc27a19b..3d088a88f832 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -11,7 +11,7 @@ /* * Default IO priority. */ -#define IOPRIO_DEFAULT IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_BE_NORM) +#define IOPRIO_DEFAULT IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0) /* * Check that a priority value has a valid class. -- cgit v1.2.3 From f7eda402878b12bc0884c5bc1192a9e76ad121fb Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 23 Jun 2022 09:48:27 +0200 Subject: block: Return effective IO priority from get_current_ioprio() get_current_ioprio() is used to initialize IO priority of various requests. As such it should be returning the effective IO priority of the task (i.e., reflecting the fact that unset IO priority should get set based on task's CPU priority) so that the conversion is concentrated in one place. Reviewed-by: Damien Le Moal Tested-by: Damien Le Moal Signed-off-by: Jan Kara Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220623074840.5960-2-jack@suse.cz Signed-off-by: Jens Axboe --- include/linux/ioprio.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 3d088a88f832..61ed6bb4998e 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -53,10 +53,17 @@ static inline int task_nice_ioclass(struct task_struct *task) static inline int get_current_ioprio(void) { struct io_context *ioc = current->io_context; + int prio; if (ioc) - return ioc->ioprio; - return IOPRIO_DEFAULT; + prio = ioc->ioprio; + else + prio = IOPRIO_DEFAULT; + + if (IOPRIO_PRIO_CLASS(prio) == IOPRIO_CLASS_NONE) + prio = IOPRIO_PRIO_VALUE(task_nice_ioclass(current), + task_nice_ioprio(current)); + return prio; } /* -- cgit v1.2.3 From 893e5d32d5832674bcf6465f27958e883b72b346 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 23 Jun 2022 09:48:28 +0200 Subject: block: Generalize get_current_ioprio() for any task get_current_ioprio() operates only on current task. We will need the same functionality for other tasks as well. Generalize get_current_ioprio() for that and also move the bulk out of the header file because it is large enough. Reviewed-by: Damien Le Moal Tested-by: Damien Le Moal Signed-off-by: Jan Kara Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220623074840.5960-3-jack@suse.cz Signed-off-by: Jens Axboe --- block/ioprio.c | 26 ++++++++++++++++++++++++++ include/linux/ioprio.h | 26 ++++++++++---------------- 2 files changed, 36 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/block/ioprio.c b/block/ioprio.c index 2a34cbca18ae..c4e3476155a1 100644 --- a/block/ioprio.c +++ b/block/ioprio.c @@ -138,6 +138,32 @@ out: return ret; } +/* + * If the task has set an I/O priority, use that. Otherwise, return + * the default I/O priority. + * + * Expected to be called for current task or with task_lock() held to keep + * io_context stable. + */ +int __get_task_ioprio(struct task_struct *p) +{ + struct io_context *ioc = p->io_context; + int prio; + + if (p != current) + lockdep_assert_held(&p->alloc_lock); + if (ioc) + prio = ioc->ioprio; + else + prio = IOPRIO_DEFAULT; + + if (IOPRIO_PRIO_CLASS(prio) == IOPRIO_CLASS_NONE) + prio = IOPRIO_PRIO_VALUE(task_nice_ioclass(p), + task_nice_ioprio(p)); + return prio; +} +EXPORT_SYMBOL_GPL(__get_task_ioprio); + static int get_task_ioprio(struct task_struct *p) { int ret; diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 61ed6bb4998e..9752cf4a9c7c 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -46,24 +46,18 @@ static inline int task_nice_ioclass(struct task_struct *task) return IOPRIO_CLASS_BE; } -/* - * If the calling process has set an I/O priority, use that. Otherwise, return - * the default I/O priority. - */ -static inline int get_current_ioprio(void) +#ifdef CONFIG_BLOCK +int __get_task_ioprio(struct task_struct *p); +#else +static inline int __get_task_ioprio(struct task_struct *p) { - struct io_context *ioc = current->io_context; - int prio; - - if (ioc) - prio = ioc->ioprio; - else - prio = IOPRIO_DEFAULT; + return IOPRIO_DEFAULT; +} +#endif /* CONFIG_BLOCK */ - if (IOPRIO_PRIO_CLASS(prio) == IOPRIO_CLASS_NONE) - prio = IOPRIO_PRIO_VALUE(task_nice_ioclass(current), - task_nice_ioprio(current)); - return prio; +static inline int get_current_ioprio(void) +{ + return __get_task_ioprio(current); } /* -- cgit v1.2.3 From fc25545e17bd74befe0b8ab2c65ac84936be5066 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 23 Jun 2022 09:48:29 +0200 Subject: block: Make ioprio_best() static Nobody outside of block/ioprio.c uses it. Reviewed-by: Damien Le Moal Tested-by: Damien Le Moal Signed-off-by: Jan Kara Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220623074840.5960-4-jack@suse.cz Signed-off-by: Jens Axboe --- block/ioprio.c | 2 +- include/linux/ioprio.h | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'include/linux') diff --git a/block/ioprio.c b/block/ioprio.c index c4e3476155a1..8c46f672a0ba 100644 --- a/block/ioprio.c +++ b/block/ioprio.c @@ -180,7 +180,7 @@ out: return ret; } -int ioprio_best(unsigned short aprio, unsigned short bprio) +static int ioprio_best(unsigned short aprio, unsigned short bprio) { if (!ioprio_valid(aprio)) aprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_BE_NORM); diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 9752cf4a9c7c..7578d4f6a969 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -60,11 +60,6 @@ static inline int get_current_ioprio(void) return __get_task_ioprio(current); } -/* - * For inheritance, return the highest of the two given priorities - */ -extern int ioprio_best(unsigned short aprio, unsigned short bprio); - extern int set_task_ioprio(struct task_struct *task, int ioprio); #ifdef CONFIG_BLOCK -- cgit v1.2.3 From ab24a01b276508dc884761bcb8e2759c36702377 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Tue, 14 Jun 2022 10:50:54 +0300 Subject: tty: Add closing marker into comment in tty_ldisc.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The closing `` is missing. Add it. Fixes: 6bb6fa6908eb ("tty: Implement lookahead to process XON/XOFF timely") Reported-by: Stephen Rothwell Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/9bc6d45d-48c8-519-1646-78ba22505b1f@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_ldisc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index 33678e1936f6..ede6f2157f32 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -187,7 +187,7 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * function for automatic flow control. * * @lookahead_buf: [DRV] ``void ()(struct tty_struct *tty, - * const unsigned char *cp, const char *fp, int count) + * const unsigned char *cp, const char *fp, int count)`` * * This function is called by the low-level tty driver for characters * not eaten by ->receive_buf() or ->receive_buf2(). It is useful for -- cgit v1.2.3 From f9008285bb69e4713918a665250ab2d356b731ba Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Mon, 13 Jun 2022 14:39:05 +0300 Subject: serial: Drop timeout from uart_port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 31f6bd7fad3b ("serial: Store character timing information to uart_port"), per frame timing information is available on uart_port. Uart port's timeout can be derived from frame_time by multiplying with fifosize. Most callers of uart_poll_timeout are not made under port's lock. To be on the safe side, make sure frame_time is only accessed once. As fifo_size is effectively a constant, it shouldn't cause any issues. Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220613113905.22962-1-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/serial/driver.rst | 5 +++-- drivers/tty/serial/mux.c | 6 ------ drivers/tty/serial/serial_core.c | 25 ++++++++++--------------- include/linux/serial_core.h | 16 ++++++++++++++-- 4 files changed, 27 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/serial/driver.rst b/Documentation/driver-api/serial/driver.rst index 7ef83fd3917b..1e7ab4142d49 100644 --- a/Documentation/driver-api/serial/driver.rst +++ b/Documentation/driver-api/serial/driver.rst @@ -422,8 +422,9 @@ Other functions --------------- uart_update_timeout(port,cflag,baud) - Update the FIFO drain timeout, port->timeout, according to the - number of bits, parity, stop bits and baud rate. + Update the frame timing information according to the number of bits, + parity, stop bits and baud rate. The FIFO drain timeout is derived + from the frame timing information. Locking: caller is expected to take port->lock diff --git a/drivers/tty/serial/mux.c b/drivers/tty/serial/mux.c index 643dfbcc43f9..0ba0f4d9459d 100644 --- a/drivers/tty/serial/mux.c +++ b/drivers/tty/serial/mux.c @@ -481,12 +481,6 @@ static int __init mux_probe(struct parisc_device *dev) port->line = port_cnt; port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_MUX_CONSOLE); - /* The port->timeout needs to match what is present in - * uart_wait_until_sent in serial_core.c. Otherwise - * the time spent in msleep_interruptable will be very - * long, causing the appearance of a console hang. - */ - port->timeout = HZ / 50; spin_lock_init(&port->lock); status = uart_add_one_port(&mux_driver, port); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 1368b0ef7d7f..75ece750bedc 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -327,13 +327,14 @@ static void uart_shutdown(struct tty_struct *tty, struct uart_state *state) } /** - * uart_update_timeout - update per-port FIFO timeout. + * uart_update_timeout - update per-port frame timing information. * @port: uart_port structure describing the port * @cflag: termios cflag value * @baud: speed of the port * - * Set the port FIFO timeout value. The @cflag value should - * reflect the actual hardware settings. + * Set the port frame timing information from which the FIFO timeout + * value is derived. The @cflag value should reflect the actual hardware + * settings. */ void uart_update_timeout(struct uart_port *port, unsigned int cflag, @@ -343,13 +344,6 @@ uart_update_timeout(struct uart_port *port, unsigned int cflag, u64 frame_time; frame_time = (u64)size * NSEC_PER_SEC; - size *= port->fifosize; - - /* - * Figure the timeout to send the above number of bits. - * Add .02 seconds of slop - */ - port->timeout = (HZ * size) / baud + HZ/50; port->frame_time = DIV64_U64_ROUND_UP(frame_time, baud); } EXPORT_SYMBOL(uart_update_timeout); @@ -1698,7 +1692,7 @@ static void uart_wait_until_sent(struct tty_struct *tty, int timeout) { struct uart_state *state = tty->driver_data; struct uart_port *port; - unsigned long char_time, expire; + unsigned long char_time, expire, fifo_timeout; port = uart_port_ref(state); if (!port) @@ -1728,12 +1722,13 @@ static void uart_wait_until_sent(struct tty_struct *tty, int timeout) * amount of time to send the entire FIFO, it probably won't * ever clear. This assumes the UART isn't doing flow * control, which is currently the case. Hence, if it ever - * takes longer than port->timeout, this is probably due to a + * takes longer than FIFO timeout, this is probably due to a * UART bug of some kind. So, we clamp the timeout parameter at - * 2*port->timeout. + * 2 * FIFO timeout. */ - if (timeout == 0 || timeout > 2 * port->timeout) - timeout = 2 * port->timeout; + fifo_timeout = uart_fifo_timeout(port); + if (timeout == 0 || timeout > 2 * fifo_timeout) + timeout = 2 * fifo_timeout; } expire = jiffies + timeout; diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 8032ffa741ed..faaf2372c60d 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -232,7 +232,6 @@ struct uart_port { int hw_stopped; /* sw-assisted CTS flow state */ unsigned int mctrl; /* current modem ctrl settings */ - unsigned int timeout; /* character-based timeout */ unsigned int frame_time; /* frame timing in ns */ unsigned int type; /* port type */ const struct uart_ops *ops; @@ -335,10 +334,23 @@ unsigned int uart_get_baud_rate(struct uart_port *port, struct ktermios *termios unsigned int max); unsigned int uart_get_divisor(struct uart_port *port, unsigned int baud); +/* + * Calculates FIFO drain time. + */ +static inline unsigned long uart_fifo_timeout(struct uart_port *port) +{ + u64 fifo_timeout = (u64)READ_ONCE(port->frame_time) * port->fifosize; + + /* Add .02 seconds of slop */ + fifo_timeout += 20 * NSEC_PER_MSEC; + + return max(nsecs_to_jiffies(fifo_timeout), 1UL); +} + /* Base timer interval for polling */ static inline int uart_poll_timeout(struct uart_port *port) { - int timeout = port->timeout; + int timeout = uart_fifo_timeout(port); return timeout > 6 ? (timeout / 2 - 2) : 1; } -- cgit v1.2.3 From eb47b59afb7e46c952d7b03884245364990d4910 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Fri, 24 Jun 2022 23:54:23 +0300 Subject: serial: Convert SERIAL_XMIT_SIZE to UART_XMIT_SIZE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both UART_XMIT_SIZE and SERIAL_XMIT_SIZE are defined. Make them all UART_XMIT_SIZE. Reviewed-by: Jiri Slaby Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220624205424.12686-6-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/amiserial.c | 18 +++++++++--------- drivers/tty/mips_ejtag_fdc.c | 2 +- drivers/tty/serial/meson_uart.c | 2 +- drivers/tty/serial/owl-uart.c | 2 +- drivers/tty/serial/rda-uart.c | 2 +- include/linux/serial.h | 6 ------ 6 files changed, 13 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c index afb2d373dd47..5458e2b1c125 100644 --- a/drivers/tty/amiserial.c +++ b/drivers/tty/amiserial.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -283,12 +284,12 @@ static void transmit_chars(struct serial_state *info) amiga_custom.serdat = info->xmit.buf[info->xmit.tail++] | 0x100; mb(); - info->xmit.tail = info->xmit.tail & (SERIAL_XMIT_SIZE-1); + info->xmit.tail = info->xmit.tail & (UART_XMIT_SIZE - 1); info->icount.tx++; if (CIRC_CNT(info->xmit.head, info->xmit.tail, - SERIAL_XMIT_SIZE) < WAKEUP_CHARS) + UART_XMIT_SIZE) < WAKEUP_CHARS) tty_wakeup(info->tport.tty); #ifdef SERIAL_DEBUG_INTR @@ -708,13 +709,13 @@ static int rs_put_char(struct tty_struct *tty, unsigned char ch) local_irq_save(flags); if (CIRC_SPACE(info->xmit.head, info->xmit.tail, - SERIAL_XMIT_SIZE) == 0) { + UART_XMIT_SIZE) == 0) { local_irq_restore(flags); return 0; } info->xmit.buf[info->xmit.head++] = ch; - info->xmit.head &= SERIAL_XMIT_SIZE-1; + info->xmit.head &= UART_XMIT_SIZE - 1; local_irq_restore(flags); return 1; } @@ -753,15 +754,14 @@ static int rs_write(struct tty_struct * tty, const unsigned char *buf, int count while (1) { c = CIRC_SPACE_TO_END(info->xmit.head, info->xmit.tail, - SERIAL_XMIT_SIZE); + UART_XMIT_SIZE); if (count < c) c = count; if (c <= 0) { break; } memcpy(info->xmit.buf + info->xmit.head, buf, c); - info->xmit.head = ((info->xmit.head + c) & - (SERIAL_XMIT_SIZE-1)); + info->xmit.head = (info->xmit.head + c) & (UART_XMIT_SIZE - 1); buf += c; count -= c; ret += c; @@ -788,14 +788,14 @@ static unsigned int rs_write_room(struct tty_struct *tty) { struct serial_state *info = tty->driver_data; - return CIRC_SPACE(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE); + return CIRC_SPACE(info->xmit.head, info->xmit.tail, UART_XMIT_SIZE); } static unsigned int rs_chars_in_buffer(struct tty_struct *tty) { struct serial_state *info = tty->driver_data; - return CIRC_CNT(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE); + return CIRC_CNT(info->xmit.head, info->xmit.tail, UART_XMIT_SIZE); } static void rs_flush_buffer(struct tty_struct *tty) diff --git a/drivers/tty/mips_ejtag_fdc.c b/drivers/tty/mips_ejtag_fdc.c index 49907427a165..e81701a66429 100644 --- a/drivers/tty/mips_ejtag_fdc.c +++ b/drivers/tty/mips_ejtag_fdc.c @@ -916,7 +916,7 @@ static int mips_ejtag_fdc_tty_probe(struct mips_cdmm_device *dev) mips_ejtag_fdc_write(priv, REG_FDCFG, cfg); /* Make each port's xmit FIFO big enough to fill FDC TX FIFO */ - priv->xmit_size = min(tx_fifo * 4, (unsigned int)SERIAL_XMIT_SIZE); + priv->xmit_size = min(tx_fifo * 4, (unsigned int)UART_XMIT_SIZE); driver = tty_alloc_driver(NUM_TTY_CHANNELS, TTY_DRIVER_REAL_RAW); if (IS_ERR(driver)) diff --git a/drivers/tty/serial/meson_uart.c b/drivers/tty/serial/meson_uart.c index 4869c0059c98..6c8db19fd572 100644 --- a/drivers/tty/serial/meson_uart.c +++ b/drivers/tty/serial/meson_uart.c @@ -162,7 +162,7 @@ static void meson_uart_start_tx(struct uart_port *port) ch = xmit->buf[xmit->tail]; writel(ch, port->membase + AML_UART_WFIFO); - xmit->tail = (xmit->tail+1) & (SERIAL_XMIT_SIZE - 1); + xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); port->icount.tx++; } diff --git a/drivers/tty/serial/owl-uart.c b/drivers/tty/serial/owl-uart.c index 44d20e5a7dd3..888e17e3f25f 100644 --- a/drivers/tty/serial/owl-uart.c +++ b/drivers/tty/serial/owl-uart.c @@ -201,7 +201,7 @@ static void owl_uart_send_chars(struct uart_port *port) ch = xmit->buf[xmit->tail]; owl_uart_write(port, ch, OWL_UART_TXDAT); - xmit->tail = (xmit->tail + 1) & (SERIAL_XMIT_SIZE - 1); + xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); port->icount.tx++; } diff --git a/drivers/tty/serial/rda-uart.c b/drivers/tty/serial/rda-uart.c index f556b4955f59..feb2054aba37 100644 --- a/drivers/tty/serial/rda-uart.c +++ b/drivers/tty/serial/rda-uart.c @@ -353,7 +353,7 @@ static void rda_uart_send_chars(struct uart_port *port) ch = xmit->buf[xmit->tail]; rda_uart_write(port, ch, RDA_UART_RXTX_BUFFER); - xmit->tail = (xmit->tail + 1) & (SERIAL_XMIT_SIZE - 1); + xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); port->icount.tx++; } diff --git a/include/linux/serial.h b/include/linux/serial.h index 0b8b7d7c8f33..70a9866e4abb 100644 --- a/include/linux/serial.h +++ b/include/linux/serial.h @@ -9,7 +9,6 @@ #ifndef _LINUX_SERIAL_H #define _LINUX_SERIAL_H -#include #include /* Helper for dealing with UART_LCR_WLEN* defines */ @@ -25,11 +24,6 @@ struct async_icount { __u32 buf_overrun; }; -/* - * The size of the serial xmit buffer is 1 page, or 4096 bytes - */ -#define SERIAL_XMIT_SIZE PAGE_SIZE - #include #endif /* _LINUX_SERIAL_H */ -- cgit v1.2.3 From 34619de1b8cb52afa90bbeb3b4fbad34c28f19cf Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Fri, 24 Jun 2022 23:54:24 +0300 Subject: serial: Consolidate BOTH_EMPTY use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per file BOTH_EMPTY defines are littering our source code here and there. Define once in serial.h and create helper for the check too. Reviewed-by: Jiri Slaby Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220624205424.12686-7-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- arch/mips/ath79/early_printk.c | 9 +++++---- drivers/accessibility/speakup/serialio.h | 3 +-- drivers/tty/serial/8250/8250_early.c | 4 +--- drivers/tty/serial/8250/8250_port.c | 12 +++++------- drivers/tty/serial/omap-serial.c | 7 +++---- drivers/tty/serial/pch_uart.c | 7 +++---- drivers/tty/serial/pxa.c | 5 ++--- drivers/tty/serial/sunsu.c | 4 +--- drivers/tty/serial/vr41xx_siu.c | 4 +--- include/linux/serial.h | 9 +++++++++ 10 files changed, 31 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/arch/mips/ath79/early_printk.c b/arch/mips/ath79/early_printk.c index 8751d067f98f..f6d02b425a10 100644 --- a/arch/mips/ath79/early_printk.c +++ b/arch/mips/ath79/early_printk.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -29,15 +30,15 @@ static inline void prom_putchar_wait(void __iomem *reg, u32 mask, u32 val) } while (1); } -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - static void prom_putchar_ar71xx(char ch) { void __iomem *base = (void __iomem *)(KSEG1ADDR(AR71XX_UART_BASE)); - prom_putchar_wait(base + UART_LSR * 4, BOTH_EMPTY, BOTH_EMPTY); + prom_putchar_wait(base + UART_LSR * 4, UART_LSR_BOTH_EMPTY, + UART_LSR_BOTH_EMPTY); __raw_writel((unsigned char)ch, base + UART_TX * 4); - prom_putchar_wait(base + UART_LSR * 4, BOTH_EMPTY, BOTH_EMPTY); + prom_putchar_wait(base + UART_LSR * 4, UART_LSR_BOTH_EMPTY, + UART_LSR_BOTH_EMPTY); } static void prom_putchar_ar933x(char ch) diff --git a/drivers/accessibility/speakup/serialio.h b/drivers/accessibility/speakup/serialio.h index 6f8f86f161bb..b4f9a1925b81 100644 --- a/drivers/accessibility/speakup/serialio.h +++ b/drivers/accessibility/speakup/serialio.h @@ -33,9 +33,8 @@ struct old_serial_port { #define NUM_DISABLE_TIMEOUTS 3 /* buffer timeout in ms */ #define SPK_TIMEOUT 100 -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) #define spk_serial_tx_busy() \ - ((inb(speakup_info.port_tts + UART_LSR) & BOTH_EMPTY) != BOTH_EMPTY) + (!uart_lsr_tx_empty(inb(speakup_info.port_tts + UART_LSR))) #endif diff --git a/drivers/tty/serial/8250/8250_early.c b/drivers/tty/serial/8250/8250_early.c index e52585064565..f271becfc46c 100644 --- a/drivers/tty/serial/8250/8250_early.c +++ b/drivers/tty/serial/8250/8250_early.c @@ -84,8 +84,6 @@ static void serial8250_early_out(struct uart_port *port, int offset, int value) } } -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - static void serial_putc(struct uart_port *port, unsigned char c) { unsigned int status; @@ -94,7 +92,7 @@ static void serial_putc(struct uart_port *port, unsigned char c) for (;;) { status = serial8250_early_in(port, UART_LSR); - if ((status & BOTH_EMPTY) == BOTH_EMPTY) + if (uart_lsr_tx_empty(status)) break; cpu_relax(); } diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 1311b00f8194..55b252954a92 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -50,8 +50,6 @@ #define DEBUG_AUTOCONF(fmt...) do { } while (0) #endif -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - /* * Here we define the default xmit fifo size used for each type of UART. */ @@ -1843,7 +1841,7 @@ void serial8250_tx_chars(struct uart_8250_port *up) if (uart_circ_empty(xmit)) break; if ((up->capabilities & UART_CAP_HFIFO) && - (serial_in(up, UART_LSR) & BOTH_EMPTY) != BOTH_EMPTY) + !uart_lsr_tx_empty(serial_in(up, UART_LSR))) break; /* The BCM2835 MINI UART THRE bit is really a not-full bit. */ if ((up->capabilities & UART_CAP_MINI) && @@ -2003,7 +2001,7 @@ static unsigned int serial8250_tx_empty(struct uart_port *port) serial8250_rpm_put(up); - return (lsr & BOTH_EMPTY) == BOTH_EMPTY ? TIOCSER_TEMT : 0; + return uart_lsr_tx_empty(lsr) ? TIOCSER_TEMT : 0; } unsigned int serial8250_do_get_mctrl(struct uart_port *port) @@ -2151,7 +2149,7 @@ static void serial8250_put_poll_char(struct uart_port *port, else serial_port_out(port, UART_IER, 0); - wait_for_xmitr(up, BOTH_EMPTY); + wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); /* * Send the character out. */ @@ -2161,7 +2159,7 @@ static void serial8250_put_poll_char(struct uart_port *port, * Finally, wait for transmitter to become empty * and restore the IER */ - wait_for_xmitr(up, BOTH_EMPTY); + wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); serial_port_out(port, UART_IER, ier); serial8250_rpm_put(up); } @@ -3431,7 +3429,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, * Finally, wait for transmitter to become empty * and restore the IER */ - wait_for_xmitr(up, BOTH_EMPTY); + wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); if (em485) { mdelay(port->rs485.delay_rts_after_send); diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 98622c35d896..52cb1a68b053 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -1102,8 +1103,6 @@ serial_omap_type(struct uart_port *port) return up->name; } -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - static void __maybe_unused wait_for_xmitr(struct uart_omap_port *up) { unsigned int status, tmout = 10000; @@ -1118,7 +1117,7 @@ static void __maybe_unused wait_for_xmitr(struct uart_omap_port *up) if (--tmout == 0) break; udelay(1); - } while ((status & BOTH_EMPTY) != BOTH_EMPTY); + } while (!uart_lsr_tx_empty(status)); /* Wait up to 1s for flow control if necessary */ if (up->port.flags & UPF_CONS_FLOW) { @@ -1186,7 +1185,7 @@ static void omap_serial_early_putc(struct uart_port *port, unsigned char c) for (;;) { status = omap_serial_early_in(port, UART_LSR); - if ((status & BOTH_EMPTY) == BOTH_EMPTY) + if (uart_lsr_tx_empty(status)) break; cpu_relax(); } diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c index 3b26524d48e3..8a9065e4a903 100644 --- a/drivers/tty/serial/pch_uart.c +++ b/drivers/tty/serial/pch_uart.c @@ -3,6 +3,7 @@ *Copyright (C) 2011 LAPIS Semiconductor Co., Ltd. */ #include +#include #include #include #include @@ -189,8 +190,6 @@ enum { #define PCH_UART_HAL_LOOP (PCH_UART_MCR_LOOP) #define PCH_UART_HAL_AFE (PCH_UART_MCR_AFE) -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - #define DEFAULT_UARTCLK 1843200 /* 1.8432 MHz */ #define CMITC_UARTCLK 192000000 /* 192.0000 MHz */ #define FRI2_64_UARTCLK 64000000 /* 64.0000 MHz */ @@ -1516,7 +1515,7 @@ static void pch_uart_put_poll_char(struct uart_port *port, * Finally, wait for transmitter to become empty * and restore the IER */ - wait_for_xmitr(priv, BOTH_EMPTY); + wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY); iowrite8(ier, priv->membase + UART_IER); } #endif /* CONFIG_CONSOLE_POLL */ @@ -1602,7 +1601,7 @@ pch_console_write(struct console *co, const char *s, unsigned int count) * Finally, wait for transmitter to become empty * and restore the IER */ - wait_for_xmitr(priv, BOTH_EMPTY); + wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY); iowrite8(ier, priv->membase + UART_IER); if (port_locked) diff --git a/drivers/tty/serial/pxa.c b/drivers/tty/serial/pxa.c index e80ba8e10407..9309ffd87c8e 100644 --- a/drivers/tty/serial/pxa.c +++ b/drivers/tty/serial/pxa.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -575,8 +576,6 @@ static struct uart_driver serial_pxa_reg; #ifdef CONFIG_SERIAL_PXA_CONSOLE -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - /* * Wait for transmitter & holding register to empty */ @@ -594,7 +593,7 @@ static void wait_for_xmitr(struct uart_pxa_port *up) if (--tmout == 0) break; udelay(1); - } while ((status & BOTH_EMPTY) != BOTH_EMPTY); + } while (!uart_lsr_tx_empty(status)); /* Wait up to 1s for flow control if necessary */ if (up->port.flags & UPF_CONS_FLOW) { diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c index fff50b5b82eb..84d545e5a8c7 100644 --- a/drivers/tty/serial/sunsu.c +++ b/drivers/tty/serial/sunsu.c @@ -1249,8 +1249,6 @@ static int sunsu_kbd_ms_init(struct uart_sunsu_port *up) #ifdef CONFIG_SERIAL_SUNSU_CONSOLE -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - /* * Wait for transmitter & holding register to empty */ @@ -1268,7 +1266,7 @@ static void wait_for_xmitr(struct uart_sunsu_port *up) if (--tmout == 0) break; udelay(1); - } while ((status & BOTH_EMPTY) != BOTH_EMPTY); + } while (!uart_lsr_tx_empty(status)); /* Wait up to 1s for flow control if necessary */ if (up->port.flags & UPF_CONS_FLOW) { diff --git a/drivers/tty/serial/vr41xx_siu.c b/drivers/tty/serial/vr41xx_siu.c index e0bf003ca3a1..1ba689a81abd 100644 --- a/drivers/tty/serial/vr41xx_siu.c +++ b/drivers/tty/serial/vr41xx_siu.c @@ -703,8 +703,6 @@ static int siu_init_ports(struct platform_device *pdev) #ifdef CONFIG_SERIAL_VR41XX_CONSOLE -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - static void wait_for_xmitr(struct uart_port *port) { int timeout = 10000; @@ -715,7 +713,7 @@ static void wait_for_xmitr(struct uart_port *port) if (lsr & UART_LSR_BI) lsr_break_flag[port->line] = UART_LSR_BI; - if ((lsr & BOTH_EMPTY) == BOTH_EMPTY) + if (uart_lsr_tx_empty(lsr)) break; } while (timeout-- > 0); diff --git a/include/linux/serial.h b/include/linux/serial.h index 70a9866e4abb..3d6fe3ef92cf 100644 --- a/include/linux/serial.h +++ b/include/linux/serial.h @@ -10,10 +10,19 @@ #define _LINUX_SERIAL_H #include +#include /* Helper for dealing with UART_LCR_WLEN* defines */ #define UART_LCR_WLEN(x) ((x) - 5) +/* FIFO and shifting register empty */ +#define UART_LSR_BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) + +static inline bool uart_lsr_tx_empty(u16 lsr) +{ + return (lsr & UART_LSR_BOTH_EMPTY) == UART_LSR_BOTH_EMPTY; +} + /* * Counters of the input lines (CTS, DSR, RI, CD) interrupts */ -- cgit v1.2.3 From f8ba5680a56be696b3f4343ed0a591abab807da4 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Fri, 24 Jun 2022 23:42:05 +0300 Subject: serial: 8250: make saved LSR larger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DW flags address received as BIT(8) in LSR. In order to not lose that on read, enlarge lsr_saved_flags to u16. Adjust lsr/status variables and related call chains to use u16. Technically, some of these type conversion would not be needed but it doesn't hurt to be consistent. Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220624204210.11112-2-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250.h | 4 ++-- drivers/tty/serial/8250/8250_exar.c | 2 +- drivers/tty/serial/8250/8250_fsl.c | 2 +- drivers/tty/serial/8250/8250_ingenic.c | 2 +- drivers/tty/serial/8250/8250_omap.c | 7 +++---- drivers/tty/serial/8250/8250_port.c | 17 +++++++++-------- include/linux/serial_8250.h | 6 +++--- 7 files changed, 20 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index b120da57c61f..0ff5688ba90c 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -133,9 +133,9 @@ static inline void serial_out(struct uart_8250_port *up, int offset, int value) * * Returns LSR value or'ed with the preserved flags (if any). */ -static inline unsigned int serial_lsr_in(struct uart_8250_port *up) +static inline u16 serial_lsr_in(struct uart_8250_port *up) { - unsigned int lsr = up->lsr_saved_flags; + u16 lsr = up->lsr_saved_flags; lsr |= serial_in(up, UART_LSR); up->lsr_saved_flags = lsr & LSR_SAVE_FLAGS; diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c index 528779b40049..3d999eec4087 100644 --- a/drivers/tty/serial/8250/8250_exar.c +++ b/drivers/tty/serial/8250/8250_exar.c @@ -195,11 +195,11 @@ static int xr17v35x_startup(struct uart_port *port) static void exar_shutdown(struct uart_port *port) { - unsigned char lsr; bool tx_complete = false; struct uart_8250_port *up = up_to_u8250p(port); struct circ_buf *xmit = &port->state->xmit; int i = 0; + u16 lsr; do { lsr = serial_in(up, UART_LSR); diff --git a/drivers/tty/serial/8250/8250_fsl.c b/drivers/tty/serial/8250/8250_fsl.c index 9c01c531349d..fd4005fcd0d6 100644 --- a/drivers/tty/serial/8250/8250_fsl.c +++ b/drivers/tty/serial/8250/8250_fsl.c @@ -25,8 +25,8 @@ int fsl8250_handle_irq(struct uart_port *port) { - unsigned char lsr, orig_lsr; unsigned long flags; + u16 lsr, orig_lsr; unsigned int iir; struct uart_8250_port *up = up_to_u8250p(port); diff --git a/drivers/tty/serial/8250/8250_ingenic.c b/drivers/tty/serial/8250/8250_ingenic.c index cff91aa03f29..2b2f5d8d24b9 100644 --- a/drivers/tty/serial/8250/8250_ingenic.c +++ b/drivers/tty/serial/8250/8250_ingenic.c @@ -54,7 +54,7 @@ static void early_out(struct uart_port *port, int offset, uint8_t value) static void ingenic_early_console_putc(struct uart_port *port, unsigned char c) { - uint8_t lsr; + u16 lsr; do { lsr = early_in(port, UART_LSR); diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index ac8bfa042391..0dcecbbc3967 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1115,8 +1115,7 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir) return omap_8250_rx_dma(up); } -static unsigned char omap_8250_handle_rx_dma(struct uart_8250_port *up, - u8 iir, unsigned char status) +static u16 omap_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status) { if ((status & (UART_LSR_DR | UART_LSR_BI)) && (iir & UART_IIR_RDI)) { @@ -1130,7 +1129,7 @@ static unsigned char omap_8250_handle_rx_dma(struct uart_8250_port *up, } static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, - unsigned char status) + u16 status) { /* * Queue a new transfer if FIFO has data. @@ -1164,7 +1163,7 @@ static int omap_8250_dma_handle_irq(struct uart_port *port) { struct uart_8250_port *up = up_to_u8250p(port); struct omap8250_priv *priv = up->port.private_data; - unsigned char status; + u16 status; u8 iir; serial8250_rpm_get(up); diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 55b252954a92..c8ae0e8376d4 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -1502,7 +1502,7 @@ static inline void __stop_tx(struct uart_8250_port *p) struct uart_8250_em485 *em485 = p->em485; if (em485) { - unsigned char lsr = serial_lsr_in(p); + u16 lsr = serial_lsr_in(p); u64 stop_delay = 0; p->lsr_saved_flags |= lsr & LSR_SAVE_FLAGS; @@ -1563,7 +1563,7 @@ static inline void __start_tx(struct uart_port *port) if (serial8250_set_THRI(up)) { if (up->bugs & UART_BUG_TXEN) { - unsigned char lsr = serial_lsr_in(up); + u16 lsr = serial_lsr_in(up); if (lsr & UART_LSR_THRE) serial8250_tx_chars(up); @@ -1716,7 +1716,7 @@ static void serial8250_enable_ms(struct uart_port *port) serial8250_rpm_put(up); } -void serial8250_read_char(struct uart_8250_port *up, unsigned char lsr) +void serial8250_read_char(struct uart_8250_port *up, u16 lsr) { struct uart_port *port = &up->port; unsigned char ch; @@ -1785,7 +1785,7 @@ EXPORT_SYMBOL_GPL(serial8250_read_char); * (such as THRE) because the LSR value might come from an already consumed * character. */ -unsigned char serial8250_rx_chars(struct uart_8250_port *up, unsigned char lsr) +u16 serial8250_rx_chars(struct uart_8250_port *up, u16 lsr) { struct uart_port *port = &up->port; int max_count = 256; @@ -1905,10 +1905,10 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir) */ int serial8250_handle_irq(struct uart_port *port, unsigned int iir) { - unsigned char status; struct uart_8250_port *up = up_to_u8250p(port); bool skip_rx = false; unsigned long flags; + u16 status; if (iir & UART_IIR_NO_INT) return 0; @@ -1991,7 +1991,7 @@ static unsigned int serial8250_tx_empty(struct uart_port *port) { struct uart_8250_port *up = up_to_u8250p(port); unsigned long flags; - unsigned int lsr; + u16 lsr; serial8250_rpm_get(up); @@ -2114,8 +2114,8 @@ static void wait_for_xmitr(struct uart_8250_port *up, int bits) static int serial8250_get_poll_char(struct uart_port *port) { struct uart_8250_port *up = up_to_u8250p(port); - unsigned char lsr; int status; + u16 lsr; serial8250_rpm_get(up); @@ -2170,8 +2170,9 @@ int serial8250_do_startup(struct uart_port *port) { struct uart_8250_port *up = up_to_u8250p(port); unsigned long flags; - unsigned char lsr, iir; + unsigned char iir; int retval; + u16 lsr; if (!port->fifosize) port->fifosize = uart_config[port->type].fifo_size; diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index ff84a3ed10ea..4565f25ba9a2 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -119,7 +119,7 @@ struct uart_8250_port { * be immediately processed. */ #define LSR_SAVE_FLAGS UART_LSR_BRK_ERROR_BITS - unsigned char lsr_saved_flags; + u16 lsr_saved_flags; #define MSR_SAVE_FLAGS UART_MSR_ANY_DELTA unsigned char msr_saved_flags; @@ -170,8 +170,8 @@ extern void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud, unsigned int quot_frac); extern int fsl8250_handle_irq(struct uart_port *port); int serial8250_handle_irq(struct uart_port *port, unsigned int iir); -unsigned char serial8250_rx_chars(struct uart_8250_port *up, unsigned char lsr); -void serial8250_read_char(struct uart_8250_port *up, unsigned char lsr); +u16 serial8250_rx_chars(struct uart_8250_port *up, u16 lsr); +void serial8250_read_char(struct uart_8250_port *up, u16 lsr); void serial8250_tx_chars(struct uart_8250_port *up); unsigned int serial8250_modem_status(struct uart_8250_port *up); void serial8250_init_port(struct uart_8250_port *up); -- cgit v1.2.3 From 507bd6fbaaefcb8dd89bd00baddf00b439d30c51 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Fri, 24 Jun 2022 23:42:06 +0300 Subject: serial: 8250: create lsr_save_mask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow drivers to alter LSR save mask. Reviewed-by: Andy Shevchenko Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220624204210.11112-3-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250.h | 2 +- drivers/tty/serial/8250/8250_core.c | 4 ++++ drivers/tty/serial/8250/8250_dw.c | 2 +- include/linux/serial_8250.h | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index 0ff5688ba90c..5cc967fe3b59 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -138,7 +138,7 @@ static inline u16 serial_lsr_in(struct uart_8250_port *up) u16 lsr = up->lsr_saved_flags; lsr |= serial_in(up, UART_LSR); - up->lsr_saved_flags = lsr & LSR_SAVE_FLAGS; + up->lsr_saved_flags = lsr & up->lsr_save_mask; return lsr; } diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index 90ddc8924811..57e86133af4f 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -1007,6 +1007,7 @@ int serial8250_register_8250_port(const struct uart_8250_port *up) uart->port.rs485 = up->port.rs485; uart->rs485_start_tx = up->rs485_start_tx; uart->rs485_stop_tx = up->rs485_stop_tx; + uart->lsr_save_mask = up->lsr_save_mask; uart->dma = up->dma; /* Take tx_loadsz from fifosize if it wasn't set separately */ @@ -1094,6 +1095,9 @@ int serial8250_register_8250_port(const struct uart_8250_port *up) ret = 0; } + if (!uart->lsr_save_mask) + uart->lsr_save_mask = LSR_SAVE_FLAGS; /* Use default LSR mask */ + /* Initialise interrupt backoff work if required */ if (up->overrun_backoff_time_ms > 0) { uart->overrun_backoff_time_ms = diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index 4cc69bb612ab..167a691c7b19 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -129,7 +129,7 @@ static void dw8250_tx_wait_empty(struct uart_port *p) while (tries--) { lsr = readb (p->membase + (UART_LSR << p->regshift)); - up->lsr_saved_flags |= lsr & LSR_SAVE_FLAGS; + up->lsr_saved_flags |= lsr & up->lsr_save_mask; if (lsr & UART_LSR_TEMT) break; diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index 4565f25ba9a2..8c7b793aa4d7 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -120,6 +120,7 @@ struct uart_8250_port { */ #define LSR_SAVE_FLAGS UART_LSR_BRK_ERROR_BITS u16 lsr_saved_flags; + u16 lsr_save_mask; #define MSR_SAVE_FLAGS UART_MSR_ANY_DELTA unsigned char msr_saved_flags; -- cgit v1.2.3 From ae50bb2752836277ae15aa4e9d99074d6d947946 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Fri, 24 Jun 2022 23:42:08 +0300 Subject: serial: take termios_rwsem for ->rs485_config() & pass termios as param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To be able to alter ADDRB within ->rs485_config(), take termios_rwsem before calling ->rs485_config() and pass termios. Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220624204210.11112-5-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250.h | 3 ++- drivers/tty/serial/8250/8250_dwlib.c | 3 ++- drivers/tty/serial/8250/8250_exar.c | 9 +++++---- drivers/tty/serial/8250/8250_fintek.c | 2 +- drivers/tty/serial/8250/8250_lpc18xx.c | 2 +- drivers/tty/serial/8250/8250_pci.c | 2 +- drivers/tty/serial/8250/8250_port.c | 3 ++- drivers/tty/serial/amba-pl011.c | 2 +- drivers/tty/serial/ar933x_uart.c | 2 +- drivers/tty/serial/atmel_serial.c | 2 +- drivers/tty/serial/fsl_lpuart.c | 4 ++-- drivers/tty/serial/imx.c | 2 +- drivers/tty/serial/max310x.c | 2 +- drivers/tty/serial/mcf.c | 3 ++- drivers/tty/serial/omap-serial.c | 3 ++- drivers/tty/serial/sc16is7xx.c | 2 +- drivers/tty/serial/serial_core.c | 14 ++++++++++---- drivers/tty/serial/stm32-usart.c | 2 +- include/linux/serial_core.h | 1 + 19 files changed, 38 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index 5cc967fe3b59..287153d32536 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -203,7 +203,8 @@ void serial8250_rpm_put(struct uart_8250_port *p); void serial8250_rpm_get_tx(struct uart_8250_port *p); void serial8250_rpm_put_tx(struct uart_8250_port *p); -int serial8250_em485_config(struct uart_port *port, struct serial_rs485 *rs485); +int serial8250_em485_config(struct uart_port *port, struct ktermios *termios, + struct serial_rs485 *rs485); void serial8250_em485_start_tx(struct uart_8250_port *p); void serial8250_em485_stop_tx(struct uart_8250_port *p); void serial8250_em485_destroy(struct uart_8250_port *p); diff --git a/drivers/tty/serial/8250/8250_dwlib.c b/drivers/tty/serial/8250/8250_dwlib.c index c83e7eaf3877..d1ff3daeb0ba 100644 --- a/drivers/tty/serial/8250/8250_dwlib.c +++ b/drivers/tty/serial/8250/8250_dwlib.c @@ -85,7 +85,8 @@ void dw8250_do_set_termios(struct uart_port *p, struct ktermios *termios, struct } EXPORT_SYMBOL_GPL(dw8250_do_set_termios); -static int dw8250_rs485_config(struct uart_port *p, struct serial_rs485 *rs485) +static int dw8250_rs485_config(struct uart_port *p, struct ktermios *termios, + struct serial_rs485 *rs485) { u32 tcr; diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c index 3d999eec4087..f5344cfe303c 100644 --- a/drivers/tty/serial/8250/8250_exar.c +++ b/drivers/tty/serial/8250/8250_exar.c @@ -112,7 +112,8 @@ struct exar8250; struct exar8250_platform { - int (*rs485_config)(struct uart_port *, struct serial_rs485 *); + int (*rs485_config)(struct uart_port *port, struct ktermios *termios, + struct serial_rs485 *rs485); const struct serial_rs485 *rs485_supported; int (*register_gpio)(struct pci_dev *, struct uart_8250_port *); void (*unregister_gpio)(struct uart_8250_port *); @@ -409,7 +410,7 @@ static void xr17v35x_unregister_gpio(struct uart_8250_port *port) port->port.private_data = NULL; } -static int generic_rs485_config(struct uart_port *port, +static int generic_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { bool is_rs485 = !!(rs485->flags & SER_RS485_ENABLED); @@ -441,7 +442,7 @@ static const struct exar8250_platform exar8250_default_platform = { .rs485_supported = &generic_rs485_supported, }; -static int iot2040_rs485_config(struct uart_port *port, +static int iot2040_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { bool is_rs485 = !!(rs485->flags & SER_RS485_ENABLED); @@ -471,7 +472,7 @@ static int iot2040_rs485_config(struct uart_port *port, value |= mode; writeb(value, p + UART_EXAR_MPIOLVL_7_0); - return generic_rs485_config(port, rs485); + return generic_rs485_config(port, termios, rs485); } static const struct serial_rs485 iot2040_rs485_supported = { diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c index 1fb86c73786c..eea693f5b577 100644 --- a/drivers/tty/serial/8250/8250_fintek.c +++ b/drivers/tty/serial/8250/8250_fintek.c @@ -191,7 +191,7 @@ static int fintek_8250_get_ldn_range(struct fintek_8250 *pdata, int *min, return -ENODEV; } -static int fintek_8250_rs485_config(struct uart_port *port, +static int fintek_8250_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { uint8_t config = 0; diff --git a/drivers/tty/serial/8250/8250_lpc18xx.c b/drivers/tty/serial/8250/8250_lpc18xx.c index 3a1cb51cbc91..d7cb3bb52069 100644 --- a/drivers/tty/serial/8250/8250_lpc18xx.c +++ b/drivers/tty/serial/8250/8250_lpc18xx.c @@ -32,7 +32,7 @@ struct lpc18xx_uart_data { int line; }; -static int lpc18xx_rs485_config(struct uart_port *port, +static int lpc18xx_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { struct uart_8250_port *up = up_to_u8250p(port); diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index b6d71268aa7d..d31d2350a9db 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -1553,7 +1553,7 @@ pci_brcm_trumanage_setup(struct serial_private *priv, #define FINTEK_RTS_INVERT BIT(5) /* We should do proper H/W transceiver setting before change to RS485 mode */ -static int pci_fintek_rs485_config(struct uart_port *port, +static int pci_fintek_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { struct pci_dev *pci_dev = to_pci_dev(port->dev); diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index c8ae0e8376d4..d4337d8346c8 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -664,7 +664,8 @@ EXPORT_SYMBOL_GPL(serial8250_em485_supported); * if the uart is incapable of driving RTS as a Transmit Enable signal in * hardware, relying on software emulation instead. */ -int serial8250_em485_config(struct uart_port *port, struct serial_rs485 *rs485) +int serial8250_em485_config(struct uart_port *port, struct ktermios *termios, + struct serial_rs485 *rs485) { struct uart_8250_port *up = up_to_u8250p(port); diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index eccd66625d25..c8f52945a4aa 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2197,7 +2197,7 @@ static int pl011_verify_port(struct uart_port *port, struct serial_struct *ser) return ret; } -static int pl011_rs485_config(struct uart_port *port, +static int pl011_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { struct uart_amba_port *uap = diff --git a/drivers/tty/serial/ar933x_uart.c b/drivers/tty/serial/ar933x_uart.c index ab2c5b2a1ce8..b73ce13683db 100644 --- a/drivers/tty/serial/ar933x_uart.c +++ b/drivers/tty/serial/ar933x_uart.c @@ -580,7 +580,7 @@ static const struct uart_ops ar933x_uart_ops = { .verify_port = ar933x_uart_verify_port, }; -static int ar933x_config_rs485(struct uart_port *port, +static int ar933x_config_rs485(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485conf) { struct ar933x_uart_port *up = diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c index 3a94c2bdda72..bc6004679585 100644 --- a/drivers/tty/serial/atmel_serial.c +++ b/drivers/tty/serial/atmel_serial.c @@ -283,7 +283,7 @@ static void atmel_tasklet_schedule(struct atmel_uart_port *atmel_port, } /* Enable or disable the rs485 support */ -static int atmel_config_rs485(struct uart_port *port, +static int atmel_config_rs485(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485conf) { struct atmel_uart_port *atmel_port = to_atmel_uart_port(port); diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c index d35414cb3e4e..8fe0494d4057 100644 --- a/drivers/tty/serial/fsl_lpuart.c +++ b/drivers/tty/serial/fsl_lpuart.c @@ -1355,7 +1355,7 @@ static void lpuart_dma_rx_free(struct uart_port *port) sport->dma_rx_cookie = -EINVAL; } -static int lpuart_config_rs485(struct uart_port *port, +static int lpuart_config_rs485(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { struct lpuart_port *sport = container_of(port, @@ -1385,7 +1385,7 @@ static int lpuart_config_rs485(struct uart_port *port, return 0; } -static int lpuart32_config_rs485(struct uart_port *port, +static int lpuart32_config_rs485(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { struct lpuart_port *sport = container_of(port, diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index f4edde54175f..3457006cea3f 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -1907,7 +1907,7 @@ static void imx_uart_poll_put_char(struct uart_port *port, unsigned char c) #endif /* called with port.lock taken and irqs off or from .probe without locking */ -static int imx_uart_rs485_config(struct uart_port *port, +static int imx_uart_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485conf) { struct imx_port *sport = (struct imx_port *)port; diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c index 4915a786e315..e162bfb44080 100644 --- a/drivers/tty/serial/max310x.c +++ b/drivers/tty/serial/max310x.c @@ -1036,7 +1036,7 @@ static void max310x_rs_proc(struct work_struct *ws) MAX310X_MODE2_ECHOSUPR_BIT, mode2); } -static int max310x_rs485_config(struct uart_port *port, +static int max310x_rs485_config(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { struct max310x_one *one = to_max310x_port(port); diff --git a/drivers/tty/serial/mcf.c b/drivers/tty/serial/mcf.c index 036f178e3d66..73c5287b8e5e 100644 --- a/drivers/tty/serial/mcf.c +++ b/drivers/tty/serial/mcf.c @@ -431,7 +431,8 @@ static int mcf_verify_port(struct uart_port *port, struct serial_struct *ser) /****************************************************************************/ /* Enable or disable the RS485 support */ -static int mcf_config_rs485(struct uart_port *port, struct serial_rs485 *rs485) +static int mcf_config_rs485(struct uart_port *port, struct ktermios *termios, + struct serial_rs485 *rs485) { unsigned char mr1, mr2; diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 52cb1a68b053..196bae704f85 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -1324,7 +1324,8 @@ static inline void serial_omap_add_console_port(struct uart_omap_port *up) /* Enable or disable the rs485 support */ static int -serial_omap_config_rs485(struct uart_port *port, struct serial_rs485 *rs485) +serial_omap_config_rs485(struct uart_port *port, struct ktermios *termios, + struct serial_rs485 *rs485) { struct uart_omap_port *up = to_uart_omap_port(port); unsigned int mode; diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index 2ceecaa4a478..8cb92a3b3fb8 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -1127,7 +1127,7 @@ static void sc16is7xx_set_termios(struct uart_port *port, spin_unlock_irqrestore(&port->lock, flags); } -static int sc16is7xx_config_rs485(struct uart_port *port, +static int sc16is7xx_config_rs485(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485) { struct sc16is7xx_port *s = dev_get_drvdata(port->dev); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 75ece750bedc..2529153c8979 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -1353,7 +1353,7 @@ int uart_rs485_config(struct uart_port *port) uart_sanitize_serial_rs485(port, rs485); - ret = port->rs485_config(port, rs485); + ret = port->rs485_config(port, NULL, rs485); if (ret) memset(rs485, 0, sizeof(*rs485)); @@ -1377,7 +1377,7 @@ static int uart_get_rs485_config(struct uart_port *port, return 0; } -static int uart_set_rs485_config(struct uart_port *port, +static int uart_set_rs485_config(struct tty_struct *tty, struct uart_port *port, struct serial_rs485 __user *rs485_user) { struct serial_rs485 rs485; @@ -1396,7 +1396,7 @@ static int uart_set_rs485_config(struct uart_port *port, uart_sanitize_serial_rs485(port, &rs485); spin_lock_irqsave(&port->lock, flags); - ret = port->rs485_config(port, &rs485); + ret = port->rs485_config(port, &tty->termios, &rs485); if (!ret) port->rs485 = rs485; spin_unlock_irqrestore(&port->lock, flags); @@ -1505,6 +1505,10 @@ uart_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) if (ret != -ENOIOCTLCMD) goto out; + /* rs485_config requires more locking than others */ + if (cmd == TIOCGRS485) + down_write(&tty->termios_rwsem); + mutex_lock(&port->mutex); uport = uart_port_check(state); @@ -1528,7 +1532,7 @@ uart_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) break; case TIOCSRS485: - ret = uart_set_rs485_config(uport, uarg); + ret = uart_set_rs485_config(tty, uport, uarg); break; case TIOCSISO7816: @@ -1545,6 +1549,8 @@ uart_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) } out_up: mutex_unlock(&port->mutex); + if (cmd == TIOCGRS485) + up_write(&tty->termios_rwsem); out: return ret; } diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c index db3dd9731ee1..13992e64a7df 100644 --- a/drivers/tty/serial/stm32-usart.c +++ b/drivers/tty/serial/stm32-usart.c @@ -97,7 +97,7 @@ static void stm32_usart_config_reg_rs485(u32 *cr1, u32 *cr3, u32 delay_ADE, *cr1 |= rs485_deat_dedt; } -static int stm32_usart_config_rs485(struct uart_port *port, +static int stm32_usart_config_rs485(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485conf) { struct stm32_port *stm32_port = to_stm32_port(port); diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index faaf2372c60d..b7b86ee3cb12 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -133,6 +133,7 @@ struct uart_port { unsigned int old); void (*handle_break)(struct uart_port *); int (*rs485_config)(struct uart_port *, + struct ktermios *termios, struct serial_rs485 *rs485); int (*iso7816_config)(struct uart_port *, struct serial_iso7816 *iso7816); -- cgit v1.2.3 From 38a523a2946d3a0961d141d477a1ee2b1f3bdbb1 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 27 Jun 2022 16:36:57 +0200 Subject: Revert "devcoredump: remove the useless gfp_t parameter in dev_coredumpv and dev_coredumpm" This reverts commit 77515ebaf01920e2db49e04672ef669a7c2907f2 as it causes build problems in linux-next. It needs to be reintroduced in a way that can allow the api to evolve and not require a "flag day" to catch all users. Link: https://lore.kernel.org/r/20220623160723.7a44b573@canb.auug.org.au Cc: Duoming Zhou Cc: Brian Norris Cc: Johannes Berg Reported-by: Stephen Rothwell Signed-off-by: Greg Kroah-Hartman --- drivers/base/devcoredump.c | 16 ++++++++++------ drivers/bluetooth/btmrvl_sdio.c | 2 +- drivers/bluetooth/hci_qca.c | 2 +- drivers/gpu/drm/etnaviv/etnaviv_dump.c | 2 +- drivers/gpu/drm/msm/disp/msm_disp_snapshot.c | 4 ++-- drivers/gpu/drm/msm/msm_gpu.c | 4 ++-- drivers/media/platform/qcom/venus/core.c | 2 +- drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c | 2 +- drivers/net/wireless/ath/ath10k/coredump.c | 2 +- drivers/net/wireless/ath/wil6210/wil_crash_dump.c | 2 +- drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c | 2 +- drivers/net/wireless/intel/iwlwifi/fw/dbg.c | 6 ++++-- drivers/net/wireless/marvell/mwifiex/main.c | 3 ++- drivers/net/wireless/mediatek/mt76/mt7615/mac.c | 3 ++- drivers/net/wireless/mediatek/mt76/mt7921/mac.c | 3 ++- drivers/net/wireless/realtek/rtw88/main.c | 2 +- drivers/net/wireless/realtek/rtw89/ser.c | 2 +- drivers/remoteproc/qcom_q6v5_mss.c | 2 +- drivers/remoteproc/remoteproc_coredump.c | 8 ++++---- include/drm/drm_print.h | 2 +- include/linux/devcoredump.h | 13 +++++++------ sound/soc/intel/avs/apl.c | 2 +- sound/soc/intel/avs/skl.c | 2 +- sound/soc/intel/catpt/dsp.c | 2 +- 24 files changed, 50 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c index 8535f0bd5dfb..f4d794d6bb85 100644 --- a/drivers/base/devcoredump.c +++ b/drivers/base/devcoredump.c @@ -173,13 +173,15 @@ static void devcd_freev(void *data) * @dev: the struct device for the crashed device * @data: vmalloc data containing the device coredump * @datalen: length of the data + * @gfp: allocation flags * * This function takes ownership of the vmalloc'ed data and will free * it when it is no longer used. See dev_coredumpm() for more information. */ -void dev_coredumpv(struct device *dev, void *data, size_t datalen) +void dev_coredumpv(struct device *dev, void *data, size_t datalen, + gfp_t gfp) { - dev_coredumpm(dev, NULL, data, datalen, devcd_readv, devcd_freev); + dev_coredumpm(dev, NULL, data, datalen, gfp, devcd_readv, devcd_freev); } EXPORT_SYMBOL_GPL(dev_coredumpv); @@ -234,6 +236,7 @@ static ssize_t devcd_read_from_sgtable(char *buffer, loff_t offset, * @owner: the module that contains the read/free functions, use %THIS_MODULE * @data: data cookie for the @read/@free functions * @datalen: length of the data + * @gfp: allocation flags * @read: function to read from the given buffer * @free: function to free the given buffer * @@ -243,7 +246,7 @@ static ssize_t devcd_read_from_sgtable(char *buffer, loff_t offset, * function will be called to free the data. */ void dev_coredumpm(struct device *dev, struct module *owner, - void *data, size_t datalen, + void *data, size_t datalen, gfp_t gfp, ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen), void (*free)(void *data)) @@ -265,7 +268,7 @@ void dev_coredumpm(struct device *dev, struct module *owner, if (!try_module_get(owner)) goto free; - devcd = kzalloc(sizeof(*devcd), GFP_KERNEL); + devcd = kzalloc(sizeof(*devcd), gfp); if (!devcd) goto put_module; @@ -315,6 +318,7 @@ EXPORT_SYMBOL_GPL(dev_coredumpm); * @dev: the struct device for the crashed device * @table: the dump data * @datalen: length of the data + * @gfp: allocation flags * * Creates a new device coredump for the given device. If a previous one hasn't * been read yet, the new coredump is discarded. The data lifetime is determined @@ -322,9 +326,9 @@ EXPORT_SYMBOL_GPL(dev_coredumpm); * it will free the data. */ void dev_coredumpsg(struct device *dev, struct scatterlist *table, - size_t datalen) + size_t datalen, gfp_t gfp) { - dev_coredumpm(dev, NULL, table, datalen, devcd_read_from_sgtable, + dev_coredumpm(dev, NULL, table, datalen, gfp, devcd_read_from_sgtable, devcd_free_sgtable); } EXPORT_SYMBOL_GPL(dev_coredumpsg); diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c index 9b9728719db2..b8ef66f89fc1 100644 --- a/drivers/bluetooth/btmrvl_sdio.c +++ b/drivers/bluetooth/btmrvl_sdio.c @@ -1515,7 +1515,7 @@ done: /* fw_dump_data will be free in device coredump release function * after 5 min */ - dev_coredumpv(&card->func->dev, fw_dump_data, fw_dump_len); + dev_coredumpv(&card->func->dev, fw_dump_data, fw_dump_len, GFP_KERNEL); BT_INFO("== btmrvl firmware dump to /sys/class/devcoredump end"); } diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 2e4074211ae9..eab34e24d944 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -1120,7 +1120,7 @@ static void qca_controller_memdump(struct work_struct *work) qca_memdump->ram_dump_size); memdump_buf = qca_memdump->memdump_buf_head; dev_coredumpv(&hu->serdev->dev, memdump_buf, - qca_memdump->received_dump); + qca_memdump->received_dump, GFP_KERNEL); cancel_delayed_work(&qca->ctrl_memdump_timeout); kfree(qca->qca_memdump); qca->qca_memdump = NULL; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index 519fcb234b3e..f418e0b75772 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -225,5 +225,5 @@ void etnaviv_core_dump(struct etnaviv_gem_submit *submit) etnaviv_core_dump_header(&iter, ETDUMP_BUF_END, iter.data); - dev_coredumpv(gpu->dev, iter.start, iter.data - iter.start); + dev_coredumpv(gpu->dev, iter.start, iter.data - iter.start, GFP_KERNEL); } diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c index f057d294c30b..e75b97127c0d 100644 --- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c +++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c @@ -74,8 +74,8 @@ static void _msm_disp_snapshot_work(struct kthread_work *work) * If there is a codedump pending for the device, the dev_coredumpm() * will also free new coredump state. */ - dev_coredumpm(disp_state->dev, THIS_MODULE, disp_state, 0, - disp_devcoredump_read, msm_disp_state_free); + dev_coredumpm(disp_state->dev, THIS_MODULE, disp_state, 0, GFP_KERNEL, + disp_devcoredump_read, msm_disp_state_free); } void msm_disp_snapshot_state(struct drm_device *drm_dev) diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 30576ced0a0a..eb8a6663f309 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -317,8 +317,8 @@ static void msm_gpu_crashstate_capture(struct msm_gpu *gpu, gpu->crashstate = state; /* FIXME: Release the crashstate if this errors out? */ - dev_coredumpm(gpu->dev->dev, THIS_MODULE, gpu, 0, - msm_gpu_devcoredump_read, msm_gpu_devcoredump_free); + dev_coredumpm(gpu->dev->dev, THIS_MODULE, gpu, 0, GFP_KERNEL, + msm_gpu_devcoredump_read, msm_gpu_devcoredump_free); } #else static void msm_gpu_crashstate_capture(struct msm_gpu *gpu, diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index db84dfb3fb11..877eca125803 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -49,7 +49,7 @@ static void venus_coredump(struct venus_core *core) memcpy(data, mem_va, mem_size); memunmap(mem_va); - dev_coredumpv(dev, data, mem_size); + dev_coredumpv(dev, data, mem_size, GFP_KERNEL); } static void venus_event_notify(struct venus_core *core, u32 event) diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c index fa520ab7c960..c991b30bc9f0 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-dump.c @@ -281,5 +281,5 @@ void mcp251xfd_dump(const struct mcp251xfd_priv *priv) mcp251xfd_dump_end(priv, &iter); dev_coredumpv(&priv->spi->dev, iter.start, - iter.data - iter.start); + iter.data - iter.start, GFP_KERNEL); } diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c index dc9237069921..fe6b6f97a916 100644 --- a/drivers/net/wireless/ath/ath10k/coredump.c +++ b/drivers/net/wireless/ath/ath10k/coredump.c @@ -1607,7 +1607,7 @@ int ath10k_coredump_submit(struct ath10k *ar) return -ENODATA; } - dev_coredumpv(ar->dev, dump, le32_to_cpu(dump->len)); + dev_coredumpv(ar->dev, dump, le32_to_cpu(dump->len), GFP_KERNEL); return 0; } diff --git a/drivers/net/wireless/ath/wil6210/wil_crash_dump.c b/drivers/net/wireless/ath/wil6210/wil_crash_dump.c index 79299609dd62..89c12cb2aaab 100644 --- a/drivers/net/wireless/ath/wil6210/wil_crash_dump.c +++ b/drivers/net/wireless/ath/wil6210/wil_crash_dump.c @@ -117,6 +117,6 @@ void wil_fw_core_dump(struct wil6210_priv *wil) /* fw_dump_data will be free in device coredump release function * after 5 min */ - dev_coredumpv(wil_to_dev(wil), fw_dump_data, fw_dump_size); + dev_coredumpv(wil_to_dev(wil), fw_dump_data, fw_dump_size, GFP_KERNEL); wil_info(wil, "fw core dumped, size %d bytes\n", fw_dump_size); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c index 87f3652ef3bd..eecf8a38d94a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c @@ -37,7 +37,7 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, return err; } - dev_coredumpv(bus->dev, dump, len + ramsize); + dev_coredumpv(bus->dev, dump, len + ramsize, GFP_KERNEL); return 0; } diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c index f2f7cf494a8c..abf49022edbe 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c @@ -2601,7 +2601,8 @@ static void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt, fw_error_dump.trans_ptr->data, fw_error_dump.trans_ptr->len, fw_error_dump.fwrt_len); - dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len); + dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len, + GFP_KERNEL); } vfree(fw_error_dump.fwrt_ptr); vfree(fw_error_dump.trans_ptr); @@ -2646,7 +2647,8 @@ static void iwl_fw_error_ini_dump(struct iwl_fw_runtime *fwrt, entry->data, entry->size, offs); offs += entry->size; } - dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len); + dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len, + GFP_KERNEL); } iwl_dump_ini_list_free(&dump_list); } diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c index 26fef0ab1b0d..ace7371c4773 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.c +++ b/drivers/net/wireless/marvell/mwifiex/main.c @@ -1115,7 +1115,8 @@ void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter) */ mwifiex_dbg(adapter, MSG, "== mwifiex dump information to /sys/class/devcoredump start\n"); - dev_coredumpv(adapter->dev, adapter->devdump_data, adapter->devdump_len); + dev_coredumpv(adapter->dev, adapter->devdump_data, adapter->devdump_len, + GFP_KERNEL); mwifiex_dbg(adapter, MSG, "== mwifiex dump information to /sys/class/devcoredump end\n"); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c index 5336fe8c668d..bd687f7de628 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c @@ -2421,5 +2421,6 @@ void mt7615_coredump_work(struct work_struct *work) dev_kfree_skb(skb); } - dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ); + dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ, + GFP_KERNEL); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index cac284f95ce0..a630ddbf19e5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -1630,7 +1630,8 @@ void mt7921_coredump_work(struct work_struct *work) } if (dump) - dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ); + dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ, + GFP_KERNEL); mt7921_reset(&dev->mt76); } diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index a276544cecdd..efabd5b1bf5b 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -414,7 +414,7 @@ static void rtw_fwcd_dump(struct rtw_dev *rtwdev) * framework. Note that a new dump will be discarded if a previous one * hasn't been released yet. */ - dev_coredumpv(rtwdev->dev, desc->data, desc->size); + dev_coredumpv(rtwdev->dev, desc->data, desc->size, GFP_KERNEL); } static void rtw_fwcd_free(struct rtw_dev *rtwdev, bool free_self) diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index d28fe01ad729..9e95ed972710 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -127,7 +127,7 @@ static void rtw89_ser_cd_send(struct rtw89_dev *rtwdev, * will be discarded if a previous one hasn't been released by * framework yet. */ - dev_coredumpv(rtwdev->dev, buf, sizeof(*buf)); + dev_coredumpv(rtwdev->dev, buf, sizeof(*buf), GFP_KERNEL); } static void rtw89_ser_cd_free(struct rtw89_dev *rtwdev, diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 813d87faef6c..af217de75e4d 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -597,7 +597,7 @@ static void q6v5_dump_mba_logs(struct q6v5 *qproc) data = vmalloc(MBA_LOG_SIZE); if (data) { memcpy(data, mba_region, MBA_LOG_SIZE); - dev_coredumpv(&rproc->dev, data, MBA_LOG_SIZE); + dev_coredumpv(&rproc->dev, data, MBA_LOG_SIZE, GFP_KERNEL); } memunmap(mba_region); } diff --git a/drivers/remoteproc/remoteproc_coredump.c b/drivers/remoteproc/remoteproc_coredump.c index cd55c2abd227..4b093420d98a 100644 --- a/drivers/remoteproc/remoteproc_coredump.c +++ b/drivers/remoteproc/remoteproc_coredump.c @@ -309,7 +309,7 @@ void rproc_coredump(struct rproc *rproc) phdr += elf_size_of_phdr(class); } if (dump_conf == RPROC_COREDUMP_ENABLED) { - dev_coredumpv(&rproc->dev, data, data_size); + dev_coredumpv(&rproc->dev, data, data_size, GFP_KERNEL); return; } @@ -318,7 +318,7 @@ void rproc_coredump(struct rproc *rproc) dump_state.header = data; init_completion(&dump_state.dump_done); - dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, + dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, GFP_KERNEL, rproc_coredump_read, rproc_coredump_free); /* @@ -449,7 +449,7 @@ void rproc_coredump_using_sections(struct rproc *rproc) } if (dump_conf == RPROC_COREDUMP_ENABLED) { - dev_coredumpv(&rproc->dev, data, data_size); + dev_coredumpv(&rproc->dev, data, data_size, GFP_KERNEL); return; } @@ -458,7 +458,7 @@ void rproc_coredump_using_sections(struct rproc *rproc) dump_state.header = data; init_completion(&dump_state.dump_done); - dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, + dev_coredumpm(&rproc->dev, NULL, &dump_state, data_size, GFP_KERNEL, rproc_coredump_read, rproc_coredump_free); /* Wait until the dump is read and free is called. Data is freed diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h index b41850366bcc..22fabdeed297 100644 --- a/include/drm/drm_print.h +++ b/include/drm/drm_print.h @@ -162,7 +162,7 @@ struct drm_print_iterator { * void makecoredump(...) * { * ... - * dev_coredumpm(dev, THIS_MODULE, data, 0, + * dev_coredumpm(dev, THIS_MODULE, data, 0, GFP_KERNEL, * coredump_read, ...) * } * diff --git a/include/linux/devcoredump.h b/include/linux/devcoredump.h index c7d840d824c3..c008169ed2c6 100644 --- a/include/linux/devcoredump.h +++ b/include/linux/devcoredump.h @@ -52,26 +52,27 @@ static inline void _devcd_free_sgtable(struct scatterlist *table) #ifdef CONFIG_DEV_COREDUMP -void dev_coredumpv(struct device *dev, void *data, size_t datalen); +void dev_coredumpv(struct device *dev, void *data, size_t datalen, + gfp_t gfp); void dev_coredumpm(struct device *dev, struct module *owner, - void *data, size_t datalen, + void *data, size_t datalen, gfp_t gfp, ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen), void (*free)(void *data)); void dev_coredumpsg(struct device *dev, struct scatterlist *table, - size_t datalen); + size_t datalen, gfp_t gfp); #else static inline void dev_coredumpv(struct device *dev, void *data, - size_t datalen) + size_t datalen, gfp_t gfp) { vfree(data); } static inline void dev_coredumpm(struct device *dev, struct module *owner, - void *data, size_t datalen, + void *data, size_t datalen, gfp_t gfp, ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen), void (*free)(void *data)) @@ -80,7 +81,7 @@ dev_coredumpm(struct device *dev, struct module *owner, } static inline void dev_coredumpsg(struct device *dev, struct scatterlist *table, - size_t datalen) + size_t datalen, gfp_t gfp) { _devcd_free_sgtable(table); } diff --git a/sound/soc/intel/avs/apl.c b/sound/soc/intel/avs/apl.c index 1ff57f1a483d..b8e2b23c9f64 100644 --- a/sound/soc/intel/avs/apl.c +++ b/sound/soc/intel/avs/apl.c @@ -164,7 +164,7 @@ static int apl_coredump(struct avs_dev *adev, union avs_notify_msg *msg) } while (offset < msg->ext.coredump.stack_dump_size); exit: - dev_coredumpv(adev->dev, dump, dump_size); + dev_coredumpv(adev->dev, dump, dump_size, GFP_KERNEL); return 0; } diff --git a/sound/soc/intel/avs/skl.c b/sound/soc/intel/avs/skl.c index 3413162768dc..bda5ec7510fe 100644 --- a/sound/soc/intel/avs/skl.c +++ b/sound/soc/intel/avs/skl.c @@ -88,7 +88,7 @@ static int skl_coredump(struct avs_dev *adev, union avs_notify_msg *msg) return -ENOMEM; memcpy_fromio(dump, avs_sram_addr(adev, AVS_FW_REGS_WINDOW), AVS_FW_REGS_SIZE); - dev_coredumpv(adev->dev, dump, AVS_FW_REGS_SIZE); + dev_coredumpv(adev->dev, dump, AVS_FW_REGS_SIZE, GFP_KERNEL); return 0; } diff --git a/sound/soc/intel/catpt/dsp.c b/sound/soc/intel/catpt/dsp.c index d2afe9ff1e3a..346bec000306 100644 --- a/sound/soc/intel/catpt/dsp.c +++ b/sound/soc/intel/catpt/dsp.c @@ -539,7 +539,7 @@ int catpt_coredump(struct catpt_dev *cdev) pos += CATPT_DMA_REGS_SIZE; } - dev_coredumpv(cdev->dev, dump, dump_size); + dev_coredumpv(cdev->dev, dump, dump_size, GFP_KERNEL); return 0; } -- cgit v1.2.3 From 086c00c71fc8d47db6983f419a45f9ee167de03f Mon Sep 17 00:00:00 2001 From: Imran Khan Date: Wed, 15 Jun 2022 12:10:56 +1000 Subject: kernfs: make ->attr.open RCU protected. After removal of kernfs_open_node->refcnt in the previous patch, kernfs_open_node_lock can be removed as well by making ->attr.open RCU protected. kernfs_put_open_node can delegate freeing to ->attr.open to RCU and other readers of ->attr.open can do so under rcu_read_(un)lock. Suggested by: Al Viro Acked-by: Tejun Heo Signed-off-by: Imran Khan Link: https://lore.kernel.org/r/20220615021059.862643-2-imran.f.khan@oracle.com Signed-off-by: Greg Kroah-Hartman --- fs/kernfs/file.c | 147 +++++++++++++++++++++++++++++++++---------------- include/linux/kernfs.h | 2 +- 2 files changed, 102 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 54b2a13ac9a2..22e7481e7b63 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -23,16 +23,16 @@ * for each kernfs_node with one or more open files. * * kernfs_node->attr.open points to kernfs_open_node. attr.open is - * protected by kernfs_open_node_lock. + * RCU protected. * * filp->private_data points to seq_file whose ->private points to * kernfs_open_file. kernfs_open_files are chained at * kernfs_open_node->files, which is protected by kernfs_open_file_mutex. */ -static DEFINE_SPINLOCK(kernfs_open_node_lock); static DEFINE_MUTEX(kernfs_open_file_mutex); struct kernfs_open_node { + struct rcu_head rcu_head; atomic_t event; wait_queue_head_t poll; struct list_head files; /* goes through kernfs_open_file.list */ @@ -51,6 +51,52 @@ struct kernfs_open_node { static DEFINE_SPINLOCK(kernfs_notify_lock); static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL; +/** + * kernfs_deref_open_node - Get kernfs_open_node corresponding to @kn. + * + * @of: associated kernfs_open_file instance. + * @kn: target kernfs_node. + * + * Fetch and return ->attr.open of @kn if @of->list is non empty. + * If @of->list is not empty we can safely assume that @of is on + * @kn->attr.open->files list and this guarantees that @kn->attr.open + * will not vanish i.e. dereferencing outside RCU read-side critical + * section is safe here. + * + * The caller needs to make sure that @of->list is not empty. + */ +static struct kernfs_open_node * +kernfs_deref_open_node(struct kernfs_open_file *of, struct kernfs_node *kn) +{ + struct kernfs_open_node *on; + + on = rcu_dereference_check(kn->attr.open, !list_empty(&of->list)); + + return on; +} + +/** + * kernfs_deref_open_node_protected - Get kernfs_open_node corresponding to @kn + * + * @kn: target kernfs_node. + * + * Fetch and return ->attr.open of @kn when caller holds the + * kernfs_open_file_mutex. + * + * Update of ->attr.open happens under kernfs_open_file_mutex. So when + * the caller guarantees that this mutex is being held, other updaters can't + * change ->attr.open and this means that we can safely deref ->attr.open + * outside RCU read-side critical section. + * + * The caller needs to make sure that kernfs_open_file_mutex is held. + */ +static struct kernfs_open_node * +kernfs_deref_open_node_protected(struct kernfs_node *kn) +{ + return rcu_dereference_protected(kn->attr.open, + lockdep_is_held(&kernfs_open_file_mutex)); +} + static struct kernfs_open_file *kernfs_of(struct file *file) { return ((struct seq_file *)file->private_data)->private; @@ -156,8 +202,12 @@ static void kernfs_seq_stop(struct seq_file *sf, void *v) static int kernfs_seq_show(struct seq_file *sf, void *v) { struct kernfs_open_file *of = sf->private; + struct kernfs_open_node *on = kernfs_deref_open_node(of, of->kn); - of->event = atomic_read(&of->kn->attr.open->event); + if (!on) + return -EINVAL; + + of->event = atomic_read(&on->event); return of->kn->attr.ops->seq_show(sf, v); } @@ -180,6 +230,7 @@ static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) struct kernfs_open_file *of = kernfs_of(iocb->ki_filp); ssize_t len = min_t(size_t, iov_iter_count(iter), PAGE_SIZE); const struct kernfs_ops *ops; + struct kernfs_open_node *on; char *buf; buf = of->prealloc_buf; @@ -201,7 +252,15 @@ static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) goto out_free; } - of->event = atomic_read(&of->kn->attr.open->event); + on = kernfs_deref_open_node(of, of->kn); + if (!on) { + len = -EINVAL; + mutex_unlock(&of->mutex); + goto out_free; + } + + of->event = atomic_read(&on->event); + ops = kernfs_ops(of->kn); if (ops->read) len = ops->read(of, buf, len, iocb->ki_pos); @@ -518,36 +577,29 @@ static int kernfs_get_open_node(struct kernfs_node *kn, { struct kernfs_open_node *on, *new_on = NULL; - retry: mutex_lock(&kernfs_open_file_mutex); - spin_lock_irq(&kernfs_open_node_lock); - - if (!kn->attr.open && new_on) { - kn->attr.open = new_on; - new_on = NULL; - } - - on = kn->attr.open; - if (on) - list_add_tail(&of->list, &on->files); - - spin_unlock_irq(&kernfs_open_node_lock); - mutex_unlock(&kernfs_open_file_mutex); + on = kernfs_deref_open_node_protected(kn); if (on) { - kfree(new_on); + list_add_tail(&of->list, &on->files); + mutex_unlock(&kernfs_open_file_mutex); return 0; + } else { + /* not there, initialize a new one */ + new_on = kmalloc(sizeof(*new_on), GFP_KERNEL); + if (!new_on) { + mutex_unlock(&kernfs_open_file_mutex); + return -ENOMEM; + } + atomic_set(&new_on->event, 1); + init_waitqueue_head(&new_on->poll); + INIT_LIST_HEAD(&new_on->files); + list_add_tail(&of->list, &new_on->files); + rcu_assign_pointer(kn->attr.open, new_on); } + mutex_unlock(&kernfs_open_file_mutex); - /* not there, initialize a new one and retry */ - new_on = kmalloc(sizeof(*new_on), GFP_KERNEL); - if (!new_on) - return -ENOMEM; - - atomic_set(&new_on->event, 1); - init_waitqueue_head(&new_on->poll); - INIT_LIST_HEAD(&new_on->files); - goto retry; + return 0; } /** @@ -566,24 +618,25 @@ static int kernfs_get_open_node(struct kernfs_node *kn, static void kernfs_unlink_open_file(struct kernfs_node *kn, struct kernfs_open_file *of) { - struct kernfs_open_node *on = kn->attr.open; - unsigned long flags; + struct kernfs_open_node *on; mutex_lock(&kernfs_open_file_mutex); - spin_lock_irqsave(&kernfs_open_node_lock, flags); + + on = kernfs_deref_open_node_protected(kn); + if (!on) { + mutex_unlock(&kernfs_open_file_mutex); + return; + } if (of) list_del(&of->list); - if (list_empty(&on->files)) - kn->attr.open = NULL; - else - on = NULL; + if (list_empty(&on->files)) { + rcu_assign_pointer(kn->attr.open, NULL); + kfree_rcu(on, rcu_head); + } - spin_unlock_irqrestore(&kernfs_open_node_lock, flags); mutex_unlock(&kernfs_open_file_mutex); - - kfree(on); } static int kernfs_fop_open(struct inode *inode, struct file *file) @@ -773,17 +826,16 @@ void kernfs_drain_open_files(struct kernfs_node *kn) * check under kernfs_open_file_mutex will ensure bailing out if * ->attr.open became NULL while waiting for the mutex. */ - if (!kn->attr.open) + if (!rcu_access_pointer(kn->attr.open)) return; mutex_lock(&kernfs_open_file_mutex); - if (!kn->attr.open) { + on = kernfs_deref_open_node_protected(kn); + if (!on) { mutex_unlock(&kernfs_open_file_mutex); return; } - on = kn->attr.open; - list_for_each_entry(of, &on->files, list) { struct inode *inode = file_inode(of->file); @@ -814,7 +866,10 @@ void kernfs_drain_open_files(struct kernfs_node *kn) __poll_t kernfs_generic_poll(struct kernfs_open_file *of, poll_table *wait) { struct kernfs_node *kn = kernfs_dentry_node(of->file->f_path.dentry); - struct kernfs_open_node *on = kn->attr.open; + struct kernfs_open_node *on = kernfs_deref_open_node(of, kn); + + if (!on) + return EPOLLERR; poll_wait(of->file, &on->poll, wait); @@ -921,13 +976,13 @@ void kernfs_notify(struct kernfs_node *kn) return; /* kick poll immediately */ - spin_lock_irqsave(&kernfs_open_node_lock, flags); - on = kn->attr.open; + rcu_read_lock(); + on = rcu_dereference(kn->attr.open); if (on) { atomic_inc(&on->event); wake_up_interruptible(&on->poll); } - spin_unlock_irqrestore(&kernfs_open_node_lock, flags); + rcu_read_unlock(); /* schedule work to kick fsnotify */ spin_lock_irqsave(&kernfs_notify_lock, flags); diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index e2ae15a6225e..13f54f078a52 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -114,7 +114,7 @@ struct kernfs_elem_symlink { struct kernfs_elem_attr { const struct kernfs_ops *ops; - struct kernfs_open_node *open; + struct kernfs_open_node __rcu *open; loff_t size; struct kernfs_node *notify_next; /* for kernfs_notify() */ }; -- cgit v1.2.3 From b8f35fa1188b84035c59d4842826c4e93a1b1c9f Mon Sep 17 00:00:00 2001 From: Imran Khan Date: Wed, 15 Jun 2022 12:10:57 +1000 Subject: kernfs: Change kernfs_notify_list to llist. At present kernfs_notify_list is implemented as a singly linked list of kernfs_node(s), where last element points to itself and value of ->attr.next tells if node is present on the list or not. Both addition and deletion to list happen under kernfs_notify_lock. Change kernfs_notify_list to llist so that addition to list can heppen locklessly. Suggested by: Al Viro Acked-by: Tejun Heo Signed-off-by: Imran Khan Link: https://lore.kernel.org/r/20220615021059.862643-3-imran.f.khan@oracle.com Signed-off-by: Greg Kroah-Hartman --- fs/kernfs/file.c | 47 ++++++++++++++++++++--------------------------- include/linux/kernfs.h | 2 +- 2 files changed, 21 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 22e7481e7b63..7d9138d2be3b 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -38,18 +38,16 @@ struct kernfs_open_node { struct list_head files; /* goes through kernfs_open_file.list */ }; -/* - * kernfs_notify() may be called from any context and bounces notifications - * through a work item. To minimize space overhead in kernfs_node, the - * pending queue is implemented as a singly linked list of kernfs_nodes. - * The list is terminated with the self pointer so that whether a - * kernfs_node is on the list or not can be determined by testing the next - * pointer for NULL. +/** + * attribute_to_node - get kernfs_node object corresponding to a kernfs attribute + * @ptr: &struct kernfs_elem_attr + * @type: struct kernfs_node + * @member: name of member (i.e attr) */ -#define KERNFS_NOTIFY_EOL ((void *)&kernfs_notify_list) +#define attribute_to_node(ptr, type, member) \ + container_of(ptr, type, member) -static DEFINE_SPINLOCK(kernfs_notify_lock); -static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL; +static LLIST_HEAD(kernfs_notify_list); /** * kernfs_deref_open_node - Get kernfs_open_node corresponding to @kn. @@ -902,18 +900,16 @@ static void kernfs_notify_workfn(struct work_struct *work) struct kernfs_node *kn; struct kernfs_super_info *info; struct kernfs_root *root; + struct llist_node *free; + struct kernfs_elem_attr *attr; repeat: /* pop one off the notify_list */ - spin_lock_irq(&kernfs_notify_lock); - kn = kernfs_notify_list; - if (kn == KERNFS_NOTIFY_EOL) { - spin_unlock_irq(&kernfs_notify_lock); + free = llist_del_first(&kernfs_notify_list); + if (free == NULL) return; - } - kernfs_notify_list = kn->attr.notify_next; - kn->attr.notify_next = NULL; - spin_unlock_irq(&kernfs_notify_lock); + attr = llist_entry(free, struct kernfs_elem_attr, notify_next); + kn = attribute_to_node(attr, struct kernfs_node, attr); root = kernfs_root(kn); /* kick fsnotify */ down_write(&root->kernfs_rwsem); @@ -969,12 +965,14 @@ repeat: void kernfs_notify(struct kernfs_node *kn) { static DECLARE_WORK(kernfs_notify_work, kernfs_notify_workfn); - unsigned long flags; struct kernfs_open_node *on; if (WARN_ON(kernfs_type(kn) != KERNFS_FILE)) return; + /* Because we are using llist for kernfs_notify_list */ + WARN_ON_ONCE(in_nmi()); + /* kick poll immediately */ rcu_read_lock(); on = rcu_dereference(kn->attr.open); @@ -985,14 +983,9 @@ void kernfs_notify(struct kernfs_node *kn) rcu_read_unlock(); /* schedule work to kick fsnotify */ - spin_lock_irqsave(&kernfs_notify_lock, flags); - if (!kn->attr.notify_next) { - kernfs_get(kn); - kn->attr.notify_next = kernfs_notify_list; - kernfs_notify_list = kn; - schedule_work(&kernfs_notify_work); - } - spin_unlock_irqrestore(&kernfs_notify_lock, flags); + kernfs_get(kn); + llist_add(&kn->attr.notify_next, &kernfs_notify_list); + schedule_work(&kernfs_notify_work); } EXPORT_SYMBOL_GPL(kernfs_notify); diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 13f54f078a52..2dd9c8df0f4f 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -116,7 +116,7 @@ struct kernfs_elem_attr { const struct kernfs_ops *ops; struct kernfs_open_node __rcu *open; loff_t size; - struct kernfs_node *notify_next; /* for kernfs_notify() */ + struct llist_node notify_next; /* for kernfs_notify() */ }; /* -- cgit v1.2.3 From 1d25b84e444ad66313c473407979ea9cd33deb3f Mon Sep 17 00:00:00 2001 From: Imran Khan Date: Wed, 15 Jun 2022 12:10:59 +1000 Subject: kernfs: Replace global kernfs_open_file_mutex with hashed mutexes. In current kernfs design a single mutex, kernfs_open_file_mutex, protects the list of kernfs_open_file instances corresponding to a sysfs attribute. So even if different tasks are opening or closing different sysfs files they can contend on osq_lock of this mutex. The contention is more apparent in large scale systems with few hundred CPUs where most of the CPUs have running tasks that are opening, accessing or closing sysfs files at any point of time. Using hashed mutexes in place of a single global mutex, can significantly reduce contention around global mutex and hence can provide better scalability. Moreover as these hashed mutexes are not part of kernfs_node objects we will not see any singnificant change in memory utilization of kernfs based file systems like sysfs, cgroupfs etc. Modify interface introduced in previous patch to make use of hashed mutexes. Use kernfs_node address as hashing key. Acked-by: Tejun Heo Signed-off-by: Imran Khan Link: https://lore.kernel.org/r/20220615021059.862643-5-imran.f.khan@oracle.com Signed-off-by: Greg Kroah-Hartman --- fs/kernfs/file.c | 17 +++----------- fs/kernfs/kernfs-internal.h | 4 ++++ fs/kernfs/mount.c | 19 +++++++++++++++ include/linux/kernfs.h | 57 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 3b354caad6b5..bb933221b4ba 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -18,19 +18,6 @@ #include "kernfs-internal.h" -/* - * There's one kernfs_open_file for each open file and one kernfs_open_node - * for each kernfs_node with one or more open files. - * - * kernfs_node->attr.open points to kernfs_open_node. attr.open is - * RCU protected. - * - * filp->private_data points to seq_file whose ->private points to - * kernfs_open_file. kernfs_open_files are chained at - * kernfs_open_node->files, which is protected by kernfs_open_file_mutex. - */ -static DEFINE_MUTEX(kernfs_open_file_mutex); - struct kernfs_open_node { struct rcu_head rcu_head; atomic_t event; @@ -51,7 +38,9 @@ static LLIST_HEAD(kernfs_notify_list); static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn) { - return &kernfs_open_file_mutex; + int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS); + + return &kernfs_locks->open_file_mutex[idx]; } static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn) diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index eeaa779b929c..3ae214d02d44 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -164,4 +164,8 @@ void kernfs_drain_open_files(struct kernfs_node *kn); */ extern const struct inode_operations kernfs_symlink_iops; +/* + * kernfs locks + */ +extern struct kernfs_global_locks *kernfs_locks; #endif /* __KERNFS_INTERNAL_H */ diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index cfa79715fc1a..d0859f72d2d6 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -20,6 +20,7 @@ #include "kernfs-internal.h" struct kmem_cache *kernfs_node_cache, *kernfs_iattrs_cache; +struct kernfs_global_locks *kernfs_locks; static int kernfs_sop_show_options(struct seq_file *sf, struct dentry *dentry) { @@ -387,6 +388,22 @@ void kernfs_kill_sb(struct super_block *sb) kfree(info); } +static void __init kernfs_mutex_init(void) +{ + int count; + + for (count = 0; count < NR_KERNFS_LOCKS; count++) + mutex_init(&kernfs_locks->open_file_mutex[count]); +} + +static void __init kernfs_lock_init(void) +{ + kernfs_locks = kmalloc(sizeof(struct kernfs_global_locks), GFP_KERNEL); + WARN_ON(!kernfs_locks); + + kernfs_mutex_init(); +} + void __init kernfs_init(void) { kernfs_node_cache = kmem_cache_create("kernfs_node_cache", @@ -397,4 +414,6 @@ void __init kernfs_init(void) kernfs_iattrs_cache = kmem_cache_create("kernfs_iattrs_cache", sizeof(struct kernfs_iattrs), 0, SLAB_PANIC, NULL); + + kernfs_lock_init(); } diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 2dd9c8df0f4f..13e703f615f7 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -18,6 +18,7 @@ #include #include #include +#include struct file; struct dentry; @@ -34,6 +35,62 @@ struct kernfs_fs_context; struct kernfs_open_node; struct kernfs_iattrs; +/* + * NR_KERNFS_LOCK_BITS determines size (NR_KERNFS_LOCKS) of hash + * table of locks. + * Having a small hash table would impact scalability, since + * more and more kernfs_node objects will end up using same lock + * and having a very large hash table would waste memory. + * + * At the moment size of hash table of locks is being set based on + * the number of CPUs as follows: + * + * NR_CPU NR_KERNFS_LOCK_BITS NR_KERNFS_LOCKS + * 1 1 2 + * 2-3 2 4 + * 4-7 4 16 + * 8-15 6 64 + * 16-31 8 256 + * 32 and more 10 1024 + * + * The above relation between NR_CPU and number of locks is based + * on some internal experimentation which involved booting qemu + * with different values of smp, performing some sysfs operations + * on all CPUs and observing how increase in number of locks impacts + * completion time of these sysfs operations on each CPU. + */ +#ifdef CONFIG_SMP +#define NR_KERNFS_LOCK_BITS (2 * (ilog2(NR_CPUS < 32 ? NR_CPUS : 32))) +#else +#define NR_KERNFS_LOCK_BITS 1 +#endif + +#define NR_KERNFS_LOCKS (1 << NR_KERNFS_LOCK_BITS) + +/* + * There's one kernfs_open_file for each open file and one kernfs_open_node + * for each kernfs_node with one or more open files. + * + * filp->private_data points to seq_file whose ->private points to + * kernfs_open_file. + * + * kernfs_open_files are chained at kernfs_open_node->files, which is + * protected by kernfs_global_locks.open_file_mutex[i]. + * + * To reduce possible contention in sysfs access, arising due to single + * locks, use an array of locks (e.g. open_file_mutex) and use kernfs_node + * object address as hash keys to get the index of these locks. + * + * Hashed mutexes are safe to use here because operations using these don't + * rely on global exclusion. + * + * In future we intend to replace other global locks with hashed ones as well. + * kernfs_global_locks acts as a holder for all such hash tables. + */ +struct kernfs_global_locks { + struct mutex open_file_mutex[NR_KERNFS_LOCKS]; +}; + enum kernfs_node_type { KERNFS_DIR = 0x0001, KERNFS_FILE = 0x0002, -- cgit v1.2.3 From 8f486cab263ca1b761c1aab9b36975afd355b799 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Thu, 23 Jun 2022 01:03:42 -0700 Subject: driver core: fw_devlink: Allow firmware to mark devices as best effort When firmware sets the FWNODE_FLAG_BEST_EFFORT flag for a fwnode, fw_devlink will do a best effort ordering for that device where it'll only enforce the probe/suspend/resume ordering of that device with suppliers that have drivers. The driver of that device can then decide if it wants to defer probe or probe without the suppliers. This will be useful for avoid probe delays of the console device that were caused by commit 71066545b48e ("driver core: Set fw_devlink.strict=1 by default"). Fixes: 71066545b48e ("driver core: Set fw_devlink.strict=1 by default") Reported-by: Sascha Hauer Reported-by: Peng Fan Tested-by: Peng Fan Signed-off-by: Saravana Kannan Link: https://lore.kernel.org/r/20220623080344.783549-2-saravanak@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 3 ++- include/linux/fwnode.h | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 839f64485a55..ccdd5b4295de 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -968,7 +968,8 @@ static void device_links_missing_supplier(struct device *dev) static bool dev_is_best_effort(struct device *dev) { - return fw_devlink_best_effort && dev->can_match; + return (fw_devlink_best_effort && dev->can_match) || + (dev->fwnode && (dev->fwnode->flags & FWNODE_FLAG_BEST_EFFORT)); } /** diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 9a81c4410b9f..89b9bdfca925 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -27,11 +27,15 @@ struct device; * driver needs its child devices to be bound with * their respective drivers as soon as they are * added. + * BEST_EFFORT: The fwnode/device needs to probe early and might be missing some + * suppliers. Only enforce ordering with suppliers that have + * drivers. */ #define FWNODE_FLAG_LINKS_ADDED BIT(0) #define FWNODE_FLAG_NOT_DEVICE BIT(1) #define FWNODE_FLAG_INITIALIZED BIT(2) #define FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD BIT(3) +#define FWNODE_FLAG_BEST_EFFORT BIT(4) struct fwnode_handle { struct fwnode_handle *secondary; -- cgit v1.2.3 From d1877e639bc6bf1c3131eda3f9ede73f8da96c22 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Wed, 8 Jun 2022 12:55:13 -0600 Subject: vfio: de-extern-ify function prototypes The use of 'extern' in function prototypes has been disrecommended in the kernel coding style for several years now, remove them from all vfio related files so contributors no longer need to decide between style and consistency. Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Reviewed-by: Eric Farman Reviewed-by: Eric Auger Reviewed-by: Cornelia Huck Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/165471414407.203056.474032786990662279.stgit@omen Signed-off-by: Alex Williamson --- Documentation/driver-api/vfio-mediated-device.rst | 10 ++-- drivers/s390/cio/vfio_ccw_cp.h | 12 ++-- drivers/s390/cio/vfio_ccw_private.h | 6 +- drivers/vfio/fsl-mc/vfio_fsl_mc_private.h | 2 +- drivers/vfio/platform/vfio_platform_private.h | 21 ++++--- include/linux/vfio.h | 70 +++++++++++------------ include/linux/vfio_pci_core.h | 65 +++++++++++---------- 7 files changed, 91 insertions(+), 95 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst index eded8719180f..1c57815619fd 100644 --- a/Documentation/driver-api/vfio-mediated-device.rst +++ b/Documentation/driver-api/vfio-mediated-device.rst @@ -114,11 +114,11 @@ to register and unregister itself with the core driver: * Register:: - extern int mdev_register_driver(struct mdev_driver *drv); + int mdev_register_driver(struct mdev_driver *drv); * Unregister:: - extern void mdev_unregister_driver(struct mdev_driver *drv); + void mdev_unregister_driver(struct mdev_driver *drv); The mediated bus driver's probe function should create a vfio_device on top of the mdev_device and connect it to an appropriate implementation of @@ -127,8 +127,8 @@ vfio_device_ops. When a driver wants to add the GUID creation sysfs to an existing device it has probe'd to then it should call:: - extern int mdev_register_device(struct device *dev, - struct mdev_driver *mdev_driver); + int mdev_register_device(struct device *dev, + struct mdev_driver *mdev_driver); This will provide the 'mdev_supported_types/XX/create' files which can then be used to trigger the creation of a mdev_device. The created mdev_device will be @@ -136,7 +136,7 @@ attached to the specified driver. When the driver needs to remove itself it calls:: - extern void mdev_unregister_device(struct device *dev); + void mdev_unregister_device(struct device *dev); Which will unbind and destroy all the created mdevs and remove the sysfs files. diff --git a/drivers/s390/cio/vfio_ccw_cp.h b/drivers/s390/cio/vfio_ccw_cp.h index e4c436199b4c..3194d887e08e 100644 --- a/drivers/s390/cio/vfio_ccw_cp.h +++ b/drivers/s390/cio/vfio_ccw_cp.h @@ -41,11 +41,11 @@ struct channel_program { struct ccw1 *guest_cp; }; -extern int cp_init(struct channel_program *cp, union orb *orb); -extern void cp_free(struct channel_program *cp); -extern int cp_prefetch(struct channel_program *cp); -extern union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm); -extern void cp_update_scsw(struct channel_program *cp, union scsw *scsw); -extern bool cp_iova_pinned(struct channel_program *cp, u64 iova); +int cp_init(struct channel_program *cp, union orb *orb); +void cp_free(struct channel_program *cp); +int cp_prefetch(struct channel_program *cp); +union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm); +void cp_update_scsw(struct channel_program *cp, union scsw *scsw); +bool cp_iova_pinned(struct channel_program *cp, u64 iova); #endif diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index 7272eb788612..b7163bac8cc7 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -119,10 +119,10 @@ struct vfio_ccw_private { struct work_struct crw_work; } __aligned(8); -extern int vfio_ccw_mdev_reg(struct subchannel *sch); -extern void vfio_ccw_mdev_unreg(struct subchannel *sch); +int vfio_ccw_mdev_reg(struct subchannel *sch); +void vfio_ccw_mdev_unreg(struct subchannel *sch); -extern int vfio_ccw_sch_quiesce(struct subchannel *sch); +int vfio_ccw_sch_quiesce(struct subchannel *sch); extern struct mdev_driver vfio_ccw_mdev_driver; diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h b/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h index 4ad63ececb91..7a29f572f93d 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h @@ -39,7 +39,7 @@ struct vfio_fsl_mc_device { struct vfio_fsl_mc_irq *mc_irqs; }; -extern int vfio_fsl_mc_set_irqs_ioctl(struct vfio_fsl_mc_device *vdev, +int vfio_fsl_mc_set_irqs_ioctl(struct vfio_fsl_mc_device *vdev, u32 flags, unsigned int index, unsigned int start, unsigned int count, void *data); diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h index 520d2a8e8375..691b43f4b2b2 100644 --- a/drivers/vfio/platform/vfio_platform_private.h +++ b/drivers/vfio/platform/vfio_platform_private.h @@ -78,21 +78,20 @@ struct vfio_platform_reset_node { vfio_platform_reset_fn_t of_reset; }; -extern int vfio_platform_probe_common(struct vfio_platform_device *vdev, - struct device *dev); +int vfio_platform_probe_common(struct vfio_platform_device *vdev, + struct device *dev); void vfio_platform_remove_common(struct vfio_platform_device *vdev); -extern int vfio_platform_irq_init(struct vfio_platform_device *vdev); -extern void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev); +int vfio_platform_irq_init(struct vfio_platform_device *vdev); +void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev); -extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, - uint32_t flags, unsigned index, - unsigned start, unsigned count, - void *data); +int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, + uint32_t flags, unsigned index, + unsigned start, unsigned count, void *data); -extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n); -extern void vfio_platform_unregister_reset(const char *compat, - vfio_platform_reset_fn_t fn); +void __vfio_platform_register_reset(struct vfio_platform_reset_node *n); +void vfio_platform_unregister_reset(const char *compat, + vfio_platform_reset_fn_t fn); #define vfio_platform_register_reset(__compat, __reset) \ static struct vfio_platform_reset_node __reset ## _node = { \ .owner = THIS_MODULE, \ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index aa888cc51757..49580fa2073a 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -140,19 +140,19 @@ int vfio_mig_get_next_state(struct vfio_device *device, /* * External user API */ -extern struct iommu_group *vfio_file_iommu_group(struct file *file); -extern bool vfio_file_enforced_coherent(struct file *file); -extern void vfio_file_set_kvm(struct file *file, struct kvm *kvm); -extern bool vfio_file_has_dev(struct file *file, struct vfio_device *device); +struct iommu_group *vfio_file_iommu_group(struct file *file); +bool vfio_file_enforced_coherent(struct file *file); +void vfio_file_set_kvm(struct file *file, struct kvm *kvm); +bool vfio_file_has_dev(struct file *file, struct vfio_device *device); #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) -extern int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, - int npage, int prot, unsigned long *phys_pfn); -extern int vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, - int npage); -extern int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, - void *data, size_t len, bool write); +int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, + int npage, int prot, unsigned long *phys_pfn); +int vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, + int npage); +int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, + void *data, size_t len, bool write); /* each type has independent events */ enum vfio_notify_type { @@ -162,13 +162,13 @@ enum vfio_notify_type { /* events for VFIO_IOMMU_NOTIFY */ #define VFIO_IOMMU_NOTIFY_DMA_UNMAP BIT(0) -extern int vfio_register_notifier(struct vfio_device *device, - enum vfio_notify_type type, - unsigned long *required_events, - struct notifier_block *nb); -extern int vfio_unregister_notifier(struct vfio_device *device, - enum vfio_notify_type type, - struct notifier_block *nb); +int vfio_register_notifier(struct vfio_device *device, + enum vfio_notify_type type, + unsigned long *required_events, + struct notifier_block *nb); +int vfio_unregister_notifier(struct vfio_device *device, + enum vfio_notify_type type, + struct notifier_block *nb); /* @@ -178,25 +178,24 @@ struct vfio_info_cap { struct vfio_info_cap_header *buf; size_t size; }; -extern struct vfio_info_cap_header *vfio_info_cap_add( - struct vfio_info_cap *caps, size_t size, u16 id, u16 version); -extern void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset); +struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, + size_t size, u16 id, + u16 version); +void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset); -extern int vfio_info_add_capability(struct vfio_info_cap *caps, - struct vfio_info_cap_header *cap, - size_t size); +int vfio_info_add_capability(struct vfio_info_cap *caps, + struct vfio_info_cap_header *cap, size_t size); -extern int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, - int num_irqs, int max_irq_type, - size_t *data_size); +int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, + int num_irqs, int max_irq_type, + size_t *data_size); struct pci_dev; #if IS_ENABLED(CONFIG_VFIO_SPAPR_EEH) -extern void vfio_spapr_pci_eeh_open(struct pci_dev *pdev); -extern void vfio_spapr_pci_eeh_release(struct pci_dev *pdev); -extern long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group, - unsigned int cmd, - unsigned long arg); +void vfio_spapr_pci_eeh_open(struct pci_dev *pdev); +void vfio_spapr_pci_eeh_release(struct pci_dev *pdev); +long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group, unsigned int cmd, + unsigned long arg); #else static inline void vfio_spapr_pci_eeh_open(struct pci_dev *pdev) { @@ -230,10 +229,9 @@ struct virqfd { struct virqfd **pvirqfd; }; -extern int vfio_virqfd_enable(void *opaque, - int (*handler)(void *, void *), - void (*thread)(void *, void *), - void *data, struct virqfd **pvirqfd, int fd); -extern void vfio_virqfd_disable(struct virqfd **pvirqfd); +int vfio_virqfd_enable(void *opaque, int (*handler)(void *, void *), + void (*thread)(void *, void *), void *data, + struct virqfd **pvirqfd, int fd); +void vfio_virqfd_disable(struct virqfd **pvirqfd); #endif /* VFIO_H */ diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 23c176d4b073..22de2bce6394 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -147,23 +147,23 @@ struct vfio_pci_core_device { #define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev))) #define irq_is(vdev, type) (vdev->irq_type == type) -extern void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev); -extern void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev); +void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev); +void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev); -extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, - uint32_t flags, unsigned index, - unsigned start, unsigned count, void *data); +int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, + uint32_t flags, unsigned index, + unsigned start, unsigned count, void *data); -extern ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, - char __user *buf, size_t count, - loff_t *ppos, bool iswrite); +ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, + char __user *buf, size_t count, + loff_t *ppos, bool iswrite); -extern ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, - size_t count, loff_t *ppos, bool iswrite); +ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); #ifdef CONFIG_VFIO_PCI_VGA -extern ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, - size_t count, loff_t *ppos, bool iswrite); +ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); #else static inline ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, size_t count, @@ -173,32 +173,31 @@ static inline ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, } #endif -extern long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, - uint64_t data, int count, int fd); +long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, + uint64_t data, int count, int fd); -extern int vfio_pci_init_perm_bits(void); -extern void vfio_pci_uninit_perm_bits(void); +int vfio_pci_init_perm_bits(void); +void vfio_pci_uninit_perm_bits(void); -extern int vfio_config_init(struct vfio_pci_core_device *vdev); -extern void vfio_config_free(struct vfio_pci_core_device *vdev); +int vfio_config_init(struct vfio_pci_core_device *vdev); +void vfio_config_free(struct vfio_pci_core_device *vdev); -extern int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, - unsigned int type, unsigned int subtype, - const struct vfio_pci_regops *ops, - size_t size, u32 flags, void *data); +int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, + unsigned int type, unsigned int subtype, + const struct vfio_pci_regops *ops, + size_t size, u32 flags, void *data); -extern int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, - pci_power_t state); +int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, + pci_power_t state); -extern bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev); -extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device - *vdev); -extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev); -extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, - u16 cmd); +bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev); +void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev); +u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev); +void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, + u16 cmd); #ifdef CONFIG_VFIO_PCI_IGD -extern int vfio_pci_igd_init(struct vfio_pci_core_device *vdev); +int vfio_pci_igd_init(struct vfio_pci_core_device *vdev); #else static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) { @@ -207,8 +206,8 @@ static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) #endif #ifdef CONFIG_S390 -extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, - struct vfio_info_cap *caps); +int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, + struct vfio_info_cap *caps); #else static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, struct vfio_info_cap *caps) -- cgit v1.2.3 From ee65728e103bb7dd99d8604bf6c7aa89c7d7e446 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 27 Jun 2022 09:00:26 +0300 Subject: docs: rename Documentation/vm to Documentation/mm so it will be consistent with code mm directory and with Documentation/admin-guide/mm and won't be confused with virtual machines. Signed-off-by: Mike Rapoport Suggested-by: Matthew Wilcox Tested-by: Ira Weiny Acked-by: Jonathan Corbet Acked-by: Wu XiangCheng --- Documentation/ABI/testing/sysfs-kernel-mm-ksm | 2 +- Documentation/ABI/testing/sysfs-kernel-slab | 4 +- Documentation/admin-guide/kernel-parameters.txt | 10 +- Documentation/admin-guide/mm/concepts.rst | 2 +- Documentation/admin-guide/mm/damon/index.rst | 2 +- Documentation/admin-guide/mm/damon/reclaim.rst | 2 +- Documentation/admin-guide/mm/damon/usage.rst | 8 +- Documentation/admin-guide/sysctl/vm.rst | 2 +- Documentation/core-api/index.rst | 2 +- Documentation/filesystems/proc.rst | 2 +- Documentation/index.rst | 2 +- Documentation/mm/active_mm.rst | 91 ++++ Documentation/mm/arch_pgtable_helpers.rst | 260 +++++++++ Documentation/mm/balance.rst | 102 ++++ Documentation/mm/bootmem.rst | 5 + Documentation/mm/damon/api.rst | 20 + Documentation/mm/damon/design.rst | 176 ++++++ Documentation/mm/damon/faq.rst | 50 ++ Documentation/mm/damon/index.rst | 29 + Documentation/mm/free_page_reporting.rst | 40 ++ Documentation/mm/frontswap.rst | 266 +++++++++ Documentation/mm/highmem.rst | 167 ++++++ Documentation/mm/hmm.rst | 452 ++++++++++++++++ Documentation/mm/hugetlbfs_reserv.rst | 596 +++++++++++++++++++++ Documentation/mm/hwpoison.rst | 184 +++++++ Documentation/mm/index.rst | 68 +++ Documentation/mm/ksm.rst | 87 +++ Documentation/mm/memory-model.rst | 177 ++++++ Documentation/mm/mmu_notifier.rst | 99 ++++ Documentation/mm/numa.rst | 150 ++++++ Documentation/mm/oom.rst | 5 + Documentation/mm/overcommit-accounting.rst | 88 +++ Documentation/mm/page_allocation.rst | 5 + Documentation/mm/page_cache.rst | 5 + Documentation/mm/page_frags.rst | 45 ++ Documentation/mm/page_migration.rst | 288 ++++++++++ Documentation/mm/page_owner.rst | 196 +++++++ Documentation/mm/page_reclaim.rst | 5 + Documentation/mm/page_table_check.rst | 56 ++ Documentation/mm/page_tables.rst | 5 + Documentation/mm/physical_memory.rst | 5 + Documentation/mm/process_addrs.rst | 5 + Documentation/mm/remap_file_pages.rst | 33 ++ Documentation/mm/shmfs.rst | 5 + Documentation/mm/slab.rst | 5 + Documentation/mm/slub.rst | 452 ++++++++++++++++ Documentation/mm/split_page_table_lock.rst | 100 ++++ Documentation/mm/swap.rst | 5 + Documentation/mm/transhuge.rst | 187 +++++++ Documentation/mm/unevictable-lru.rst | 554 +++++++++++++++++++ Documentation/mm/vmalloc.rst | 5 + Documentation/mm/vmalloced-kernel-stacks.rst | 153 ++++++ Documentation/mm/vmemmap_dedup.rst | 223 ++++++++ Documentation/mm/z3fold.rst | 30 ++ Documentation/mm/zsmalloc.rst | 82 +++ .../zh_CN/admin-guide/mm/damon/index.rst | 2 +- .../zh_CN/admin-guide/mm/damon/reclaim.rst | 2 +- .../zh_CN/admin-guide/mm/damon/usage.rst | 8 +- .../translations/zh_CN/core-api/index.rst | 2 +- Documentation/translations/zh_CN/index.rst | 2 +- Documentation/translations/zh_CN/mm/active_mm.rst | 85 +++ Documentation/translations/zh_CN/mm/balance.rst | 81 +++ Documentation/translations/zh_CN/mm/damon/api.rst | 32 ++ .../translations/zh_CN/mm/damon/design.rst | 140 +++++ Documentation/translations/zh_CN/mm/damon/faq.rst | 48 ++ .../translations/zh_CN/mm/damon/index.rst | 32 ++ .../translations/zh_CN/mm/free_page_reporting.rst | 38 ++ Documentation/translations/zh_CN/mm/frontswap.rst | 196 +++++++ Documentation/translations/zh_CN/mm/highmem.rst | 128 +++++ Documentation/translations/zh_CN/mm/hmm.rst | 361 +++++++++++++ .../translations/zh_CN/mm/hugetlbfs_reserv.rst | 436 +++++++++++++++ Documentation/translations/zh_CN/mm/hwpoison.rst | 166 ++++++ Documentation/translations/zh_CN/mm/index.rst | 54 ++ Documentation/translations/zh_CN/mm/ksm.rst | 70 +++ .../translations/zh_CN/mm/memory-model.rst | 135 +++++ .../translations/zh_CN/mm/mmu_notifier.rst | 97 ++++ Documentation/translations/zh_CN/mm/numa.rst | 101 ++++ .../zh_CN/mm/overcommit-accounting.rst | 86 +++ Documentation/translations/zh_CN/mm/page_frags.rst | 38 ++ Documentation/translations/zh_CN/mm/page_owner.rst | 116 ++++ .../translations/zh_CN/mm/page_table_check.rst | 56 ++ .../translations/zh_CN/mm/remap_file_pages.rst | 32 ++ .../zh_CN/mm/split_page_table_lock.rst | 96 ++++ Documentation/translations/zh_CN/mm/z3fold.rst | 31 ++ Documentation/translations/zh_CN/mm/zsmalloc.rst | 78 +++ Documentation/translations/zh_CN/vm/active_mm.rst | 85 --- Documentation/translations/zh_CN/vm/balance.rst | 81 --- Documentation/translations/zh_CN/vm/damon/api.rst | 32 -- .../translations/zh_CN/vm/damon/design.rst | 140 ----- Documentation/translations/zh_CN/vm/damon/faq.rst | 48 -- .../translations/zh_CN/vm/damon/index.rst | 33 -- .../translations/zh_CN/vm/free_page_reporting.rst | 38 -- Documentation/translations/zh_CN/vm/frontswap.rst | 196 ------- Documentation/translations/zh_CN/vm/highmem.rst | 128 ----- Documentation/translations/zh_CN/vm/hmm.rst | 361 ------------- .../translations/zh_CN/vm/hugetlbfs_reserv.rst | 436 --------------- Documentation/translations/zh_CN/vm/hwpoison.rst | 166 ------ Documentation/translations/zh_CN/vm/index.rst | 54 -- Documentation/translations/zh_CN/vm/ksm.rst | 70 --- .../translations/zh_CN/vm/memory-model.rst | 135 ----- .../translations/zh_CN/vm/mmu_notifier.rst | 97 ---- Documentation/translations/zh_CN/vm/numa.rst | 101 ---- .../zh_CN/vm/overcommit-accounting.rst | 86 --- Documentation/translations/zh_CN/vm/page_frags.rst | 38 -- Documentation/translations/zh_CN/vm/page_owner.rst | 116 ---- .../translations/zh_CN/vm/page_table_check.rst | 56 -- .../translations/zh_CN/vm/remap_file_pages.rst | 32 -- .../zh_CN/vm/split_page_table_lock.rst | 96 ---- Documentation/translations/zh_CN/vm/z3fold.rst | 31 -- Documentation/translations/zh_CN/vm/zsmalloc.rst | 78 --- Documentation/translations/zh_TW/index.rst | 2 +- Documentation/vm/.gitignore | 3 - Documentation/vm/active_mm.rst | 91 ---- Documentation/vm/arch_pgtable_helpers.rst | 260 --------- Documentation/vm/balance.rst | 102 ---- Documentation/vm/bootmem.rst | 5 - Documentation/vm/damon/api.rst | 20 - Documentation/vm/damon/design.rst | 176 ------ Documentation/vm/damon/faq.rst | 50 -- Documentation/vm/damon/index.rst | 29 - Documentation/vm/free_page_reporting.rst | 40 -- Documentation/vm/frontswap.rst | 266 --------- Documentation/vm/highmem.rst | 167 ------ Documentation/vm/hmm.rst | 452 ---------------- Documentation/vm/hugetlbfs_reserv.rst | 596 --------------------- Documentation/vm/hwpoison.rst | 184 ------- Documentation/vm/index.rst | 68 --- Documentation/vm/ksm.rst | 87 --- Documentation/vm/memory-model.rst | 177 ------ Documentation/vm/mmu_notifier.rst | 99 ---- Documentation/vm/numa.rst | 150 ------ Documentation/vm/oom.rst | 5 - Documentation/vm/overcommit-accounting.rst | 88 --- Documentation/vm/page_allocation.rst | 5 - Documentation/vm/page_cache.rst | 5 - Documentation/vm/page_frags.rst | 45 -- Documentation/vm/page_migration.rst | 288 ---------- Documentation/vm/page_owner.rst | 196 ------- Documentation/vm/page_reclaim.rst | 5 - Documentation/vm/page_table_check.rst | 56 -- Documentation/vm/page_tables.rst | 5 - Documentation/vm/physical_memory.rst | 5 - Documentation/vm/process_addrs.rst | 5 - Documentation/vm/remap_file_pages.rst | 33 -- Documentation/vm/shmfs.rst | 5 - Documentation/vm/slab.rst | 5 - Documentation/vm/slub.rst | 452 ---------------- Documentation/vm/split_page_table_lock.rst | 100 ---- Documentation/vm/swap.rst | 5 - Documentation/vm/transhuge.rst | 187 ------- Documentation/vm/unevictable-lru.rst | 554 ------------------- Documentation/vm/vmalloc.rst | 5 - Documentation/vm/vmalloced-kernel-stacks.rst | 153 ------ Documentation/vm/vmemmap_dedup.rst | 223 -------- Documentation/vm/z3fold.rst | 30 -- Documentation/vm/zsmalloc.rst | 82 --- MAINTAINERS | 12 +- arch/loongarch/Kconfig | 2 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 +- include/linux/hmm.h | 4 +- include/linux/memremap.h | 2 +- include/linux/mmu_notifier.h | 2 +- include/linux/sched/mm.h | 4 +- include/linux/swap.h | 2 +- mm/Kconfig | 2 +- mm/debug_vm_pgtable.c | 2 +- mm/frontswap.c | 2 +- mm/huge_memory.c | 2 +- mm/hugetlb.c | 6 +- mm/hugetlb_vmemmap.c | 2 +- mm/ksm.c | 4 +- mm/mmap.c | 2 +- mm/rmap.c | 8 +- mm/sparse-vmemmap.c | 2 +- mm/util.c | 2 +- tools/vm/page_owner_sort.c | 2 +- 176 files changed, 8355 insertions(+), 8359 deletions(-) create mode 100644 Documentation/mm/active_mm.rst create mode 100644 Documentation/mm/arch_pgtable_helpers.rst create mode 100644 Documentation/mm/balance.rst create mode 100644 Documentation/mm/bootmem.rst create mode 100644 Documentation/mm/damon/api.rst create mode 100644 Documentation/mm/damon/design.rst create mode 100644 Documentation/mm/damon/faq.rst create mode 100644 Documentation/mm/damon/index.rst create mode 100644 Documentation/mm/free_page_reporting.rst create mode 100644 Documentation/mm/frontswap.rst create mode 100644 Documentation/mm/highmem.rst create mode 100644 Documentation/mm/hmm.rst create mode 100644 Documentation/mm/hugetlbfs_reserv.rst create mode 100644 Documentation/mm/hwpoison.rst create mode 100644 Documentation/mm/index.rst create mode 100644 Documentation/mm/ksm.rst create mode 100644 Documentation/mm/memory-model.rst create mode 100644 Documentation/mm/mmu_notifier.rst create mode 100644 Documentation/mm/numa.rst create mode 100644 Documentation/mm/oom.rst create mode 100644 Documentation/mm/overcommit-accounting.rst create mode 100644 Documentation/mm/page_allocation.rst create mode 100644 Documentation/mm/page_cache.rst create mode 100644 Documentation/mm/page_frags.rst create mode 100644 Documentation/mm/page_migration.rst create mode 100644 Documentation/mm/page_owner.rst create mode 100644 Documentation/mm/page_reclaim.rst create mode 100644 Documentation/mm/page_table_check.rst create mode 100644 Documentation/mm/page_tables.rst create mode 100644 Documentation/mm/physical_memory.rst create mode 100644 Documentation/mm/process_addrs.rst create mode 100644 Documentation/mm/remap_file_pages.rst create mode 100644 Documentation/mm/shmfs.rst create mode 100644 Documentation/mm/slab.rst create mode 100644 Documentation/mm/slub.rst create mode 100644 Documentation/mm/split_page_table_lock.rst create mode 100644 Documentation/mm/swap.rst create mode 100644 Documentation/mm/transhuge.rst create mode 100644 Documentation/mm/unevictable-lru.rst create mode 100644 Documentation/mm/vmalloc.rst create mode 100644 Documentation/mm/vmalloced-kernel-stacks.rst create mode 100644 Documentation/mm/vmemmap_dedup.rst create mode 100644 Documentation/mm/z3fold.rst create mode 100644 Documentation/mm/zsmalloc.rst create mode 100644 Documentation/translations/zh_CN/mm/active_mm.rst create mode 100644 Documentation/translations/zh_CN/mm/balance.rst create mode 100644 Documentation/translations/zh_CN/mm/damon/api.rst create mode 100644 Documentation/translations/zh_CN/mm/damon/design.rst create mode 100644 Documentation/translations/zh_CN/mm/damon/faq.rst create mode 100644 Documentation/translations/zh_CN/mm/damon/index.rst create mode 100644 Documentation/translations/zh_CN/mm/free_page_reporting.rst create mode 100644 Documentation/translations/zh_CN/mm/frontswap.rst create mode 100644 Documentation/translations/zh_CN/mm/highmem.rst create mode 100644 Documentation/translations/zh_CN/mm/hmm.rst create mode 100644 Documentation/translations/zh_CN/mm/hugetlbfs_reserv.rst create mode 100644 Documentation/translations/zh_CN/mm/hwpoison.rst create mode 100644 Documentation/translations/zh_CN/mm/index.rst create mode 100644 Documentation/translations/zh_CN/mm/ksm.rst create mode 100644 Documentation/translations/zh_CN/mm/memory-model.rst create mode 100644 Documentation/translations/zh_CN/mm/mmu_notifier.rst create mode 100644 Documentation/translations/zh_CN/mm/numa.rst create mode 100644 Documentation/translations/zh_CN/mm/overcommit-accounting.rst create mode 100644 Documentation/translations/zh_CN/mm/page_frags.rst create mode 100644 Documentation/translations/zh_CN/mm/page_owner.rst create mode 100644 Documentation/translations/zh_CN/mm/page_table_check.rst create mode 100644 Documentation/translations/zh_CN/mm/remap_file_pages.rst create mode 100644 Documentation/translations/zh_CN/mm/split_page_table_lock.rst create mode 100644 Documentation/translations/zh_CN/mm/z3fold.rst create mode 100644 Documentation/translations/zh_CN/mm/zsmalloc.rst delete mode 100644 Documentation/translations/zh_CN/vm/active_mm.rst delete mode 100644 Documentation/translations/zh_CN/vm/balance.rst delete mode 100644 Documentation/translations/zh_CN/vm/damon/api.rst delete mode 100644 Documentation/translations/zh_CN/vm/damon/design.rst delete mode 100644 Documentation/translations/zh_CN/vm/damon/faq.rst delete mode 100644 Documentation/translations/zh_CN/vm/damon/index.rst delete mode 100644 Documentation/translations/zh_CN/vm/free_page_reporting.rst delete mode 100644 Documentation/translations/zh_CN/vm/frontswap.rst delete mode 100644 Documentation/translations/zh_CN/vm/highmem.rst delete mode 100644 Documentation/translations/zh_CN/vm/hmm.rst delete mode 100644 Documentation/translations/zh_CN/vm/hugetlbfs_reserv.rst delete mode 100644 Documentation/translations/zh_CN/vm/hwpoison.rst delete mode 100644 Documentation/translations/zh_CN/vm/index.rst delete mode 100644 Documentation/translations/zh_CN/vm/ksm.rst delete mode 100644 Documentation/translations/zh_CN/vm/memory-model.rst delete mode 100644 Documentation/translations/zh_CN/vm/mmu_notifier.rst delete mode 100644 Documentation/translations/zh_CN/vm/numa.rst delete mode 100644 Documentation/translations/zh_CN/vm/overcommit-accounting.rst delete mode 100644 Documentation/translations/zh_CN/vm/page_frags.rst delete mode 100644 Documentation/translations/zh_CN/vm/page_owner.rst delete mode 100644 Documentation/translations/zh_CN/vm/page_table_check.rst delete mode 100644 Documentation/translations/zh_CN/vm/remap_file_pages.rst delete mode 100644 Documentation/translations/zh_CN/vm/split_page_table_lock.rst delete mode 100644 Documentation/translations/zh_CN/vm/z3fold.rst delete mode 100644 Documentation/translations/zh_CN/vm/zsmalloc.rst delete mode 100644 Documentation/vm/.gitignore delete mode 100644 Documentation/vm/active_mm.rst delete mode 100644 Documentation/vm/arch_pgtable_helpers.rst delete mode 100644 Documentation/vm/balance.rst delete mode 100644 Documentation/vm/bootmem.rst delete mode 100644 Documentation/vm/damon/api.rst delete mode 100644 Documentation/vm/damon/design.rst delete mode 100644 Documentation/vm/damon/faq.rst delete mode 100644 Documentation/vm/damon/index.rst delete mode 100644 Documentation/vm/free_page_reporting.rst delete mode 100644 Documentation/vm/frontswap.rst delete mode 100644 Documentation/vm/highmem.rst delete mode 100644 Documentation/vm/hmm.rst delete mode 100644 Documentation/vm/hugetlbfs_reserv.rst delete mode 100644 Documentation/vm/hwpoison.rst delete mode 100644 Documentation/vm/index.rst delete mode 100644 Documentation/vm/ksm.rst delete mode 100644 Documentation/vm/memory-model.rst delete mode 100644 Documentation/vm/mmu_notifier.rst delete mode 100644 Documentation/vm/numa.rst delete mode 100644 Documentation/vm/oom.rst delete mode 100644 Documentation/vm/overcommit-accounting.rst delete mode 100644 Documentation/vm/page_allocation.rst delete mode 100644 Documentation/vm/page_cache.rst delete mode 100644 Documentation/vm/page_frags.rst delete mode 100644 Documentation/vm/page_migration.rst delete mode 100644 Documentation/vm/page_owner.rst delete mode 100644 Documentation/vm/page_reclaim.rst delete mode 100644 Documentation/vm/page_table_check.rst delete mode 100644 Documentation/vm/page_tables.rst delete mode 100644 Documentation/vm/physical_memory.rst delete mode 100644 Documentation/vm/process_addrs.rst delete mode 100644 Documentation/vm/remap_file_pages.rst delete mode 100644 Documentation/vm/shmfs.rst delete mode 100644 Documentation/vm/slab.rst delete mode 100644 Documentation/vm/slub.rst delete mode 100644 Documentation/vm/split_page_table_lock.rst delete mode 100644 Documentation/vm/swap.rst delete mode 100644 Documentation/vm/transhuge.rst delete mode 100644 Documentation/vm/unevictable-lru.rst delete mode 100644 Documentation/vm/vmalloc.rst delete mode 100644 Documentation/vm/vmalloced-kernel-stacks.rst delete mode 100644 Documentation/vm/vmemmap_dedup.rst delete mode 100644 Documentation/vm/z3fold.rst delete mode 100644 Documentation/vm/zsmalloc.rst (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-ksm b/Documentation/ABI/testing/sysfs-kernel-mm-ksm index 1c9bed5595f5..d244674a9480 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-ksm +++ b/Documentation/ABI/testing/sysfs-kernel-mm-ksm @@ -41,7 +41,7 @@ Description: Kernel Samepage Merging daemon sysfs interface sleep_millisecs: how many milliseconds ksm should sleep between scans. - See Documentation/vm/ksm.rst for more information. + See Documentation/mm/ksm.rst for more information. What: /sys/kernel/mm/ksm/merge_across_nodes Date: January 2013 diff --git a/Documentation/ABI/testing/sysfs-kernel-slab b/Documentation/ABI/testing/sysfs-kernel-slab index c440f4946e12..cd5fb8fa3ddf 100644 --- a/Documentation/ABI/testing/sysfs-kernel-slab +++ b/Documentation/ABI/testing/sysfs-kernel-slab @@ -37,7 +37,7 @@ Description: The alloc_calls file is read-only and lists the kernel code locations from which allocations for this cache were performed. The alloc_calls file only contains information if debugging is - enabled for that cache (see Documentation/vm/slub.rst). + enabled for that cache (see Documentation/mm/slub.rst). What: /sys/kernel/slab//alloc_fastpath Date: February 2008 @@ -219,7 +219,7 @@ Contact: Pekka Enberg , Description: The free_calls file is read-only and lists the locations of object frees if slab debugging is enabled (see - Documentation/vm/slub.rst). + Documentation/mm/slub.rst). What: /sys/kernel/slab//free_fastpath Date: February 2008 diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2522b11e593f..8c0ea6b6c6a9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5442,7 +5442,7 @@ cache (risks via metadata attacks are mostly unchanged). Debug options disable merging on their own. - For more information see Documentation/vm/slub.rst. + For more information see Documentation/mm/slub.rst. slab_max_order= [MM, SLAB] Determines the maximum allowed order for slabs. @@ -5456,13 +5456,13 @@ slub_debug can create guard zones around objects and may poison objects when not in use. Also tracks the last alloc / free. For more information see - Documentation/vm/slub.rst. + Documentation/mm/slub.rst. slub_max_order= [MM, SLUB] Determines the maximum allowed order for slabs. A high setting may cause OOMs due to memory fragmentation. For more information see - Documentation/vm/slub.rst. + Documentation/mm/slub.rst. slub_min_objects= [MM, SLUB] The minimum number of objects per slab. SLUB will @@ -5471,12 +5471,12 @@ the number of objects indicated. The higher the number of objects the smaller the overhead of tracking slabs and the less frequently locks need to be acquired. - For more information see Documentation/vm/slub.rst. + For more information see Documentation/mm/slub.rst. slub_min_order= [MM, SLUB] Determines the minimum page order for slabs. Must be lower than slub_max_order. - For more information see Documentation/vm/slub.rst. + For more information see Documentation/mm/slub.rst. slub_merge [MM, SLUB] Same with slab_merge. diff --git a/Documentation/admin-guide/mm/concepts.rst b/Documentation/admin-guide/mm/concepts.rst index b966fcff993b..c79f1e336222 100644 --- a/Documentation/admin-guide/mm/concepts.rst +++ b/Documentation/admin-guide/mm/concepts.rst @@ -125,7 +125,7 @@ processor. Each bank is referred to as a `node` and for each node Linux constructs an independent memory management subsystem. A node has its own set of zones, lists of free and used pages and various statistics counters. You can find more details about NUMA in -:ref:`Documentation/vm/numa.rst ` and in +:ref:`Documentation/mm/numa.rst ` and in :ref:`Documentation/admin-guide/mm/numa_memory_policy.rst `. Page cache diff --git a/Documentation/admin-guide/mm/damon/index.rst b/Documentation/admin-guide/mm/damon/index.rst index 61aff88347f3..c4681fa69b9c 100644 --- a/Documentation/admin-guide/mm/damon/index.rst +++ b/Documentation/admin-guide/mm/damon/index.rst @@ -4,7 +4,7 @@ Monitoring Data Accesses ======================== -:doc:`DAMON ` allows light-weight data access monitoring. +:doc:`DAMON ` allows light-weight data access monitoring. Using DAMON, users can analyze the memory access patterns of their systems and optimize those. diff --git a/Documentation/admin-guide/mm/damon/reclaim.rst b/Documentation/admin-guide/mm/damon/reclaim.rst index 46306f1f34b1..a8bd3bd29959 100644 --- a/Documentation/admin-guide/mm/damon/reclaim.rst +++ b/Documentation/admin-guide/mm/damon/reclaim.rst @@ -268,4 +268,4 @@ granularity reclamation. :: .. [1] https://research.google/pubs/pub48551/ .. [2] https://lwn.net/Articles/787611/ -.. [3] https://www.kernel.org/doc/html/latest/vm/free_page_reporting.html +.. [3] https://www.kernel.org/doc/html/latest/mm/free_page_reporting.html diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 1bb7b72414b2..5540a3a40fc9 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -30,11 +30,11 @@ DAMON provides below interfaces for different users. `. This will be removed after next LTS kernel is released, so users should move to the :ref:`sysfs interface `. - *Kernel Space Programming Interface.* - :doc:`This ` is for kernel space programmers. Using this, + :doc:`This ` is for kernel space programmers. Using this, users can utilize every feature of DAMON most flexibly and efficiently by writing kernel space DAMON application programs for you. You can even extend DAMON for various address spaces. For detail, please refer to the interface - :doc:`document `. + :doc:`document `. .. _sysfs_interface: @@ -185,7 +185,7 @@ controls the monitoring overhead, exist. You can set and get the values by writing to and rading from the files. For more details about the intervals and monitoring regions range, please refer -to the Design document (:doc:`/vm/damon/design`). +to the Design document (:doc:`/mm/damon/design`). contexts//targets/ --------------------- @@ -402,7 +402,7 @@ Attributes Users can get and set the ``sampling interval``, ``aggregation interval``, ``update interval``, and min/max number of monitoring target regions by reading from and writing to the ``attrs`` file. To know about the monitoring -attributes in detail, please refer to the :doc:`/vm/damon/design`. For +attributes in detail, please refer to the :doc:`/mm/damon/design`. For example, below commands set those values to 5 ms, 100 ms, 1,000 ms, 10 and 1000, and then check it again:: diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 5c9aa171a0d3..4a440a7cfeb0 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -760,7 +760,7 @@ and don't use much of it. The default value is 0. -See Documentation/vm/overcommit-accounting.rst and +See Documentation/mm/overcommit-accounting.rst and mm/util.c::__vm_enough_memory() for more information. diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index dedd4d853329..5b1188494bcd 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -87,7 +87,7 @@ Memory management ================= How to allocate and use memory in the kernel. Note that there is a lot -more memory-management documentation in Documentation/vm/index.rst. +more memory-management documentation in Documentation/mm/index.rst. .. toctree:: :maxdepth: 1 diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 1bc91fb8c321..8543a59f288f 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -1109,7 +1109,7 @@ CommitLimit yield a CommitLimit of 7.3G. For more details, see the memory overcommit documentation - in vm/overcommit-accounting. + in mm/overcommit-accounting. Committed_AS The amount of memory presently allocated on the system. The committed memory is a sum of all of the memory which diff --git a/Documentation/index.rst b/Documentation/index.rst index 67036a05b771..4737c18c97ff 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -128,7 +128,7 @@ needed). sound/index crypto/index filesystems/index - vm/index + mm/index bpf/index usb/index PCI/index diff --git a/Documentation/mm/active_mm.rst b/Documentation/mm/active_mm.rst new file mode 100644 index 000000000000..6f8269c284ed --- /dev/null +++ b/Documentation/mm/active_mm.rst @@ -0,0 +1,91 @@ +.. _active_mm: + +========= +Active MM +========= + +:: + + List: linux-kernel + Subject: Re: active_mm + From: Linus Torvalds + Date: 1999-07-30 21:36:24 + + Cc'd to linux-kernel, because I don't write explanations all that often, + and when I do I feel better about more people reading them. + + On Fri, 30 Jul 1999, David Mosberger wrote: + > + > Is there a brief description someplace on how "mm" vs. "active_mm" in + > the task_struct are supposed to be used? (My apologies if this was + > discussed on the mailing lists---I just returned from vacation and + > wasn't able to follow linux-kernel for a while). + + Basically, the new setup is: + + - we have "real address spaces" and "anonymous address spaces". The + difference is that an anonymous address space doesn't care about the + user-level page tables at all, so when we do a context switch into an + anonymous address space we just leave the previous address space + active. + + The obvious use for a "anonymous address space" is any thread that + doesn't need any user mappings - all kernel threads basically fall into + this category, but even "real" threads can temporarily say that for + some amount of time they are not going to be interested in user space, + and that the scheduler might as well try to avoid wasting time on + switching the VM state around. Currently only the old-style bdflush + sync does that. + + - "tsk->mm" points to the "real address space". For an anonymous process, + tsk->mm will be NULL, for the logical reason that an anonymous process + really doesn't _have_ a real address space at all. + + - however, we obviously need to keep track of which address space we + "stole" for such an anonymous user. For that, we have "tsk->active_mm", + which shows what the currently active address space is. + + The rule is that for a process with a real address space (ie tsk->mm is + non-NULL) the active_mm obviously always has to be the same as the real + one. + + For a anonymous process, tsk->mm == NULL, and tsk->active_mm is the + "borrowed" mm while the anonymous process is running. When the + anonymous process gets scheduled away, the borrowed address space is + returned and cleared. + + To support all that, the "struct mm_struct" now has two counters: a + "mm_users" counter that is how many "real address space users" there are, + and a "mm_count" counter that is the number of "lazy" users (ie anonymous + users) plus one if there are any real users. + + Usually there is at least one real user, but it could be that the real + user exited on another CPU while a lazy user was still active, so you do + actually get cases where you have a address space that is _only_ used by + lazy users. That is often a short-lived state, because once that thread + gets scheduled away in favour of a real thread, the "zombie" mm gets + released because "mm_count" becomes zero. + + Also, a new rule is that _nobody_ ever has "init_mm" as a real MM any + more. "init_mm" should be considered just a "lazy context when no other + context is available", and in fact it is mainly used just at bootup when + no real VM has yet been created. So code that used to check + + if (current->mm == &init_mm) + + should generally just do + + if (!current->mm) + + instead (which makes more sense anyway - the test is basically one of "do + we have a user context", and is generally done by the page fault handler + and things like that). + + Anyway, I put a pre-patch-2.3.13-1 on ftp.kernel.org just a moment ago, + because it slightly changes the interfaces to accommodate the alpha (who + would have thought it, but the alpha actually ends up having one of the + ugliest context switch codes - unlike the other architectures where the MM + and register state is separate, the alpha PALcode joins the two, and you + need to switch both together). + + (From http://marc.info/?l=linux-kernel&m=93337278602211&w=2) diff --git a/Documentation/mm/arch_pgtable_helpers.rst b/Documentation/mm/arch_pgtable_helpers.rst new file mode 100644 index 000000000000..cbaee9e59241 --- /dev/null +++ b/Documentation/mm/arch_pgtable_helpers.rst @@ -0,0 +1,260 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _arch_page_table_helpers: + +=============================== +Architecture Page Table Helpers +=============================== + +Generic MM expects architectures (with MMU) to provide helpers to create, access +and modify page table entries at various level for different memory functions. +These page table helpers need to conform to a common semantics across platforms. +Following tables describe the expected semantics which can also be tested during +boot via CONFIG_DEBUG_VM_PGTABLE option. All future changes in here or the debug +test need to be in sync. + + +PTE Page Table Helpers +====================== + ++---------------------------+--------------------------------------------------+ +| pte_same | Tests whether both PTE entries are the same | ++---------------------------+--------------------------------------------------+ +| pte_bad | Tests a non-table mapped PTE | ++---------------------------+--------------------------------------------------+ +| pte_present | Tests a valid mapped PTE | ++---------------------------+--------------------------------------------------+ +| pte_young | Tests a young PTE | ++---------------------------+--------------------------------------------------+ +| pte_dirty | Tests a dirty PTE | ++---------------------------+--------------------------------------------------+ +| pte_write | Tests a writable PTE | ++---------------------------+--------------------------------------------------+ +| pte_special | Tests a special PTE | ++---------------------------+--------------------------------------------------+ +| pte_protnone | Tests a PROT_NONE PTE | ++---------------------------+--------------------------------------------------+ +| pte_devmap | Tests a ZONE_DEVICE mapped PTE | ++---------------------------+--------------------------------------------------+ +| pte_soft_dirty | Tests a soft dirty PTE | ++---------------------------+--------------------------------------------------+ +| pte_swp_soft_dirty | Tests a soft dirty swapped PTE | ++---------------------------+--------------------------------------------------+ +| pte_mkyoung | Creates a young PTE | ++---------------------------+--------------------------------------------------+ +| pte_mkold | Creates an old PTE | ++---------------------------+--------------------------------------------------+ +| pte_mkdirty | Creates a dirty PTE | ++---------------------------+--------------------------------------------------+ +| pte_mkclean | Creates a clean PTE | ++---------------------------+--------------------------------------------------+ +| pte_mkwrite | Creates a writable PTE | ++---------------------------+--------------------------------------------------+ +| pte_wrprotect | Creates a write protected PTE | ++---------------------------+--------------------------------------------------+ +| pte_mkspecial | Creates a special PTE | ++---------------------------+--------------------------------------------------+ +| pte_mkdevmap | Creates a ZONE_DEVICE mapped PTE | ++---------------------------+--------------------------------------------------+ +| pte_mksoft_dirty | Creates a soft dirty PTE | ++---------------------------+--------------------------------------------------+ +| pte_clear_soft_dirty | Clears a soft dirty PTE | ++---------------------------+--------------------------------------------------+ +| pte_swp_mksoft_dirty | Creates a soft dirty swapped PTE | ++---------------------------+--------------------------------------------------+ +| pte_swp_clear_soft_dirty | Clears a soft dirty swapped PTE | ++---------------------------+--------------------------------------------------+ +| pte_mknotpresent | Invalidates a mapped PTE | ++---------------------------+--------------------------------------------------+ +| ptep_clear | Clears a PTE | ++---------------------------+--------------------------------------------------+ +| ptep_get_and_clear | Clears and returns PTE | ++---------------------------+--------------------------------------------------+ +| ptep_get_and_clear_full | Clears and returns PTE (batched PTE unmap) | ++---------------------------+--------------------------------------------------+ +| ptep_test_and_clear_young | Clears young from a PTE | ++---------------------------+--------------------------------------------------+ +| ptep_set_wrprotect | Converts into a write protected PTE | ++---------------------------+--------------------------------------------------+ +| ptep_set_access_flags | Converts into a more permissive PTE | ++---------------------------+--------------------------------------------------+ + + +PMD Page Table Helpers +====================== + ++---------------------------+--------------------------------------------------+ +| pmd_same | Tests whether both PMD entries are the same | ++---------------------------+--------------------------------------------------+ +| pmd_bad | Tests a non-table mapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_leaf | Tests a leaf mapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_huge | Tests a HugeTLB mapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_trans_huge | Tests a Transparent Huge Page (THP) at PMD | ++---------------------------+--------------------------------------------------+ +| pmd_present | Tests a valid mapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_young | Tests a young PMD | ++---------------------------+--------------------------------------------------+ +| pmd_dirty | Tests a dirty PMD | ++---------------------------+--------------------------------------------------+ +| pmd_write | Tests a writable PMD | ++---------------------------+--------------------------------------------------+ +| pmd_special | Tests a special PMD | ++---------------------------+--------------------------------------------------+ +| pmd_protnone | Tests a PROT_NONE PMD | ++---------------------------+--------------------------------------------------+ +| pmd_devmap | Tests a ZONE_DEVICE mapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_soft_dirty | Tests a soft dirty PMD | ++---------------------------+--------------------------------------------------+ +| pmd_swp_soft_dirty | Tests a soft dirty swapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkyoung | Creates a young PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkold | Creates an old PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkdirty | Creates a dirty PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkclean | Creates a clean PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkwrite | Creates a writable PMD | ++---------------------------+--------------------------------------------------+ +| pmd_wrprotect | Creates a write protected PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkspecial | Creates a special PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkdevmap | Creates a ZONE_DEVICE mapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mksoft_dirty | Creates a soft dirty PMD | ++---------------------------+--------------------------------------------------+ +| pmd_clear_soft_dirty | Clears a soft dirty PMD | ++---------------------------+--------------------------------------------------+ +| pmd_swp_mksoft_dirty | Creates a soft dirty swapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_swp_clear_soft_dirty | Clears a soft dirty swapped PMD | ++---------------------------+--------------------------------------------------+ +| pmd_mkinvalid | Invalidates a mapped PMD [1] | ++---------------------------+--------------------------------------------------+ +| pmd_set_huge | Creates a PMD huge mapping | ++---------------------------+--------------------------------------------------+ +| pmd_clear_huge | Clears a PMD huge mapping | ++---------------------------+--------------------------------------------------+ +| pmdp_get_and_clear | Clears a PMD | ++---------------------------+--------------------------------------------------+ +| pmdp_get_and_clear_full | Clears a PMD | ++---------------------------+--------------------------------------------------+ +| pmdp_test_and_clear_young | Clears young from a PMD | ++---------------------------+--------------------------------------------------+ +| pmdp_set_wrprotect | Converts into a write protected PMD | ++---------------------------+--------------------------------------------------+ +| pmdp_set_access_flags | Converts into a more permissive PMD | ++---------------------------+--------------------------------------------------+ + + +PUD Page Table Helpers +====================== + ++---------------------------+--------------------------------------------------+ +| pud_same | Tests whether both PUD entries are the same | ++---------------------------+--------------------------------------------------+ +| pud_bad | Tests a non-table mapped PUD | ++---------------------------+--------------------------------------------------+ +| pud_leaf | Tests a leaf mapped PUD | ++---------------------------+--------------------------------------------------+ +| pud_huge | Tests a HugeTLB mapped PUD | ++---------------------------+--------------------------------------------------+ +| pud_trans_huge | Tests a Transparent Huge Page (THP) at PUD | ++---------------------------+--------------------------------------------------+ +| pud_present | Tests a valid mapped PUD | ++---------------------------+--------------------------------------------------+ +| pud_young | Tests a young PUD | ++---------------------------+--------------------------------------------------+ +| pud_dirty | Tests a dirty PUD | ++---------------------------+--------------------------------------------------+ +| pud_write | Tests a writable PUD | ++---------------------------+--------------------------------------------------+ +| pud_devmap | Tests a ZONE_DEVICE mapped PUD | ++---------------------------+--------------------------------------------------+ +| pud_mkyoung | Creates a young PUD | ++---------------------------+--------------------------------------------------+ +| pud_mkold | Creates an old PUD | ++---------------------------+--------------------------------------------------+ +| pud_mkdirty | Creates a dirty PUD | ++---------------------------+--------------------------------------------------+ +| pud_mkclean | Creates a clean PUD | ++---------------------------+--------------------------------------------------+ +| pud_mkwrite | Creates a writable PUD | ++---------------------------+--------------------------------------------------+ +| pud_wrprotect | Creates a write protected PUD | ++---------------------------+--------------------------------------------------+ +| pud_mkdevmap | Creates a ZONE_DEVICE mapped PUD | ++---------------------------+--------------------------------------------------+ +| pud_mkinvalid | Invalidates a mapped PUD [1] | ++---------------------------+--------------------------------------------------+ +| pud_set_huge | Creates a PUD huge mapping | ++---------------------------+--------------------------------------------------+ +| pud_clear_huge | Clears a PUD huge mapping | ++---------------------------+--------------------------------------------------+ +| pudp_get_and_clear | Clears a PUD | ++---------------------------+--------------------------------------------------+ +| pudp_get_and_clear_full | Clears a PUD | ++---------------------------+--------------------------------------------------+ +| pudp_test_and_clear_young | Clears young from a PUD | ++---------------------------+--------------------------------------------------+ +| pudp_set_wrprotect | Converts into a write protected PUD | ++---------------------------+--------------------------------------------------+ +| pudp_set_access_flags | Converts into a more permissive PUD | ++---------------------------+--------------------------------------------------+ + + +HugeTLB Page Table Helpers +========================== + ++---------------------------+--------------------------------------------------+ +| pte_huge | Tests a HugeTLB | ++---------------------------+--------------------------------------------------+ +| pte_mkhuge | Creates a HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_pte_dirty | Tests a dirty HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_pte_write | Tests a writable HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_pte_mkdirty | Creates a dirty HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_pte_mkwrite | Creates a writable HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_pte_wrprotect | Creates a write protected HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_ptep_get_and_clear | Clears a HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_ptep_set_wrprotect | Converts into a write protected HugeTLB | ++---------------------------+--------------------------------------------------+ +| huge_ptep_set_access_flags | Converts into a more permissive HugeTLB | ++---------------------------+--------------------------------------------------+ + + +SWAP Page Table Helpers +======================== + ++---------------------------+--------------------------------------------------+ +| __pte_to_swp_entry | Creates a swapped entry (arch) from a mapped PTE | ++---------------------------+--------------------------------------------------+ +| __swp_to_pte_entry | Creates a mapped PTE from a swapped entry (arch) | ++---------------------------+--------------------------------------------------+ +| __pmd_to_swp_entry | Creates a swapped entry (arch) from a mapped PMD | ++---------------------------+--------------------------------------------------+ +| __swp_to_pmd_entry | Creates a mapped PMD from a swapped entry (arch) | ++---------------------------+--------------------------------------------------+ +| is_migration_entry | Tests a migration (read or write) swapped entry | ++-------------------------------+----------------------------------------------+ +| is_writable_migration_entry | Tests a write migration swapped entry | ++-------------------------------+----------------------------------------------+ +| make_readable_migration_entry | Creates a read migration swapped entry | ++-------------------------------+----------------------------------------------+ +| make_writable_migration_entry | Creates a write migration swapped entry | ++-------------------------------+----------------------------------------------+ + +[1] https://lore.kernel.org/linux-mm/20181017020930.GN30832@redhat.com/ diff --git a/Documentation/mm/balance.rst b/Documentation/mm/balance.rst new file mode 100644 index 000000000000..6a1fadf3e173 --- /dev/null +++ b/Documentation/mm/balance.rst @@ -0,0 +1,102 @@ +.. _balance: + +================ +Memory Balancing +================ + +Started Jan 2000 by Kanoj Sarcar + +Memory balancing is needed for !__GFP_ATOMIC and !__GFP_KSWAPD_RECLAIM as +well as for non __GFP_IO allocations. + +The first reason why a caller may avoid reclaim is that the caller can not +sleep due to holding a spinlock or is in interrupt context. The second may +be that the caller is willing to fail the allocation without incurring the +overhead of page reclaim. This may happen for opportunistic high-order +allocation requests that have order-0 fallback options. In such cases, +the caller may also wish to avoid waking kswapd. + +__GFP_IO allocation requests are made to prevent file system deadlocks. + +In the absence of non sleepable allocation requests, it seems detrimental +to be doing balancing. Page reclamation can be kicked off lazily, that +is, only when needed (aka zone free memory is 0), instead of making it +a proactive process. + +That being said, the kernel should try to fulfill requests for direct +mapped pages from the direct mapped pool, instead of falling back on +the dma pool, so as to keep the dma pool filled for dma requests (atomic +or not). A similar argument applies to highmem and direct mapped pages. +OTOH, if there is a lot of free dma pages, it is preferable to satisfy +regular memory requests by allocating one from the dma pool, instead +of incurring the overhead of regular zone balancing. + +In 2.2, memory balancing/page reclamation would kick off only when the +_total_ number of free pages fell below 1/64 th of total memory. With the +right ratio of dma and regular memory, it is quite possible that balancing +would not be done even when the dma zone was completely empty. 2.2 has +been running production machines of varying memory sizes, and seems to be +doing fine even with the presence of this problem. In 2.3, due to +HIGHMEM, this problem is aggravated. + +In 2.3, zone balancing can be done in one of two ways: depending on the +zone size (and possibly of the size of lower class zones), we can decide +at init time how many free pages we should aim for while balancing any +zone. The good part is, while balancing, we do not need to look at sizes +of lower class zones, the bad part is, we might do too frequent balancing +due to ignoring possibly lower usage in the lower class zones. Also, +with a slight change in the allocation routine, it is possible to reduce +the memclass() macro to be a simple equality. + +Another possible solution is that we balance only when the free memory +of a zone _and_ all its lower class zones falls below 1/64th of the +total memory in the zone and its lower class zones. This fixes the 2.2 +balancing problem, and stays as close to 2.2 behavior as possible. Also, +the balancing algorithm works the same way on the various architectures, +which have different numbers and types of zones. If we wanted to get +fancy, we could assign different weights to free pages in different +zones in the future. + +Note that if the size of the regular zone is huge compared to dma zone, +it becomes less significant to consider the free dma pages while +deciding whether to balance the regular zone. The first solution +becomes more attractive then. + +The appended patch implements the second solution. It also "fixes" two +problems: first, kswapd is woken up as in 2.2 on low memory conditions +for non-sleepable allocations. Second, the HIGHMEM zone is also balanced, +so as to give a fighting chance for replace_with_highmem() to get a +HIGHMEM page, as well as to ensure that HIGHMEM allocations do not +fall back into regular zone. This also makes sure that HIGHMEM pages +are not leaked (for example, in situations where a HIGHMEM page is in +the swapcache but is not being used by anyone) + +kswapd also needs to know about the zones it should balance. kswapd is +primarily needed in a situation where balancing can not be done, +probably because all allocation requests are coming from intr context +and all process contexts are sleeping. For 2.3, kswapd does not really +need to balance the highmem zone, since intr context does not request +highmem pages. kswapd looks at the zone_wake_kswapd field in the zone +structure to decide whether a zone needs balancing. + +Page stealing from process memory and shm is done if stealing the page would +alleviate memory pressure on any zone in the page's node that has fallen below +its watermark. + +watemark[WMARK_MIN/WMARK_LOW/WMARK_HIGH]/low_on_memory/zone_wake_kswapd: These +are per-zone fields, used to determine when a zone needs to be balanced. When +the number of pages falls below watermark[WMARK_MIN], the hysteric field +low_on_memory gets set. This stays set till the number of free pages becomes +watermark[WMARK_HIGH]. When low_on_memory is set, page allocation requests will +try to free some pages in the zone (providing GFP_WAIT is set in the request). +Orthogonal to this, is the decision to poke kswapd to free some zone pages. +That decision is not hysteresis based, and is done when the number of free +pages is below watermark[WMARK_LOW]; in which case zone_wake_kswapd is also set. + + +(Good) Ideas that I have heard: + +1. Dynamic experience should influence balancing: number of failed requests + for a zone can be tracked and fed into the balancing scheme (jalvo@mbay.net) +2. Implement a replace_with_highmem()-like replace_with_regular() to preserve + dma pages. (lkd@tantalophile.demon.co.uk) diff --git a/Documentation/mm/bootmem.rst b/Documentation/mm/bootmem.rst new file mode 100644 index 000000000000..eb2b31eedfa1 --- /dev/null +++ b/Documentation/mm/bootmem.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========== +Boot Memory +=========== diff --git a/Documentation/mm/damon/api.rst b/Documentation/mm/damon/api.rst new file mode 100644 index 000000000000..08f34df45523 --- /dev/null +++ b/Documentation/mm/damon/api.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +API Reference +============= + +Kernel space programs can use every feature of DAMON using below APIs. All you +need to do is including ``damon.h``, which is located in ``include/linux/`` of +the source tree. + +Structures +========== + +.. kernel-doc:: include/linux/damon.h + + +Functions +========= + +.. kernel-doc:: mm/damon/core.c diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst new file mode 100644 index 000000000000..0cff6fac6b7e --- /dev/null +++ b/Documentation/mm/damon/design.rst @@ -0,0 +1,176 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====== +Design +====== + +Configurable Layers +=================== + +DAMON provides data access monitoring functionality while making the accuracy +and the overhead controllable. The fundamental access monitorings require +primitives that dependent on and optimized for the target address space. On +the other hand, the accuracy and overhead tradeoff mechanism, which is the core +of DAMON, is in the pure logic space. DAMON separates the two parts in +different layers and defines its interface to allow various low level +primitives implementations configurable with the core logic. We call the low +level primitives implementations monitoring operations. + +Due to this separated design and the configurable interface, users can extend +DAMON for any address space by configuring the core logics with appropriate +monitoring operations. If appropriate one is not provided, users can implement +the operations on their own. + +For example, physical memory, virtual memory, swap space, those for specific +processes, NUMA nodes, files, and backing memory devices would be supportable. +Also, if some architectures or devices support special optimized access check +primitives, those will be easily configurable. + + +Reference Implementations of Address Space Specific Monitoring Operations +========================================================================= + +The monitoring operations are defined in two parts: + +1. Identification of the monitoring target address range for the address space. +2. Access check of specific address range in the target space. + +DAMON currently provides the implementations of the operations for the physical +and virtual address spaces. Below two subsections describe how those work. + + +VMA-based Target Address Range Construction +------------------------------------------- + +This is only for the virtual address space monitoring operations +implementation. That for the physical address space simply asks users to +manually set the monitoring target address ranges. + +Only small parts in the super-huge virtual address space of the processes are +mapped to the physical memory and accessed. Thus, tracking the unmapped +address regions is just wasteful. However, because DAMON can deal with some +level of noise using the adaptive regions adjustment mechanism, tracking every +mapping is not strictly required but could even incur a high overhead in some +cases. That said, too huge unmapped areas inside the monitoring target should +be removed to not take the time for the adaptive mechanism. + +For the reason, this implementation converts the complex mappings to three +distinct regions that cover every mapped area of the address space. The two +gaps between the three regions are the two biggest unmapped areas in the given +address space. The two biggest unmapped areas would be the gap between the +heap and the uppermost mmap()-ed region, and the gap between the lowermost +mmap()-ed region and the stack in most of the cases. Because these gaps are +exceptionally huge in usual address spaces, excluding these will be sufficient +to make a reasonable trade-off. Below shows this in detail:: + + + + + (small mmap()-ed regions and munmap()-ed regions) + + + + + +PTE Accessed-bit Based Access Check +----------------------------------- + +Both of the implementations for physical and virtual address spaces use PTE +Accessed-bit for basic access checks. Only one difference is the way of +finding the relevant PTE Accessed bit(s) from the address. While the +implementation for the virtual address walks the page table for the target task +of the address, the implementation for the physical address walks every page +table having a mapping to the address. In this way, the implementations find +and clear the bit(s) for next sampling target address and checks whether the +bit(s) set again after one sampling period. This could disturb other kernel +subsystems using the Accessed bits, namely Idle page tracking and the reclaim +logic. DAMON does nothing to avoid disturbing Idle page tracking, so handling +the interference is the responsibility of sysadmins. However, it solves the +conflict with the reclaim logic using ``PG_idle`` and ``PG_young`` page flags, +as Idle page tracking does. + + +Address Space Independent Core Mechanisms +========================================= + +Below four sections describe each of the DAMON core mechanisms and the five +monitoring attributes, ``sampling interval``, ``aggregation interval``, +``update interval``, ``minimum number of regions``, and ``maximum number of +regions``. + + +Access Frequency Monitoring +--------------------------- + +The output of DAMON says what pages are how frequently accessed for a given +duration. The resolution of the access frequency is controlled by setting +``sampling interval`` and ``aggregation interval``. In detail, DAMON checks +access to each page per ``sampling interval`` and aggregates the results. In +other words, counts the number of the accesses to each page. After each +``aggregation interval`` passes, DAMON calls callback functions that previously +registered by users so that users can read the aggregated results and then +clears the results. This can be described in below simple pseudo-code:: + + while monitoring_on: + for page in monitoring_target: + if accessed(page): + nr_accesses[page] += 1 + if time() % aggregation_interval == 0: + for callback in user_registered_callbacks: + callback(monitoring_target, nr_accesses) + for page in monitoring_target: + nr_accesses[page] = 0 + sleep(sampling interval) + +The monitoring overhead of this mechanism will arbitrarily increase as the +size of the target workload grows. + + +Region Based Sampling +--------------------- + +To avoid the unbounded increase of the overhead, DAMON groups adjacent pages +that assumed to have the same access frequencies into a region. As long as the +assumption (pages in a region have the same access frequencies) is kept, only +one page in the region is required to be checked. Thus, for each ``sampling +interval``, DAMON randomly picks one page in each region, waits for one +``sampling interval``, checks whether the page is accessed meanwhile, and +increases the access frequency of the region if so. Therefore, the monitoring +overhead is controllable by setting the number of regions. DAMON allows users +to set the minimum and the maximum number of regions for the trade-off. + +This scheme, however, cannot preserve the quality of the output if the +assumption is not guaranteed. + + +Adaptive Regions Adjustment +--------------------------- + +Even somehow the initial monitoring target regions are well constructed to +fulfill the assumption (pages in same region have similar access frequencies), +the data access pattern can be dynamically changed. This will result in low +monitoring quality. To keep the assumption as much as possible, DAMON +adaptively merges and splits each region based on their access frequency. + +For each ``aggregation interval``, it compares the access frequencies of +adjacent regions and merges those if the frequency difference is small. Then, +after it reports and clears the aggregated access frequency of each region, it +splits each region into two or three regions if the total number of regions +will not exceed the user-specified maximum number of regions after the split. + +In this way, DAMON provides its best-effort quality and minimal overhead while +keeping the bounds users set for their trade-off. + + +Dynamic Target Space Updates Handling +------------------------------------- + +The monitoring target address range could dynamically changed. For example, +virtual memory could be dynamically mapped and unmapped. Physical memory could +be hot-plugged. + +As the changes could be quite frequent in some cases, DAMON allows the +monitoring operations to check dynamic changes including memory mapping changes +and applies it to monitoring operations-related data structures such as the +abstracted monitoring target memory area only for each of a user-specified time +interval (``update interval``). diff --git a/Documentation/mm/damon/faq.rst b/Documentation/mm/damon/faq.rst new file mode 100644 index 000000000000..dde7e2414ee6 --- /dev/null +++ b/Documentation/mm/damon/faq.rst @@ -0,0 +1,50 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================== +Frequently Asked Questions +========================== + +Why a new subsystem, instead of extending perf or other user space tools? +========================================================================= + +First, because it needs to be lightweight as much as possible so that it can be +used online, any unnecessary overhead such as kernel - user space context +switching cost should be avoided. Second, DAMON aims to be used by other +programs including the kernel. Therefore, having a dependency on specific +tools like perf is not desirable. These are the two biggest reasons why DAMON +is implemented in the kernel space. + + +Can 'idle pages tracking' or 'perf mem' substitute DAMON? +========================================================= + +Idle page tracking is a low level primitive for access check of the physical +address space. 'perf mem' is similar, though it can use sampling to minimize +the overhead. On the other hand, DAMON is a higher-level framework for the +monitoring of various address spaces. It is focused on memory management +optimization and provides sophisticated accuracy/overhead handling mechanisms. +Therefore, 'idle pages tracking' and 'perf mem' could provide a subset of +DAMON's output, but cannot substitute DAMON. + + +Does DAMON support virtual memory only? +======================================= + +No. The core of the DAMON is address space independent. The address space +specific monitoring operations including monitoring target regions +constructions and actual access checks can be implemented and configured on the +DAMON core by the users. In this way, DAMON users can monitor any address +space with any access check technique. + +Nonetheless, DAMON provides vma/rmap tracking and PTE Accessed bit check based +implementations of the address space dependent functions for the virtual memory +and the physical memory by default, for a reference and convenient use. + + +Can I simply monitor page granularity? +====================================== + +Yes. You can do so by setting the ``min_nr_regions`` attribute higher than the +working set size divided by the page size. Because the monitoring target +regions size is forced to be ``>=page size``, the region split will make no +effect. diff --git a/Documentation/mm/damon/index.rst b/Documentation/mm/damon/index.rst new file mode 100644 index 000000000000..48c0bbff98b2 --- /dev/null +++ b/Documentation/mm/damon/index.rst @@ -0,0 +1,29 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================== +DAMON: Data Access MONitor +========================== + +DAMON is a data access monitoring framework subsystem for the Linux kernel. +The core mechanisms of DAMON (refer to :doc:`design` for the detail) make it + + - *accurate* (the monitoring output is useful enough for DRAM level memory + management; It might not appropriate for CPU Cache levels, though), + - *light-weight* (the monitoring overhead is low enough to be applied online), + and + - *scalable* (the upper-bound of the overhead is in constant range regardless + of the size of target workloads). + +Using this framework, therefore, the kernel's memory management mechanisms can +make advanced decisions. Experimental memory management optimization works +that incurring high data accesses monitoring overhead could implemented again. +In user space, meanwhile, users who have some special workloads can write +personalized applications for better understanding and optimizations of their +workloads and systems. + +.. toctree:: + :maxdepth: 2 + + faq + design + api diff --git a/Documentation/mm/free_page_reporting.rst b/Documentation/mm/free_page_reporting.rst new file mode 100644 index 000000000000..8c05e62d8b2b --- /dev/null +++ b/Documentation/mm/free_page_reporting.rst @@ -0,0 +1,40 @@ +.. _free_page_reporting: + +===================== +Free Page Reporting +===================== + +Free page reporting is an API by which a device can register to receive +lists of pages that are currently unused by the system. This is useful in +the case of virtualization where a guest is then able to use this data to +notify the hypervisor that it is no longer using certain pages in memory. + +For the driver, typically a balloon driver, to use of this functionality +it will allocate and initialize a page_reporting_dev_info structure. The +field within the structure it will populate is the "report" function +pointer used to process the scatterlist. It must also guarantee that it can +handle at least PAGE_REPORTING_CAPACITY worth of scatterlist entries per +call to the function. A call to page_reporting_register will register the +page reporting interface with the reporting framework assuming no other +page reporting devices are already registered. + +Once registered the page reporting API will begin reporting batches of +pages to the driver. The API will start reporting pages 2 seconds after +the interface is registered and will continue to do so 2 seconds after any +page of a sufficiently high order is freed. + +Pages reported will be stored in the scatterlist passed to the reporting +function with the final entry having the end bit set in entry nent - 1. +While pages are being processed by the report function they will not be +accessible to the allocator. Once the report function has been completed +the pages will be returned to the free area from which they were obtained. + +Prior to removing a driver that is making use of free page reporting it +is necessary to call page_reporting_unregister to have the +page_reporting_dev_info structure that is currently in use by free page +reporting removed. Doing this will prevent further reports from being +issued via the interface. If another driver or the same driver is +registered it is possible for it to resume where the previous driver had +left off in terms of reporting free pages. + +Alexander Duyck, Dec 04, 2019 diff --git a/Documentation/mm/frontswap.rst b/Documentation/mm/frontswap.rst new file mode 100644 index 000000000000..feecc5e24477 --- /dev/null +++ b/Documentation/mm/frontswap.rst @@ -0,0 +1,266 @@ +.. _frontswap: + +========= +Frontswap +========= + +Frontswap provides a "transcendent memory" interface for swap pages. +In some environments, dramatic performance savings may be obtained because +swapped pages are saved in RAM (or a RAM-like device) instead of a swap disk. + +.. _Transcendent memory in a nutshell: https://lwn.net/Articles/454795/ + +Frontswap is so named because it can be thought of as the opposite of +a "backing" store for a swap device. The storage is assumed to be +a synchronous concurrency-safe page-oriented "pseudo-RAM device" conforming +to the requirements of transcendent memory (such as Xen's "tmem", or +in-kernel compressed memory, aka "zcache", or future RAM-like devices); +this pseudo-RAM device is not directly accessible or addressable by the +kernel and is of unknown and possibly time-varying size. The driver +links itself to frontswap by calling frontswap_register_ops to set the +frontswap_ops funcs appropriately and the functions it provides must +conform to certain policies as follows: + +An "init" prepares the device to receive frontswap pages associated +with the specified swap device number (aka "type"). A "store" will +copy the page to transcendent memory and associate it with the type and +offset associated with the page. A "load" will copy the page, if found, +from transcendent memory into kernel memory, but will NOT remove the page +from transcendent memory. An "invalidate_page" will remove the page +from transcendent memory and an "invalidate_area" will remove ALL pages +associated with the swap type (e.g., like swapoff) and notify the "device" +to refuse further stores with that swap type. + +Once a page is successfully stored, a matching load on the page will normally +succeed. So when the kernel finds itself in a situation where it needs +to swap out a page, it first attempts to use frontswap. If the store returns +success, the data has been successfully saved to transcendent memory and +a disk write and, if the data is later read back, a disk read are avoided. +If a store returns failure, transcendent memory has rejected the data, and the +page can be written to swap as usual. + +Note that if a page is stored and the page already exists in transcendent memory +(a "duplicate" store), either the store succeeds and the data is overwritten, +or the store fails AND the page is invalidated. This ensures stale data may +never be obtained from frontswap. + +If properly configured, monitoring of frontswap is done via debugfs in +the `/sys/kernel/debug/frontswap` directory. The effectiveness of +frontswap can be measured (across all swap devices) with: + +``failed_stores`` + how many store attempts have failed + +``loads`` + how many loads were attempted (all should succeed) + +``succ_stores`` + how many store attempts have succeeded + +``invalidates`` + how many invalidates were attempted + +A backend implementation may provide additional metrics. + +FAQ +=== + +* Where's the value? + +When a workload starts swapping, performance falls through the floor. +Frontswap significantly increases performance in many such workloads by +providing a clean, dynamic interface to read and write swap pages to +"transcendent memory" that is otherwise not directly addressable to the kernel. +This interface is ideal when data is transformed to a different form +and size (such as with compression) or secretly moved (as might be +useful for write-balancing for some RAM-like devices). Swap pages (and +evicted page-cache pages) are a great use for this kind of slower-than-RAM- +but-much-faster-than-disk "pseudo-RAM device". + +Frontswap with a fairly small impact on the kernel, +provides a huge amount of flexibility for more dynamic, flexible RAM +utilization in various system configurations: + +In the single kernel case, aka "zcache", pages are compressed and +stored in local memory, thus increasing the total anonymous pages +that can be safely kept in RAM. Zcache essentially trades off CPU +cycles used in compression/decompression for better memory utilization. +Benchmarks have shown little or no impact when memory pressure is +low while providing a significant performance improvement (25%+) +on some workloads under high memory pressure. + +"RAMster" builds on zcache by adding "peer-to-peer" transcendent memory +support for clustered systems. Frontswap pages are locally compressed +as in zcache, but then "remotified" to another system's RAM. This +allows RAM to be dynamically load-balanced back-and-forth as needed, +i.e. when system A is overcommitted, it can swap to system B, and +vice versa. RAMster can also be configured as a memory server so +many servers in a cluster can swap, dynamically as needed, to a single +server configured with a large amount of RAM... without pre-configuring +how much of the RAM is available for each of the clients! + +In the virtual case, the whole point of virtualization is to statistically +multiplex physical resources across the varying demands of multiple +virtual machines. This is really hard to do with RAM and efforts to do +it well with no kernel changes have essentially failed (except in some +well-publicized special-case workloads). +Specifically, the Xen Transcendent Memory backend allows otherwise +"fallow" hypervisor-owned RAM to not only be "time-shared" between multiple +virtual machines, but the pages can be compressed and deduplicated to +optimize RAM utilization. And when guest OS's are induced to surrender +underutilized RAM (e.g. with "selfballooning"), sudden unexpected +memory pressure may result in swapping; frontswap allows those pages +to be swapped to and from hypervisor RAM (if overall host system memory +conditions allow), thus mitigating the potentially awful performance impact +of unplanned swapping. + +A KVM implementation is underway and has been RFC'ed to lkml. And, +using frontswap, investigation is also underway on the use of NVM as +a memory extension technology. + +* Sure there may be performance advantages in some situations, but + what's the space/time overhead of frontswap? + +If CONFIG_FRONTSWAP is disabled, every frontswap hook compiles into +nothingness and the only overhead is a few extra bytes per swapon'ed +swap device. If CONFIG_FRONTSWAP is enabled but no frontswap "backend" +registers, there is one extra global variable compared to zero for +every swap page read or written. If CONFIG_FRONTSWAP is enabled +AND a frontswap backend registers AND the backend fails every "store" +request (i.e. provides no memory despite claiming it might), +CPU overhead is still negligible -- and since every frontswap fail +precedes a swap page write-to-disk, the system is highly likely +to be I/O bound and using a small fraction of a percent of a CPU +will be irrelevant anyway. + +As for space, if CONFIG_FRONTSWAP is enabled AND a frontswap backend +registers, one bit is allocated for every swap page for every swap +device that is swapon'd. This is added to the EIGHT bits (which +was sixteen until about 2.6.34) that the kernel already allocates +for every swap page for every swap device that is swapon'd. (Hugh +Dickins has observed that frontswap could probably steal one of +the existing eight bits, but let's worry about that minor optimization +later.) For very large swap disks (which are rare) on a standard +4K pagesize, this is 1MB per 32GB swap. + +When swap pages are stored in transcendent memory instead of written +out to disk, there is a side effect that this may create more memory +pressure that can potentially outweigh the other advantages. A +backend, such as zcache, must implement policies to carefully (but +dynamically) manage memory limits to ensure this doesn't happen. + +* OK, how about a quick overview of what this frontswap patch does + in terms that a kernel hacker can grok? + +Let's assume that a frontswap "backend" has registered during +kernel initialization; this registration indicates that this +frontswap backend has access to some "memory" that is not directly +accessible by the kernel. Exactly how much memory it provides is +entirely dynamic and random. + +Whenever a swap-device is swapon'd frontswap_init() is called, +passing the swap device number (aka "type") as a parameter. +This notifies frontswap to expect attempts to "store" swap pages +associated with that number. + +Whenever the swap subsystem is readying a page to write to a swap +device (c.f swap_writepage()), frontswap_store is called. Frontswap +consults with the frontswap backend and if the backend says it does NOT +have room, frontswap_store returns -1 and the kernel swaps the page +to the swap device as normal. Note that the response from the frontswap +backend is unpredictable to the kernel; it may choose to never accept a +page, it could accept every ninth page, or it might accept every +page. But if the backend does accept a page, the data from the page +has already been copied and associated with the type and offset, +and the backend guarantees the persistence of the data. In this case, +frontswap sets a bit in the "frontswap_map" for the swap device +corresponding to the page offset on the swap device to which it would +otherwise have written the data. + +When the swap subsystem needs to swap-in a page (swap_readpage()), +it first calls frontswap_load() which checks the frontswap_map to +see if the page was earlier accepted by the frontswap backend. If +it was, the page of data is filled from the frontswap backend and +the swap-in is complete. If not, the normal swap-in code is +executed to obtain the page of data from the real swap device. + +So every time the frontswap backend accepts a page, a swap device read +and (potentially) a swap device write are replaced by a "frontswap backend +store" and (possibly) a "frontswap backend loads", which are presumably much +faster. + +* Can't frontswap be configured as a "special" swap device that is + just higher priority than any real swap device (e.g. like zswap, + or maybe swap-over-nbd/NFS)? + +No. First, the existing swap subsystem doesn't allow for any kind of +swap hierarchy. Perhaps it could be rewritten to accommodate a hierarchy, +but this would require fairly drastic changes. Even if it were +rewritten, the existing swap subsystem uses the block I/O layer which +assumes a swap device is fixed size and any page in it is linearly +addressable. Frontswap barely touches the existing swap subsystem, +and works around the constraints of the block I/O subsystem to provide +a great deal of flexibility and dynamicity. + +For example, the acceptance of any swap page by the frontswap backend is +entirely unpredictable. This is critical to the definition of frontswap +backends because it grants completely dynamic discretion to the +backend. In zcache, one cannot know a priori how compressible a page is. +"Poorly" compressible pages can be rejected, and "poorly" can itself be +defined dynamically depending on current memory constraints. + +Further, frontswap is entirely synchronous whereas a real swap +device is, by definition, asynchronous and uses block I/O. The +block I/O layer is not only unnecessary, but may perform "optimizations" +that are inappropriate for a RAM-oriented device including delaying +the write of some pages for a significant amount of time. Synchrony is +required to ensure the dynamicity of the backend and to avoid thorny race +conditions that would unnecessarily and greatly complicate frontswap +and/or the block I/O subsystem. That said, only the initial "store" +and "load" operations need be synchronous. A separate asynchronous thread +is free to manipulate the pages stored by frontswap. For example, +the "remotification" thread in RAMster uses standard asynchronous +kernel sockets to move compressed frontswap pages to a remote machine. +Similarly, a KVM guest-side implementation could do in-guest compression +and use "batched" hypercalls. + +In a virtualized environment, the dynamicity allows the hypervisor +(or host OS) to do "intelligent overcommit". For example, it can +choose to accept pages only until host-swapping might be imminent, +then force guests to do their own swapping. + +There is a downside to the transcendent memory specifications for +frontswap: Since any "store" might fail, there must always be a real +slot on a real swap device to swap the page. Thus frontswap must be +implemented as a "shadow" to every swapon'd device with the potential +capability of holding every page that the swap device might have held +and the possibility that it might hold no pages at all. This means +that frontswap cannot contain more pages than the total of swapon'd +swap devices. For example, if NO swap device is configured on some +installation, frontswap is useless. Swapless portable devices +can still use frontswap but a backend for such devices must configure +some kind of "ghost" swap device and ensure that it is never used. + +* Why this weird definition about "duplicate stores"? If a page + has been previously successfully stored, can't it always be + successfully overwritten? + +Nearly always it can, but no, sometimes it cannot. Consider an example +where data is compressed and the original 4K page has been compressed +to 1K. Now an attempt is made to overwrite the page with data that +is non-compressible and so would take the entire 4K. But the backend +has no more space. In this case, the store must be rejected. Whenever +frontswap rejects a store that would overwrite, it also must invalidate +the old data and ensure that it is no longer accessible. Since the +swap subsystem then writes the new data to the read swap device, +this is the correct course of action to ensure coherency. + +* Why does the frontswap patch create the new include file swapfile.h? + +The frontswap code depends on some swap-subsystem-internal data +structures that have, over the years, moved back and forth between +static and global. This seemed a reasonable compromise: Define +them as global but declare them in a new include file that isn't +included by the large number of source files that include swap.h. + +Dan Magenheimer, last updated April 9, 2012 diff --git a/Documentation/mm/highmem.rst b/Documentation/mm/highmem.rst new file mode 100644 index 000000000000..c9887f241c6c --- /dev/null +++ b/Documentation/mm/highmem.rst @@ -0,0 +1,167 @@ +.. _highmem: + +==================== +High Memory Handling +==================== + +By: Peter Zijlstra + +.. contents:: :local: + +What Is High Memory? +==================== + +High memory (highmem) is used when the size of physical memory approaches or +exceeds the maximum size of virtual memory. At that point it becomes +impossible for the kernel to keep all of the available physical memory mapped +at all times. This means the kernel needs to start using temporary mappings of +the pieces of physical memory that it wants to access. + +The part of (physical) memory not covered by a permanent mapping is what we +refer to as 'highmem'. There are various architecture dependent constraints on +where exactly that border lies. + +In the i386 arch, for example, we choose to map the kernel into every process's +VM space so that we don't have to pay the full TLB invalidation costs for +kernel entry/exit. This means the available virtual memory space (4GiB on +i386) has to be divided between user and kernel space. + +The traditional split for architectures using this approach is 3:1, 3GiB for +userspace and the top 1GiB for kernel space:: + + +--------+ 0xffffffff + | Kernel | + +--------+ 0xc0000000 + | | + | User | + | | + +--------+ 0x00000000 + +This means that the kernel can at most map 1GiB of physical memory at any one +time, but because we need virtual address space for other things - including +temporary maps to access the rest of the physical memory - the actual direct +map will typically be less (usually around ~896MiB). + +Other architectures that have mm context tagged TLBs can have separate kernel +and user maps. Some hardware (like some ARMs), however, have limited virtual +space when they use mm context tags. + + +Temporary Virtual Mappings +========================== + +The kernel contains several ways of creating temporary mappings. The following +list shows them in order of preference of use. + +* kmap_local_page(). This function is used to require short term mappings. + It can be invoked from any context (including interrupts) but the mappings + can only be used in the context which acquired them. + + This function should be preferred, where feasible, over all the others. + + These mappings are thread-local and CPU-local, meaning that the mapping + can only be accessed from within this thread and the thread is bound the + CPU while the mapping is active. Even if the thread is preempted (since + preemption is never disabled by the function) the CPU can not be + unplugged from the system via CPU-hotplug until the mapping is disposed. + + It's valid to take pagefaults in a local kmap region, unless the context + in which the local mapping is acquired does not allow it for other reasons. + + kmap_local_page() always returns a valid virtual address and it is assumed + that kunmap_local() will never fail. + + Nesting kmap_local_page() and kmap_atomic() mappings is allowed to a certain + extent (up to KMAP_TYPE_NR) but their invocations have to be strictly ordered + because the map implementation is stack based. See kmap_local_page() kdocs + (included in the "Functions" section) for details on how to manage nested + mappings. + +* kmap_atomic(). This permits a very short duration mapping of a single + page. Since the mapping is restricted to the CPU that issued it, it + performs well, but the issuing task is therefore required to stay on that + CPU until it has finished, lest some other task displace its mappings. + + kmap_atomic() may also be used by interrupt contexts, since it does not + sleep and the callers too may not sleep until after kunmap_atomic() is + called. + + Each call of kmap_atomic() in the kernel creates a non-preemptible section + and disable pagefaults. This could be a source of unwanted latency. Therefore + users should prefer kmap_local_page() instead of kmap_atomic(). + + It is assumed that k[un]map_atomic() won't fail. + +* kmap(). This should be used to make short duration mapping of a single + page with no restrictions on preemption or migration. It comes with an + overhead as mapping space is restricted and protected by a global lock + for synchronization. When mapping is no longer needed, the address that + the page was mapped to must be released with kunmap(). + + Mapping changes must be propagated across all the CPUs. kmap() also + requires global TLB invalidation when the kmap's pool wraps and it might + block when the mapping space is fully utilized until a slot becomes + available. Therefore, kmap() is only callable from preemptible context. + + All the above work is necessary if a mapping must last for a relatively + long time but the bulk of high-memory mappings in the kernel are + short-lived and only used in one place. This means that the cost of + kmap() is mostly wasted in such cases. kmap() was not intended for long + term mappings but it has morphed in that direction and its use is + strongly discouraged in newer code and the set of the preceding functions + should be preferred. + + On 64-bit systems, calls to kmap_local_page(), kmap_atomic() and kmap() have + no real work to do because a 64-bit address space is more than sufficient to + address all the physical memory whose pages are permanently mapped. + +* vmap(). This can be used to make a long duration mapping of multiple + physical pages into a contiguous virtual space. It needs global + synchronization to unmap. + + +Cost of Temporary Mappings +========================== + +The cost of creating temporary mappings can be quite high. The arch has to +manipulate the kernel's page tables, the data TLB and/or the MMU's registers. + +If CONFIG_HIGHMEM is not set, then the kernel will try and create a mapping +simply with a bit of arithmetic that will convert the page struct address into +a pointer to the page contents rather than juggling mappings about. In such a +case, the unmap operation may be a null operation. + +If CONFIG_MMU is not set, then there can be no temporary mappings and no +highmem. In such a case, the arithmetic approach will also be used. + + +i386 PAE +======== + +The i386 arch, under some circumstances, will permit you to stick up to 64GiB +of RAM into your 32-bit machine. This has a number of consequences: + +* Linux needs a page-frame structure for each page in the system and the + pageframes need to live in the permanent mapping, which means: + +* you can have 896M/sizeof(struct page) page-frames at most; with struct + page being 32-bytes that would end up being something in the order of 112G + worth of pages; the kernel, however, needs to store more than just + page-frames in that memory... + +* PAE makes your page tables larger - which slows the system down as more + data has to be accessed to traverse in TLB fills and the like. One + advantage is that PAE has more PTE bits and can provide advanced features + like NX and PAT. + +The general recommendation is that you don't use more than 8GiB on a 32-bit +machine - although more might work for you and your workload, you're pretty +much on your own - don't expect kernel developers to really care much if things +come apart. + + +Functions +========= + +.. kernel-doc:: include/linux/highmem.h +.. kernel-doc:: include/linux/highmem-internal.h diff --git a/Documentation/mm/hmm.rst b/Documentation/mm/hmm.rst new file mode 100644 index 000000000000..f2a59ed82ed3 --- /dev/null +++ b/Documentation/mm/hmm.rst @@ -0,0 +1,452 @@ +.. _hmm: + +===================================== +Heterogeneous Memory Management (HMM) +===================================== + +Provide infrastructure and helpers to integrate non-conventional memory (device +memory like GPU on board memory) into regular kernel path, with the cornerstone +of this being specialized struct page for such memory (see sections 5 to 7 of +this document). + +HMM also provides optional helpers for SVM (Share Virtual Memory), i.e., +allowing a device to transparently access program addresses coherently with +the CPU meaning that any valid pointer on the CPU is also a valid pointer +for the device. This is becoming mandatory to simplify the use of advanced +heterogeneous computing where GPU, DSP, or FPGA are used to perform various +computations on behalf of a process. + +This document is divided as follows: in the first section I expose the problems +related to using device specific memory allocators. In the second section, I +expose the hardware limitations that are inherent to many platforms. The third +section gives an overview of the HMM design. The fourth section explains how +CPU page-table mirroring works and the purpose of HMM in this context. The +fifth section deals with how device memory is represented inside the kernel. +Finally, the last section presents a new migration helper that allows +leveraging the device DMA engine. + +.. contents:: :local: + +Problems of using a device specific memory allocator +==================================================== + +Devices with a large amount of on board memory (several gigabytes) like GPUs +have historically managed their memory through dedicated driver specific APIs. +This creates a disconnect between memory allocated and managed by a device +driver and regular application memory (private anonymous, shared memory, or +regular file backed memory). From here on I will refer to this aspect as split +address space. I use shared address space to refer to the opposite situation: +i.e., one in which any application memory region can be used by a device +transparently. + +Split address space happens because devices can only access memory allocated +through a device specific API. This implies that all memory objects in a program +are not equal from the device point of view which complicates large programs +that rely on a wide set of libraries. + +Concretely, this means that code that wants to leverage devices like GPUs needs +to copy objects between generically allocated memory (malloc, mmap private, mmap +share) and memory allocated through the device driver API (this still ends up +with an mmap but of the device file). + +For flat data sets (array, grid, image, ...) this isn't too hard to achieve but +for complex data sets (list, tree, ...) it's hard to get right. Duplicating a +complex data set needs to re-map all the pointer relations between each of its +elements. This is error prone and programs get harder to debug because of the +duplicate data set and addresses. + +Split address space also means that libraries cannot transparently use data +they are getting from the core program or another library and thus each library +might have to duplicate its input data set using the device specific memory +allocator. Large projects suffer from this and waste resources because of the +various memory copies. + +Duplicating each library API to accept as input or output memory allocated by +each device specific allocator is not a viable option. It would lead to a +combinatorial explosion in the library entry points. + +Finally, with the advance of high level language constructs (in C++ but in +other languages too) it is now possible for the compiler to leverage GPUs and +other devices without programmer knowledge. Some compiler identified patterns +are only do-able with a shared address space. It is also more reasonable to use +a shared address space for all other patterns. + + +I/O bus, device memory characteristics +====================================== + +I/O buses cripple shared address spaces due to a few limitations. Most I/O +buses only allow basic memory access from device to main memory; even cache +coherency is often optional. Access to device memory from a CPU is even more +limited. More often than not, it is not cache coherent. + +If we only consider the PCIE bus, then a device can access main memory (often +through an IOMMU) and be cache coherent with the CPUs. However, it only allows +a limited set of atomic operations from the device on main memory. This is worse +in the other direction: the CPU can only access a limited range of the device +memory and cannot perform atomic operations on it. Thus device memory cannot +be considered the same as regular memory from the kernel point of view. + +Another crippling factor is the limited bandwidth (~32GBytes/s with PCIE 4.0 +and 16 lanes). This is 33 times less than the fastest GPU memory (1 TBytes/s). +The final limitation is latency. Access to main memory from the device has an +order of magnitude higher latency than when the device accesses its own memory. + +Some platforms are developing new I/O buses or additions/modifications to PCIE +to address some of these limitations (OpenCAPI, CCIX). They mainly allow +two-way cache coherency between CPU and device and allow all atomic operations the +architecture supports. Sadly, not all platforms are following this trend and +some major architectures are left without hardware solutions to these problems. + +So for shared address space to make sense, not only must we allow devices to +access any memory but we must also permit any memory to be migrated to device +memory while the device is using it (blocking CPU access while it happens). + + +Shared address space and migration +================================== + +HMM intends to provide two main features. The first one is to share the address +space by duplicating the CPU page table in the device page table so the same +address points to the same physical memory for any valid main memory address in +the process address space. + +To achieve this, HMM offers a set of helpers to populate the device page table +while keeping track of CPU page table updates. Device page table updates are +not as easy as CPU page table updates. To update the device page table, you must +allocate a buffer (or use a pool of pre-allocated buffers) and write GPU +specific commands in it to perform the update (unmap, cache invalidations, and +flush, ...). This cannot be done through common code for all devices. Hence +why HMM provides helpers to factor out everything that can be while leaving the +hardware specific details to the device driver. + +The second mechanism HMM provides is a new kind of ZONE_DEVICE memory that +allows allocating a struct page for each page of device memory. Those pages +are special because the CPU cannot map them. However, they allow migrating +main memory to device memory using existing migration mechanisms and everything +looks like a page that is swapped out to disk from the CPU point of view. Using a +struct page gives the easiest and cleanest integration with existing mm +mechanisms. Here again, HMM only provides helpers, first to hotplug new ZONE_DEVICE +memory for the device memory and second to perform migration. Policy decisions +of what and when to migrate is left to the device driver. + +Note that any CPU access to a device page triggers a page fault and a migration +back to main memory. For example, when a page backing a given CPU address A is +migrated from a main memory page to a device page, then any CPU access to +address A triggers a page fault and initiates a migration back to main memory. + +With these two features, HMM not only allows a device to mirror process address +space and keeps both CPU and device page tables synchronized, but also +leverages device memory by migrating the part of the data set that is actively being +used by the device. + + +Address space mirroring implementation and API +============================================== + +Address space mirroring's main objective is to allow duplication of a range of +CPU page table into a device page table; HMM helps keep both synchronized. A +device driver that wants to mirror a process address space must start with the +registration of a mmu_interval_notifier:: + + int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub, + struct mm_struct *mm, unsigned long start, + unsigned long length, + const struct mmu_interval_notifier_ops *ops); + +During the ops->invalidate() callback the device driver must perform the +update action to the range (mark range read only, or fully unmap, etc.). The +device must complete the update before the driver callback returns. + +When the device driver wants to populate a range of virtual addresses, it can +use:: + + int hmm_range_fault(struct hmm_range *range); + +It will trigger a page fault on missing or read-only entries if write access is +requested (see below). Page faults use the generic mm page fault code path just +like a CPU page fault. + +Both functions copy CPU page table entries into their pfns array argument. Each +entry in that array corresponds to an address in the virtual range. HMM +provides a set of flags to help the driver identify special CPU page table +entries. + +Locking within the sync_cpu_device_pagetables() callback is the most important +aspect the driver must respect in order to keep things properly synchronized. +The usage pattern is:: + + int driver_populate_range(...) + { + struct hmm_range range; + ... + + range.notifier = &interval_sub; + range.start = ...; + range.end = ...; + range.hmm_pfns = ...; + + if (!mmget_not_zero(interval_sub->notifier.mm)) + return -EFAULT; + + again: + range.notifier_seq = mmu_interval_read_begin(&interval_sub); + mmap_read_lock(mm); + ret = hmm_range_fault(&range); + if (ret) { + mmap_read_unlock(mm); + if (ret == -EBUSY) + goto again; + return ret; + } + mmap_read_unlock(mm); + + take_lock(driver->update); + if (mmu_interval_read_retry(&ni, range.notifier_seq) { + release_lock(driver->update); + goto again; + } + + /* Use pfns array content to update device page table, + * under the update lock */ + + release_lock(driver->update); + return 0; + } + +The driver->update lock is the same lock that the driver takes inside its +invalidate() callback. That lock must be held before calling +mmu_interval_read_retry() to avoid any race with a concurrent CPU page table +update. + +Leverage default_flags and pfn_flags_mask +========================================= + +The hmm_range struct has 2 fields, default_flags and pfn_flags_mask, that specify +fault or snapshot policy for the whole range instead of having to set them +for each entry in the pfns array. + +For instance if the device driver wants pages for a range with at least read +permission, it sets:: + + range->default_flags = HMM_PFN_REQ_FAULT; + range->pfn_flags_mask = 0; + +and calls hmm_range_fault() as described above. This will fill fault all pages +in the range with at least read permission. + +Now let's say the driver wants to do the same except for one page in the range for +which it wants to have write permission. Now driver set:: + + range->default_flags = HMM_PFN_REQ_FAULT; + range->pfn_flags_mask = HMM_PFN_REQ_WRITE; + range->pfns[index_of_write] = HMM_PFN_REQ_WRITE; + +With this, HMM will fault in all pages with at least read (i.e., valid) and for the +address == range->start + (index_of_write << PAGE_SHIFT) it will fault with +write permission i.e., if the CPU pte does not have write permission set then HMM +will call handle_mm_fault(). + +After hmm_range_fault completes the flag bits are set to the current state of +the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be set if the page is +writable. + + +Represent and manage device memory from core kernel point of view +================================================================= + +Several different designs were tried to support device memory. The first one +used a device specific data structure to keep information about migrated memory +and HMM hooked itself in various places of mm code to handle any access to +addresses that were backed by device memory. It turns out that this ended up +replicating most of the fields of struct page and also needed many kernel code +paths to be updated to understand this new kind of memory. + +Most kernel code paths never try to access the memory behind a page +but only care about struct page contents. Because of this, HMM switched to +directly using struct page for device memory which left most kernel code paths +unaware of the difference. We only need to make sure that no one ever tries to +map those pages from the CPU side. + +Migration to and from device memory +=================================== + +Because the CPU cannot access device memory directly, the device driver must +use hardware DMA or device specific load/store instructions to migrate data. +The migrate_vma_setup(), migrate_vma_pages(), and migrate_vma_finalize() +functions are designed to make drivers easier to write and to centralize common +code across drivers. + +Before migrating pages to device private memory, special device private +``struct page`` need to be created. These will be used as special "swap" +page table entries so that a CPU process will fault if it tries to access +a page that has been migrated to device private memory. + +These can be allocated and freed with:: + + struct resource *res; + struct dev_pagemap pagemap; + + res = request_free_mem_region(&iomem_resource, /* number of bytes */, + "name of driver resource"); + pagemap.type = MEMORY_DEVICE_PRIVATE; + pagemap.range.start = res->start; + pagemap.range.end = res->end; + pagemap.nr_range = 1; + pagemap.ops = &device_devmem_ops; + memremap_pages(&pagemap, numa_node_id()); + + memunmap_pages(&pagemap); + release_mem_region(pagemap.range.start, range_len(&pagemap.range)); + +There are also devm_request_free_mem_region(), devm_memremap_pages(), +devm_memunmap_pages(), and devm_release_mem_region() when the resources can +be tied to a ``struct device``. + +The overall migration steps are similar to migrating NUMA pages within system +memory (see :ref:`Page migration `) but the steps are split +between device driver specific code and shared common code: + +1. ``mmap_read_lock()`` + + The device driver has to pass a ``struct vm_area_struct`` to + migrate_vma_setup() so the mmap_read_lock() or mmap_write_lock() needs to + be held for the duration of the migration. + +2. ``migrate_vma_setup(struct migrate_vma *args)`` + + The device driver initializes the ``struct migrate_vma`` fields and passes + the pointer to migrate_vma_setup(). The ``args->flags`` field is used to + filter which source pages should be migrated. For example, setting + ``MIGRATE_VMA_SELECT_SYSTEM`` will only migrate system memory and + ``MIGRATE_VMA_SELECT_DEVICE_PRIVATE`` will only migrate pages residing in + device private memory. If the latter flag is set, the ``args->pgmap_owner`` + field is used to identify device private pages owned by the driver. This + avoids trying to migrate device private pages residing in other devices. + Currently only anonymous private VMA ranges can be migrated to or from + system memory and device private memory. + + One of the first steps migrate_vma_setup() does is to invalidate other + device's MMUs with the ``mmu_notifier_invalidate_range_start(()`` and + ``mmu_notifier_invalidate_range_end()`` calls around the page table + walks to fill in the ``args->src`` array with PFNs to be migrated. + The ``invalidate_range_start()`` callback is passed a + ``struct mmu_notifier_range`` with the ``event`` field set to + ``MMU_NOTIFY_MIGRATE`` and the ``owner`` field set to + the ``args->pgmap_owner`` field passed to migrate_vma_setup(). This is + allows the device driver to skip the invalidation callback and only + invalidate device private MMU mappings that are actually migrating. + This is explained more in the next section. + + While walking the page tables, a ``pte_none()`` or ``is_zero_pfn()`` + entry results in a valid "zero" PFN stored in the ``args->src`` array. + This lets the driver allocate device private memory and clear it instead + of copying a page of zeros. Valid PTE entries to system memory or + device private struct pages will be locked with ``lock_page()``, isolated + from the LRU (if system memory since device private pages are not on + the LRU), unmapped from the process, and a special migration PTE is + inserted in place of the original PTE. + migrate_vma_setup() also clears the ``args->dst`` array. + +3. The device driver allocates destination pages and copies source pages to + destination pages. + + The driver checks each ``src`` entry to see if the ``MIGRATE_PFN_MIGRATE`` + bit is set and skips entries that are not migrating. The device driver + can also choose to skip migrating a page by not filling in the ``dst`` + array for that page. + + The driver then allocates either a device private struct page or a + system memory page, locks the page with ``lock_page()``, and fills in the + ``dst`` array entry with:: + + dst[i] = migrate_pfn(page_to_pfn(dpage)); + + Now that the driver knows that this page is being migrated, it can + invalidate device private MMU mappings and copy device private memory + to system memory or another device private page. The core Linux kernel + handles CPU page table invalidations so the device driver only has to + invalidate its own MMU mappings. + + The driver can use ``migrate_pfn_to_page(src[i])`` to get the + ``struct page`` of the source and either copy the source page to the + destination or clear the destination device private memory if the pointer + is ``NULL`` meaning the source page was not populated in system memory. + +4. ``migrate_vma_pages()`` + + This step is where the migration is actually "committed". + + If the source page was a ``pte_none()`` or ``is_zero_pfn()`` page, this + is where the newly allocated page is inserted into the CPU's page table. + This can fail if a CPU thread faults on the same page. However, the page + table is locked and only one of the new pages will be inserted. + The device driver will see that the ``MIGRATE_PFN_MIGRATE`` bit is cleared + if it loses the race. + + If the source page was locked, isolated, etc. the source ``struct page`` + information is now copied to destination ``struct page`` finalizing the + migration on the CPU side. + +5. Device driver updates device MMU page tables for pages still migrating, + rolling back pages not migrating. + + If the ``src`` entry still has ``MIGRATE_PFN_MIGRATE`` bit set, the device + driver can update the device MMU and set the write enable bit if the + ``MIGRATE_PFN_WRITE`` bit is set. + +6. ``migrate_vma_finalize()`` + + This step replaces the special migration page table entry with the new + page's page table entry and releases the reference to the source and + destination ``struct page``. + +7. ``mmap_read_unlock()`` + + The lock can now be released. + +Exclusive access memory +======================= + +Some devices have features such as atomic PTE bits that can be used to implement +atomic access to system memory. To support atomic operations to a shared virtual +memory page such a device needs access to that page which is exclusive of any +userspace access from the CPU. The ``make_device_exclusive_range()`` function +can be used to make a memory range inaccessible from userspace. + +This replaces all mappings for pages in the given range with special swap +entries. Any attempt to access the swap entry results in a fault which is +resovled by replacing the entry with the original mapping. A driver gets +notified that the mapping has been changed by MMU notifiers, after which point +it will no longer have exclusive access to the page. Exclusive access is +guranteed to last until the driver drops the page lock and page reference, at +which point any CPU faults on the page may proceed as described. + +Memory cgroup (memcg) and rss accounting +======================================== + +For now, device memory is accounted as any regular page in rss counters (either +anonymous if device page is used for anonymous, file if device page is used for +file backed page, or shmem if device page is used for shared memory). This is a +deliberate choice to keep existing applications, that might start using device +memory without knowing about it, running unimpacted. + +A drawback is that the OOM killer might kill an application using a lot of +device memory and not a lot of regular system memory and thus not freeing much +system memory. We want to gather more real world experience on how applications +and system react under memory pressure in the presence of device memory before +deciding to account device memory differently. + + +Same decision was made for memory cgroup. Device memory pages are accounted +against same memory cgroup a regular page would be accounted to. This does +simplify migration to and from device memory. This also means that migration +back from device memory to regular memory cannot fail because it would +go above memory cgroup limit. We might revisit this choice latter on once we +get more experience in how device memory is used and its impact on memory +resource control. + + +Note that device memory can never be pinned by a device driver nor through GUP +and thus such memory is always free upon process exit. Or when last reference +is dropped in case of shared memory or file backed memory. diff --git a/Documentation/mm/hugetlbfs_reserv.rst b/Documentation/mm/hugetlbfs_reserv.rst new file mode 100644 index 000000000000..f143954e0d05 --- /dev/null +++ b/Documentation/mm/hugetlbfs_reserv.rst @@ -0,0 +1,596 @@ +.. _hugetlbfs_reserve: + +===================== +Hugetlbfs Reservation +===================== + +Overview +======== + +Huge pages as described at :ref:`hugetlbpage` are typically +preallocated for application use. These huge pages are instantiated in a +task's address space at page fault time if the VMA indicates huge pages are +to be used. If no huge page exists at page fault time, the task is sent +a SIGBUS and often dies an unhappy death. Shortly after huge page support +was added, it was determined that it would be better to detect a shortage +of huge pages at mmap() time. The idea is that if there were not enough +huge pages to cover the mapping, the mmap() would fail. This was first +done with a simple check in the code at mmap() time to determine if there +were enough free huge pages to cover the mapping. Like most things in the +kernel, the code has evolved over time. However, the basic idea was to +'reserve' huge pages at mmap() time to ensure that huge pages would be +available for page faults in that mapping. The description below attempts to +describe how huge page reserve processing is done in the v4.10 kernel. + + +Audience +======== +This description is primarily targeted at kernel developers who are modifying +hugetlbfs code. + + +The Data Structures +=================== + +resv_huge_pages + This is a global (per-hstate) count of reserved huge pages. Reserved + huge pages are only available to the task which reserved them. + Therefore, the number of huge pages generally available is computed + as (``free_huge_pages - resv_huge_pages``). +Reserve Map + A reserve map is described by the structure:: + + struct resv_map { + struct kref refs; + spinlock_t lock; + struct list_head regions; + long adds_in_progress; + struct list_head region_cache; + long region_cache_count; + }; + + There is one reserve map for each huge page mapping in the system. + The regions list within the resv_map describes the regions within + the mapping. A region is described as:: + + struct file_region { + struct list_head link; + long from; + long to; + }; + + The 'from' and 'to' fields of the file region structure are huge page + indices into the mapping. Depending on the type of mapping, a + region in the reserv_map may indicate reservations exist for the + range, or reservations do not exist. +Flags for MAP_PRIVATE Reservations + These are stored in the bottom bits of the reservation map pointer. + + ``#define HPAGE_RESV_OWNER (1UL << 0)`` + Indicates this task is the owner of the reservations + associated with the mapping. + ``#define HPAGE_RESV_UNMAPPED (1UL << 1)`` + Indicates task originally mapping this range (and creating + reserves) has unmapped a page from this task (the child) + due to a failed COW. +Page Flags + The PagePrivate page flag is used to indicate that a huge page + reservation must be restored when the huge page is freed. More + details will be discussed in the "Freeing huge pages" section. + + +Reservation Map Location (Private or Shared) +============================================ + +A huge page mapping or segment is either private or shared. If private, +it is typically only available to a single address space (task). If shared, +it can be mapped into multiple address spaces (tasks). The location and +semantics of the reservation map is significantly different for the two types +of mappings. Location differences are: + +- For private mappings, the reservation map hangs off the VMA structure. + Specifically, vma->vm_private_data. This reserve map is created at the + time the mapping (mmap(MAP_PRIVATE)) is created. +- For shared mappings, the reservation map hangs off the inode. Specifically, + inode->i_mapping->private_data. Since shared mappings are always backed + by files in the hugetlbfs filesystem, the hugetlbfs code ensures each inode + contains a reservation map. As a result, the reservation map is allocated + when the inode is created. + + +Creating Reservations +===================== +Reservations are created when a huge page backed shared memory segment is +created (shmget(SHM_HUGETLB)) or a mapping is created via mmap(MAP_HUGETLB). +These operations result in a call to the routine hugetlb_reserve_pages():: + + int hugetlb_reserve_pages(struct inode *inode, + long from, long to, + struct vm_area_struct *vma, + vm_flags_t vm_flags) + +The first thing hugetlb_reserve_pages() does is check if the NORESERVE +flag was specified in either the shmget() or mmap() call. If NORESERVE +was specified, then this routine returns immediately as no reservations +are desired. + +The arguments 'from' and 'to' are huge page indices into the mapping or +underlying file. For shmget(), 'from' is always 0 and 'to' corresponds to +the length of the segment/mapping. For mmap(), the offset argument could +be used to specify the offset into the underlying file. In such a case, +the 'from' and 'to' arguments have been adjusted by this offset. + +One of the big differences between PRIVATE and SHARED mappings is the way +in which reservations are represented in the reservation map. + +- For shared mappings, an entry in the reservation map indicates a reservation + exists or did exist for the corresponding page. As reservations are + consumed, the reservation map is not modified. +- For private mappings, the lack of an entry in the reservation map indicates + a reservation exists for the corresponding page. As reservations are + consumed, entries are added to the reservation map. Therefore, the + reservation map can also be used to determine which reservations have + been consumed. + +For private mappings, hugetlb_reserve_pages() creates the reservation map and +hangs it off the VMA structure. In addition, the HPAGE_RESV_OWNER flag is set +to indicate this VMA owns the reservations. + +The reservation map is consulted to determine how many huge page reservations +are needed for the current mapping/segment. For private mappings, this is +always the value (to - from). However, for shared mappings it is possible that +some reservations may already exist within the range (to - from). See the +section :ref:`Reservation Map Modifications ` +for details on how this is accomplished. + +The mapping may be associated with a subpool. If so, the subpool is consulted +to ensure there is sufficient space for the mapping. It is possible that the +subpool has set aside reservations that can be used for the mapping. See the +section :ref:`Subpool Reservations ` for more details. + +After consulting the reservation map and subpool, the number of needed new +reservations is known. The routine hugetlb_acct_memory() is called to check +for and take the requested number of reservations. hugetlb_acct_memory() +calls into routines that potentially allocate and adjust surplus page counts. +However, within those routines the code is simply checking to ensure there +are enough free huge pages to accommodate the reservation. If there are, +the global reservation count resv_huge_pages is adjusted something like the +following:: + + if (resv_needed <= (resv_huge_pages - free_huge_pages)) + resv_huge_pages += resv_needed; + +Note that the global lock hugetlb_lock is held when checking and adjusting +these counters. + +If there were enough free huge pages and the global count resv_huge_pages +was adjusted, then the reservation map associated with the mapping is +modified to reflect the reservations. In the case of a shared mapping, a +file_region will exist that includes the range 'from' - 'to'. For private +mappings, no modifications are made to the reservation map as lack of an +entry indicates a reservation exists. + +If hugetlb_reserve_pages() was successful, the global reservation count and +reservation map associated with the mapping will be modified as required to +ensure reservations exist for the range 'from' - 'to'. + +.. _consume_resv: + +Consuming Reservations/Allocating a Huge Page +============================================= + +Reservations are consumed when huge pages associated with the reservations +are allocated and instantiated in the corresponding mapping. The allocation +is performed within the routine alloc_huge_page():: + + struct page *alloc_huge_page(struct vm_area_struct *vma, + unsigned long addr, int avoid_reserve) + +alloc_huge_page is passed a VMA pointer and a virtual address, so it can +consult the reservation map to determine if a reservation exists. In addition, +alloc_huge_page takes the argument avoid_reserve which indicates reserves +should not be used even if it appears they have been set aside for the +specified address. The avoid_reserve argument is most often used in the case +of Copy on Write and Page Migration where additional copies of an existing +page are being allocated. + +The helper routine vma_needs_reservation() is called to determine if a +reservation exists for the address within the mapping(vma). See the section +:ref:`Reservation Map Helper Routines ` for detailed +information on what this routine does. +The value returned from vma_needs_reservation() is generally +0 or 1. 0 if a reservation exists for the address, 1 if no reservation exists. +If a reservation does not exist, and there is a subpool associated with the +mapping the subpool is consulted to determine if it contains reservations. +If the subpool contains reservations, one can be used for this allocation. +However, in every case the avoid_reserve argument overrides the use of +a reservation for the allocation. After determining whether a reservation +exists and can be used for the allocation, the routine dequeue_huge_page_vma() +is called. This routine takes two arguments related to reservations: + +- avoid_reserve, this is the same value/argument passed to alloc_huge_page() +- chg, even though this argument is of type long only the values 0 or 1 are + passed to dequeue_huge_page_vma. If the value is 0, it indicates a + reservation exists (see the section "Memory Policy and Reservations" for + possible issues). If the value is 1, it indicates a reservation does not + exist and the page must be taken from the global free pool if possible. + +The free lists associated with the memory policy of the VMA are searched for +a free page. If a page is found, the value free_huge_pages is decremented +when the page is removed from the free list. If there was a reservation +associated with the page, the following adjustments are made:: + + SetPagePrivate(page); /* Indicates allocating this page consumed + * a reservation, and if an error is + * encountered such that the page must be + * freed, the reservation will be restored. */ + resv_huge_pages--; /* Decrement the global reservation count */ + +Note, if no huge page can be found that satisfies the VMA's memory policy +an attempt will be made to allocate one using the buddy allocator. This +brings up the issue of surplus huge pages and overcommit which is beyond +the scope reservations. Even if a surplus page is allocated, the same +reservation based adjustments as above will be made: SetPagePrivate(page) and +resv_huge_pages--. + +After obtaining a new huge page, (page)->private is set to the value of +the subpool associated with the page if it exists. This will be used for +subpool accounting when the page is freed. + +The routine vma_commit_reservation() is then called to adjust the reserve +map based on the consumption of the reservation. In general, this involves +ensuring the page is represented within a file_region structure of the region +map. For shared mappings where the reservation was present, an entry +in the reserve map already existed so no change is made. However, if there +was no reservation in a shared mapping or this was a private mapping a new +entry must be created. + +It is possible that the reserve map could have been changed between the call +to vma_needs_reservation() at the beginning of alloc_huge_page() and the +call to vma_commit_reservation() after the page was allocated. This would +be possible if hugetlb_reserve_pages was called for the same page in a shared +mapping. In such cases, the reservation count and subpool free page count +will be off by one. This rare condition can be identified by comparing the +return value from vma_needs_reservation and vma_commit_reservation. If such +a race is detected, the subpool and global reserve counts are adjusted to +compensate. See the section +:ref:`Reservation Map Helper Routines ` for more +information on these routines. + + +Instantiate Huge Pages +====================== + +After huge page allocation, the page is typically added to the page tables +of the allocating task. Before this, pages in a shared mapping are added +to the page cache and pages in private mappings are added to an anonymous +reverse mapping. In both cases, the PagePrivate flag is cleared. Therefore, +when a huge page that has been instantiated is freed no adjustment is made +to the global reservation count (resv_huge_pages). + + +Freeing Huge Pages +================== + +Huge page freeing is performed by the routine free_huge_page(). This routine +is the destructor for hugetlbfs compound pages. As a result, it is only +passed a pointer to the page struct. When a huge page is freed, reservation +accounting may need to be performed. This would be the case if the page was +associated with a subpool that contained reserves, or the page is being freed +on an error path where a global reserve count must be restored. + +The page->private field points to any subpool associated with the page. +If the PagePrivate flag is set, it indicates the global reserve count should +be adjusted (see the section +:ref:`Consuming Reservations/Allocating a Huge Page ` +for information on how these are set). + +The routine first calls hugepage_subpool_put_pages() for the page. If this +routine returns a value of 0 (which does not equal the value passed 1) it +indicates reserves are associated with the subpool, and this newly free page +must be used to keep the number of subpool reserves above the minimum size. +Therefore, the global resv_huge_pages counter is incremented in this case. + +If the PagePrivate flag was set in the page, the global resv_huge_pages counter +will always be incremented. + +.. _sub_pool_resv: + +Subpool Reservations +==================== + +There is a struct hstate associated with each huge page size. The hstate +tracks all huge pages of the specified size. A subpool represents a subset +of pages within a hstate that is associated with a mounted hugetlbfs +filesystem. + +When a hugetlbfs filesystem is mounted a min_size option can be specified +which indicates the minimum number of huge pages required by the filesystem. +If this option is specified, the number of huge pages corresponding to +min_size are reserved for use by the filesystem. This number is tracked in +the min_hpages field of a struct hugepage_subpool. At mount time, +hugetlb_acct_memory(min_hpages) is called to reserve the specified number of +huge pages. If they can not be reserved, the mount fails. + +The routines hugepage_subpool_get/put_pages() are called when pages are +obtained from or released back to a subpool. They perform all subpool +accounting, and track any reservations associated with the subpool. +hugepage_subpool_get/put_pages are passed the number of huge pages by which +to adjust the subpool 'used page' count (down for get, up for put). Normally, +they return the same value that was passed or an error if not enough pages +exist in the subpool. + +However, if reserves are associated with the subpool a return value less +than the passed value may be returned. This return value indicates the +number of additional global pool adjustments which must be made. For example, +suppose a subpool contains 3 reserved huge pages and someone asks for 5. +The 3 reserved pages associated with the subpool can be used to satisfy part +of the request. But, 2 pages must be obtained from the global pools. To +relay this information to the caller, the value 2 is returned. The caller +is then responsible for attempting to obtain the additional two pages from +the global pools. + + +COW and Reservations +==================== + +Since shared mappings all point to and use the same underlying pages, the +biggest reservation concern for COW is private mappings. In this case, +two tasks can be pointing at the same previously allocated page. One task +attempts to write to the page, so a new page must be allocated so that each +task points to its own page. + +When the page was originally allocated, the reservation for that page was +consumed. When an attempt to allocate a new page is made as a result of +COW, it is possible that no free huge pages are free and the allocation +will fail. + +When the private mapping was originally created, the owner of the mapping +was noted by setting the HPAGE_RESV_OWNER bit in the pointer to the reservation +map of the owner. Since the owner created the mapping, the owner owns all +the reservations associated with the mapping. Therefore, when a write fault +occurs and there is no page available, different action is taken for the owner +and non-owner of the reservation. + +In the case where the faulting task is not the owner, the fault will fail and +the task will typically receive a SIGBUS. + +If the owner is the faulting task, we want it to succeed since it owned the +original reservation. To accomplish this, the page is unmapped from the +non-owning task. In this way, the only reference is from the owning task. +In addition, the HPAGE_RESV_UNMAPPED bit is set in the reservation map pointer +of the non-owning task. The non-owning task may receive a SIGBUS if it later +faults on a non-present page. But, the original owner of the +mapping/reservation will behave as expected. + + +.. _resv_map_modifications: + +Reservation Map Modifications +============================= + +The following low level routines are used to make modifications to a +reservation map. Typically, these routines are not called directly. Rather, +a reservation map helper routine is called which calls one of these low level +routines. These low level routines are fairly well documented in the source +code (mm/hugetlb.c). These routines are:: + + long region_chg(struct resv_map *resv, long f, long t); + long region_add(struct resv_map *resv, long f, long t); + void region_abort(struct resv_map *resv, long f, long t); + long region_count(struct resv_map *resv, long f, long t); + +Operations on the reservation map typically involve two operations: + +1) region_chg() is called to examine the reserve map and determine how + many pages in the specified range [f, t) are NOT currently represented. + + The calling code performs global checks and allocations to determine if + there are enough huge pages for the operation to succeed. + +2) + a) If the operation can succeed, region_add() is called to actually modify + the reservation map for the same range [f, t) previously passed to + region_chg(). + b) If the operation can not succeed, region_abort is called for the same + range [f, t) to abort the operation. + +Note that this is a two step process where region_add() and region_abort() +are guaranteed to succeed after a prior call to region_chg() for the same +range. region_chg() is responsible for pre-allocating any data structures +necessary to ensure the subsequent operations (specifically region_add())) +will succeed. + +As mentioned above, region_chg() determines the number of pages in the range +which are NOT currently represented in the map. This number is returned to +the caller. region_add() returns the number of pages in the range added to +the map. In most cases, the return value of region_add() is the same as the +return value of region_chg(). However, in the case of shared mappings it is +possible for changes to the reservation map to be made between the calls to +region_chg() and region_add(). In this case, the return value of region_add() +will not match the return value of region_chg(). It is likely that in such +cases global counts and subpool accounting will be incorrect and in need of +adjustment. It is the responsibility of the caller to check for this condition +and make the appropriate adjustments. + +The routine region_del() is called to remove regions from a reservation map. +It is typically called in the following situations: + +- When a file in the hugetlbfs filesystem is being removed, the inode will + be released and the reservation map freed. Before freeing the reservation + map, all the individual file_region structures must be freed. In this case + region_del is passed the range [0, LONG_MAX). +- When a hugetlbfs file is being truncated. In this case, all allocated pages + after the new file size must be freed. In addition, any file_region entries + in the reservation map past the new end of file must be deleted. In this + case, region_del is passed the range [new_end_of_file, LONG_MAX). +- When a hole is being punched in a hugetlbfs file. In this case, huge pages + are removed from the middle of the file one at a time. As the pages are + removed, region_del() is called to remove the corresponding entry from the + reservation map. In this case, region_del is passed the range + [page_idx, page_idx + 1). + +In every case, region_del() will return the number of pages removed from the +reservation map. In VERY rare cases, region_del() can fail. This can only +happen in the hole punch case where it has to split an existing file_region +entry and can not allocate a new structure. In this error case, region_del() +will return -ENOMEM. The problem here is that the reservation map will +indicate that there is a reservation for the page. However, the subpool and +global reservation counts will not reflect the reservation. To handle this +situation, the routine hugetlb_fix_reserve_counts() is called to adjust the +counters so that they correspond with the reservation map entry that could +not be deleted. + +region_count() is called when unmapping a private huge page mapping. In +private mappings, the lack of a entry in the reservation map indicates that +a reservation exists. Therefore, by counting the number of entries in the +reservation map we know how many reservations were consumed and how many are +outstanding (outstanding = (end - start) - region_count(resv, start, end)). +Since the mapping is going away, the subpool and global reservation counts +are decremented by the number of outstanding reservations. + +.. _resv_map_helpers: + +Reservation Map Helper Routines +=============================== + +Several helper routines exist to query and modify the reservation maps. +These routines are only interested with reservations for a specific huge +page, so they just pass in an address instead of a range. In addition, +they pass in the associated VMA. From the VMA, the type of mapping (private +or shared) and the location of the reservation map (inode or VMA) can be +determined. These routines simply call the underlying routines described +in the section "Reservation Map Modifications". However, they do take into +account the 'opposite' meaning of reservation map entries for private and +shared mappings and hide this detail from the caller:: + + long vma_needs_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +This routine calls region_chg() for the specified page. If no reservation +exists, 1 is returned. If a reservation exists, 0 is returned:: + + long vma_commit_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +This calls region_add() for the specified page. As in the case of region_chg +and region_add, this routine is to be called after a previous call to +vma_needs_reservation. It will add a reservation entry for the page. It +returns 1 if the reservation was added and 0 if not. The return value should +be compared with the return value of the previous call to +vma_needs_reservation. An unexpected difference indicates the reservation +map was modified between calls:: + + void vma_end_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +This calls region_abort() for the specified page. As in the case of region_chg +and region_abort, this routine is to be called after a previous call to +vma_needs_reservation. It will abort/end the in progress reservation add +operation:: + + long vma_add_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +This is a special wrapper routine to help facilitate reservation cleanup +on error paths. It is only called from the routine restore_reserve_on_error(). +This routine is used in conjunction with vma_needs_reservation in an attempt +to add a reservation to the reservation map. It takes into account the +different reservation map semantics for private and shared mappings. Hence, +region_add is called for shared mappings (as an entry present in the map +indicates a reservation), and region_del is called for private mappings (as +the absence of an entry in the map indicates a reservation). See the section +"Reservation cleanup in error paths" for more information on what needs to +be done on error paths. + + +Reservation Cleanup in Error Paths +================================== + +As mentioned in the section +:ref:`Reservation Map Helper Routines `, reservation +map modifications are performed in two steps. First vma_needs_reservation +is called before a page is allocated. If the allocation is successful, +then vma_commit_reservation is called. If not, vma_end_reservation is called. +Global and subpool reservation counts are adjusted based on success or failure +of the operation and all is well. + +Additionally, after a huge page is instantiated the PagePrivate flag is +cleared so that accounting when the page is ultimately freed is correct. + +However, there are several instances where errors are encountered after a huge +page is allocated but before it is instantiated. In this case, the page +allocation has consumed the reservation and made the appropriate subpool, +reservation map and global count adjustments. If the page is freed at this +time (before instantiation and clearing of PagePrivate), then free_huge_page +will increment the global reservation count. However, the reservation map +indicates the reservation was consumed. This resulting inconsistent state +will cause the 'leak' of a reserved huge page. The global reserve count will +be higher than it should and prevent allocation of a pre-allocated page. + +The routine restore_reserve_on_error() attempts to handle this situation. It +is fairly well documented. The intention of this routine is to restore +the reservation map to the way it was before the page allocation. In this +way, the state of the reservation map will correspond to the global reservation +count after the page is freed. + +The routine restore_reserve_on_error itself may encounter errors while +attempting to restore the reservation map entry. In this case, it will +simply clear the PagePrivate flag of the page. In this way, the global +reserve count will not be incremented when the page is freed. However, the +reservation map will continue to look as though the reservation was consumed. +A page can still be allocated for the address, but it will not use a reserved +page as originally intended. + +There is some code (most notably userfaultfd) which can not call +restore_reserve_on_error. In this case, it simply modifies the PagePrivate +so that a reservation will not be leaked when the huge page is freed. + + +Reservations and Memory Policy +============================== +Per-node huge page lists existed in struct hstate when git was first used +to manage Linux code. The concept of reservations was added some time later. +When reservations were added, no attempt was made to take memory policy +into account. While cpusets are not exactly the same as memory policy, this +comment in hugetlb_acct_memory sums up the interaction between reservations +and cpusets/memory policy:: + + /* + * When cpuset is configured, it breaks the strict hugetlb page + * reservation as the accounting is done on a global variable. Such + * reservation is completely rubbish in the presence of cpuset because + * the reservation is not checked against page availability for the + * current cpuset. Application can still potentially OOM'ed by kernel + * with lack of free htlb page in cpuset that the task is in. + * Attempt to enforce strict accounting with cpuset is almost + * impossible (or too ugly) because cpuset is too fluid that + * task or memory node can be dynamically moved between cpusets. + * + * The change of semantics for shared hugetlb mapping with cpuset is + * undesirable. However, in order to preserve some of the semantics, + * we fall back to check against current free page availability as + * a best attempt and hopefully to minimize the impact of changing + * semantics that cpuset has. + */ + +Huge page reservations were added to prevent unexpected page allocation +failures (OOM) at page fault time. However, if an application makes use +of cpusets or memory policy there is no guarantee that huge pages will be +available on the required nodes. This is true even if there are a sufficient +number of global reservations. + +Hugetlbfs regression testing +============================ + +The most complete set of hugetlb tests are in the libhugetlbfs repository. +If you modify any hugetlb related code, use the libhugetlbfs test suite +to check for regressions. In addition, if you add any new hugetlb +functionality, please add appropriate tests to libhugetlbfs. + +-- +Mike Kravetz, 7 April 2017 diff --git a/Documentation/mm/hwpoison.rst b/Documentation/mm/hwpoison.rst new file mode 100644 index 000000000000..b9d5253c1305 --- /dev/null +++ b/Documentation/mm/hwpoison.rst @@ -0,0 +1,184 @@ +.. hwpoison: + +======== +hwpoison +======== + +What is hwpoison? +================= + +Upcoming Intel CPUs have support for recovering from some memory errors +(``MCA recovery``). This requires the OS to declare a page "poisoned", +kill the processes associated with it and avoid using it in the future. + +This patchkit implements the necessary infrastructure in the VM. + +To quote the overview comment:: + + High level machine check handler. Handles pages reported by the + hardware as being corrupted usually due to a 2bit ECC memory or cache + failure. + + This focusses on pages detected as corrupted in the background. + When the current CPU tries to consume corruption the currently + running process can just be killed directly instead. This implies + that if the error cannot be handled for some reason it's safe to + just ignore it because no corruption has been consumed yet. Instead + when that happens another machine check will happen. + + Handles page cache pages in various states. The tricky part + here is that we can access any page asynchronous to other VM + users, because memory failures could happen anytime and anywhere, + possibly violating some of their assumptions. This is why this code + has to be extremely careful. Generally it tries to use normal locking + rules, as in get the standard locks, even if that means the + error handling takes potentially a long time. + + Some of the operations here are somewhat inefficient and have non + linear algorithmic complexity, because the data structures have not + been optimized for this case. This is in particular the case + for the mapping from a vma to a process. Since this case is expected + to be rare we hope we can get away with this. + +The code consists of a the high level handler in mm/memory-failure.c, +a new page poison bit and various checks in the VM to handle poisoned +pages. + +The main target right now is KVM guests, but it works for all kinds +of applications. KVM support requires a recent qemu-kvm release. + +For the KVM use there was need for a new signal type so that +KVM can inject the machine check into the guest with the proper +address. This in theory allows other applications to handle +memory failures too. The expection is that near all applications +won't do that, but some very specialized ones might. + +Failure recovery modes +====================== + +There are two (actually three) modes memory failure recovery can be in: + +vm.memory_failure_recovery sysctl set to zero: + All memory failures cause a panic. Do not attempt recovery. + +early kill + (can be controlled globally and per process) + Send SIGBUS to the application as soon as the error is detected + This allows applications who can process memory errors in a gentle + way (e.g. drop affected object) + This is the mode used by KVM qemu. + +late kill + Send SIGBUS when the application runs into the corrupted page. + This is best for memory error unaware applications and default + Note some pages are always handled as late kill. + +User control +============ + +vm.memory_failure_recovery + See sysctl.txt + +vm.memory_failure_early_kill + Enable early kill mode globally + +PR_MCE_KILL + Set early/late kill mode/revert to system default + + arg1: PR_MCE_KILL_CLEAR: + Revert to system default + arg1: PR_MCE_KILL_SET: + arg2 defines thread specific mode + + PR_MCE_KILL_EARLY: + Early kill + PR_MCE_KILL_LATE: + Late kill + PR_MCE_KILL_DEFAULT + Use system global default + + Note that if you want to have a dedicated thread which handles + the SIGBUS(BUS_MCEERR_AO) on behalf of the process, you should + call prctl(PR_MCE_KILL_EARLY) on the designated thread. Otherwise, + the SIGBUS is sent to the main thread. + +PR_MCE_KILL_GET + return current mode + +Testing +======= + +* madvise(MADV_HWPOISON, ....) (as root) - Poison a page in the + process for testing + +* hwpoison-inject module through debugfs ``/sys/kernel/debug/hwpoison/`` + + corrupt-pfn + Inject hwpoison fault at PFN echoed into this file. This does + some early filtering to avoid corrupted unintended pages in test suites. + + unpoison-pfn + Software-unpoison page at PFN echoed into this file. This way + a page can be reused again. This only works for Linux + injected failures, not for real memory failures. Once any hardware + memory failure happens, this feature is disabled. + + Note these injection interfaces are not stable and might change between + kernel versions + + corrupt-filter-dev-major, corrupt-filter-dev-minor + Only handle memory failures to pages associated with the file + system defined by block device major/minor. -1U is the + wildcard value. This should be only used for testing with + artificial injection. + + corrupt-filter-memcg + Limit injection to pages owned by memgroup. Specified by inode + number of the memcg. + + Example:: + + mkdir /sys/fs/cgroup/mem/hwpoison + + usemem -m 100 -s 1000 & + echo `jobs -p` > /sys/fs/cgroup/mem/hwpoison/tasks + + memcg_ino=$(ls -id /sys/fs/cgroup/mem/hwpoison | cut -f1 -d' ') + echo $memcg_ino > /debug/hwpoison/corrupt-filter-memcg + + page-types -p `pidof init` --hwpoison # shall do nothing + page-types -p `pidof usemem` --hwpoison # poison its pages + + corrupt-filter-flags-mask, corrupt-filter-flags-value + When specified, only poison pages if ((page_flags & mask) == + value). This allows stress testing of many kinds of + pages. The page_flags are the same as in /proc/kpageflags. The + flag bits are defined in include/linux/kernel-page-flags.h and + documented in Documentation/admin-guide/mm/pagemap.rst + +* Architecture specific MCE injector + + x86 has mce-inject, mce-test + + Some portable hwpoison test programs in mce-test, see below. + +References +========== + +http://halobates.de/mce-lc09-2.pdf + Overview presentation from LinuxCon 09 + +git://git.kernel.org/pub/scm/utils/cpu/mce/mce-test.git + Test suite (hwpoison specific portable tests in tsrc) + +git://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git + x86 specific injector + + +Limitations +=========== +- Not all page types are supported and never will. Most kernel internal + objects cannot be recovered, only LRU pages for now. + +--- +Andi Kleen, Oct 2009 diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst new file mode 100644 index 000000000000..575ccd40e30c --- /dev/null +++ b/Documentation/mm/index.rst @@ -0,0 +1,68 @@ +===================================== +Linux Memory Management Documentation +===================================== + +Memory Management Guide +======================= + +This is a guide to understanding the memory management subsystem +of Linux. If you are looking for advice on simply allocating memory, +see the :ref:`memory_allocation`. For controlling and tuning guides, +see the :doc:`admin guide <../admin-guide/mm/index>`. + +.. toctree:: + :maxdepth: 1 + + physical_memory + page_tables + process_addrs + bootmem + page_allocation + vmalloc + slab + highmem + page_reclaim + swap + page_cache + shmfs + oom + +Legacy Documentation +==================== + +This is a collection of older documents about the Linux memory management +(MM) subsystem internals with different level of details ranging from +notes and mailing list responses for elaborating descriptions of data +structures and algorithms. It should all be integrated nicely into the +above structured documentation, or deleted if it has served its purpose. + +.. toctree:: + :maxdepth: 1 + + active_mm + arch_pgtable_helpers + balance + damon/index + free_page_reporting + frontswap + hmm + hwpoison + hugetlbfs_reserv + ksm + memory-model + mmu_notifier + numa + overcommit-accounting + page_migration + page_frags + page_owner + page_table_check + remap_file_pages + slub + split_page_table_lock + transhuge + unevictable-lru + vmalloced-kernel-stacks + vmemmap_dedup + z3fold + zsmalloc diff --git a/Documentation/mm/ksm.rst b/Documentation/mm/ksm.rst new file mode 100644 index 000000000000..9e37add068e6 --- /dev/null +++ b/Documentation/mm/ksm.rst @@ -0,0 +1,87 @@ +.. _ksm: + +======================= +Kernel Samepage Merging +======================= + +KSM is a memory-saving de-duplication feature, enabled by CONFIG_KSM=y, +added to the Linux kernel in 2.6.32. See ``mm/ksm.c`` for its implementation, +and http://lwn.net/Articles/306704/ and https://lwn.net/Articles/330589/ + +The userspace interface of KSM is described in :ref:`Documentation/admin-guide/mm/ksm.rst ` + +Design +====== + +Overview +-------- + +.. kernel-doc:: mm/ksm.c + :DOC: Overview + +Reverse mapping +--------------- +KSM maintains reverse mapping information for KSM pages in the stable +tree. + +If a KSM page is shared between less than ``max_page_sharing`` VMAs, +the node of the stable tree that represents such KSM page points to a +list of struct rmap_item and the ``page->mapping`` of the +KSM page points to the stable tree node. + +When the sharing passes this threshold, KSM adds a second dimension to +the stable tree. The tree node becomes a "chain" that links one or +more "dups". Each "dup" keeps reverse mapping information for a KSM +page with ``page->mapping`` pointing to that "dup". + +Every "chain" and all "dups" linked into a "chain" enforce the +invariant that they represent the same write protected memory content, +even if each "dup" will be pointed by a different KSM page copy of +that content. + +This way the stable tree lookup computational complexity is unaffected +if compared to an unlimited list of reverse mappings. It is still +enforced that there cannot be KSM page content duplicates in the +stable tree itself. + +The deduplication limit enforced by ``max_page_sharing`` is required +to avoid the virtual memory rmap lists to grow too large. The rmap +walk has O(N) complexity where N is the number of rmap_items +(i.e. virtual mappings) that are sharing the page, which is in turn +capped by ``max_page_sharing``. So this effectively spreads the linear +O(N) computational complexity from rmap walk context over different +KSM pages. The ksmd walk over the stable_node "chains" is also O(N), +but N is the number of stable_node "dups", not the number of +rmap_items, so it has not a significant impact on ksmd performance. In +practice the best stable_node "dup" candidate will be kept and found +at the head of the "dups" list. + +High values of ``max_page_sharing`` result in faster memory merging +(because there will be fewer stable_node dups queued into the +stable_node chain->hlist to check for pruning) and higher +deduplication factor at the expense of slower worst case for rmap +walks for any KSM page which can happen during swapping, compaction, +NUMA balancing and page migration. + +The ``stable_node_dups/stable_node_chains`` ratio is also affected by the +``max_page_sharing`` tunable, and an high ratio may indicate fragmentation +in the stable_node dups, which could be solved by introducing +fragmentation algorithms in ksmd which would refile rmap_items from +one stable_node dup to another stable_node dup, in order to free up +stable_node "dups" with few rmap_items in them, but that may increase +the ksmd CPU usage and possibly slowdown the readonly computations on +the KSM pages of the applications. + +The whole list of stable_node "dups" linked in the stable_node +"chains" is scanned periodically in order to prune stale stable_nodes. +The frequency of such scans is defined by +``stable_node_chains_prune_millisecs`` sysfs tunable. + +Reference +--------- +.. kernel-doc:: mm/ksm.c + :functions: mm_slot ksm_scan stable_node rmap_item + +-- +Izik Eidus, +Hugh Dickins, 17 Nov 2009 diff --git a/Documentation/mm/memory-model.rst b/Documentation/mm/memory-model.rst new file mode 100644 index 000000000000..3779e562dc76 --- /dev/null +++ b/Documentation/mm/memory-model.rst @@ -0,0 +1,177 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _physical_memory_model: + +===================== +Physical Memory Model +===================== + +Physical memory in a system may be addressed in different ways. The +simplest case is when the physical memory starts at address 0 and +spans a contiguous range up to the maximal address. It could be, +however, that this range contains small holes that are not accessible +for the CPU. Then there could be several contiguous ranges at +completely distinct addresses. And, don't forget about NUMA, where +different memory banks are attached to different CPUs. + +Linux abstracts this diversity using one of the two memory models: +FLATMEM and SPARSEMEM. Each architecture defines what +memory models it supports, what the default memory model is and +whether it is possible to manually override that default. + +All the memory models track the status of physical page frames using +struct page arranged in one or more arrays. + +Regardless of the selected memory model, there exists one-to-one +mapping between the physical page frame number (PFN) and the +corresponding `struct page`. + +Each memory model defines :c:func:`pfn_to_page` and :c:func:`page_to_pfn` +helpers that allow the conversion from PFN to `struct page` and vice +versa. + +FLATMEM +======= + +The simplest memory model is FLATMEM. This model is suitable for +non-NUMA systems with contiguous, or mostly contiguous, physical +memory. + +In the FLATMEM memory model, there is a global `mem_map` array that +maps the entire physical memory. For most architectures, the holes +have entries in the `mem_map` array. The `struct page` objects +corresponding to the holes are never fully initialized. + +To allocate the `mem_map` array, architecture specific setup code should +call :c:func:`free_area_init` function. Yet, the mappings array is not +usable until the call to :c:func:`memblock_free_all` that hands all the +memory to the page allocator. + +An architecture may free parts of the `mem_map` array that do not cover the +actual physical pages. In such case, the architecture specific +:c:func:`pfn_valid` implementation should take the holes in the +`mem_map` into account. + +With FLATMEM, the conversion between a PFN and the `struct page` is +straightforward: `PFN - ARCH_PFN_OFFSET` is an index to the +`mem_map` array. + +The `ARCH_PFN_OFFSET` defines the first page frame number for +systems with physical memory starting at address different from 0. + +SPARSEMEM +========= + +SPARSEMEM is the most versatile memory model available in Linux and it +is the only memory model that supports several advanced features such +as hot-plug and hot-remove of the physical memory, alternative memory +maps for non-volatile memory devices and deferred initialization of +the memory map for larger systems. + +The SPARSEMEM model presents the physical memory as a collection of +sections. A section is represented with struct mem_section +that contains `section_mem_map` that is, logically, a pointer to an +array of struct pages. However, it is stored with some other magic +that aids the sections management. The section size and maximal number +of section is specified using `SECTION_SIZE_BITS` and +`MAX_PHYSMEM_BITS` constants defined by each architecture that +supports SPARSEMEM. While `MAX_PHYSMEM_BITS` is an actual width of a +physical address that an architecture supports, the +`SECTION_SIZE_BITS` is an arbitrary value. + +The maximal number of sections is denoted `NR_MEM_SECTIONS` and +defined as + +.. math:: + + NR\_MEM\_SECTIONS = 2 ^ {(MAX\_PHYSMEM\_BITS - SECTION\_SIZE\_BITS)} + +The `mem_section` objects are arranged in a two-dimensional array +called `mem_sections`. The size and placement of this array depend +on `CONFIG_SPARSEMEM_EXTREME` and the maximal possible number of +sections: + +* When `CONFIG_SPARSEMEM_EXTREME` is disabled, the `mem_sections` + array is static and has `NR_MEM_SECTIONS` rows. Each row holds a + single `mem_section` object. +* When `CONFIG_SPARSEMEM_EXTREME` is enabled, the `mem_sections` + array is dynamically allocated. Each row contains PAGE_SIZE worth of + `mem_section` objects and the number of rows is calculated to fit + all the memory sections. + +The architecture setup code should call sparse_init() to +initialize the memory sections and the memory maps. + +With SPARSEMEM there are two possible ways to convert a PFN to the +corresponding `struct page` - a "classic sparse" and "sparse +vmemmap". The selection is made at build time and it is determined by +the value of `CONFIG_SPARSEMEM_VMEMMAP`. + +The classic sparse encodes the section number of a page in page->flags +and uses high bits of a PFN to access the section that maps that page +frame. Inside a section, the PFN is the index to the array of pages. + +The sparse vmemmap uses a virtually mapped memory map to optimize +pfn_to_page and page_to_pfn operations. There is a global `struct +page *vmemmap` pointer that points to a virtually contiguous array of +`struct page` objects. A PFN is an index to that array and the +offset of the `struct page` from `vmemmap` is the PFN of that +page. + +To use vmemmap, an architecture has to reserve a range of virtual +addresses that will map the physical pages containing the memory +map and make sure that `vmemmap` points to that range. In addition, +the architecture should implement :c:func:`vmemmap_populate` method +that will allocate the physical memory and create page tables for the +virtual memory map. If an architecture does not have any special +requirements for the vmemmap mappings, it can use default +:c:func:`vmemmap_populate_basepages` provided by the generic memory +management. + +The virtually mapped memory map allows storing `struct page` objects +for persistent memory devices in pre-allocated storage on those +devices. This storage is represented with struct vmem_altmap +that is eventually passed to vmemmap_populate() through a long chain +of function calls. The vmemmap_populate() implementation may use the +`vmem_altmap` along with :c:func:`vmemmap_alloc_block_buf` helper to +allocate memory map on the persistent memory device. + +ZONE_DEVICE +=========== +The `ZONE_DEVICE` facility builds upon `SPARSEMEM_VMEMMAP` to offer +`struct page` `mem_map` services for device driver identified physical +address ranges. The "device" aspect of `ZONE_DEVICE` relates to the fact +that the page objects for these address ranges are never marked online, +and that a reference must be taken against the device, not just the page +to keep the memory pinned for active use. `ZONE_DEVICE`, via +:c:func:`devm_memremap_pages`, performs just enough memory hotplug to +turn on :c:func:`pfn_to_page`, :c:func:`page_to_pfn`, and +:c:func:`get_user_pages` service for the given range of pfns. Since the +page reference count never drops below 1 the page is never tracked as +free memory and the page's `struct list_head lru` space is repurposed +for back referencing to the host device / driver that mapped the memory. + +While `SPARSEMEM` presents memory as a collection of sections, +optionally collected into memory blocks, `ZONE_DEVICE` users have a need +for smaller granularity of populating the `mem_map`. Given that +`ZONE_DEVICE` memory is never marked online it is subsequently never +subject to its memory ranges being exposed through the sysfs memory +hotplug api on memory block boundaries. The implementation relies on +this lack of user-api constraint to allow sub-section sized memory +ranges to be specified to :c:func:`arch_add_memory`, the top-half of +memory hotplug. Sub-section support allows for 2MB as the cross-arch +common alignment granularity for :c:func:`devm_memremap_pages`. + +The users of `ZONE_DEVICE` are: + +* pmem: Map platform persistent memory to be used as a direct-I/O target + via DAX mappings. + +* hmm: Extend `ZONE_DEVICE` with `->page_fault()` and `->page_free()` + event callbacks to allow a device-driver to coordinate memory management + events related to device-memory, typically GPU memory. See + Documentation/mm/hmm.rst. + +* p2pdma: Create `struct page` objects to allow peer devices in a + PCI/-E topology to coordinate direct-DMA operations between themselves, + i.e. bypass host memory. diff --git a/Documentation/mm/mmu_notifier.rst b/Documentation/mm/mmu_notifier.rst new file mode 100644 index 000000000000..df5d7777fc6b --- /dev/null +++ b/Documentation/mm/mmu_notifier.rst @@ -0,0 +1,99 @@ +.. _mmu_notifier: + +When do you need to notify inside page table lock ? +=================================================== + +When clearing a pte/pmd we are given a choice to notify the event through +(notify version of \*_clear_flush call mmu_notifier_invalidate_range) under +the page table lock. But that notification is not necessary in all cases. + +For secondary TLB (non CPU TLB) like IOMMU TLB or device TLB (when device use +thing like ATS/PASID to get the IOMMU to walk the CPU page table to access a +process virtual address space). There is only 2 cases when you need to notify +those secondary TLB while holding page table lock when clearing a pte/pmd: + + A) page backing address is free before mmu_notifier_invalidate_range_end() + B) a page table entry is updated to point to a new page (COW, write fault + on zero page, __replace_page(), ...) + +Case A is obvious you do not want to take the risk for the device to write to +a page that might now be used by some completely different task. + +Case B is more subtle. For correctness it requires the following sequence to +happen: + + - take page table lock + - clear page table entry and notify ([pmd/pte]p_huge_clear_flush_notify()) + - set page table entry to point to new page + +If clearing the page table entry is not followed by a notify before setting +the new pte/pmd value then you can break memory model like C11 or C++11 for +the device. + +Consider the following scenario (device use a feature similar to ATS/PASID): + +Two address addrA and addrB such that \|addrA - addrB\| >= PAGE_SIZE we assume +they are write protected for COW (other case of B apply too). + +:: + + [Time N] -------------------------------------------------------------------- + CPU-thread-0 {try to write to addrA} + CPU-thread-1 {try to write to addrB} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {read addrA and populate device TLB} + DEV-thread-2 {read addrB and populate device TLB} + [Time N+1] ------------------------------------------------------------------ + CPU-thread-0 {COW_step0: {mmu_notifier_invalidate_range_start(addrA)}} + CPU-thread-1 {COW_step0: {mmu_notifier_invalidate_range_start(addrB)}} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+2] ------------------------------------------------------------------ + CPU-thread-0 {COW_step1: {update page table to point to new page for addrA}} + CPU-thread-1 {COW_step1: {update page table to point to new page for addrB}} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+3] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {preempted} + CPU-thread-2 {write to addrA which is a write to new page} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+3] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {preempted} + CPU-thread-2 {} + CPU-thread-3 {write to addrB which is a write to new page} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+4] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {COW_step3: {mmu_notifier_invalidate_range_end(addrB)}} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+5] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {read addrA from old page} + DEV-thread-2 {read addrB from new page} + +So here because at time N+2 the clear page table entry was not pair with a +notification to invalidate the secondary TLB, the device see the new value for +addrB before seeing the new value for addrA. This break total memory ordering +for the device. + +When changing a pte to write protect or to point to a new write protected page +with same content (KSM) it is fine to delay the mmu_notifier_invalidate_range +call to mmu_notifier_invalidate_range_end() outside the page table lock. This +is true even if the thread doing the page table update is preempted right after +releasing page table lock but before call mmu_notifier_invalidate_range_end(). diff --git a/Documentation/mm/numa.rst b/Documentation/mm/numa.rst new file mode 100644 index 000000000000..99fdeca917ca --- /dev/null +++ b/Documentation/mm/numa.rst @@ -0,0 +1,150 @@ +.. _numa: + +Started Nov 1999 by Kanoj Sarcar + +============= +What is NUMA? +============= + +This question can be answered from a couple of perspectives: the +hardware view and the Linux software view. + +From the hardware perspective, a NUMA system is a computer platform that +comprises multiple components or assemblies each of which may contain 0 +or more CPUs, local memory, and/or IO buses. For brevity and to +disambiguate the hardware view of these physical components/assemblies +from the software abstraction thereof, we'll call the components/assemblies +'cells' in this document. + +Each of the 'cells' may be viewed as an SMP [symmetric multi-processor] subset +of the system--although some components necessary for a stand-alone SMP system +may not be populated on any given cell. The cells of the NUMA system are +connected together with some sort of system interconnect--e.g., a crossbar or +point-to-point link are common types of NUMA system interconnects. Both of +these types of interconnects can be aggregated to create NUMA platforms with +cells at multiple distances from other cells. + +For Linux, the NUMA platforms of interest are primarily what is known as Cache +Coherent NUMA or ccNUMA systems. With ccNUMA systems, all memory is visible +to and accessible from any CPU attached to any cell and cache coherency +is handled in hardware by the processor caches and/or the system interconnect. + +Memory access time and effective memory bandwidth varies depending on how far +away the cell containing the CPU or IO bus making the memory access is from the +cell containing the target memory. For example, access to memory by CPUs +attached to the same cell will experience faster access times and higher +bandwidths than accesses to memory on other, remote cells. NUMA platforms +can have cells at multiple remote distances from any given cell. + +Platform vendors don't build NUMA systems just to make software developers' +lives interesting. Rather, this architecture is a means to provide scalable +memory bandwidth. However, to achieve scalable memory bandwidth, system and +application software must arrange for a large majority of the memory references +[cache misses] to be to "local" memory--memory on the same cell, if any--or +to the closest cell with memory. + +This leads to the Linux software view of a NUMA system: + +Linux divides the system's hardware resources into multiple software +abstractions called "nodes". Linux maps the nodes onto the physical cells +of the hardware platform, abstracting away some of the details for some +architectures. As with physical cells, software nodes may contain 0 or more +CPUs, memory and/or IO buses. And, again, memory accesses to memory on +"closer" nodes--nodes that map to closer cells--will generally experience +faster access times and higher effective bandwidth than accesses to more +remote cells. + +For some architectures, such as x86, Linux will "hide" any node representing a +physical cell that has no memory attached, and reassign any CPUs attached to +that cell to a node representing a cell that does have memory. Thus, on +these architectures, one cannot assume that all CPUs that Linux associates with +a given node will see the same local memory access times and bandwidth. + +In addition, for some architectures, again x86 is an example, Linux supports +the emulation of additional nodes. For NUMA emulation, linux will carve up +the existing nodes--or the system memory for non-NUMA platforms--into multiple +nodes. Each emulated node will manage a fraction of the underlying cells' +physical memory. NUMA emluation is useful for testing NUMA kernel and +application features on non-NUMA platforms, and as a sort of memory resource +management mechanism when used together with cpusets. +[see Documentation/admin-guide/cgroup-v1/cpusets.rst] + +For each node with memory, Linux constructs an independent memory management +subsystem, complete with its own free page lists, in-use page lists, usage +statistics and locks to mediate access. In addition, Linux constructs for +each memory zone [one or more of DMA, DMA32, NORMAL, HIGH_MEMORY, MOVABLE], +an ordered "zonelist". A zonelist specifies the zones/nodes to visit when a +selected zone/node cannot satisfy the allocation request. This situation, +when a zone has no available memory to satisfy a request, is called +"overflow" or "fallback". + +Because some nodes contain multiple zones containing different types of +memory, Linux must decide whether to order the zonelists such that allocations +fall back to the same zone type on a different node, or to a different zone +type on the same node. This is an important consideration because some zones, +such as DMA or DMA32, represent relatively scarce resources. Linux chooses +a default Node ordered zonelist. This means it tries to fallback to other zones +from the same node before using remote nodes which are ordered by NUMA distance. + +By default, Linux will attempt to satisfy memory allocation requests from the +node to which the CPU that executes the request is assigned. Specifically, +Linux will attempt to allocate from the first node in the appropriate zonelist +for the node where the request originates. This is called "local allocation." +If the "local" node cannot satisfy the request, the kernel will examine other +nodes' zones in the selected zonelist looking for the first zone in the list +that can satisfy the request. + +Local allocation will tend to keep subsequent access to the allocated memory +"local" to the underlying physical resources and off the system interconnect-- +as long as the task on whose behalf the kernel allocated some memory does not +later migrate away from that memory. The Linux scheduler is aware of the +NUMA topology of the platform--embodied in the "scheduling domains" data +structures [see Documentation/scheduler/sched-domains.rst]--and the scheduler +attempts to minimize task migration to distant scheduling domains. However, +the scheduler does not take a task's NUMA footprint into account directly. +Thus, under sufficient imbalance, tasks can migrate between nodes, remote +from their initial node and kernel data structures. + +System administrators and application designers can restrict a task's migration +to improve NUMA locality using various CPU affinity command line interfaces, +such as taskset(1) and numactl(1), and program interfaces such as +sched_setaffinity(2). Further, one can modify the kernel's default local +allocation behavior using Linux NUMA memory policy. [see +:ref:`Documentation/admin-guide/mm/numa_memory_policy.rst `]. + +System administrators can restrict the CPUs and nodes' memories that a non- +privileged user can specify in the scheduling or NUMA commands and functions +using control groups and CPUsets. [see Documentation/admin-guide/cgroup-v1/cpusets.rst] + +On architectures that do not hide memoryless nodes, Linux will include only +zones [nodes] with memory in the zonelists. This means that for a memoryless +node the "local memory node"--the node of the first zone in CPU's node's +zonelist--will not be the node itself. Rather, it will be the node that the +kernel selected as the nearest node with memory when it built the zonelists. +So, default, local allocations will succeed with the kernel supplying the +closest available memory. This is a consequence of the same mechanism that +allows such allocations to fallback to other nearby nodes when a node that +does contain memory overflows. + +Some kernel allocations do not want or cannot tolerate this allocation fallback +behavior. Rather they want to be sure they get memory from the specified node +or get notified that the node has no free memory. This is usually the case when +a subsystem allocates per CPU memory resources, for example. + +A typical model for making such an allocation is to obtain the node id of the +node to which the "current CPU" is attached using one of the kernel's +numa_node_id() or CPU_to_node() functions and then request memory from only +the node id returned. When such an allocation fails, the requesting subsystem +may revert to its own fallback path. The slab kernel memory allocator is an +example of this. Or, the subsystem may choose to disable or not to enable +itself on allocation failure. The kernel profiling subsystem is an example of +this. + +If the architecture supports--does not hide--memoryless nodes, then CPUs +attached to memoryless nodes would always incur the fallback path overhead +or some subsystems would fail to initialize if they attempted to allocated +memory exclusively from a node without memory. To support such +architectures transparently, kernel subsystems can use the numa_mem_id() +or cpu_to_mem() function to locate the "local memory node" for the calling or +specified CPU. Again, this is the same node from which default, local page +allocations will be attempted. diff --git a/Documentation/mm/oom.rst b/Documentation/mm/oom.rst new file mode 100644 index 000000000000..18e9e40c1ec1 --- /dev/null +++ b/Documentation/mm/oom.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================== +Out Of Memory Handling +====================== diff --git a/Documentation/mm/overcommit-accounting.rst b/Documentation/mm/overcommit-accounting.rst new file mode 100644 index 000000000000..1addb0c374a4 --- /dev/null +++ b/Documentation/mm/overcommit-accounting.rst @@ -0,0 +1,88 @@ +.. _overcommit_accounting: + +===================== +Overcommit Accounting +===================== + +The Linux kernel supports the following overcommit handling modes + +0 + Heuristic overcommit handling. Obvious overcommits of address + space are refused. Used for a typical system. It ensures a + seriously wild allocation fails while allowing overcommit to + reduce swap usage. root is allowed to allocate slightly more + memory in this mode. This is the default. + +1 + Always overcommit. Appropriate for some scientific + applications. Classic example is code using sparse arrays and + just relying on the virtual memory consisting almost entirely + of zero pages. + +2 + Don't overcommit. The total address space commit for the + system is not permitted to exceed swap + a configurable amount + (default is 50%) of physical RAM. Depending on the amount you + use, in most situations this means a process will not be + killed while accessing pages but will receive errors on memory + allocation as appropriate. + + Useful for applications that want to guarantee their memory + allocations will be available in the future without having to + initialize every page. + +The overcommit policy is set via the sysctl ``vm.overcommit_memory``. + +The overcommit amount can be set via ``vm.overcommit_ratio`` (percentage) +or ``vm.overcommit_kbytes`` (absolute value). These only have an effect +when ``vm.overcommit_memory`` is set to 2. + +The current overcommit limit and amount committed are viewable in +``/proc/meminfo`` as CommitLimit and Committed_AS respectively. + +Gotchas +======= + +The C language stack growth does an implicit mremap. If you want absolute +guarantees and run close to the edge you MUST mmap your stack for the +largest size you think you will need. For typical stack usage this does +not matter much but it's a corner case if you really really care + +In mode 2 the MAP_NORESERVE flag is ignored. + + +How It Works +============ + +The overcommit is based on the following rules + +For a file backed map + | SHARED or READ-only - 0 cost (the file is the map not swap) + | PRIVATE WRITABLE - size of mapping per instance + +For an anonymous or ``/dev/zero`` map + | SHARED - size of mapping + | PRIVATE READ-only - 0 cost (but of little use) + | PRIVATE WRITABLE - size of mapping per instance + +Additional accounting + | Pages made writable copies by mmap + | shmfs memory drawn from the same pool + +Status +====== + +* We account mmap memory mappings +* We account mprotect changes in commit +* We account mremap changes in size +* We account brk +* We account munmap +* We report the commit status in /proc +* Account and check on fork +* Review stack handling/building on exec +* SHMfs accounting +* Implement actual limit enforcement + +To Do +===== +* Account ptrace pages (this is hard) diff --git a/Documentation/mm/page_allocation.rst b/Documentation/mm/page_allocation.rst new file mode 100644 index 000000000000..d9b4495561f1 --- /dev/null +++ b/Documentation/mm/page_allocation.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +Page Allocation +=============== diff --git a/Documentation/mm/page_cache.rst b/Documentation/mm/page_cache.rst new file mode 100644 index 000000000000..75eba7c431b2 --- /dev/null +++ b/Documentation/mm/page_cache.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========== +Page Cache +========== diff --git a/Documentation/mm/page_frags.rst b/Documentation/mm/page_frags.rst new file mode 100644 index 000000000000..7d6f9385d129 --- /dev/null +++ b/Documentation/mm/page_frags.rst @@ -0,0 +1,45 @@ +.. _page_frags: + +============== +Page fragments +============== + +A page fragment is an arbitrary-length arbitrary-offset area of memory +which resides within a 0 or higher order compound page. Multiple +fragments within that page are individually refcounted, in the page's +reference counter. + +The page_frag functions, page_frag_alloc and page_frag_free, provide a +simple allocation framework for page fragments. This is used by the +network stack and network device drivers to provide a backing region of +memory for use as either an sk_buff->head, or to be used in the "frags" +portion of skb_shared_info. + +In order to make use of the page fragment APIs a backing page fragment +cache is needed. This provides a central point for the fragment allocation +and tracks allows multiple calls to make use of a cached page. The +advantage to doing this is that multiple calls to get_page can be avoided +which can be expensive at allocation time. However due to the nature of +this caching it is required that any calls to the cache be protected by +either a per-cpu limitation, or a per-cpu limitation and forcing interrupts +to be disabled when executing the fragment allocation. + +The network stack uses two separate caches per CPU to handle fragment +allocation. The netdev_alloc_cache is used by callers making use of the +netdev_alloc_frag and __netdev_alloc_skb calls. The napi_alloc_cache is +used by callers of the __napi_alloc_frag and __napi_alloc_skb calls. The +main difference between these two calls is the context in which they may be +called. The "netdev" prefixed functions are usable in any context as these +functions will disable interrupts, while the "napi" prefixed functions are +only usable within the softirq context. + +Many network device drivers use a similar methodology for allocating page +fragments, but the page fragments are cached at the ring or descriptor +level. In order to enable these cases it is necessary to provide a generic +way of tearing down a page cache. For this reason __page_frag_cache_drain +was implemented. It allows for freeing multiple references from a single +page via a single call. The advantage to doing this is that it allows for +cleaning up the multiple references that were added to a page in order to +avoid calling get_page per allocation. + +Alexander Duyck, Nov 29, 2016. diff --git a/Documentation/mm/page_migration.rst b/Documentation/mm/page_migration.rst new file mode 100644 index 000000000000..8c5cb8147e55 --- /dev/null +++ b/Documentation/mm/page_migration.rst @@ -0,0 +1,288 @@ +.. _page_migration: + +============== +Page migration +============== + +Page migration allows moving the physical location of pages between +nodes in a NUMA system while the process is running. This means that the +virtual addresses that the process sees do not change. However, the +system rearranges the physical location of those pages. + +Also see :ref:`Heterogeneous Memory Management (HMM) ` +for migrating pages to or from device private memory. + +The main intent of page migration is to reduce the latency of memory accesses +by moving pages near to the processor where the process accessing that memory +is running. + +Page migration allows a process to manually relocate the node on which its +pages are located through the MF_MOVE and MF_MOVE_ALL options while setting +a new memory policy via mbind(). The pages of a process can also be relocated +from another process using the sys_migrate_pages() function call. The +migrate_pages() function call takes two sets of nodes and moves pages of a +process that are located on the from nodes to the destination nodes. +Page migration functions are provided by the numactl package by Andi Kleen +(a version later than 0.9.3 is required. Get it from +https://github.com/numactl/numactl.git). numactl provides libnuma +which provides an interface similar to other NUMA functionality for page +migration. cat ``/proc//numa_maps`` allows an easy review of where the +pages of a process are located. See also the numa_maps documentation in the +proc(5) man page. + +Manual migration is useful if for example the scheduler has relocated +a process to a processor on a distant node. A batch scheduler or an +administrator may detect the situation and move the pages of the process +nearer to the new processor. The kernel itself only provides +manual page migration support. Automatic page migration may be implemented +through user space processes that move pages. A special function call +"move_pages" allows the moving of individual pages within a process. +For example, A NUMA profiler may obtain a log showing frequent off-node +accesses and may use the result to move pages to more advantageous +locations. + +Larger installations usually partition the system using cpusets into +sections of nodes. Paul Jackson has equipped cpusets with the ability to +move pages when a task is moved to another cpuset (See +:ref:`CPUSETS `). +Cpusets allow the automation of process locality. If a task is moved to +a new cpuset then also all its pages are moved with it so that the +performance of the process does not sink dramatically. Also the pages +of processes in a cpuset are moved if the allowed memory nodes of a +cpuset are changed. + +Page migration allows the preservation of the relative location of pages +within a group of nodes for all migration techniques which will preserve a +particular memory allocation pattern generated even after migrating a +process. This is necessary in order to preserve the memory latencies. +Processes will run with similar performance after migration. + +Page migration occurs in several steps. First a high level +description for those trying to use migrate_pages() from the kernel +(for userspace usage see the Andi Kleen's numactl package mentioned above) +and then a low level description of how the low level details work. + +In kernel use of migrate_pages() +================================ + +1. Remove pages from the LRU. + + Lists of pages to be migrated are generated by scanning over + pages and moving them into lists. This is done by + calling isolate_lru_page(). + Calling isolate_lru_page() increases the references to the page + so that it cannot vanish while the page migration occurs. + It also prevents the swapper or other scans from encountering + the page. + +2. We need to have a function of type new_page_t that can be + passed to migrate_pages(). This function should figure out + how to allocate the correct new page given the old page. + +3. The migrate_pages() function is called which attempts + to do the migration. It will call the function to allocate + the new page for each page that is considered for + moving. + +How migrate_pages() works +========================= + +migrate_pages() does several passes over its list of pages. A page is moved +if all references to a page are removable at the time. The page has +already been removed from the LRU via isolate_lru_page() and the refcount +is increased so that the page cannot be freed while page migration occurs. + +Steps: + +1. Lock the page to be migrated. + +2. Ensure that writeback is complete. + +3. Lock the new page that we want to move to. It is locked so that accesses to + this (not yet up-to-date) page immediately block while the move is in progress. + +4. All the page table references to the page are converted to migration + entries. This decreases the mapcount of a page. If the resulting + mapcount is not zero then we do not migrate the page. All user space + processes that attempt to access the page will now wait on the page lock + or wait for the migration page table entry to be removed. + +5. The i_pages lock is taken. This will cause all processes trying + to access the page via the mapping to block on the spinlock. + +6. The refcount of the page is examined and we back out if references remain. + Otherwise, we know that we are the only one referencing this page. + +7. The radix tree is checked and if it does not contain the pointer to this + page then we back out because someone else modified the radix tree. + +8. The new page is prepped with some settings from the old page so that + accesses to the new page will discover a page with the correct settings. + +9. The radix tree is changed to point to the new page. + +10. The reference count of the old page is dropped because the address space + reference is gone. A reference to the new page is established because + the new page is referenced by the address space. + +11. The i_pages lock is dropped. With that lookups in the mapping + become possible again. Processes will move from spinning on the lock + to sleeping on the locked new page. + +12. The page contents are copied to the new page. + +13. The remaining page flags are copied to the new page. + +14. The old page flags are cleared to indicate that the page does + not provide any information anymore. + +15. Queued up writeback on the new page is triggered. + +16. If migration entries were inserted into the page table, then replace them + with real ptes. Doing so will enable access for user space processes not + already waiting for the page lock. + +17. The page locks are dropped from the old and new page. + Processes waiting on the page lock will redo their page faults + and will reach the new page. + +18. The new page is moved to the LRU and can be scanned by the swapper, + etc. again. + +Non-LRU page migration +====================== + +Although migration originally aimed for reducing the latency of memory accesses +for NUMA, compaction also uses migration to create high-order pages. + +Current problem of the implementation is that it is designed to migrate only +*LRU* pages. However, there are potential non-LRU pages which can be migrated +in drivers, for example, zsmalloc, virtio-balloon pages. + +For virtio-balloon pages, some parts of migration code path have been hooked +up and added virtio-balloon specific functions to intercept migration logics. +It's too specific to a driver so other drivers who want to make their pages +movable would have to add their own specific hooks in the migration path. + +To overcome the problem, VM supports non-LRU page migration which provides +generic functions for non-LRU movable pages without driver specific hooks +in the migration path. + +If a driver wants to make its pages movable, it should define three functions +which are function pointers of struct address_space_operations. + +1. ``bool (*isolate_page) (struct page *page, isolate_mode_t mode);`` + + What VM expects from isolate_page() function of driver is to return *true* + if driver isolates the page successfully. On returning true, VM marks the page + as PG_isolated so concurrent isolation in several CPUs skip the page + for isolation. If a driver cannot isolate the page, it should return *false*. + + Once page is successfully isolated, VM uses page.lru fields so driver + shouldn't expect to preserve values in those fields. + +2. ``int (*migratepage) (struct address_space *mapping,`` +| ``struct page *newpage, struct page *oldpage, enum migrate_mode);`` + + After isolation, VM calls migratepage() of driver with the isolated page. + The function of migratepage() is to move the contents of the old page to the + new page + and set up fields of struct page newpage. Keep in mind that you should + indicate to the VM the oldpage is no longer movable via __ClearPageMovable() + under page_lock if you migrated the oldpage successfully and returned + MIGRATEPAGE_SUCCESS. If driver cannot migrate the page at the moment, driver + can return -EAGAIN. On -EAGAIN, VM will retry page migration in a short time + because VM interprets -EAGAIN as "temporary migration failure". On returning + any error except -EAGAIN, VM will give up the page migration without + retrying. + + Driver shouldn't touch the page.lru field while in the migratepage() function. + +3. ``void (*putback_page)(struct page *);`` + + If migration fails on the isolated page, VM should return the isolated page + to the driver so VM calls the driver's putback_page() with the isolated page. + In this function, the driver should put the isolated page back into its own data + structure. + +Non-LRU movable page flags + + There are two page flags for supporting non-LRU movable page. + + * PG_movable + + Driver should use the function below to make page movable under page_lock:: + + void __SetPageMovable(struct page *page, struct address_space *mapping) + + It needs argument of address_space for registering migration + family functions which will be called by VM. Exactly speaking, + PG_movable is not a real flag of struct page. Rather, VM + reuses the page->mapping's lower bits to represent it:: + + #define PAGE_MAPPING_MOVABLE 0x2 + page->mapping = page->mapping | PAGE_MAPPING_MOVABLE; + + so driver shouldn't access page->mapping directly. Instead, driver should + use page_mapping() which masks off the low two bits of page->mapping under + page lock so it can get the right struct address_space. + + For testing of non-LRU movable pages, VM supports __PageMovable() function. + However, it doesn't guarantee to identify non-LRU movable pages because + the page->mapping field is unified with other variables in struct page. + If the driver releases the page after isolation by VM, page->mapping + doesn't have a stable value although it has PAGE_MAPPING_MOVABLE set + (look at __ClearPageMovable). But __PageMovable() is cheap to call whether + page is LRU or non-LRU movable once the page has been isolated because LRU + pages can never have PAGE_MAPPING_MOVABLE set in page->mapping. It is also + good for just peeking to test non-LRU movable pages before more expensive + checking with lock_page() in pfn scanning to select a victim. + + For guaranteeing non-LRU movable page, VM provides PageMovable() function. + Unlike __PageMovable(), PageMovable() validates page->mapping and + mapping->a_ops->isolate_page under lock_page(). The lock_page() prevents + sudden destroying of page->mapping. + + Drivers using __SetPageMovable() should clear the flag via + __ClearMovablePage() under page_lock() before the releasing the page. + + * PG_isolated + + To prevent concurrent isolation among several CPUs, VM marks isolated page + as PG_isolated under lock_page(). So if a CPU encounters PG_isolated + non-LRU movable page, it can skip it. Driver doesn't need to manipulate the + flag because VM will set/clear it automatically. Keep in mind that if the + driver sees a PG_isolated page, it means the page has been isolated by the + VM so it shouldn't touch the page.lru field. + The PG_isolated flag is aliased with the PG_reclaim flag so drivers + shouldn't use PG_isolated for its own purposes. + +Monitoring Migration +===================== + +The following events (counters) can be used to monitor page migration. + +1. PGMIGRATE_SUCCESS: Normal page migration success. Each count means that a + page was migrated. If the page was a non-THP and non-hugetlb page, then + this counter is increased by one. If the page was a THP or hugetlb, then + this counter is increased by the number of THP or hugetlb subpages. + For example, migration of a single 2MB THP that has 4KB-size base pages + (subpages) will cause this counter to increase by 512. + +2. PGMIGRATE_FAIL: Normal page migration failure. Same counting rules as for + PGMIGRATE_SUCCESS, above: this will be increased by the number of subpages, + if it was a THP or hugetlb. + +3. THP_MIGRATION_SUCCESS: A THP was migrated without being split. + +4. THP_MIGRATION_FAIL: A THP could not be migrated nor it could be split. + +5. THP_MIGRATION_SPLIT: A THP was migrated, but not as such: first, the THP had + to be split. After splitting, a migration retry was used for it's sub-pages. + +THP_MIGRATION_* events also update the appropriate PGMIGRATE_SUCCESS or +PGMIGRATE_FAIL events. For example, a THP migration failure will cause both +THP_MIGRATION_FAIL and PGMIGRATE_FAIL to increase. + +Christoph Lameter, May 8, 2006. +Minchan Kim, Mar 28, 2016. diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst new file mode 100644 index 000000000000..f5c954afe97c --- /dev/null +++ b/Documentation/mm/page_owner.rst @@ -0,0 +1,196 @@ +.. _page_owner: + +================================================== +page owner: Tracking about who allocated each page +================================================== + +Introduction +============ + +page owner is for the tracking about who allocated each page. +It can be used to debug memory leak or to find a memory hogger. +When allocation happens, information about allocation such as call stack +and order of pages is stored into certain storage for each page. +When we need to know about status of all pages, we can get and analyze +this information. + +Although we already have tracepoint for tracing page allocation/free, +using it for analyzing who allocate each page is rather complex. We need +to enlarge the trace buffer for preventing overlapping until userspace +program launched. And, launched program continually dump out the trace +buffer for later analysis and it would change system behaviour with more +possibility rather than just keeping it in memory, so bad for debugging. + +page owner can also be used for various purposes. For example, accurate +fragmentation statistics can be obtained through gfp flag information of +each page. It is already implemented and activated if page owner is +enabled. Other usages are more than welcome. + +page owner is disabled by default. So, if you'd like to use it, you need +to add "page_owner=on" to your boot cmdline. If the kernel is built +with page owner and page owner is disabled in runtime due to not enabling +boot option, runtime overhead is marginal. If disabled in runtime, it +doesn't require memory to store owner information, so there is no runtime +memory overhead. And, page owner inserts just two unlikely branches into +the page allocator hotpath and if not enabled, then allocation is done +like as the kernel without page owner. These two unlikely branches should +not affect to allocation performance, especially if the static keys jump +label patching functionality is available. Following is the kernel's code +size change due to this facility. + +- Without page owner:: + + text data bss dec hex filename + 48392 2333 644 51369 c8a9 mm/page_alloc.o + +- With page owner:: + + text data bss dec hex filename + 48800 2445 644 51889 cab1 mm/page_alloc.o + 6662 108 29 6799 1a8f mm/page_owner.o + 1025 8 8 1041 411 mm/page_ext.o + +Although, roughly, 8 KB code is added in total, page_alloc.o increase by +520 bytes and less than half of it is in hotpath. Building the kernel with +page owner and turning it on if needed would be great option to debug +kernel memory problem. + +There is one notice that is caused by implementation detail. page owner +stores information into the memory from struct page extension. This memory +is initialized some time later than that page allocator starts in sparse +memory system, so, until initialization, many pages can be allocated and +they would have no owner information. To fix it up, these early allocated +pages are investigated and marked as allocated in initialization phase. +Although it doesn't mean that they have the right owner information, +at least, we can tell whether the page is allocated or not, +more accurately. On 2GB memory x86-64 VM box, 13343 early allocated pages +are catched and marked, although they are mostly allocated from struct +page extension feature. Anyway, after that, no page is left in +un-tracking state. + +Usage +===== + +1) Build user-space helper:: + + cd tools/vm + make page_owner_sort + +2) Enable page owner: add "page_owner=on" to boot cmdline. + +3) Do the job that you want to debug. + +4) Analyze information from page owner:: + + cat /sys/kernel/debug/page_owner > page_owner_full.txt + ./page_owner_sort page_owner_full.txt sorted_page_owner.txt + + The general output of ``page_owner_full.txt`` is as follows:: + + Page allocated via order XXX, ... + PFN XXX ... + // Detailed stack + + Page allocated via order XXX, ... + PFN XXX ... + // Detailed stack + + The ``page_owner_sort`` tool ignores ``PFN`` rows, puts the remaining rows + in buf, uses regexp to extract the page order value, counts the times + and pages of buf, and finally sorts them according to the parameter(s). + + See the result about who allocated each page + in the ``sorted_page_owner.txt``. General output:: + + XXX times, XXX pages: + Page allocated via order XXX, ... + // Detailed stack + + By default, ``page_owner_sort`` is sorted according to the times of buf. + If you want to sort by the page nums of buf, use the ``-m`` parameter. + The detailed parameters are: + + fundamental function:: + + Sort: + -a Sort by memory allocation time. + -m Sort by total memory. + -p Sort by pid. + -P Sort by tgid. + -n Sort by task command name. + -r Sort by memory release time. + -s Sort by stack trace. + -t Sort by times (default). + --sort Specify sorting order. Sorting syntax is [+|-]key[,[+|-]key[,...]]. + Choose a key from the **STANDARD FORMAT SPECIFIERS** section. The "+" is + optional since default direction is increasing numerical or lexicographic + order. Mixed use of abbreviated and complete-form of keys is allowed. + + Examples: + ./page_owner_sort --sort=n,+pid,-tgid + ./page_owner_sort --sort=at + + additional function:: + + Cull: + --cull + Specify culling rules.Culling syntax is key[,key[,...]].Choose a + multi-letter key from the **STANDARD FORMAT SPECIFIERS** section. + + is a single argument in the form of a comma-separated list, + which offers a way to specify individual culling rules. The recognized + keywords are described in the **STANDARD FORMAT SPECIFIERS** section below. + can be specified by the sequence of keys k1,k2, ..., as described in + the STANDARD SORT KEYS section below. Mixed use of abbreviated and + complete-form of keys is allowed. + + Examples: + ./page_owner_sort --cull=stacktrace + ./page_owner_sort --cull=st,pid,name + ./page_owner_sort --cull=n,f + + Filter: + -f Filter out the information of blocks whose memory has been released. + + Select: + --pid Select by pid. This selects the blocks whose process ID + numbers appear in . + --tgid Select by tgid. This selects the blocks whose thread + group ID numbers appear in . + --name Select by task command name. This selects the blocks whose + task command name appear in . + + , , are single arguments in the form of a comma-separated list, + which offers a way to specify individual selecting rules. + + + Examples: + ./page_owner_sort --pid=1 + ./page_owner_sort --tgid=1,2,3 + ./page_owner_sort --name name1,name2 + +STANDARD FORMAT SPECIFIERS +========================== +:: + + For --sort option: + + KEY LONG DESCRIPTION + p pid process ID + tg tgid thread group ID + n name task command name + st stacktrace stack trace of the page allocation + T txt full text of block + ft free_ts timestamp of the page when it was released + at alloc_ts timestamp of the page when it was allocated + ator allocator memory allocator for pages + + For --curl option: + + KEY LONG DESCRIPTION + p pid process ID + tg tgid thread group ID + n name task command name + f free whether the page has been released or not + st stacktrace stack trace of the page allocation + ator allocator memory allocator for pages diff --git a/Documentation/mm/page_reclaim.rst b/Documentation/mm/page_reclaim.rst new file mode 100644 index 000000000000..50a30b7f8ac3 --- /dev/null +++ b/Documentation/mm/page_reclaim.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============ +Page Reclaim +============ diff --git a/Documentation/mm/page_table_check.rst b/Documentation/mm/page_table_check.rst new file mode 100644 index 000000000000..1a09472f10a3 --- /dev/null +++ b/Documentation/mm/page_table_check.rst @@ -0,0 +1,56 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _page_table_check: + +================ +Page Table Check +================ + +Introduction +============ + +Page table check allows to harden the kernel by ensuring that some types of +the memory corruptions are prevented. + +Page table check performs extra verifications at the time when new pages become +accessible from the userspace by getting their page table entries (PTEs PMDs +etc.) added into the table. + +In case of detected corruption, the kernel is crashed. There is a small +performance and memory overhead associated with the page table check. Therefore, +it is disabled by default, but can be optionally enabled on systems where the +extra hardening outweighs the performance costs. Also, because page table check +is synchronous, it can help with debugging double map memory corruption issues, +by crashing kernel at the time wrong mapping occurs instead of later which is +often the case with memory corruptions bugs. + +Double mapping detection logic +============================== + ++-------------------+-------------------+-------------------+------------------+ +| Current Mapping | New mapping | Permissions | Rule | ++===================+===================+===================+==================+ +| Anonymous | Anonymous | Read | Allow | ++-------------------+-------------------+-------------------+------------------+ +| Anonymous | Anonymous | Read / Write | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Anonymous | Named | Any | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Named | Anonymous | Any | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Named | Named | Any | Allow | ++-------------------+-------------------+-------------------+------------------+ + +Enabling Page Table Check +========================= + +Build kernel with: + +- PAGE_TABLE_CHECK=y + Note, it can only be enabled on platforms where ARCH_SUPPORTS_PAGE_TABLE_CHECK + is available. + +- Boot with 'page_table_check=on' kernel parameter. + +Optionally, build kernel with PAGE_TABLE_CHECK_ENFORCED in order to have page +table support without extra kernel parameter. diff --git a/Documentation/mm/page_tables.rst b/Documentation/mm/page_tables.rst new file mode 100644 index 000000000000..96939571d7bc --- /dev/null +++ b/Documentation/mm/page_tables.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========== +Page Tables +=========== diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst new file mode 100644 index 000000000000..2ab7b8c1c863 --- /dev/null +++ b/Documentation/mm/physical_memory.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +Physical Memory +=============== diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst new file mode 100644 index 000000000000..e8618fbc62c9 --- /dev/null +++ b/Documentation/mm/process_addrs.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================= +Process Addresses +================= diff --git a/Documentation/mm/remap_file_pages.rst b/Documentation/mm/remap_file_pages.rst new file mode 100644 index 000000000000..7bef6718e3a9 --- /dev/null +++ b/Documentation/mm/remap_file_pages.rst @@ -0,0 +1,33 @@ +.. _remap_file_pages: + +============================== +remap_file_pages() system call +============================== + +The remap_file_pages() system call is used to create a nonlinear mapping, +that is, a mapping in which the pages of the file are mapped into a +nonsequential order in memory. The advantage of using remap_file_pages() +over using repeated calls to mmap(2) is that the former approach does not +require the kernel to create additional VMA (Virtual Memory Area) data +structures. + +Supporting of nonlinear mapping requires significant amount of non-trivial +code in kernel virtual memory subsystem including hot paths. Also to get +nonlinear mapping work kernel need a way to distinguish normal page table +entries from entries with file offset (pte_file). Kernel reserves flag in +PTE for this purpose. PTE flags are scarce resource especially on some CPU +architectures. It would be nice to free up the flag for other usage. + +Fortunately, there are not many users of remap_file_pages() in the wild. +It's only known that one enterprise RDBMS implementation uses the syscall +on 32-bit systems to map files bigger than can linearly fit into 32-bit +virtual address space. This use-case is not critical anymore since 64-bit +systems are widely available. + +The syscall is deprecated and replaced it with an emulation now. The +emulation creates new VMAs instead of nonlinear mappings. It's going to +work slower for rare users of remap_file_pages() but ABI is preserved. + +One side effect of emulation (apart from performance) is that user can hit +vm.max_map_count limit more easily due to additional VMAs. See comment for +DEFAULT_MAX_MAP_COUNT for more details on the limit. diff --git a/Documentation/mm/shmfs.rst b/Documentation/mm/shmfs.rst new file mode 100644 index 000000000000..8b01ebb4c30e --- /dev/null +++ b/Documentation/mm/shmfs.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================== +Shared Memory Filesystem +======================== diff --git a/Documentation/mm/slab.rst b/Documentation/mm/slab.rst new file mode 100644 index 000000000000..87d5a5bb172f --- /dev/null +++ b/Documentation/mm/slab.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +Slab Allocation +=============== diff --git a/Documentation/mm/slub.rst b/Documentation/mm/slub.rst new file mode 100644 index 000000000000..43063ade737a --- /dev/null +++ b/Documentation/mm/slub.rst @@ -0,0 +1,452 @@ +.. _slub: + +========================== +Short users guide for SLUB +========================== + +The basic philosophy of SLUB is very different from SLAB. SLAB +requires rebuilding the kernel to activate debug options for all +slab caches. SLUB always includes full debugging but it is off by default. +SLUB can enable debugging only for selected slabs in order to avoid +an impact on overall system performance which may make a bug more +difficult to find. + +In order to switch debugging on one can add an option ``slub_debug`` +to the kernel command line. That will enable full debugging for +all slabs. + +Typically one would then use the ``slabinfo`` command to get statistical +data and perform operation on the slabs. By default ``slabinfo`` only lists +slabs that have data in them. See "slabinfo -h" for more options when +running the command. ``slabinfo`` can be compiled with +:: + + gcc -o slabinfo tools/vm/slabinfo.c + +Some of the modes of operation of ``slabinfo`` require that slub debugging +be enabled on the command line. F.e. no tracking information will be +available without debugging on and validation can only partially +be performed if debugging was not switched on. + +Some more sophisticated uses of slub_debug: +------------------------------------------- + +Parameters may be given to ``slub_debug``. If none is specified then full +debugging is enabled. Format: + +slub_debug= + Enable options for all slabs + +slub_debug=,,,... + Enable options only for select slabs (no spaces + after a comma) + +Multiple blocks of options for all slabs or selected slabs can be given, with +blocks of options delimited by ';'. The last of "all slabs" blocks is applied +to all slabs except those that match one of the "select slabs" block. Options +of the first "select slabs" blocks that matches the slab's name are applied. + +Possible debug options are:: + + F Sanity checks on (enables SLAB_DEBUG_CONSISTENCY_CHECKS + Sorry SLAB legacy issues) + Z Red zoning + P Poisoning (object and padding) + U User tracking (free and alloc) + T Trace (please only use on single slabs) + A Enable failslab filter mark for the cache + O Switch debugging off for caches that would have + caused higher minimum slab orders + - Switch all debugging off (useful if the kernel is + configured with CONFIG_SLUB_DEBUG_ON) + +F.e. in order to boot just with sanity checks and red zoning one would specify:: + + slub_debug=FZ + +Trying to find an issue in the dentry cache? Try:: + + slub_debug=,dentry + +to only enable debugging on the dentry cache. You may use an asterisk at the +end of the slab name, in order to cover all slabs with the same prefix. For +example, here's how you can poison the dentry cache as well as all kmalloc +slabs:: + + slub_debug=P,kmalloc-*,dentry + +Red zoning and tracking may realign the slab. We can just apply sanity checks +to the dentry cache with:: + + slub_debug=F,dentry + +Debugging options may require the minimum possible slab order to increase as +a result of storing the metadata (for example, caches with PAGE_SIZE object +sizes). This has a higher liklihood of resulting in slab allocation errors +in low memory situations or if there's high fragmentation of memory. To +switch off debugging for such caches by default, use:: + + slub_debug=O + +You can apply different options to different list of slab names, using blocks +of options. This will enable red zoning for dentry and user tracking for +kmalloc. All other slabs will not get any debugging enabled:: + + slub_debug=Z,dentry;U,kmalloc-* + +You can also enable options (e.g. sanity checks and poisoning) for all caches +except some that are deemed too performance critical and don't need to be +debugged by specifying global debug options followed by a list of slab names +with "-" as options:: + + slub_debug=FZ;-,zs_handle,zspage + +The state of each debug option for a slab can be found in the respective files +under:: + + /sys/kernel/slab// + +If the file contains 1, the option is enabled, 0 means disabled. The debug +options from the ``slub_debug`` parameter translate to the following files:: + + F sanity_checks + Z red_zone + P poison + U store_user + T trace + A failslab + +Careful with tracing: It may spew out lots of information and never stop if +used on the wrong slab. + +Slab merging +============ + +If no debug options are specified then SLUB may merge similar slabs together +in order to reduce overhead and increase cache hotness of objects. +``slabinfo -a`` displays which slabs were merged together. + +Slab validation +=============== + +SLUB can validate all object if the kernel was booted with slub_debug. In +order to do so you must have the ``slabinfo`` tool. Then you can do +:: + + slabinfo -v + +which will test all objects. Output will be generated to the syslog. + +This also works in a more limited way if boot was without slab debug. +In that case ``slabinfo -v`` simply tests all reachable objects. Usually +these are in the cpu slabs and the partial slabs. Full slabs are not +tracked by SLUB in a non debug situation. + +Getting more performance +======================== + +To some degree SLUB's performance is limited by the need to take the +list_lock once in a while to deal with partial slabs. That overhead is +governed by the order of the allocation for each slab. The allocations +can be influenced by kernel parameters: + +.. slub_min_objects=x (default 4) +.. slub_min_order=x (default 0) +.. slub_max_order=x (default 3 (PAGE_ALLOC_COSTLY_ORDER)) + +``slub_min_objects`` + allows to specify how many objects must at least fit into one + slab in order for the allocation order to be acceptable. In + general slub will be able to perform this number of + allocations on a slab without consulting centralized resources + (list_lock) where contention may occur. + +``slub_min_order`` + specifies a minimum order of slabs. A similar effect like + ``slub_min_objects``. + +``slub_max_order`` + specified the order at which ``slub_min_objects`` should no + longer be checked. This is useful to avoid SLUB trying to + generate super large order pages to fit ``slub_min_objects`` + of a slab cache with large object sizes into one high order + page. Setting command line parameter + ``debug_guardpage_minorder=N`` (N > 0), forces setting + ``slub_max_order`` to 0, what cause minimum possible order of + slabs allocation. + +SLUB Debug output +================= + +Here is a sample of slub debug output:: + + ==================================================================== + BUG kmalloc-8: Right Redzone overwritten + -------------------------------------------------------------------- + + INFO: 0xc90f6d28-0xc90f6d2b. First byte 0x00 instead of 0xcc + INFO: Slab 0xc528c530 flags=0x400000c3 inuse=61 fp=0xc90f6d58 + INFO: Object 0xc90f6d20 @offset=3360 fp=0xc90f6d58 + INFO: Allocated in get_modalias+0x61/0xf5 age=53 cpu=1 pid=554 + + Bytes b4 (0xc90f6d10): 00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ + Object (0xc90f6d20): 31 30 31 39 2e 30 30 35 1019.005 + Redzone (0xc90f6d28): 00 cc cc cc . + Padding (0xc90f6d50): 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZ + + [] dump_trace+0x63/0x1eb + [] show_trace_log_lvl+0x1a/0x2f + [] show_trace+0x12/0x14 + [] dump_stack+0x16/0x18 + [] object_err+0x143/0x14b + [] check_object+0x66/0x234 + [] __slab_free+0x239/0x384 + [] kfree+0xa6/0xc6 + [] get_modalias+0xb9/0xf5 + [] dmi_dev_uevent+0x27/0x3c + [] dev_uevent+0x1ad/0x1da + [] kobject_uevent_env+0x20a/0x45b + [] kobject_uevent+0xa/0xf + [] store_uevent+0x4f/0x58 + [] dev_attr_store+0x29/0x2f + [] sysfs_write_file+0x16e/0x19c + [] vfs_write+0xd1/0x15a + [] sys_write+0x3d/0x72 + [] sysenter_past_esp+0x5f/0x99 + [] 0xb7f7b410 + ======================= + + FIX kmalloc-8: Restoring Redzone 0xc90f6d28-0xc90f6d2b=0xcc + +If SLUB encounters a corrupted object (full detection requires the kernel +to be booted with slub_debug) then the following output will be dumped +into the syslog: + +1. Description of the problem encountered + + This will be a message in the system log starting with:: + + =============================================== + BUG : + ----------------------------------------------- + + INFO: - + INFO: Slab
+ INFO: Object
+ INFO: Allocated in age= cpu= pid= + INFO: Freed in age= cpu= + pid= + + (Object allocation / free information is only available if SLAB_STORE_USER is + set for the slab. slub_debug sets that option) + +2. The object contents if an object was involved. + + Various types of lines can follow the BUG SLUB line: + + Bytes b4
: + Shows a few bytes before the object where the problem was detected. + Can be useful if the corruption does not stop with the start of the + object. + + Object
: + The bytes of the object. If the object is inactive then the bytes + typically contain poison values. Any non-poison value shows a + corruption by a write after free. + + Redzone
: + The Redzone following the object. The Redzone is used to detect + writes after the object. All bytes should always have the same + value. If there is any deviation then it is due to a write after + the object boundary. + + (Redzone information is only available if SLAB_RED_ZONE is set. + slub_debug sets that option) + + Padding
: + Unused data to fill up the space in order to get the next object + properly aligned. In the debug case we make sure that there are + at least 4 bytes of padding. This allows the detection of writes + before the object. + +3. A stackdump + + The stackdump describes the location where the error was detected. The cause + of the corruption is may be more likely found by looking at the function that + allocated or freed the object. + +4. Report on how the problem was dealt with in order to ensure the continued + operation of the system. + + These are messages in the system log beginning with:: + + FIX : + + In the above sample SLUB found that the Redzone of an active object has + been overwritten. Here a string of 8 characters was written into a slab that + has the length of 8 characters. However, a 8 character string needs a + terminating 0. That zero has overwritten the first byte of the Redzone field. + After reporting the details of the issue encountered the FIX SLUB message + tells us that SLUB has restored the Redzone to its proper value and then + system operations continue. + +Emergency operations +==================== + +Minimal debugging (sanity checks alone) can be enabled by booting with:: + + slub_debug=F + +This will be generally be enough to enable the resiliency features of slub +which will keep the system running even if a bad kernel component will +keep corrupting objects. This may be important for production systems. +Performance will be impacted by the sanity checks and there will be a +continual stream of error messages to the syslog but no additional memory +will be used (unlike full debugging). + +No guarantees. The kernel component still needs to be fixed. Performance +may be optimized further by locating the slab that experiences corruption +and enabling debugging only for that cache + +I.e.:: + + slub_debug=F,dentry + +If the corruption occurs by writing after the end of the object then it +may be advisable to enable a Redzone to avoid corrupting the beginning +of other objects:: + + slub_debug=FZ,dentry + +Extended slabinfo mode and plotting +=================================== + +The ``slabinfo`` tool has a special 'extended' ('-X') mode that includes: + - Slabcache Totals + - Slabs sorted by size (up to -N slabs, default 1) + - Slabs sorted by loss (up to -N slabs, default 1) + +Additionally, in this mode ``slabinfo`` does not dynamically scale +sizes (G/M/K) and reports everything in bytes (this functionality is +also available to other slabinfo modes via '-B' option) which makes +reporting more precise and accurate. Moreover, in some sense the `-X' +mode also simplifies the analysis of slabs' behaviour, because its +output can be plotted using the ``slabinfo-gnuplot.sh`` script. So it +pushes the analysis from looking through the numbers (tons of numbers) +to something easier -- visual analysis. + +To generate plots: + +a) collect slabinfo extended records, for example:: + + while [ 1 ]; do slabinfo -X >> FOO_STATS; sleep 1; done + +b) pass stats file(-s) to ``slabinfo-gnuplot.sh`` script:: + + slabinfo-gnuplot.sh FOO_STATS [FOO_STATS2 .. FOO_STATSN] + + The ``slabinfo-gnuplot.sh`` script will pre-processes the collected records + and generates 3 png files (and 3 pre-processing cache files) per STATS + file: + - Slabcache Totals: FOO_STATS-totals.png + - Slabs sorted by size: FOO_STATS-slabs-by-size.png + - Slabs sorted by loss: FOO_STATS-slabs-by-loss.png + +Another use case, when ``slabinfo-gnuplot.sh`` can be useful, is when you +need to compare slabs' behaviour "prior to" and "after" some code +modification. To help you out there, ``slabinfo-gnuplot.sh`` script +can 'merge' the `Slabcache Totals` sections from different +measurements. To visually compare N plots: + +a) Collect as many STATS1, STATS2, .. STATSN files as you need:: + + while [ 1 ]; do slabinfo -X >> STATS; sleep 1; done + +b) Pre-process those STATS files:: + + slabinfo-gnuplot.sh STATS1 STATS2 .. STATSN + +c) Execute ``slabinfo-gnuplot.sh`` in '-t' mode, passing all of the + generated pre-processed \*-totals:: + + slabinfo-gnuplot.sh -t STATS1-totals STATS2-totals .. STATSN-totals + + This will produce a single plot (png file). + + Plots, expectedly, can be large so some fluctuations or small spikes + can go unnoticed. To deal with that, ``slabinfo-gnuplot.sh`` has two + options to 'zoom-in'/'zoom-out': + + a) ``-s %d,%d`` -- overwrites the default image width and height + b) ``-r %d,%d`` -- specifies a range of samples to use (for example, + in ``slabinfo -X >> FOO_STATS; sleep 1;`` case, using a ``-r + 40,60`` range will plot only samples collected between 40th and + 60th seconds). + + +DebugFS files for SLUB +====================== + +For more information about current state of SLUB caches with the user tracking +debug option enabled, debugfs files are available, typically under +/sys/kernel/debug/slab// (created only for caches with enabled user +tracking). There are 2 types of these files with the following debug +information: + +1. alloc_traces:: + + Prints information about unique allocation traces of the currently + allocated objects. The output is sorted by frequency of each trace. + + Information in the output: + Number of objects, allocating function, minimal/average/maximal jiffies since alloc, + pid range of the allocating processes, cpu mask of allocating cpus, and stack trace. + + Example::: + + 1085 populate_error_injection_list+0x97/0x110 age=166678/166680/166682 pid=1 cpus=1:: + __slab_alloc+0x6d/0x90 + kmem_cache_alloc_trace+0x2eb/0x300 + populate_error_injection_list+0x97/0x110 + init_error_injection+0x1b/0x71 + do_one_initcall+0x5f/0x2d0 + kernel_init_freeable+0x26f/0x2d7 + kernel_init+0xe/0x118 + ret_from_fork+0x22/0x30 + + +2. free_traces:: + + Prints information about unique freeing traces of the currently allocated + objects. The freeing traces thus come from the previous life-cycle of the + objects and are reported as not available for objects allocated for the first + time. The output is sorted by frequency of each trace. + + Information in the output: + Number of objects, freeing function, minimal/average/maximal jiffies since free, + pid range of the freeing processes, cpu mask of freeing cpus, and stack trace. + + Example::: + + 1980 age=4294912290 pid=0 cpus=0 + 51 acpi_ut_update_ref_count+0x6a6/0x782 age=236886/237027/237772 pid=1 cpus=1 + kfree+0x2db/0x420 + acpi_ut_update_ref_count+0x6a6/0x782 + acpi_ut_update_object_reference+0x1ad/0x234 + acpi_ut_remove_reference+0x7d/0x84 + acpi_rs_get_prt_method_data+0x97/0xd6 + acpi_get_irq_routing_table+0x82/0xc4 + acpi_pci_irq_find_prt_entry+0x8e/0x2e0 + acpi_pci_irq_lookup+0x3a/0x1e0 + acpi_pci_irq_enable+0x77/0x240 + pcibios_enable_device+0x39/0x40 + do_pci_enable_device.part.0+0x5d/0xe0 + pci_enable_device_flags+0xfc/0x120 + pci_enable_device+0x13/0x20 + virtio_pci_probe+0x9e/0x170 + local_pci_probe+0x48/0x80 + pci_device_probe+0x105/0x1c0 + +Christoph Lameter, May 30, 2007 +Sergey Senozhatsky, October 23, 2015 diff --git a/Documentation/mm/split_page_table_lock.rst b/Documentation/mm/split_page_table_lock.rst new file mode 100644 index 000000000000..c08919662704 --- /dev/null +++ b/Documentation/mm/split_page_table_lock.rst @@ -0,0 +1,100 @@ +.. _split_page_table_lock: + +===================== +Split page table lock +===================== + +Originally, mm->page_table_lock spinlock protected all page tables of the +mm_struct. But this approach leads to poor page fault scalability of +multi-threaded applications due high contention on the lock. To improve +scalability, split page table lock was introduced. + +With split page table lock we have separate per-table lock to serialize +access to the table. At the moment we use split lock for PTE and PMD +tables. Access to higher level tables protected by mm->page_table_lock. + +There are helpers to lock/unlock a table and other accessor functions: + + - pte_offset_map_lock() + maps pte and takes PTE table lock, returns pointer to the taken + lock; + - pte_unmap_unlock() + unlocks and unmaps PTE table; + - pte_alloc_map_lock() + allocates PTE table if needed and take the lock, returns pointer + to taken lock or NULL if allocation failed; + - pte_lockptr() + returns pointer to PTE table lock; + - pmd_lock() + takes PMD table lock, returns pointer to taken lock; + - pmd_lockptr() + returns pointer to PMD table lock; + +Split page table lock for PTE tables is enabled compile-time if +CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS. +If split lock is disabled, all tables are guarded by mm->page_table_lock. + +Split page table lock for PMD tables is enabled, if it's enabled for PTE +tables and the architecture supports it (see below). + +Hugetlb and split page table lock +================================= + +Hugetlb can support several page sizes. We use split lock only for PMD +level, but not for PUD. + +Hugetlb-specific helpers: + + - huge_pte_lock() + takes pmd split lock for PMD_SIZE page, mm->page_table_lock + otherwise; + - huge_pte_lockptr() + returns pointer to table lock; + +Support of split page table lock by an architecture +=================================================== + +There's no need in special enabling of PTE split page table lock: everything +required is done by pgtable_pte_page_ctor() and pgtable_pte_page_dtor(), which +must be called on PTE table allocation / freeing. + +Make sure the architecture doesn't use slab allocator for page table +allocation: slab uses page->slab_cache for its pages. +This field shares storage with page->ptl. + +PMD split lock only makes sense if you have more than two page table +levels. + +PMD split lock enabling requires pgtable_pmd_page_ctor() call on PMD table +allocation and pgtable_pmd_page_dtor() on freeing. + +Allocation usually happens in pmd_alloc_one(), freeing in pmd_free() and +pmd_free_tlb(), but make sure you cover all PMD table allocation / freeing +paths: i.e X86_PAE preallocate few PMDs on pgd_alloc(). + +With everything in place you can set CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK. + +NOTE: pgtable_pte_page_ctor() and pgtable_pmd_page_ctor() can fail -- it must +be handled properly. + +page->ptl +========= + +page->ptl is used to access split page table lock, where 'page' is struct +page of page containing the table. It shares storage with page->private +(and few other fields in union). + +To avoid increasing size of struct page and have best performance, we use a +trick: + + - if spinlock_t fits into long, we use page->ptr as spinlock, so we + can avoid indirect access and save a cache line. + - if size of spinlock_t is bigger then size of long, we use page->ptl as + pointer to spinlock_t and allocate it dynamically. This allows to use + split lock with enabled DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC, but costs + one more cache line for indirect access; + +The spinlock_t allocated in pgtable_pte_page_ctor() for PTE table and in +pgtable_pmd_page_ctor() for PMD table. + +Please, never access page->ptl directly -- use appropriate helper. diff --git a/Documentation/mm/swap.rst b/Documentation/mm/swap.rst new file mode 100644 index 000000000000..78819bd4d745 --- /dev/null +++ b/Documentation/mm/swap.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==== +Swap +==== diff --git a/Documentation/mm/transhuge.rst b/Documentation/mm/transhuge.rst new file mode 100644 index 000000000000..216db1d67d04 --- /dev/null +++ b/Documentation/mm/transhuge.rst @@ -0,0 +1,187 @@ +.. _transhuge: + +============================ +Transparent Hugepage Support +============================ + +This document describes design principles for Transparent Hugepage (THP) +support and its interaction with other parts of the memory management +system. + +Design principles +================= + +- "graceful fallback": mm components which don't have transparent hugepage + knowledge fall back to breaking huge pmd mapping into table of ptes and, + if necessary, split a transparent hugepage. Therefore these components + can continue working on the regular pages or regular pte mappings. + +- if a hugepage allocation fails because of memory fragmentation, + regular pages should be gracefully allocated instead and mixed in + the same vma without any failure or significant delay and without + userland noticing + +- if some task quits and more hugepages become available (either + immediately in the buddy or through the VM), guest physical memory + backed by regular pages should be relocated on hugepages + automatically (with khugepaged) + +- it doesn't require memory reservation and in turn it uses hugepages + whenever possible (the only possible reservation here is kernelcore= + to avoid unmovable pages to fragment all the memory but such a tweak + is not specific to transparent hugepage support and it's a generic + feature that applies to all dynamic high order allocations in the + kernel) + +get_user_pages and follow_page +============================== + +get_user_pages and follow_page if run on a hugepage, will return the +head or tail pages as usual (exactly as they would do on +hugetlbfs). Most GUP users will only care about the actual physical +address of the page and its temporary pinning to release after the I/O +is complete, so they won't ever notice the fact the page is huge. But +if any driver is going to mangle over the page structure of the tail +page (like for checking page->mapping or other bits that are relevant +for the head page and not the tail page), it should be updated to jump +to check head page instead. Taking a reference on any head/tail page would +prevent the page from being split by anyone. + +.. note:: + these aren't new constraints to the GUP API, and they match the + same constraints that apply to hugetlbfs too, so any driver capable + of handling GUP on hugetlbfs will also work fine on transparent + hugepage backed mappings. + +Graceful fallback +================= + +Code walking pagetables but unaware about huge pmds can simply call +split_huge_pmd(vma, pmd, addr) where the pmd is the one returned by +pmd_offset. It's trivial to make the code transparent hugepage aware +by just grepping for "pmd_offset" and adding split_huge_pmd where +missing after pmd_offset returns the pmd. Thanks to the graceful +fallback design, with a one liner change, you can avoid to write +hundreds if not thousands of lines of complex code to make your code +hugepage aware. + +If you're not walking pagetables but you run into a physical hugepage +that you can't handle natively in your code, you can split it by +calling split_huge_page(page). This is what the Linux VM does before +it tries to swapout the hugepage for example. split_huge_page() can fail +if the page is pinned and you must handle this correctly. + +Example to make mremap.c transparent hugepage aware with a one liner +change:: + + diff --git a/mm/mremap.c b/mm/mremap.c + --- a/mm/mremap.c + +++ b/mm/mremap.c + @@ -41,6 +41,7 @@ static pmd_t *get_old_pmd(struct mm_stru + return NULL; + + pmd = pmd_offset(pud, addr); + + split_huge_pmd(vma, pmd, addr); + if (pmd_none_or_clear_bad(pmd)) + return NULL; + +Locking in hugepage aware code +============================== + +We want as much code as possible hugepage aware, as calling +split_huge_page() or split_huge_pmd() has a cost. + +To make pagetable walks huge pmd aware, all you need to do is to call +pmd_trans_huge() on the pmd returned by pmd_offset. You must hold the +mmap_lock in read (or write) mode to be sure a huge pmd cannot be +created from under you by khugepaged (khugepaged collapse_huge_page +takes the mmap_lock in write mode in addition to the anon_vma lock). If +pmd_trans_huge returns false, you just fallback in the old code +paths. If instead pmd_trans_huge returns true, you have to take the +page table lock (pmd_lock()) and re-run pmd_trans_huge. Taking the +page table lock will prevent the huge pmd being converted into a +regular pmd from under you (split_huge_pmd can run in parallel to the +pagetable walk). If the second pmd_trans_huge returns false, you +should just drop the page table lock and fallback to the old code as +before. Otherwise, you can proceed to process the huge pmd and the +hugepage natively. Once finished, you can drop the page table lock. + +Refcounts and transparent huge pages +==================================== + +Refcounting on THP is mostly consistent with refcounting on other compound +pages: + + - get_page()/put_page() and GUP operate on head page's ->_refcount. + + - ->_refcount in tail pages is always zero: get_page_unless_zero() never + succeeds on tail pages. + + - map/unmap of the pages with PTE entry increment/decrement ->_mapcount + on relevant sub-page of the compound page. + + - map/unmap of the whole compound page is accounted for in compound_mapcount + (stored in first tail page). For file huge pages, we also increment + ->_mapcount of all sub-pages in order to have race-free detection of + last unmap of subpages. + +PageDoubleMap() indicates that the page is *possibly* mapped with PTEs. + +For anonymous pages, PageDoubleMap() also indicates ->_mapcount in all +subpages is offset up by one. This additional reference is required to +get race-free detection of unmap of subpages when we have them mapped with +both PMDs and PTEs. + +This optimization is required to lower the overhead of per-subpage mapcount +tracking. The alternative is to alter ->_mapcount in all subpages on each +map/unmap of the whole compound page. + +For anonymous pages, we set PG_double_map when a PMD of the page is split +for the first time, but still have a PMD mapping. The additional references +go away with the last compound_mapcount. + +File pages get PG_double_map set on the first map of the page with PTE and +goes away when the page gets evicted from the page cache. + +split_huge_page internally has to distribute the refcounts in the head +page to the tail pages before clearing all PG_head/tail bits from the page +structures. It can be done easily for refcounts taken by page table +entries, but we don't have enough information on how to distribute any +additional pins (i.e. from get_user_pages). split_huge_page() fails any +requests to split pinned huge pages: it expects page count to be equal to +the sum of mapcount of all sub-pages plus one (split_huge_page caller must +have a reference to the head page). + +split_huge_page uses migration entries to stabilize page->_refcount and +page->_mapcount of anonymous pages. File pages just get unmapped. + +We are safe against physical memory scanners too: the only legitimate way +a scanner can get a reference to a page is get_page_unless_zero(). + +All tail pages have zero ->_refcount until atomic_add(). This prevents the +scanner from getting a reference to the tail page up to that point. After the +atomic_add() we don't care about the ->_refcount value. We already know how +many references should be uncharged from the head page. + +For head page get_page_unless_zero() will succeed and we don't mind. It's +clear where references should go after split: it will stay on the head page. + +Note that split_huge_pmd() doesn't have any limitations on refcounting: +pmd can be split at any point and never fails. + +Partial unmap and deferred_split_huge_page() +============================================ + +Unmapping part of THP (with munmap() or other way) is not going to free +memory immediately. Instead, we detect that a subpage of THP is not in use +in page_remove_rmap() and queue the THP for splitting if memory pressure +comes. Splitting will free up unused subpages. + +Splitting the page right away is not an option due to locking context in +the place where we can detect partial unmap. It also might be +counterproductive since in many cases partial unmap happens during exit(2) if +a THP crosses a VMA boundary. + +The function deferred_split_huge_page() is used to queue a page for splitting. +The splitting itself will happen when we get memory pressure via shrinker +interface. diff --git a/Documentation/mm/unevictable-lru.rst b/Documentation/mm/unevictable-lru.rst new file mode 100644 index 000000000000..b280367d6a44 --- /dev/null +++ b/Documentation/mm/unevictable-lru.rst @@ -0,0 +1,554 @@ +.. _unevictable_lru: + +============================== +Unevictable LRU Infrastructure +============================== + +.. contents:: :local: + + +Introduction +============ + +This document describes the Linux memory manager's "Unevictable LRU" +infrastructure and the use of this to manage several types of "unevictable" +pages. + +The document attempts to provide the overall rationale behind this mechanism +and the rationale for some of the design decisions that drove the +implementation. The latter design rationale is discussed in the context of an +implementation description. Admittedly, one can obtain the implementation +details - the "what does it do?" - by reading the code. One hopes that the +descriptions below add value by provide the answer to "why does it do that?". + + + +The Unevictable LRU +=================== + +The Unevictable LRU facility adds an additional LRU list to track unevictable +pages and to hide these pages from vmscan. This mechanism is based on a patch +by Larry Woodman of Red Hat to address several scalability problems with page +reclaim in Linux. The problems have been observed at customer sites on large +memory x86_64 systems. + +To illustrate this with an example, a non-NUMA x86_64 platform with 128GB of +main memory will have over 32 million 4k pages in a single node. When a large +fraction of these pages are not evictable for any reason [see below], vmscan +will spend a lot of time scanning the LRU lists looking for the small fraction +of pages that are evictable. This can result in a situation where all CPUs are +spending 100% of their time in vmscan for hours or days on end, with the system +completely unresponsive. + +The unevictable list addresses the following classes of unevictable pages: + + * Those owned by ramfs. + + * Those mapped into SHM_LOCK'd shared memory regions. + + * Those mapped into VM_LOCKED [mlock()ed] VMAs. + +The infrastructure may also be able to handle other conditions that make pages +unevictable, either by definition or by circumstance, in the future. + + +The Unevictable LRU Page List +----------------------------- + +The Unevictable LRU page list is a lie. It was never an LRU-ordered list, but a +companion to the LRU-ordered anonymous and file, active and inactive page lists; +and now it is not even a page list. But following familiar convention, here in +this document and in the source, we often imagine it as a fifth LRU page list. + +The Unevictable LRU infrastructure consists of an additional, per-node, LRU list +called the "unevictable" list and an associated page flag, PG_unevictable, to +indicate that the page is being managed on the unevictable list. + +The PG_unevictable flag is analogous to, and mutually exclusive with, the +PG_active flag in that it indicates on which LRU list a page resides when +PG_lru is set. + +The Unevictable LRU infrastructure maintains unevictable pages as if they were +on an additional LRU list for a few reasons: + + (1) We get to "treat unevictable pages just like we treat other pages in the + system - which means we get to use the same code to manipulate them, the + same code to isolate them (for migrate, etc.), the same code to keep track + of the statistics, etc..." [Rik van Riel] + + (2) We want to be able to migrate unevictable pages between nodes for memory + defragmentation, workload management and memory hotplug. The Linux kernel + can only migrate pages that it can successfully isolate from the LRU + lists (or "Movable" pages: outside of consideration here). If we were to + maintain pages elsewhere than on an LRU-like list, where they can be + detected by isolate_lru_page(), we would prevent their migration. + +The unevictable list does not differentiate between file-backed and anonymous, +swap-backed pages. This differentiation is only important while the pages are, +in fact, evictable. + +The unevictable list benefits from the "arrayification" of the per-node LRU +lists and statistics originally proposed and posted by Christoph Lameter. + + +Memory Control Group Interaction +-------------------------------- + +The unevictable LRU facility interacts with the memory control group [aka +memory controller; see Documentation/admin-guide/cgroup-v1/memory.rst] by +extending the lru_list enum. + +The memory controller data structure automatically gets a per-node unevictable +list as a result of the "arrayification" of the per-node LRU lists (one per +lru_list enum element). The memory controller tracks the movement of pages to +and from the unevictable list. + +When a memory control group comes under memory pressure, the controller will +not attempt to reclaim pages on the unevictable list. This has a couple of +effects: + + (1) Because the pages are "hidden" from reclaim on the unevictable list, the + reclaim process can be more efficient, dealing only with pages that have a + chance of being reclaimed. + + (2) On the other hand, if too many of the pages charged to the control group + are unevictable, the evictable portion of the working set of the tasks in + the control group may not fit into the available memory. This can cause + the control group to thrash or to OOM-kill tasks. + + +.. _mark_addr_space_unevict: + +Marking Address Spaces Unevictable +---------------------------------- + +For facilities such as ramfs none of the pages attached to the address space +may be evicted. To prevent eviction of any such pages, the AS_UNEVICTABLE +address space flag is provided, and this can be manipulated by a filesystem +using a number of wrapper functions: + + * ``void mapping_set_unevictable(struct address_space *mapping);`` + + Mark the address space as being completely unevictable. + + * ``void mapping_clear_unevictable(struct address_space *mapping);`` + + Mark the address space as being evictable. + + * ``int mapping_unevictable(struct address_space *mapping);`` + + Query the address space, and return true if it is completely + unevictable. + +These are currently used in three places in the kernel: + + (1) By ramfs to mark the address spaces of its inodes when they are created, + and this mark remains for the life of the inode. + + (2) By SYSV SHM to mark SHM_LOCK'd address spaces until SHM_UNLOCK is called. + Note that SHM_LOCK is not required to page in the locked pages if they're + swapped out; the application must touch the pages manually if it wants to + ensure they're in memory. + + (3) By the i915 driver to mark pinned address space until it's unpinned. The + amount of unevictable memory marked by i915 driver is roughly the bounded + object size in debugfs/dri/0/i915_gem_objects. + + +Detecting Unevictable Pages +--------------------------- + +The function page_evictable() in mm/internal.h determines whether a page is +evictable or not using the query function outlined above [see section +:ref:`Marking address spaces unevictable `] +to check the AS_UNEVICTABLE flag. + +For address spaces that are so marked after being populated (as SHM regions +might be), the lock action (e.g. SHM_LOCK) can be lazy, and need not populate +the page tables for the region as does, for example, mlock(), nor need it make +any special effort to push any pages in the SHM_LOCK'd area to the unevictable +list. Instead, vmscan will do this if and when it encounters the pages during +a reclamation scan. + +On an unlock action (such as SHM_UNLOCK), the unlocker (e.g. shmctl()) must scan +the pages in the region and "rescue" them from the unevictable list if no other +condition is keeping them unevictable. If an unevictable region is destroyed, +the pages are also "rescued" from the unevictable list in the process of +freeing them. + +page_evictable() also checks for mlocked pages by testing an additional page +flag, PG_mlocked (as wrapped by PageMlocked()), which is set when a page is +faulted into a VM_LOCKED VMA, or found in a VMA being VM_LOCKED. + + +Vmscan's Handling of Unevictable Pages +-------------------------------------- + +If unevictable pages are culled in the fault path, or moved to the unevictable +list at mlock() or mmap() time, vmscan will not encounter the pages until they +have become evictable again (via munlock() for example) and have been "rescued" +from the unevictable list. However, there may be situations where we decide, +for the sake of expediency, to leave an unevictable page on one of the regular +active/inactive LRU lists for vmscan to deal with. vmscan checks for such +pages in all of the shrink_{active|inactive|page}_list() functions and will +"cull" such pages that it encounters: that is, it diverts those pages to the +unevictable list for the memory cgroup and node being scanned. + +There may be situations where a page is mapped into a VM_LOCKED VMA, but the +page is not marked as PG_mlocked. Such pages will make it all the way to +shrink_active_list() or shrink_page_list() where they will be detected when +vmscan walks the reverse map in page_referenced() or try_to_unmap(). The page +is culled to the unevictable list when it is released by the shrinker. + +To "cull" an unevictable page, vmscan simply puts the page back on the LRU list +using putback_lru_page() - the inverse operation to isolate_lru_page() - after +dropping the page lock. Because the condition which makes the page unevictable +may change once the page is unlocked, __pagevec_lru_add_fn() will recheck the +unevictable state of a page before placing it on the unevictable list. + + +MLOCKED Pages +============= + +The unevictable page list is also useful for mlock(), in addition to ramfs and +SYSV SHM. Note that mlock() is only available in CONFIG_MMU=y situations; in +NOMMU situations, all mappings are effectively mlocked. + + +History +------- + +The "Unevictable mlocked Pages" infrastructure is based on work originally +posted by Nick Piggin in an RFC patch entitled "mm: mlocked pages off LRU". +Nick posted his patch as an alternative to a patch posted by Christoph Lameter +to achieve the same objective: hiding mlocked pages from vmscan. + +In Nick's patch, he used one of the struct page LRU list link fields as a count +of VM_LOCKED VMAs that map the page (Rik van Riel had the same idea three years +earlier). But this use of the link field for a count prevented the management +of the pages on an LRU list, and thus mlocked pages were not migratable as +isolate_lru_page() could not detect them, and the LRU list link field was not +available to the migration subsystem. + +Nick resolved this by putting mlocked pages back on the LRU list before +attempting to isolate them, thus abandoning the count of VM_LOCKED VMAs. When +Nick's patch was integrated with the Unevictable LRU work, the count was +replaced by walking the reverse map when munlocking, to determine whether any +other VM_LOCKED VMAs still mapped the page. + +However, walking the reverse map for each page when munlocking was ugly and +inefficient, and could lead to catastrophic contention on a file's rmap lock, +when many processes which had it mlocked were trying to exit. In 5.18, the +idea of keeping mlock_count in Unevictable LRU list link field was revived and +put to work, without preventing the migration of mlocked pages. This is why +the "Unevictable LRU list" cannot be a linked list of pages now; but there was +no use for that linked list anyway - though its size is maintained for meminfo. + + +Basic Management +---------------- + +mlocked pages - pages mapped into a VM_LOCKED VMA - are a class of unevictable +pages. When such a page has been "noticed" by the memory management subsystem, +the page is marked with the PG_mlocked flag. This can be manipulated using the +PageMlocked() functions. + +A PG_mlocked page will be placed on the unevictable list when it is added to +the LRU. Such pages can be "noticed" by memory management in several places: + + (1) in the mlock()/mlock2()/mlockall() system call handlers; + + (2) in the mmap() system call handler when mmapping a region with the + MAP_LOCKED flag; + + (3) mmapping a region in a task that has called mlockall() with the MCL_FUTURE + flag; + + (4) in the fault path and when a VM_LOCKED stack segment is expanded; or + + (5) as mentioned above, in vmscan:shrink_page_list() when attempting to + reclaim a page in a VM_LOCKED VMA by page_referenced() or try_to_unmap(). + +mlocked pages become unlocked and rescued from the unevictable list when: + + (1) mapped in a range unlocked via the munlock()/munlockall() system calls; + + (2) munmap()'d out of the last VM_LOCKED VMA that maps the page, including + unmapping at task exit; + + (3) when the page is truncated from the last VM_LOCKED VMA of an mmapped file; + or + + (4) before a page is COW'd in a VM_LOCKED VMA. + + +mlock()/mlock2()/mlockall() System Call Handling +------------------------------------------------ + +mlock(), mlock2() and mlockall() system call handlers proceed to mlock_fixup() +for each VMA in the range specified by the call. In the case of mlockall(), +this is the entire active address space of the task. Note that mlock_fixup() +is used for both mlocking and munlocking a range of memory. A call to mlock() +an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED, is +treated as a no-op and mlock_fixup() simply returns. + +If the VMA passes some filtering as described in "Filtering Special VMAs" +below, mlock_fixup() will attempt to merge the VMA with its neighbors or split +off a subset of the VMA if the range does not cover the entire VMA. Any pages +already present in the VMA are then marked as mlocked by mlock_page() via +mlock_pte_range() via walk_page_range() via mlock_vma_pages_range(). + +Before returning from the system call, do_mlock() or mlockall() will call +__mm_populate() to fault in the remaining pages via get_user_pages() and to +mark those pages as mlocked as they are faulted. + +Note that the VMA being mlocked might be mapped with PROT_NONE. In this case, +get_user_pages() will be unable to fault in the pages. That's okay. If pages +do end up getting faulted into this VM_LOCKED VMA, they will be handled in the +fault path - which is also how mlock2()'s MLOCK_ONFAULT areas are handled. + +For each PTE (or PMD) being faulted into a VMA, the page add rmap function +calls mlock_vma_page(), which calls mlock_page() when the VMA is VM_LOCKED +(unless it is a PTE mapping of a part of a transparent huge page). Or when +it is a newly allocated anonymous page, lru_cache_add_inactive_or_unevictable() +calls mlock_new_page() instead: similar to mlock_page(), but can make better +judgments, since this page is held exclusively and known not to be on LRU yet. + +mlock_page() sets PageMlocked immediately, then places the page on the CPU's +mlock pagevec, to batch up the rest of the work to be done under lru_lock by +__mlock_page(). __mlock_page() sets PageUnevictable, initializes mlock_count +and moves the page to unevictable state ("the unevictable LRU", but with +mlock_count in place of LRU threading). Or if the page was already PageLRU +and PageUnevictable and PageMlocked, it simply increments the mlock_count. + +But in practice that may not work ideally: the page may not yet be on an LRU, or +it may have been temporarily isolated from LRU. In such cases the mlock_count +field cannot be touched, but will be set to 0 later when __pagevec_lru_add_fn() +returns the page to "LRU". Races prohibit mlock_count from being set to 1 then: +rather than risk stranding a page indefinitely as unevictable, always err with +mlock_count on the low side, so that when munlocked the page will be rescued to +an evictable LRU, then perhaps be mlocked again later if vmscan finds it in a +VM_LOCKED VMA. + + +Filtering Special VMAs +---------------------- + +mlock_fixup() filters several classes of "special" VMAs: + +1) VMAs with VM_IO or VM_PFNMAP set are skipped entirely. The pages behind + these mappings are inherently pinned, so we don't need to mark them as + mlocked. In any case, most of the pages have no struct page in which to so + mark the page. Because of this, get_user_pages() will fail for these VMAs, + so there is no sense in attempting to visit them. + +2) VMAs mapping hugetlbfs page are already effectively pinned into memory. We + neither need nor want to mlock() these pages. But __mm_populate() includes + hugetlbfs ranges, allocating the huge pages and populating the PTEs. + +3) VMAs with VM_DONTEXPAND are generally userspace mappings of kernel pages, + such as the VDSO page, relay channel pages, etc. These pages are inherently + unevictable and are not managed on the LRU lists. __mm_populate() includes + these ranges, populating the PTEs if not already populated. + +4) VMAs with VM_MIXEDMAP set are not marked VM_LOCKED, but __mm_populate() + includes these ranges, populating the PTEs if not already populated. + +Note that for all of these special VMAs, mlock_fixup() does not set the +VM_LOCKED flag. Therefore, we won't have to deal with them later during +munlock(), munmap() or task exit. Neither does mlock_fixup() account these +VMAs against the task's "locked_vm". + + +munlock()/munlockall() System Call Handling +------------------------------------------- + +The munlock() and munlockall() system calls are handled by the same +mlock_fixup() function as mlock(), mlock2() and mlockall() system calls are. +If called to munlock an already munlocked VMA, mlock_fixup() simply returns. +Because of the VMA filtering discussed above, VM_LOCKED will not be set in +any "special" VMAs. So, those VMAs will be ignored for munlock. + +If the VMA is VM_LOCKED, mlock_fixup() again attempts to merge or split off the +specified range. All pages in the VMA are then munlocked by munlock_page() via +mlock_pte_range() via walk_page_range() via mlock_vma_pages_range() - the same +function used when mlocking a VMA range, with new flags for the VMA indicating +that it is munlock() being performed. + +munlock_page() uses the mlock pagevec to batch up work to be done under +lru_lock by __munlock_page(). __munlock_page() decrements the page's +mlock_count, and when that reaches 0 it clears PageMlocked and clears +PageUnevictable, moving the page from unevictable state to inactive LRU. + +But in practice that may not work ideally: the page may not yet have reached +"the unevictable LRU", or it may have been temporarily isolated from it. In +those cases its mlock_count field is unusable and must be assumed to be 0: so +that the page will be rescued to an evictable LRU, then perhaps be mlocked +again later if vmscan finds it in a VM_LOCKED VMA. + + +Migrating MLOCKED Pages +----------------------- + +A page that is being migrated has been isolated from the LRU lists and is held +locked across unmapping of the page, updating the page's address space entry +and copying the contents and state, until the page table entry has been +replaced with an entry that refers to the new page. Linux supports migration +of mlocked pages and other unevictable pages. PG_mlocked is cleared from the +the old page when it is unmapped from the last VM_LOCKED VMA, and set when the +new page is mapped in place of migration entry in a VM_LOCKED VMA. If the page +was unevictable because mlocked, PG_unevictable follows PG_mlocked; but if the +page was unevictable for other reasons, PG_unevictable is copied explicitly. + +Note that page migration can race with mlocking or munlocking of the same page. +There is mostly no problem since page migration requires unmapping all PTEs of +the old page (including munlock where VM_LOCKED), then mapping in the new page +(including mlock where VM_LOCKED). The page table locks provide sufficient +synchronization. + +However, since mlock_vma_pages_range() starts by setting VM_LOCKED on a VMA, +before mlocking any pages already present, if one of those pages were migrated +before mlock_pte_range() reached it, it would get counted twice in mlock_count. +To prevent that, mlock_vma_pages_range() temporarily marks the VMA as VM_IO, +so that mlock_vma_page() will skip it. + +To complete page migration, we place the old and new pages back onto the LRU +afterwards. The "unneeded" page - old page on success, new page on failure - +is freed when the reference count held by the migration process is released. + + +Compacting MLOCKED Pages +------------------------ + +The memory map can be scanned for compactable regions and the default behavior +is to let unevictable pages be moved. /proc/sys/vm/compact_unevictable_allowed +controls this behavior (see Documentation/admin-guide/sysctl/vm.rst). The work +of compaction is mostly handled by the page migration code and the same work +flow as described in Migrating MLOCKED Pages will apply. + + +MLOCKING Transparent Huge Pages +------------------------------- + +A transparent huge page is represented by a single entry on an LRU list. +Therefore, we can only make unevictable an entire compound page, not +individual subpages. + +If a user tries to mlock() part of a huge page, and no user mlock()s the +whole of the huge page, we want the rest of the page to be reclaimable. + +We cannot just split the page on partial mlock() as split_huge_page() can +fail and a new intermittent failure mode for the syscall is undesirable. + +We handle this by keeping PTE-mlocked huge pages on evictable LRU lists: +the PMD on the border of a VM_LOCKED VMA will be split into a PTE table. + +This way the huge page is accessible for vmscan. Under memory pressure the +page will be split, subpages which belong to VM_LOCKED VMAs will be moved +to the unevictable LRU and the rest can be reclaimed. + +/proc/meminfo's Unevictable and Mlocked amounts do not include those parts +of a transparent huge page which are mapped only by PTEs in VM_LOCKED VMAs. + + +mmap(MAP_LOCKED) System Call Handling +------------------------------------- + +In addition to the mlock(), mlock2() and mlockall() system calls, an application +can request that a region of memory be mlocked by supplying the MAP_LOCKED flag +to the mmap() call. There is one important and subtle difference here, though. +mmap() + mlock() will fail if the range cannot be faulted in (e.g. because +mm_populate fails) and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. +The mmaped area will still have properties of the locked area - pages will not +get swapped out - but major page faults to fault memory in might still happen. + +Furthermore, any mmap() call or brk() call that expands the heap by a task +that has previously called mlockall() with the MCL_FUTURE flag will result +in the newly mapped memory being mlocked. Before the unevictable/mlock +changes, the kernel simply called make_pages_present() to allocate pages +and populate the page table. + +To mlock a range of memory under the unevictable/mlock infrastructure, +the mmap() handler and task address space expansion functions call +populate_vma_page_range() specifying the vma and the address range to mlock. + + +munmap()/exit()/exec() System Call Handling +------------------------------------------- + +When unmapping an mlocked region of memory, whether by an explicit call to +munmap() or via an internal unmap from exit() or exec() processing, we must +munlock the pages if we're removing the last VM_LOCKED VMA that maps the pages. +Before the unevictable/mlock changes, mlocking did not mark the pages in any +way, so unmapping them required no processing. + +For each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls +munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED +(unless it was a PTE mapping of a part of a transparent huge page). + +munlock_page() uses the mlock pagevec to batch up work to be done under +lru_lock by __munlock_page(). __munlock_page() decrements the page's +mlock_count, and when that reaches 0 it clears PageMlocked and clears +PageUnevictable, moving the page from unevictable state to inactive LRU. + +But in practice that may not work ideally: the page may not yet have reached +"the unevictable LRU", or it may have been temporarily isolated from it. In +those cases its mlock_count field is unusable and must be assumed to be 0: so +that the page will be rescued to an evictable LRU, then perhaps be mlocked +again later if vmscan finds it in a VM_LOCKED VMA. + + +Truncating MLOCKED Pages +------------------------ + +File truncation or hole punching forcibly unmaps the deleted pages from +userspace; truncation even unmaps and deletes any private anonymous pages +which had been Copied-On-Write from the file pages now being truncated. + +Mlocked pages can be munlocked and deleted in this way: like with munmap(), +for each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls +munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED +(unless it was a PTE mapping of a part of a transparent huge page). + +However, if there is a racing munlock(), since mlock_vma_pages_range() starts +munlocking by clearing VM_LOCKED from a VMA, before munlocking all the pages +present, if one of those pages were unmapped by truncation or hole punch before +mlock_pte_range() reached it, it would not be recognized as mlocked by this VMA, +and would not be counted out of mlock_count. In this rare case, a page may +still appear as PageMlocked after it has been fully unmapped: and it is left to +release_pages() (or __page_cache_release()) to clear it and update statistics +before freeing (this event is counted in /proc/vmstat unevictable_pgs_cleared, +which is usually 0). + + +Page Reclaim in shrink_*_list() +------------------------------- + +vmscan's shrink_active_list() culls any obviously unevictable pages - +i.e. !page_evictable(page) pages - diverting those to the unevictable list. +However, shrink_active_list() only sees unevictable pages that made it onto the +active/inactive LRU lists. Note that these pages do not have PageUnevictable +set - otherwise they would be on the unevictable list and shrink_active_list() +would never see them. + +Some examples of these unevictable pages on the LRU lists are: + + (1) ramfs pages that have been placed on the LRU lists when first allocated. + + (2) SHM_LOCK'd shared memory pages. shmctl(SHM_LOCK) does not attempt to + allocate or fault in the pages in the shared memory region. This happens + when an application accesses the page the first time after SHM_LOCK'ing + the segment. + + (3) pages still mapped into VM_LOCKED VMAs, which should be marked mlocked, + but events left mlock_count too low, so they were munlocked too early. + +vmscan's shrink_inactive_list() and shrink_page_list() also divert obviously +unevictable pages found on the inactive lists to the appropriate memory cgroup +and node unevictable list. + +rmap's page_referenced_one(), called via vmscan's shrink_active_list() or +shrink_page_list(), and rmap's try_to_unmap_one() called via shrink_page_list(), +check for (3) pages still mapped into VM_LOCKED VMAs, and call mlock_vma_page() +to correct them. Such pages are culled to the unevictable list when released +by the shrinker. diff --git a/Documentation/mm/vmalloc.rst b/Documentation/mm/vmalloc.rst new file mode 100644 index 000000000000..363fe20d6b9f --- /dev/null +++ b/Documentation/mm/vmalloc.rst @@ -0,0 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================================== +Virtually Contiguous Memory Allocation +====================================== diff --git a/Documentation/mm/vmalloced-kernel-stacks.rst b/Documentation/mm/vmalloced-kernel-stacks.rst new file mode 100644 index 000000000000..fc8c67833af6 --- /dev/null +++ b/Documentation/mm/vmalloced-kernel-stacks.rst @@ -0,0 +1,153 @@ +.. SPDX-License-Identifier: GPL-2.0 + +===================================== +Virtually Mapped Kernel Stack Support +===================================== + +:Author: Shuah Khan + +.. contents:: :local: + +Overview +-------- + +This is a compilation of information from the code and original patch +series that introduced the `Virtually Mapped Kernel Stacks feature +` + +Introduction +------------ + +Kernel stack overflows are often hard to debug and make the kernel +susceptible to exploits. Problems could show up at a later time making +it difficult to isolate and root-cause. + +Virtually-mapped kernel stacks with guard pages causes kernel stack +overflows to be caught immediately rather than causing difficult to +diagnose corruptions. + +HAVE_ARCH_VMAP_STACK and VMAP_STACK configuration options enable +support for virtually mapped stacks with guard pages. This feature +causes reliable faults when the stack overflows. The usability of +the stack trace after overflow and response to the overflow itself +is architecture dependent. + +.. note:: + As of this writing, arm64, powerpc, riscv, s390, um, and x86 have + support for VMAP_STACK. + +HAVE_ARCH_VMAP_STACK +-------------------- + +Architectures that can support Virtually Mapped Kernel Stacks should +enable this bool configuration option. The requirements are: + +- vmalloc space must be large enough to hold many kernel stacks. This + may rule out many 32-bit architectures. +- Stacks in vmalloc space need to work reliably. For example, if + vmap page tables are created on demand, either this mechanism + needs to work while the stack points to a virtual address with + unpopulated page tables or arch code (switch_to() and switch_mm(), + most likely) needs to ensure that the stack's page table entries + are populated before running on a possibly unpopulated stack. +- If the stack overflows into a guard page, something reasonable + should happen. The definition of "reasonable" is flexible, but + instantly rebooting without logging anything would be unfriendly. + +VMAP_STACK +---------- + +VMAP_STACK bool configuration option when enabled allocates virtually +mapped task stacks. This option depends on HAVE_ARCH_VMAP_STACK. + +- Enable this if you want the use virtually-mapped kernel stacks + with guard pages. This causes kernel stack overflows to be caught + immediately rather than causing difficult-to-diagnose corruption. + +.. note:: + + Using this feature with KASAN requires architecture support + for backing virtual mappings with real shadow memory, and + KASAN_VMALLOC must be enabled. + +.. note:: + + VMAP_STACK is enabled, it is not possible to run DMA on stack + allocated data. + +Kernel configuration options and dependencies keep changing. Refer to +the latest code base: + +`Kconfig ` + +Allocation +----------- + +When a new kernel thread is created, thread stack is allocated from +virtually contiguous memory pages from the page level allocator. These +pages are mapped into contiguous kernel virtual space with PAGE_KERNEL +protections. + +alloc_thread_stack_node() calls __vmalloc_node_range() to allocate stack +with PAGE_KERNEL protections. + +- Allocated stacks are cached and later reused by new threads, so memcg + accounting is performed manually on assigning/releasing stacks to tasks. + Hence, __vmalloc_node_range is called without __GFP_ACCOUNT. +- vm_struct is cached to be able to find when thread free is initiated + in interrupt context. free_thread_stack() can be called in interrupt + context. +- On arm64, all VMAP's stacks need to have the same alignment to ensure + that VMAP'd stack overflow detection works correctly. Arch specific + vmap stack allocator takes care of this detail. +- This does not address interrupt stacks - according to the original patch + +Thread stack allocation is initiated from clone(), fork(), vfork(), +kernel_thread() via kernel_clone(). Leaving a few hints for searching +the code base to understand when and how thread stack is allocated. + +Bulk of the code is in: +`kernel/fork.c `. + +stack_vm_area pointer in task_struct keeps track of the virtually allocated +stack and a non-null stack_vm_area pointer serves as a indication that the +virtually mapped kernel stacks are enabled. + +:: + + struct vm_struct *stack_vm_area; + +Stack overflow handling +----------------------- + +Leading and trailing guard pages help detect stack overflows. When stack +overflows into the guard pages, handlers have to be careful not overflow +the stack again. When handlers are called, it is likely that very little +stack space is left. + +On x86, this is done by handling the page fault indicating the kernel +stack overflow on the double-fault stack. + +Testing VMAP allocation with guard pages +---------------------------------------- + +How do we ensure that VMAP_STACK is actually allocating with a leading +and trailing guard page? The following lkdtm tests can help detect any +regressions. + +:: + + void lkdtm_STACK_GUARD_PAGE_LEADING() + void lkdtm_STACK_GUARD_PAGE_TRAILING() + +Conclusions +----------- + +- A percpu cache of vmalloced stacks appears to be a bit faster than a + high-order stack allocation, at least when the cache hits. +- THREAD_INFO_IN_TASK gets rid of arch-specific thread_info entirely and + simply embed the thread_info (containing only flags) and 'int cpu' into + task_struct. +- The thread stack can be free'ed as soon as the task is dead (without + waiting for RCU) and then, if vmapped stacks are in use, cache the + entire stack for reuse on the same cpu. diff --git a/Documentation/mm/vmemmap_dedup.rst b/Documentation/mm/vmemmap_dedup.rst new file mode 100644 index 000000000000..c9c495f62d12 --- /dev/null +++ b/Documentation/mm/vmemmap_dedup.rst @@ -0,0 +1,223 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================================= +A vmemmap diet for HugeTLB and Device DAX +========================================= + +HugeTLB +======= + +The struct page structures (page structs) are used to describe a physical +page frame. By default, there is a one-to-one mapping from a page frame to +it's corresponding page struct. + +HugeTLB pages consist of multiple base page size pages and is supported by many +architectures. See Documentation/admin-guide/mm/hugetlbpage.rst for more +details. On the x86-64 architecture, HugeTLB pages of size 2MB and 1GB are +currently supported. Since the base page size on x86 is 4KB, a 2MB HugeTLB page +consists of 512 base pages and a 1GB HugeTLB page consists of 4096 base pages. +For each base page, there is a corresponding page struct. + +Within the HugeTLB subsystem, only the first 4 page structs are used to +contain unique information about a HugeTLB page. __NR_USED_SUBPAGE provides +this upper limit. The only 'useful' information in the remaining page structs +is the compound_head field, and this field is the same for all tail pages. + +By removing redundant page structs for HugeTLB pages, memory can be returned +to the buddy allocator for other uses. + +Different architectures support different HugeTLB pages. For example, the +following table is the HugeTLB page size supported by x86 and arm64 +architectures. Because arm64 supports 4k, 16k, and 64k base pages and +supports contiguous entries, so it supports many kinds of sizes of HugeTLB +page. + ++--------------+-----------+-----------------------------------------------+ +| Architecture | Page Size | HugeTLB Page Size | ++--------------+-----------+-----------+-----------+-----------+-----------+ +| x86-64 | 4KB | 2MB | 1GB | | | ++--------------+-----------+-----------+-----------+-----------+-----------+ +| | 4KB | 64KB | 2MB | 32MB | 1GB | +| +-----------+-----------+-----------+-----------+-----------+ +| arm64 | 16KB | 2MB | 32MB | 1GB | | +| +-----------+-----------+-----------+-----------+-----------+ +| | 64KB | 2MB | 512MB | 16GB | | ++--------------+-----------+-----------+-----------+-----------+-----------+ + +When the system boot up, every HugeTLB page has more than one struct page +structs which size is (unit: pages):: + + struct_size = HugeTLB_Size / PAGE_SIZE * sizeof(struct page) / PAGE_SIZE + +Where HugeTLB_Size is the size of the HugeTLB page. We know that the size +of the HugeTLB page is always n times PAGE_SIZE. So we can get the following +relationship:: + + HugeTLB_Size = n * PAGE_SIZE + +Then:: + + struct_size = n * PAGE_SIZE / PAGE_SIZE * sizeof(struct page) / PAGE_SIZE + = n * sizeof(struct page) / PAGE_SIZE + +We can use huge mapping at the pud/pmd level for the HugeTLB page. + +For the HugeTLB page of the pmd level mapping, then:: + + struct_size = n * sizeof(struct page) / PAGE_SIZE + = PAGE_SIZE / sizeof(pte_t) * sizeof(struct page) / PAGE_SIZE + = sizeof(struct page) / sizeof(pte_t) + = 64 / 8 + = 8 (pages) + +Where n is how many pte entries which one page can contains. So the value of +n is (PAGE_SIZE / sizeof(pte_t)). + +This optimization only supports 64-bit system, so the value of sizeof(pte_t) +is 8. And this optimization also applicable only when the size of struct page +is a power of two. In most cases, the size of struct page is 64 bytes (e.g. +x86-64 and arm64). So if we use pmd level mapping for a HugeTLB page, the +size of struct page structs of it is 8 page frames which size depends on the +size of the base page. + +For the HugeTLB page of the pud level mapping, then:: + + struct_size = PAGE_SIZE / sizeof(pmd_t) * struct_size(pmd) + = PAGE_SIZE / 8 * 8 (pages) + = PAGE_SIZE (pages) + +Where the struct_size(pmd) is the size of the struct page structs of a +HugeTLB page of the pmd level mapping. + +E.g.: A 2MB HugeTLB page on x86_64 consists in 8 page frames while 1GB +HugeTLB page consists in 4096. + +Next, we take the pmd level mapping of the HugeTLB page as an example to +show the internal implementation of this optimization. There are 8 pages +struct page structs associated with a HugeTLB page which is pmd mapped. + +Here is how things look before optimization:: + + HugeTLB struct pages(8 pages) page frame(8 pages) + +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ + | | | 0 | -------------> | 0 | + | | +-----------+ +-----------+ + | | | 1 | -------------> | 1 | + | | +-----------+ +-----------+ + | | | 2 | -------------> | 2 | + | | +-----------+ +-----------+ + | | | 3 | -------------> | 3 | + | | +-----------+ +-----------+ + | | | 4 | -------------> | 4 | + | PMD | +-----------+ +-----------+ + | level | | 5 | -------------> | 5 | + | mapping | +-----------+ +-----------+ + | | | 6 | -------------> | 6 | + | | +-----------+ +-----------+ + | | | 7 | -------------> | 7 | + | | +-----------+ +-----------+ + | | + | | + | | + +-----------+ + +The value of page->compound_head is the same for all tail pages. The first +page of page structs (page 0) associated with the HugeTLB page contains the 4 +page structs necessary to describe the HugeTLB. The only use of the remaining +pages of page structs (page 1 to page 7) is to point to page->compound_head. +Therefore, we can remap pages 1 to 7 to page 0. Only 1 page of page structs +will be used for each HugeTLB page. This will allow us to free the remaining +7 pages to the buddy allocator. + +Here is how things look after remapping:: + + HugeTLB struct pages(8 pages) page frame(8 pages) + +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ + | | | 0 | -------------> | 0 | + | | +-----------+ +-----------+ + | | | 1 | ---------------^ ^ ^ ^ ^ ^ ^ + | | +-----------+ | | | | | | + | | | 2 | -----------------+ | | | | | + | | +-----------+ | | | | | + | | | 3 | -------------------+ | | | | + | | +-----------+ | | | | + | | | 4 | ---------------------+ | | | + | PMD | +-----------+ | | | + | level | | 5 | -----------------------+ | | + | mapping | +-----------+ | | + | | | 6 | -------------------------+ | + | | +-----------+ | + | | | 7 | ---------------------------+ + | | +-----------+ + | | + | | + | | + +-----------+ + +When a HugeTLB is freed to the buddy system, we should allocate 7 pages for +vmemmap pages and restore the previous mapping relationship. + +For the HugeTLB page of the pud level mapping. It is similar to the former. +We also can use this approach to free (PAGE_SIZE - 1) vmemmap pages. + +Apart from the HugeTLB page of the pmd/pud level mapping, some architectures +(e.g. aarch64) provides a contiguous bit in the translation table entries +that hints to the MMU to indicate that it is one of a contiguous set of +entries that can be cached in a single TLB entry. + +The contiguous bit is used to increase the mapping size at the pmd and pte +(last) level. So this type of HugeTLB page can be optimized only when its +size of the struct page structs is greater than 1 page. + +Notice: The head vmemmap page is not freed to the buddy allocator and all +tail vmemmap pages are mapped to the head vmemmap page frame. So we can see +more than one struct page struct with PG_head (e.g. 8 per 2 MB HugeTLB page) +associated with each HugeTLB page. The compound_head() can handle this +correctly (more details refer to the comment above compound_head()). + +Device DAX +========== + +The device-dax interface uses the same tail deduplication technique explained +in the previous chapter, except when used with the vmemmap in +the device (altmap). + +The following page sizes are supported in DAX: PAGE_SIZE (4K on x86_64), +PMD_SIZE (2M on x86_64) and PUD_SIZE (1G on x86_64). + +The differences with HugeTLB are relatively minor. + +It only use 3 page structs for storing all information as opposed +to 4 on HugeTLB pages. + +There's no remapping of vmemmap given that device-dax memory is not part of +System RAM ranges initialized at boot. Thus the tail page deduplication +happens at a later stage when we populate the sections. HugeTLB reuses the +the head vmemmap page representing, whereas device-dax reuses the tail +vmemmap page. This results in only half of the savings compared to HugeTLB. + +Deduplicated tail pages are not mapped read-only. + +Here's how things look like on device-dax after the sections are populated:: + + +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ + | | | 0 | -------------> | 0 | + | | +-----------+ +-----------+ + | | | 1 | -------------> | 1 | + | | +-----------+ +-----------+ + | | | 2 | ----------------^ ^ ^ ^ ^ ^ + | | +-----------+ | | | | | + | | | 3 | ------------------+ | | | | + | | +-----------+ | | | | + | | | 4 | --------------------+ | | | + | PMD | +-----------+ | | | + | level | | 5 | ----------------------+ | | + | mapping | +-----------+ | | + | | | 6 | ------------------------+ | + | | +-----------+ | + | | | 7 | --------------------------+ + | | +-----------+ + | | + | | + | | + +-----------+ diff --git a/Documentation/mm/z3fold.rst b/Documentation/mm/z3fold.rst new file mode 100644 index 000000000000..224e3c61d686 --- /dev/null +++ b/Documentation/mm/z3fold.rst @@ -0,0 +1,30 @@ +.. _z3fold: + +====== +z3fold +====== + +z3fold is a special purpose allocator for storing compressed pages. +It is designed to store up to three compressed pages per physical page. +It is a zbud derivative which allows for higher compression +ratio keeping the simplicity and determinism of its predecessor. + +The main differences between z3fold and zbud are: + +* unlike zbud, z3fold allows for up to PAGE_SIZE allocations +* z3fold can hold up to 3 compressed pages in its page +* z3fold doesn't export any API itself and is thus intended to be used + via the zpool API. + +To keep the determinism and simplicity, z3fold, just like zbud, always +stores an integral number of compressed pages per page, but it can store +up to 3 pages unlike zbud which can store at most 2. Therefore the +compression ratio goes to around 2.7x while zbud's one is around 1.7x. + +Unlike zbud (but like zsmalloc for that matter) z3fold_alloc() does not +return a dereferenceable pointer. Instead, it returns an unsigned long +handle which encodes actual location of the allocated object. + +Keeping effective compression ratio close to zsmalloc's, z3fold doesn't +depend on MMU enabled and provides more predictable reclaim behavior +which makes it a better fit for small and response-critical systems. diff --git a/Documentation/mm/zsmalloc.rst b/Documentation/mm/zsmalloc.rst new file mode 100644 index 000000000000..6e79893d6132 --- /dev/null +++ b/Documentation/mm/zsmalloc.rst @@ -0,0 +1,82 @@ +.. _zsmalloc: + +======== +zsmalloc +======== + +This allocator is designed for use with zram. Thus, the allocator is +supposed to work well under low memory conditions. In particular, it +never attempts higher order page allocation which is very likely to +fail under memory pressure. On the other hand, if we just use single +(0-order) pages, it would suffer from very high fragmentation -- +any object of size PAGE_SIZE/2 or larger would occupy an entire page. +This was one of the major issues with its predecessor (xvmalloc). + +To overcome these issues, zsmalloc allocates a bunch of 0-order pages +and links them together using various 'struct page' fields. These linked +pages act as a single higher-order page i.e. an object can span 0-order +page boundaries. The code refers to these linked pages as a single entity +called zspage. + +For simplicity, zsmalloc can only allocate objects of size up to PAGE_SIZE +since this satisfies the requirements of all its current users (in the +worst case, page is incompressible and is thus stored "as-is" i.e. in +uncompressed form). For allocation requests larger than this size, failure +is returned (see zs_malloc). + +Additionally, zs_malloc() does not return a dereferenceable pointer. +Instead, it returns an opaque handle (unsigned long) which encodes actual +location of the allocated object. The reason for this indirection is that +zsmalloc does not keep zspages permanently mapped since that would cause +issues on 32-bit systems where the VA region for kernel space mappings +is very small. So, before using the allocating memory, the object has to +be mapped using zs_map_object() to get a usable pointer and subsequently +unmapped using zs_unmap_object(). + +stat +==== + +With CONFIG_ZSMALLOC_STAT, we could see zsmalloc internal information via +``/sys/kernel/debug/zsmalloc/``. Here is a sample of stat output:: + + # cat /sys/kernel/debug/zsmalloc/zram0/classes + + class size almost_full almost_empty obj_allocated obj_used pages_used pages_per_zspage + ... + ... + 9 176 0 1 186 129 8 4 + 10 192 1 0 2880 2872 135 3 + 11 208 0 1 819 795 42 2 + 12 224 0 1 219 159 12 4 + ... + ... + + +class + index +size + object size zspage stores +almost_empty + the number of ZS_ALMOST_EMPTY zspages(see below) +almost_full + the number of ZS_ALMOST_FULL zspages(see below) +obj_allocated + the number of objects allocated +obj_used + the number of objects allocated to the user +pages_used + the number of pages allocated for the class +pages_per_zspage + the number of 0-order pages to make a zspage + +We assign a zspage to ZS_ALMOST_EMPTY fullness group when n <= N / f, where + +* n = number of allocated objects +* N = total number of objects zspage can store +* f = fullness_threshold_frac(ie, 4 at the moment) + +Similarly, we assign zspage to: + +* ZS_ALMOST_FULL when n > N / f +* ZS_EMPTY when n == 0 +* ZS_FULL when n == N diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst index 0c8276109fc0..30c69e1f44fe 100644 --- a/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst +++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst @@ -13,7 +13,7 @@ 监测数据访问 ============ -:doc:`DAMON ` 允许轻量级的数据访问监测。使用DAMON， +:doc:`DAMON ` 允许轻量级的数据访问监测。使用DAMON， 用户可以分析他们系统的内存访问模式，并优化它们。 .. toctree:: diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst index 1500bdbf338a..c976f3e33ffd 100644 --- a/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst +++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst @@ -229,4 +229,4 @@ DAMON_RECLAIM再次什么都不做，这样我们就可以退回到基于LRU列 .. [1] https://research.google/pubs/pub48551/ .. [2] https://lwn.net/Articles/787611/ -.. [3] https://www.kernel.org/doc/html/latest/vm/free_page_reporting.html +.. [3] https://www.kernel.org/doc/html/latest/mm/free_page_reporting.html diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst index eee0e8c5c368..cd41ada4fdad 100644 --- a/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst +++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst @@ -33,9 +33,9 @@ DAMON 为不同的用户提供了下面这些接口。 口相同。这将在下一个LTS内核发布后被移除，所以用户应该转移到 :ref:`sysfs interface `。 - *内核空间编程接口。* - :doc:`这 ` 这是为内核空间程序员准备的。使用它，用户可以通过为你编写内 + :doc:`这 ` 这是为内核空间程序员准备的。使用它，用户可以通过为你编写内 核空间的DAMON应用程序，最灵活有效地利用DAMON的每一个功能。你甚至可以为各种地址空间扩展DAMON。 - 详细情况请参考接口 :doc:`文件 `。 + 详细情况请参考接口 :doc:`文件 `。 sysfs接口 ========= @@ -148,7 +148,7 @@ contexts//monitoring_attrs/ 在 ``nr_regions`` 目录下，有两个文件分别用于DAMON监测区域的下限和上限（``min`` 和 ``max`` ）， 这两个文件控制着监测的开销。你可以通过向这些文件的写入和读出来设置和获取这些值。 -关于间隔和监测区域范围的更多细节，请参考设计文件 (:doc:`/vm/damon/design`)。 +关于间隔和监测区域范围的更多细节，请参考设计文件 (:doc:`/mm/damon/design`)。 contexts//targets/ --------------------- @@ -318,7 +318,7 @@ DAMON导出了八个文件, ``attrs``, ``target_ids``, ``init_regions``, ---- 用户可以通过读取和写入 ``attrs`` 文件获得和设置 ``采样间隔`` 、 ``聚集间隔`` 、 ``更新间隔`` -以及监测目标区域的最小/最大数量。要详细了解监测属性，请参考 `:doc:/vm/damon/design` 。例如， +以及监测目标区域的最小/最大数量。要详细了解监测属性，请参考 `:doc:/mm/damon/design` 。例如， 下面的命令将这些值设置为5ms、100ms、1000ms、10和1000，然后再次检查:: # cd /damon diff --git a/Documentation/translations/zh_CN/core-api/index.rst b/Documentation/translations/zh_CN/core-api/index.rst index 26d9913fc8b6..b03020c8b2ab 100644 --- a/Documentation/translations/zh_CN/core-api/index.rst +++ b/Documentation/translations/zh_CN/core-api/index.rst @@ -101,7 +101,7 @@ Todolist: ======== 如何在内核中分配和使用内存。请注意，在 -:doc:`/vm/index` 中有更多的内存管理文档。 +:doc:`/mm/index` 中有更多的内存管理文档。 .. toctree:: :maxdepth: 1 diff --git a/Documentation/translations/zh_CN/index.rst b/Documentation/translations/zh_CN/index.rst index ad7bb8c17562..bf85baca8b3e 100644 --- a/Documentation/translations/zh_CN/index.rst +++ b/Documentation/translations/zh_CN/index.rst @@ -118,7 +118,7 @@ TODOList: sound/index filesystems/index scheduler/index - vm/index + mm/index peci/index TODOList: diff --git a/Documentation/translations/zh_CN/mm/active_mm.rst b/Documentation/translations/zh_CN/mm/active_mm.rst new file mode 100644 index 000000000000..c2816f523bd7 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/active_mm.rst @@ -0,0 +1,85 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/active_mm.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +========= +Active MM +========= + +这是一封linux之父回复开发者的一封邮件，所以翻译时我尽量保持邮件格式的完整。 + +:: + + List: linux-kernel + Subject: Re: active_mm + From: Linus Torvalds + Date: 1999-07-30 21:36:24 + + 因为我并不经常写解释，所以已经抄送到linux-kernel邮件列表，而当我做这些， + 且更多的人在阅读它们时，我觉得棒极了。 + + 1999年7月30日 星期五， David Mosberger 写道： + > + > 是否有一个简短的描述，说明task_struct中的 + > "mm" 和 "active_mm"应该如何使用？ (如果 + > 这个问题在邮件列表中讨论过，我表示歉意--我刚 + > 刚度假回来，有一段时间没能关注linux-kernel了）。 + + 基本上，新的设定是： + + - 我们有“真实地址空间”和“匿名地址空间”。区别在于，匿名地址空间根本不关心用 + 户级页表，所以当我们做上下文切换到匿名地址空间时，我们只是让以前的地址空间 + 处于活动状态。 + + 一个“匿名地址空间”的明显用途是任何不需要任何用户映射的线程--所有的内核线 + 程基本上都属于这一类，但即使是“真正的”线程也可以暂时说在一定时间内它们不 + 会对用户空间感兴趣，调度器不妨试着避免在切换VM状态上浪费时间。目前只有老 + 式的bdflush sync能做到这一点。 + + - “tsk->mm” 指向 “真实地址空间”。对于一个匿名进程来说，tsk->mm将是NULL， + 其逻辑原因是匿名进程实际上根本就 “没有” 真正的地址空间。 + + - 然而，我们显然需要跟踪我们为这样的匿名用户“偷用”了哪个地址空间。为此，我们 + 有 “tsk->active_mm”，它显示了当前活动的地址空间是什么。 + + 规则是，对于一个有真实地址空间的进程（即tsk->mm是 non-NULL），active_mm + 显然必须与真实的mm相同。 + + 对于一个匿名进程，tsk->mm == NULL，而tsk->active_mm是匿名进程运行时 + “借用”的mm。当匿名进程被调度走时，借用的地址空间被返回并清除。 + + 为了支持所有这些，“struct mm_struct”现在有两个计数器：一个是 “mm_users” + 计数器，即有多少 “真正的地址空间用户”，另一个是 “mm_count”计数器，即 “lazy” + 用户（即匿名用户）的数量，如果有任何真正的用户，则加1。 + + 通常情况下，至少有一个真正的用户，但也可能是真正的用户在另一个CPU上退出，而 + 一个lazy的用户仍在活动，所以你实际上得到的情况是，你有一个地址空间 **只** + 被lazy的用户使用。这通常是一个短暂的生命周期状态，因为一旦这个线程被安排给一 + 个真正的线程，这个 “僵尸” mm就会被释放，因为 “mm_count”变成了零。 + + 另外，一个新的规则是，**没有人** 再把 “init_mm” 作为一个真正的MM了。 + “init_mm”应该被认为只是一个 “没有其他上下文时的lazy上下文”，事实上，它主 + 要是在启动时使用，当时还没有真正的VM被创建。因此，用来检查的代码 + + if (current->mm == &init_mm) + + 一般来说，应该用 + + if (!current->mm) + + 取代上面的写法（这更有意义--测试基本上是 “我们是否有一个用户环境”，并且通常 + 由缺页异常处理程序和类似的东西来完成）。 + + 总之，我刚才在ftp.kernel.org上放了一个pre-patch-2.3.13-1，因为它稍微改 + 变了接口以适配alpha（谁会想到呢，但alpha体系结构上下文切换代码实际上最终是 + 最丑陋的之一--不像其他架构的MM和寄存器状态是分开的，alpha的PALcode将两者 + 连接起来，你需要同时切换两者）。 + + (文档来源 http://marc.info/?l=linux-kernel&m=93337278602211&w=2) diff --git a/Documentation/translations/zh_CN/mm/balance.rst b/Documentation/translations/zh_CN/mm/balance.rst new file mode 100644 index 000000000000..6fd79209c307 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/balance.rst @@ -0,0 +1,81 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/balance.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +======== +内存平衡 +======== + +2000年1月开始，作者：Kanoj Sarcar + +对于 !__GFP_HIGH 和 !__GFP_KSWAPD_RECLAIM 以及非 __GFP_IO 的分配，需要进行 +内存平衡。 + +调用者避免回收的第一个原因是调用者由于持有自旋锁或处于中断环境中而无法睡眠。第二个 +原因可能是，调用者愿意在不产生页面回收开销的情况下分配失败。这可能发生在有0阶回退 +选项的机会主义高阶分配请求中。在这种情况下，调用者可能也希望避免唤醒kswapd。 + +__GFP_IO分配请求是为了防止文件系统死锁。 + +在没有非睡眠分配请求的情况下，做平衡似乎是有害的。页面回收可以被懒散地启动，也就是 +说，只有在需要的时候（也就是区域的空闲内存为0），而不是让它成为一个主动的过程。 + +也就是说，内核应该尝试从直接映射池中满足对直接映射页的请求，而不是回退到dma池中， +这样就可以保持dma池为dma请求（不管是不是原子的）所填充。类似的争论也适用于高内存 +和直接映射的页面。相反，如果有很多空闲的dma页，最好是通过从dma池中分配一个来满足 +常规的内存请求，而不是产生常规区域平衡的开销。 + +在2.2中，只有当空闲页总数低于总内存的1/64时，才会启动内存平衡/页面回收。如果dma +和常规内存的比例合适，即使dma区完全空了，也很可能不会进行平衡。2.2已经在不同内存 +大小的生产机器上运行，即使有这个问题存在，似乎也做得不错。在2.3中，由于HIGHMEM的 +存在，这个问题变得更加严重。 + +在2.3中，区域平衡可以用两种方式之一来完成：根据区域的大小（可能是低级区域的大小）， +我们可以在初始化阶段决定在平衡任何区域时应该争取多少空闲页。好的方面是，在平衡的时 +候，我们不需要看低级区的大小，坏的方面是，我们可能会因为忽略低级区可能较低的使用率 +而做过于频繁的平衡。另外，只要对分配程序稍作修改，就有可能将memclass()宏简化为一 +个简单的等式。 + +另一个可能的解决方案是，我们只在一个区 **和** 其所有低级区的空闲内存低于该区及其 +低级区总内存的1/64时进行平衡。这就解决了2.2的平衡问题，并尽可能地保持了与2.2行为 +的接近。另外，平衡算法在各种架构上的工作方式也是一样的，这些架构有不同数量和类型的 +内存区。如果我们想变得更花哨一点，我们可以在未来为不同区域的自由页面分配不同的权重。 + +请注意，如果普通区的大小与dma区相比是巨大的，那么在决定是否平衡普通区的时候，考虑 +空闲的dma页就变得不那么重要了。那么第一个解决方案就变得更有吸引力。 + +所附的补丁实现了第二个解决方案。它还 “修复”了两个问题：首先，在低内存条件下，kswapd +被唤醒，就像2.2中的非睡眠分配。第二，HIGHMEM区也被平衡了，以便给replace_with_highmem() +一个争取获得HIGHMEM页的机会，同时确保HIGHMEM分配不会落回普通区。这也确保了HIGHMEM +页不会被泄露（例如，在一个HIGHMEM页在交换缓存中但没有被任何人使用的情况下）。 + +kswapd还需要知道它应该平衡哪些区。kswapd主要是在无法进行平衡的情况下需要的，可能 +是因为所有的分配请求都来自中断上下文，而所有的进程上下文都在睡眠。对于2.3， +kswapd并不真正需要平衡高内存区，因为中断上下文并不请求高内存页。kswapd看zone +结构体中的zone_wake_kswapd字段来决定一个区是否需要平衡。 + +如果从进程内存和shm中偷取页面可以减轻该页面节点中任何区的内存压力，而该区的内存压力 +已经低于其水位，则会进行偷取。 + +watemark[WMARK_MIN/WMARK_LOW/WMARK_HIGH]/low_on_memory/zone_wake_kswapd： +这些是每个区的字段，用于确定一个区何时需要平衡。当页面数低于水位[WMARK_MIN]时， +hysteric 的字段low_on_memory被设置。这个字段会一直被设置，直到空闲页数变成水位 +[WMARK_HIGH]。当low_on_memory被设置时，页面分配请求将尝试释放该区域的一些页面（如果 +请求中设置了GFP_WAIT）。与此相反的是，决定唤醒kswapd以释放一些区的页。这个决定不是基于 +hysteresis 的，而是当空闲页的数量低于watermark[WMARK_LOW]时就会进行；在这种情况下， +zone_wake_kswapd也被设置。 + + +我所听到的（超棒的）想法： + +1. 动态经历应该影响平衡：可以跟踪一个区的失败请求的数量，并反馈到平衡方案中（jalvo@mbay.net）。 + +2. 实现一个类似于replace_with_highmem()的replace_with_regular()，以保留dma页面。 + (lkd@tantalophile.demon.co.uk) diff --git a/Documentation/translations/zh_CN/mm/damon/api.rst b/Documentation/translations/zh_CN/mm/damon/api.rst new file mode 100644 index 000000000000..5593a83c86bc --- /dev/null +++ b/Documentation/translations/zh_CN/mm/damon/api.rst @@ -0,0 +1,32 @@ +.. SPDX-License-Identifier: GPL-2.0 + +:Original: Documentation/mm/damon/api.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +======= +API参考 +======= + +内核空间的程序可以使用下面的API来使用DAMON的每个功能。你所需要做的就是引用 ``damon.h`` ， +它位于源代码树的include/linux/。 + +结构体 +====== + +该API在以下内核代码中: + +include/linux/damon.h + + +函数 +==== + +该API在以下内核代码中: + +mm/damon/core.c diff --git a/Documentation/translations/zh_CN/mm/damon/design.rst b/Documentation/translations/zh_CN/mm/damon/design.rst new file mode 100644 index 000000000000..16e3db34a7dd --- /dev/null +++ b/Documentation/translations/zh_CN/mm/damon/design.rst @@ -0,0 +1,140 @@ +.. SPDX-License-Identifier: GPL-2.0 + +:Original: Documentation/mm/damon/design.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +==== +设计 +==== + +可配置的层 +========== + +DAMON提供了数据访问监控功能，同时使其准确性和开销可控。基本的访问监控需要依赖于目标地址空间 +并为之优化的基元。另一方面，作为DAMON的核心，准确性和开销的权衡机制是在纯逻辑空间中。DAMON +将这两部分分离在不同的层中，并定义了它的接口，以允许各种低层次的基元实现与核心逻辑的配置。 + +由于这种分离的设计和可配置的接口，用户可以通过配置核心逻辑和适当的低级基元实现来扩展DAMON的 +任何地址空间。如果没有提供合适的，用户可以自己实现基元。 + +例如，物理内存、虚拟内存、交换空间、那些特定的进程、NUMA节点、文件和支持的内存设备将被支持。 +另外，如果某些架构或设备支持特殊的优化访问检查基元，这些基元将很容易被配置。 + + +特定地址空间基元的参考实现 +========================== + +基本访问监测的低级基元被定义为两部分。: + +1. 确定地址空间的监测目标地址范围 +2. 目标空间中特定地址范围的访问检查。 + +DAMON目前为物理和虚拟地址空间提供了基元的实现。下面两个小节描述了这些工作的方式。 + + +基于VMA的目标地址范围构造 +------------------------- + +这仅仅是针对虚拟地址空间基元的实现。对于物理地址空间，只是要求用户手动设置监控目标地址范围。 + +在进程的超级巨大的虚拟地址空间中，只有小部分被映射到物理内存并被访问。因此，跟踪未映射的地 +址区域只是一种浪费。然而，由于DAMON可以使用自适应区域调整机制来处理一定程度的噪声，所以严 +格来说，跟踪每一个映射并不是必须的，但在某些情况下甚至会产生很高的开销。也就是说，监测目标 +内部过于巨大的未映射区域应该被移除，以不占用自适应机制的时间。 + +出于这个原因，这个实现将复杂的映射转换为三个不同的区域，覆盖地址空间的每个映射区域。这三个 +区域之间的两个空隙是给定地址空间中两个最大的未映射区域。这两个最大的未映射区域是堆和最上面 +的mmap()区域之间的间隙，以及在大多数情况下最下面的mmap()区域和堆之间的间隙。因为这些间隙 +在通常的地址空间中是异常巨大的，排除这些间隙就足以做出合理的权衡。下面详细说明了这一点:: + + + + + (small mmap()-ed regions and munmap()-ed regions) + + + + + +基于PTE访问位的访问检查 +----------------------- + +物理和虚拟地址空间的实现都使用PTE Accessed-bit进行基本访问检查。唯一的区别在于从地址中 +找到相关的PTE访问位的方式。虚拟地址的实现是为该地址的目标任务查找页表，而物理地址的实现则 +是查找与该地址有映射关系的每一个页表。通过这种方式，实现者找到并清除下一个采样目标地址的位， +并检查该位是否在一个采样周期后再次设置。这可能会干扰其他使用访问位的内核子系统，即空闲页跟 +踪和回收逻辑。为了避免这种干扰，DAMON使其与空闲页面跟踪相互排斥，并使用 ``PG_idle`` 和 +``PG_young`` 页面标志来解决与回收逻辑的冲突，就像空闲页面跟踪那样。 + + +独立于地址空间的核心机制 +======================== + +下面四个部分分别描述了DAMON的核心机制和五个监测属性，即 ``采样间隔`` 、 ``聚集间隔`` 、 +``更新间隔`` 、 ``最小区域数`` 和 ``最大区域数`` 。 + + +访问频率监测 +------------ + +DAMON的输出显示了在给定的时间内哪些页面的访问频率是多少。访问频率的分辨率是通过设置 +``采样间隔`` 和 ``聚集间隔`` 来控制的。详细地说，DAMON检查每个 ``采样间隔`` 对每 +个页面的访问，并将结果汇总。换句话说，计算每个页面的访问次数。在每个 ``聚合间隔`` 过 +去后，DAMON调用先前由用户注册的回调函数，以便用户可以阅读聚合的结果，然后再清除这些结 +果。这可以用以下简单的伪代码来描述:: + + while monitoring_on: + for page in monitoring_target: + if accessed(page): + nr_accesses[page] += 1 + if time() % aggregation_interval == 0: + for callback in user_registered_callbacks: + callback(monitoring_target, nr_accesses) + for page in monitoring_target: + nr_accesses[page] = 0 + sleep(sampling interval) + +这种机制的监测开销将随着目标工作负载规模的增长而任意增加。 + + +基于区域的抽样调查 +------------------ + +为了避免开销的无限制增加，DAMON将假定具有相同访问频率的相邻页面归入一个区域。只要保持 +这个假设（一个区域内的页面具有相同的访问频率），该区域内就只需要检查一个页面。因此，对 +于每个 ``采样间隔`` ，DAMON在每个区域中随机挑选一个页面，等待一个 ``采样间隔`` ，检 +查该页面是否同时被访问，如果被访问则增加该区域的访问频率。因此，监测开销是可以通过设置 +区域的数量来控制的。DAMON允许用户设置最小和最大的区域数量来进行权衡。 + +然而，如果假设没有得到保证，这个方案就不能保持输出的质量。 + + +适应性区域调整 +-------------- + +即使最初的监测目标区域被很好地构建以满足假设（同一区域内的页面具有相似的访问频率），数 +据访问模式也会被动态地改变。这将导致监测质量下降。为了尽可能地保持假设，DAMON根据每个 +区域的访问频率自适应地进行合并和拆分。 + +对于每个 ``聚集区间`` ，它比较相邻区域的访问频率，如果频率差异较小，就合并这些区域。 +然后，在它报告并清除每个区域的聚合接入频率后，如果区域总数不超过用户指定的最大区域数， +它将每个区域拆分为两个或三个区域。 + +通过这种方式，DAMON提供了其最佳的质量和最小的开销，同时保持了用户为其权衡设定的界限。 + + +动态目标空间更新处理 +-------------------- + +监测目标地址范围可以动态改变。例如，虚拟内存可以动态地被映射和解映射。物理内存可以被 +热插拔。 + +由于在某些情况下变化可能相当频繁，DAMON允许监控操作检查动态变化，包括内存映射变化， +并仅在用户指定的时间间隔（ ``更新间隔`` ）中的每个时间段，将其应用于监控操作相关的 +数据结构，如抽象的监控目标内存区。 \ No newline at end of file diff --git a/Documentation/translations/zh_CN/mm/damon/faq.rst b/Documentation/translations/zh_CN/mm/damon/faq.rst new file mode 100644 index 000000000000..de4be417494a --- /dev/null +++ b/Documentation/translations/zh_CN/mm/damon/faq.rst @@ -0,0 +1,48 @@ +.. SPDX-License-Identifier: GPL-2.0 + +:Original: Documentation/mm/damon/faq.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +======== +常见问题 +======== + +为什么是一个新的子系统，而不是扩展perf或其他用户空间工具？ +========================================================== + +首先，因为它需要尽可能的轻量级，以便可以在线使用，所以应该避免任何不必要的开销，如内核-用户 +空间的上下文切换成本。第二，DAMON的目标是被包括内核在内的其他程序所使用。因此，对特定工具 +（如perf）的依赖性是不可取的。这就是DAMON在内核空间实现的两个最大的原因。 + + +“闲置页面跟踪” 或 “perf mem” 可以替代DAMON吗？ +============================================== + +闲置页跟踪是物理地址空间访问检查的一个低层次的原始方法。“perf mem”也是类似的，尽管它可以 +使用采样来减少开销。另一方面，DAMON是一个更高层次的框架，用于监控各种地址空间。它专注于内 +存管理优化，并提供复杂的精度/开销处理机制。因此，“空闲页面跟踪” 和 “perf mem” 可以提供 +DAMON输出的一个子集，但不能替代DAMON。 + + +DAMON是否只支持虚拟内存？ +========================= + +不，DAMON的核心是独立于地址空间的。用户可以在DAMON核心上实现和配置特定地址空间的低级原始 +部分，包括监测目标区域的构造和实际的访问检查。通过这种方式，DAMON用户可以用任何访问检查技 +术来监测任何地址空间。 + +尽管如此，DAMON默认为虚拟内存和物理内存提供了基于vma/rmap跟踪和PTE访问位检查的地址空间 +相关功能的实现，以供参考和方便使用。 + + +我可以简单地监测页面的粒度吗？ +============================== + +是的，你可以通过设置 ``min_nr_regions`` 属性高于工作集大小除以页面大小的值来实现。 +因为监视目标区域的大小被强制为 ``>=page size`` ，所以区域分割不会产生任何影响。 diff --git a/Documentation/translations/zh_CN/mm/damon/index.rst b/Documentation/translations/zh_CN/mm/damon/index.rst new file mode 100644 index 000000000000..b03bf307204f --- /dev/null +++ b/Documentation/translations/zh_CN/mm/damon/index.rst @@ -0,0 +1,32 @@ +.. SPDX-License-Identifier: GPL-2.0 + +:Original: Documentation/mm/damon/index.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +========================== +DAMON:数据访问监视器 +========================== + +DAMON是Linux内核的一个数据访问监控框架子系统。DAMON的核心机制使其成为 +（该核心机制详见(Documentation/translations/zh_CN/mm/damon/design.rst)） + + - *准确度* （监测输出对DRAM级别的内存管理足够有用；但可能不适合CPU Cache级别）， + - *轻量级* （监控开销低到可以在线应用），以及 + - *可扩展* （无论目标工作负载的大小，开销的上限值都在恒定范围内）。 + +因此，利用这个框架，内核的内存管理机制可以做出高级决策。会导致高数据访问监控开销的实 +验性内存管理优化工作可以再次进行。同时，在用户空间，有一些特殊工作负载的用户可以编写 +个性化的应用程序，以便更好地了解和优化他们的工作负载和系统。 + +.. toctree:: + :maxdepth: 2 + + faq + design + api diff --git a/Documentation/translations/zh_CN/mm/free_page_reporting.rst b/Documentation/translations/zh_CN/mm/free_page_reporting.rst new file mode 100644 index 000000000000..83b14cce9adf --- /dev/null +++ b/Documentation/translations/zh_CN/mm/free_page_reporting.rst @@ -0,0 +1,38 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/_free_page_reporting.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +========== +空闲页报告 +========== + +空闲页报告是一个API，设备可以通过它来注册接收系统当前未使用的页面列表。这在虚拟 +化的情况下是很有用的，客户机能够使用这些数据来通知管理器它不再使用内存中的某些页 +面。 + +对于驱动，通常是气球驱动要使用这个功能，它将分配和初始化一个page_reporting_dev_info +结构体。它要填充的结构体中的字段是用于处理散点列表的 "report" 函数指针。它还必 +须保证每次调用该函数时能处理至少相当于PAGE_REPORTING_CAPACITY的散点列表条目。 +假设没有其他页面报告设备已经注册， 对page_reporting_register的调用将向报告框 +架注册页面报告接口。 + +一旦注册，页面报告API将开始向驱动报告成批的页面。API将在接口被注册后2秒开始报告 +页面，并在任何足够高的页面被释放之后2秒继续报告。 + +报告的页面将被存储在传递给报告函数的散列表中，最后一个条目的结束位被设置在条目 +nent-1中。 当页面被报告函数处理时，分配器将无法访问它们。一旦报告函数完成，这些 +页将被返回到它们所获得的自由区域。 + +在移除使用空闲页报告的驱动之前，有必要调用page_reporting_unregister，以移除 +目前被空闲页报告使用的page_reporting_dev_info结构体。这样做将阻止进一步的报 +告通过该接口发出。如果另一个驱动或同一驱动被注册，它就有可能恢复前一个驱动在报告 +空闲页方面的工作。 + + +Alexander Duyck, 2019年12月04日 diff --git a/Documentation/translations/zh_CN/mm/frontswap.rst b/Documentation/translations/zh_CN/mm/frontswap.rst new file mode 100644 index 000000000000..5c18ea2be04f --- /dev/null +++ b/Documentation/translations/zh_CN/mm/frontswap.rst @@ -0,0 +1,196 @@ +:Original: Documentation/mm/_free_page_reporting.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +========= +Frontswap +========= + +Frontswap为交换页提供了一个 “transcendent memory” 的接口。在一些环境中，由 +于交换页被保存在RAM（或类似RAM的设备）中，而不是交换磁盘，因此可以获得巨大的性能 +节省（提高）。 + +.. _Transcendent memory in a nutshell: https://lwn.net/Articles/454795/ + +Frontswap之所以这么命名，是因为它可以被认为是与swap设备的“back”存储相反。存 +储器被认为是一个同步并发安全的面向页面的“伪RAM设备”，符合transcendent memory +（如Xen的“tmem”，或内核内压缩内存，又称“zcache”，或未来的类似RAM的设备）的要 +求；这个伪RAM设备不能被内核直接访问或寻址，其大小未知且可能随时间变化。驱动程序通过 +调用frontswap_register_ops将自己与frontswap链接起来，以适当地设置frontswap_ops +的功能，它提供的功能必须符合某些策略，如下所示: + +一个 “init” 将设备准备好接收与指定的交换设备编号（又称“类型”）相关的frontswap +交换页。一个 “store” 将把该页复制到transcendent memory，并与该页的类型和偏移 +量相关联。一个 “load” 将把该页，如果找到的话，从transcendent memory复制到内核 +内存，但不会从transcendent memory中删除该页。一个 “invalidate_page” 将从 +transcendent memory中删除该页，一个 “invalidate_area” 将删除所有与交换类型 +相关的页（例如，像swapoff）并通知 “device” 拒绝进一步存储该交换类型。 + +一旦一个页面被成功存储，在该页面上的匹配加载通常会成功。因此，当内核发现自己处于需 +要交换页面的情况时，它首先尝试使用frontswap。如果存储的结果是成功的，那么数据就已 +经成功的保存到了transcendent memory中，并且避免了磁盘写入，如果后来再读回数据， +也避免了磁盘读取。如果存储返回失败，transcendent memory已经拒绝了该数据，且该页 +可以像往常一样被写入交换空间。 + +请注意，如果一个页面被存储，而该页面已经存在于transcendent memory中（一个 “重复” +的存储），要么存储成功，数据被覆盖，要么存储失败，该页面被废止。这确保了旧的数据永远 +不会从frontswap中获得。 + +如果配置正确，对frontswap的监控是通过 `/sys/kernel/debug/frontswap` 目录下的 +debugfs完成的。frontswap的有效性可以通过以下方式测量（在所有交换设备中）: + +``failed_stores`` + 有多少次存储的尝试是失败的 + +``loads`` + 尝试了多少次加载（应该全部成功） + +``succ_stores`` + 有多少次存储的尝试是成功的 + +``invalidates`` + 尝试了多少次作废 + +后台实现可以提供额外的指标。 + +经常问到的问题 +============== + +* 价值在哪里? + +当一个工作负载开始交换时，性能就会下降。Frontswap通过提供一个干净的、动态的接口来 +读取和写入交换页到 “transcendent memory”，从而大大增加了许多这样的工作负载的性 +能，否则内核是无法直接寻址的。当数据被转换为不同的形式和大小（比如压缩）或者被秘密 +移动（对于一些类似RAM的设备来说，这可能对写平衡很有用）时，这个接口是理想的。交换 +页（和被驱逐的页面缓存页）是这种比RAM慢但比磁盘快得多的“伪RAM设备”的一大用途。 + +Frontswap对内核的影响相当小，为各种系统配置中更动态、更灵活的RAM利用提供了巨大的 +灵活性： + +在单一内核的情况下，又称“zcache”，页面被压缩并存储在本地内存中，从而增加了可以安 +全保存在RAM中的匿名页面总数。Zcache本质上是用压缩/解压缩的CPU周期换取更好的内存利 +用率。Benchmarks测试显示，当内存压力较低时，几乎没有影响，而在高内存压力下的一些 +工作负载上，则有明显的性能改善（25%以上）。 + +“RAMster” 在zcache的基础上增加了对集群系统的 “peer-to-peer” transcendent memory +的支持。Frontswap页面像zcache一样被本地压缩，但随后被“remotified” 到另一个系 +统的RAM。这使得RAM可以根据需要动态地来回负载平衡，也就是说，当系统A超载时，它可以 +交换到系统B，反之亦然。RAMster也可以被配置成一个内存服务器，因此集群中的许多服务器 +可以根据需要动态地交换到配置有大量内存的单一服务器上......而不需要预先配置每个客户 +有多少内存可用 + +在虚拟情况下，虚拟化的全部意义在于统计地将物理资源在多个虚拟机的不同需求之间进行复 +用。对于RAM来说，这真的很难做到，而且在不改变内核的情况下，要做好这一点的努力基本上 +是失败的（除了一些广为人知的特殊情况下的工作负载）。具体来说，Xen Transcendent Memory +后端允许管理器拥有的RAM “fallow”，不仅可以在多个虚拟机之间进行“time-shared”， +而且页面可以被压缩和重复利用，以优化RAM的利用率。当客户操作系统被诱导交出未充分利用 +的RAM时（如 “selfballooning”），突然出现的意外内存压力可能会导致交换；frontswap +允许这些页面被交换到管理器RAM中或从管理器RAM中交换（如果整体主机系统内存条件允许）， +从而减轻计划外交换可能带来的可怕的性能影响。 + +一个KVM的实现正在进行中，并且已经被RFC'ed到lkml。而且，利用frontswap，对NVM作为 +内存扩展技术的调查也在进行中。 + +* 当然，在某些情况下可能有性能上的优势，但frontswap的空间/时间开销是多少？ + +如果 CONFIG_FRONTSWAP 被禁用，每个 frontswap 钩子都会编译成空，唯一的开销是每 +个 swapon'ed swap 设备的几个额外字节。如果 CONFIG_FRONTSWAP 被启用，但没有 +frontswap的 “backend” 寄存器，每读或写一个交换页就会有一个额外的全局变量，而不 +是零。如果 CONFIG_FRONTSWAP 被启用，并且有一个frontswap的backend寄存器，并且 +后端每次 “store” 请求都失败（即尽管声称可能，但没有提供内存），CPU 的开销仍然可以 +忽略不计 - 因为每次frontswap失败都是在交换页写到磁盘之前，系统很可能是 I/O 绑定 +的，无论如何使用一小部分的 CPU 都是不相关的。 + +至于空间，如果CONFIG_FRONTSWAP被启用，并且有一个frontswap的backend注册，那么 +每个交换设备的每个交换页都会被分配一个比特。这是在内核已经为每个交换设备的每个交换 +页分配的8位（在2.6.34之前是16位）上增加的。(Hugh Dickins观察到，frontswap可能 +会偷取现有的8个比特，但是我们以后再来担心这个小的优化问题)。对于标准的4K页面大小的 +非常大的交换盘（这很罕见），这是每32GB交换盘1MB开销。 + +当交换页存储在transcendent memory中而不是写到磁盘上时，有一个副作用，即这可能会 +产生更多的内存压力，有可能超过其他的优点。一个backend，比如zcache，必须实现策略 +来仔细（但动态地）管理内存限制，以确保这种情况不会发生。 + +* 好吧，那就用内核骇客能理解的术语来快速概述一下这个frontswap补丁的作用如何？ + +我们假设在内核初始化过程中，一个frontswap 的 “backend” 已经注册了；这个注册表 +明这个frontswap 的 “backend” 可以访问一些不被内核直接访问的“内存”。它到底提 +供了多少内存是完全动态和随机的。 + +每当一个交换设备被交换时，就会调用frontswap_init()，把交换设备的编号（又称“类 +型”）作为一个参数传给它。这就通知了frontswap，以期待 “store” 与该号码相关的交 +换页的尝试。 + +每当交换子系统准备将一个页面写入交换设备时（参见swap_writepage()），就会调用 +frontswap_store。Frontswap与frontswap backend协商，如果backend说它没有空 +间，frontswap_store返回-1，内核就会照常把页换到交换设备上。注意，来自frontswap +backend的响应对内核来说是不可预测的；它可能选择从不接受一个页面，可能接受每九个 +页面，也可能接受每一个页面。但是如果backend确实接受了一个页面，那么这个页面的数 +据已经被复制并与类型和偏移量相关联了，而且backend保证了数据的持久性。在这种情况 +下，frontswap在交换设备的“frontswap_map” 中设置了一个位，对应于交换设备上的 +页面偏移量，否则它就会将数据写入该设备。 + +当交换子系统需要交换一个页面时（swap_readpage()），它首先调用frontswap_load()， +检查frontswap_map，看这个页面是否早先被frontswap backend接受。如果是，该页 +的数据就会从frontswap后端填充，换入就完成了。如果不是，正常的交换代码将被执行， +以便从真正的交换设备上获得这一页的数据。 + +所以每次frontswap backend接受一个页面时，交换设备的读取和（可能）交换设备的写 +入都被 “frontswap backend store” 和（可能）“frontswap backend loads” +所取代，这可能会快得多。 + +* frontswap不能被配置为一个 “特殊的” 交换设备，它的优先级要高于任何真正的交换 + 设备（例如像zswap，或者可能是swap-over-nbd/NFS）？ + +首先，现有的交换子系统不允许有任何种类的交换层次结构。也许它可以被重写以适应层次 +结构，但这将需要相当大的改变。即使它被重写，现有的交换子系统也使用了块I/O层，它 +假定交换设备是固定大小的，其中的任何页面都是可线性寻址的。Frontswap几乎没有触 +及现有的交换子系统，而是围绕着块I/O子系统的限制，提供了大量的灵活性和动态性。 + +例如，frontswap backend对任何交换页的接受是完全不可预测的。这对frontswap backend +的定义至关重要，因为它赋予了backend完全动态的决定权。在zcache中，人们无法预 +先知道一个页面的可压缩性如何。可压缩性 “差” 的页面会被拒绝，而 “差” 本身也可 +以根据当前的内存限制动态地定义。 + +此外，frontswap是完全同步的，而真正的交换设备，根据定义，是异步的，并且使用 +块I/O。块I/O层不仅是不必要的，而且可能进行 “优化”，这对面向RAM的设备来说是 +不合适的，包括将一些页面的写入延迟相当长的时间。同步是必须的，以确保后端的动 +态性，并避免棘手的竞争条件，这将不必要地大大增加frontswap和/或块I/O子系统的 +复杂性。也就是说，只有最初的 “store” 和 “load” 操作是需要同步的。一个独立 +的异步线程可以自由地操作由frontswap存储的页面。例如，RAMster中的 “remotification” +线程使用标准的异步内核套接字，将压缩的frontswap页面移动到远程机器。同样， +KVM的客户方实现可以进行客户内压缩，并使用 “batched” hypercalls。 + +在虚拟化环境中，动态性允许管理程序（或主机操作系统）做“intelligent overcommit”。 +例如，它可以选择只接受页面，直到主机交换可能即将发生，然后强迫客户机做他们 +自己的交换。 + +transcendent memory规格的frontswap有一个坏处。因为任何 “store” 都可 +能失败，所以必须在一个真正的交换设备上有一个真正的插槽来交换页面。因此， +frontswap必须作为每个交换设备的 “影子” 来实现，它有可能容纳交换设备可能 +容纳的每一个页面，也有可能根本不容纳任何页面。这意味着frontswap不能包含比 +swap设备总数更多的页面。例如，如果在某些安装上没有配置交换设备，frontswap +就没有用。无交换设备的便携式设备仍然可以使用frontswap，但是这种设备的 +backend必须配置某种 “ghost” 交换设备，并确保它永远不会被使用。 + + +* 为什么会有这种关于 “重复存储” 的奇怪定义？如果一个页面以前被成功地存储过， + 难道它不能总是被成功地覆盖吗？ + +几乎总是可以的，不，有时不能。考虑一个例子，数据被压缩了，原来的4K页面被压 +缩到了1K。现在，有人试图用不可压缩的数据覆盖该页，因此会占用整个4K。但是 +backend没有更多的空间了。在这种情况下，这个存储必须被拒绝。每当frontswap +拒绝一个会覆盖的存储时，它也必须使旧的数据作废，并确保它不再被访问。因为交 +换子系统会把新的数据写到读交换设备上，这是确保一致性的正确做法。 + +* 为什么frontswap补丁会创建新的头文件swapfile.h？ + +frontswap代码依赖于一些swap子系统内部的数据结构，这些数据结构多年来一直 +在静态和全局之间来回移动。这似乎是一个合理的妥协：将它们定义为全局，但在一 +个新的包含文件中声明它们，该文件不被包含swap.h的大量源文件所包含。 + +Dan Magenheimer，最后更新于2012年4月9日 diff --git a/Documentation/translations/zh_CN/mm/highmem.rst b/Documentation/translations/zh_CN/mm/highmem.rst new file mode 100644 index 000000000000..81202c65e000 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/highmem.rst @@ -0,0 +1,128 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/highmem.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +========== +高内存处理 +========== + +作者: Peter Zijlstra + +.. contents:: :local: + +高内存是什么？ +============== + +当物理内存的大小接近或超过虚拟内存的最大大小时，就会使用高内存（highmem）。在这一点上，内 +核不可能在任何时候都保持所有可用的物理内存的映射。这意味着内核需要开始使用它想访问的物理内 +存的临时映射。 + +没有被永久映射覆盖的那部分（物理）内存就是我们所说的 "高内存"。对于这个边界的确切位置，有 +各种架构上的限制。 + +例如，在i386架构中，我们选择将内核映射到每个进程的虚拟空间，这样我们就不必为内核的进入/退 +出付出全部的TLB作废代价。这意味着可用的虚拟内存空间（i386上为4GiB）必须在用户和内核空间之 +间进行划分。 + +使用这种方法的架构的传统分配方式是3:1，3GiB用于用户空间，顶部的1GiB用于内核空间。:: + + +--------+ 0xffffffff + | Kernel | + +--------+ 0xc0000000 + | | + | User | + | | + +--------+ 0x00000000 + +这意味着内核在任何时候最多可以映射1GiB的物理内存，但是由于我们需要虚拟地址空间来做其他事 +情--包括访问其余物理内存的临时映射--实际的直接映射通常会更少（通常在~896MiB左右）。 + +其他有mm上下文标签的TLB的架构可以有独立的内核和用户映射。然而，一些硬件（如一些ARM）在使 +用mm上下文标签时，其虚拟空间有限。 + + +临时虚拟映射 +============ + +内核包含几种创建临时映射的方法。: + +* vmap(). 这可以用来将多个物理页长期映射到一个连续的虚拟空间。它需要synchronization + 来解除映射。 + +* kmap(). 这允许对单个页面进行短期映射。它需要synchronization，但在一定程度上被摊销。 + 当以嵌套方式使用时，它也很容易出现死锁，因此不建议在新代码中使用它。 + +* kmap_atomic(). 这允许对单个页面进行非常短的时间映射。由于映射被限制在发布它的CPU上， + 它表现得很好，但发布任务因此被要求留在该CPU上直到它完成，以免其他任务取代它的映射。 + + kmap_atomic() 也可以由中断上下文使用，因为它不睡眠，而且调用者可能在调用kunmap_atomic() + 之后才睡眠。 + + 可以假设k[un]map_atomic()不会失败。 + + +使用kmap_atomic +=============== + +何时何地使用 kmap_atomic() 是很直接的。当代码想要访问一个可能从高内存（见__GFP_HIGHMEM） +分配的页面的内容时，例如在页缓存中的页面，就会使用它。该API有两个函数，它们的使用方式与 +下面类似:: + + /* 找到感兴趣的页面。 */ + struct page *page = find_get_page(mapping, offset); + + /* 获得对该页内容的访问权。 */ + void *vaddr = kmap_atomic(page); + + /* 对该页的内容做一些处理。 */ + memset(vaddr, 0, PAGE_SIZE); + + /* 解除该页面的映射。 */ + kunmap_atomic(vaddr); + +注意，kunmap_atomic()调用的是kmap_atomic()调用的结果而不是参数。 + +如果你需要映射两个页面，因为你想从一个页面复制到另一个页面，你需要保持kmap_atomic调用严 +格嵌套，如:: + + vaddr1 = kmap_atomic(page1); + vaddr2 = kmap_atomic(page2); + + memcpy(vaddr1, vaddr2, PAGE_SIZE); + + kunmap_atomic(vaddr2); + kunmap_atomic(vaddr1); + + +临时映射的成本 +============== + +创建临时映射的代价可能相当高。体系架构必须操作内核的页表、数据TLB和/或MMU的寄存器。 + +如果CONFIG_HIGHMEM没有被设置，那么内核会尝试用一点计算来创建映射，将页面结构地址转换成 +指向页面内容的指针，而不是去捣鼓映射。在这种情况下，解映射操作可能是一个空操作。 + +如果CONFIG_MMU没有被设置，那么就不可能有临时映射和高内存。在这种情况下，也将使用计算方法。 + + +i386 PAE +======== + +在某些情况下，i386 架构将允许你在 32 位机器上安装多达 64GiB 的内存。但这有一些后果: + +* Linux需要为系统中的每个页面建立一个页帧结构，而且页帧需要驻在永久映射中，这意味着： + +* 你最多可以有896M/sizeof(struct page)页帧；由于页结构体是32字节的，所以最终会有 + 112G的页；然而，内核需要在内存中存储更多的页帧...... + +* PAE使你的页表变大--这使系统变慢，因为更多的数据需要在TLB填充等方面被访问。一个好处 + 是，PAE有更多的PTE位，可以提供像NX和PAT这样的高级功能。 + +一般的建议是，你不要在32位机器上使用超过8GiB的空间--尽管更多的空间可能对你和你的工作 +量有用，但你几乎是靠你自己--不要指望内核开发者真的会很关心事情的进展情况。 diff --git a/Documentation/translations/zh_CN/mm/hmm.rst b/Documentation/translations/zh_CN/mm/hmm.rst new file mode 100644 index 000000000000..5024a8a15516 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/hmm.rst @@ -0,0 +1,361 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/hmm.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +================== +异构内存管理 (HMM) +================== + +提供基础设施和帮助程序以将非常规内存（设备内存，如板上 GPU 内存）集成到常规内核路径中，其 +基石是此类内存的专用struct page（请参阅本文档的第 5 至 7 节）。 + +HMM 还为 SVM（共享虚拟内存）提供了可选的帮助程序，即允许设备透明地访问与 CPU 一致的程序 +地址，这意味着 CPU 上的任何有效指针也是该设备的有效指针。这对于简化高级异构计算的使用变得 +必不可少，其中 GPU、DSP 或 FPGA 用于代表进程执行各种计算。 + +本文档分为以下部分：在第一部分中，我揭示了与使用特定于设备的内存分配器相关的问题。在第二 +部分中，我揭示了许多平台固有的硬件限制。第三部分概述了 HMM 设计。第四部分解释了 CPU 页 +表镜像的工作原理以及 HMM 在这种情况下的目的。第五部分处理内核中如何表示设备内存。最后， +最后一节介绍了一个新的迁移助手，它允许利用设备 DMA 引擎。 + +.. contents:: :local: + +使用特定于设备的内存分配器的问题 +================================ + +具有大量板载内存（几 GB）的设备（如 GPU）历来通过专用驱动程序特定 API 管理其内存。这会 +造成设备驱动程序分配和管理的内存与常规应用程序内存（私有匿名、共享内存或常规文件支持内存） +之间的隔断。从这里开始，我将把这个方面称为分割的地址空间。我使用共享地址空间来指代相反的情况： +即，设备可以透明地使用任何应用程序内存区域。 + +分割的地址空间的发生是因为设备只能访问通过设备特定 API 分配的内存。这意味着从设备的角度来 +看，程序中的所有内存对象并不平等，这使得依赖于广泛的库的大型程序变得复杂。 + +具体来说，这意味着想要利用像 GPU 这样的设备的代码需要在通用分配的内存（malloc、mmap +私有、mmap 共享）和通过设备驱动程序 API 分配的内存之间复制对象（这仍然以 mmap 结束， +但是是设备文件）。 + +对于平面数据集（数组、网格、图像……），这并不难实现，但对于复杂数据集（列表、树……）， +很难做到正确。复制一个复杂的数据集需要重新映射其每个元素之间的所有指针关系。这很容易出错， +而且由于数据集和地址的重复，程序更难调试。 + +分割地址空间也意味着库不能透明地使用它们从核心程序或另一个库中获得的数据，因此每个库可能 +不得不使用设备特定的内存分配器来重复其输入数据集。大型项目会因此受到影响，并因为各种内存 +拷贝而浪费资源。 + +复制每个库的API以接受每个设备特定分配器分配的内存作为输入或输出，并不是一个可行的选择。 +这将导致库入口点的组合爆炸。 + +最后，随着高级语言结构（在 C++ 中，当然也在其他语言中）的进步，编译器现在有可能在没有程 +序员干预的情况下利用 GPU 和其他设备。某些编译器识别的模式仅适用于共享地址空间。对所有 +其他模式，使用共享地址空间也更合理。 + + +I/O 总线、设备内存特性 +====================== + +由于一些限制，I/O 总线削弱了共享地址空间。大多数 I/O 总线只允许从设备到主内存的基本 +内存访问；甚至缓存一致性通常是可选的。从 CPU 访问设备内存甚至更加有限。通常情况下，它 +不是缓存一致的。 + +如果我们只考虑 PCIE 总线，那么设备可以访问主内存（通常通过 IOMMU）并与 CPU 缓存一 +致。但是，它只允许设备对主存储器进行一组有限的原子操作。这在另一个方向上更糟：CPU +只能访问有限范围的设备内存，而不能对其执行原子操作。因此，从内核的角度来看，设备内存不 +能被视为与常规内存等同。 + +另一个严重的因素是带宽有限（约 32GBytes/s，PCIE 4.0 和 16 通道）。这比最快的 GPU +内存 (1 TBytes/s) 慢 33 倍。最后一个限制是延迟。从设备访问主内存的延迟比设备访问自 +己的内存时高一个数量级。 + +一些平台正在开发新的 I/O 总线或对 PCIE 的添加/修改以解决其中一些限制 +（OpenCAPI、CCIX）。它们主要允许 CPU 和设备之间的双向缓存一致性，并允许架构支持的所 +有原子操作。遗憾的是，并非所有平台都遵循这一趋势，并且一些主要架构没有针对这些问题的硬 +件解决方案。 + +因此，为了使共享地址空间有意义，我们不仅必须允许设备访问任何内存，而且还必须允许任何内 +存在设备使用时迁移到设备内存（在迁移时阻止 CPU 访问）。 + + +共享地址空间和迁移 +================== + +HMM 打算提供两个主要功能。第一个是通过复制cpu页表到设备页表中来共享地址空间，因此对 +于进程地址空间中的任何有效主内存地址，相同的地址指向相同的物理内存。 + +为了实现这一点，HMM 提供了一组帮助程序来填充设备页表，同时跟踪 CPU 页表更新。设备页表 +更新不像 CPU 页表更新那么容易。要更新设备页表，您必须分配一个缓冲区（或使用预先分配的 +缓冲区池）并在其中写入 GPU 特定命令以执行更新（取消映射、缓存失效和刷新等）。这不能通 +过所有设备的通用代码来完成。因此，为什么HMM提供了帮助器，在把硬件的具体细节留给设备驱 +动程序的同时，把一切可以考虑的因素都考虑进去了。 + +HMM 提供的第二种机制是一种新的 ZONE_DEVICE 内存，它允许为设备内存的每个页面分配一个 +struct page。这些页面很特殊，因为 CPU 无法映射它们。然而，它们允许使用现有的迁移机 +制将主内存迁移到设备内存，从 CPU 的角度来看，一切看起来都像是换出到磁盘的页面。使用 +struct page可以与现有的 mm 机制进行最简单、最干净的集成。再次，HMM 仅提供帮助程序， +首先为设备内存热插拔新的 ZONE_DEVICE 内存，然后执行迁移。迁移内容和时间的策略决定留 +给设备驱动程序。 + +请注意，任何 CPU 对设备页面的访问都会触发缺页异常并迁移回主内存。例如，当支持给定CPU +地址 A 的页面从主内存页面迁移到设备页面时，对地址 A 的任何 CPU 访问都会触发缺页异常 +并启动向主内存的迁移。 + +凭借这两个特性，HMM 不仅允许设备镜像进程地址空间并保持 CPU 和设备页表同步，而且还通 +过迁移设备正在使用的数据集部分来利用设备内存。 + + +地址空间镜像实现和API +===================== + +地址空间镜像的主要目标是允许将一定范围的 CPU 页表复制到一个设备页表中；HMM 有助于 +保持两者同步。想要镜像进程地址空间的设备驱动程序必须从注册 mmu_interval_notifier +开始:: + + int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub, + struct mm_struct *mm, unsigned long start, + unsigned long length, + const struct mmu_interval_notifier_ops *ops); + +在 ops->invalidate() 回调期间，设备驱动程序必须对范围执行更新操作（将范围标记为只 +读，或完全取消映射等）。设备必须在驱动程序回调返回之前完成更新。 + +当设备驱动程序想要填充一个虚拟地址范围时，它可以使用:: + + int hmm_range_fault(struct hmm_range *range); + +如果请求写访问，它将在丢失或只读条目上触发缺页异常（见下文）。缺页异常使用通用的 mm 缺 +页异常代码路径，就像 CPU 缺页异常一样。 + +这两个函数都将 CPU 页表条目复制到它们的 pfns 数组参数中。该数组中的每个条目对应于虚拟 +范围中的一个地址。HMM 提供了一组标志来帮助驱动程序识别特殊的 CPU 页表项。 + +在 sync_cpu_device_pagetables() 回调中锁定是驱动程序必须尊重的最重要的方面，以保 +持事物正确同步。使用模式是:: + + int driver_populate_range(...) + { + struct hmm_range range; + ... + + range.notifier = &interval_sub; + range.start = ...; + range.end = ...; + range.hmm_pfns = ...; + + if (!mmget_not_zero(interval_sub->notifier.mm)) + return -EFAULT; + + again: + range.notifier_seq = mmu_interval_read_begin(&interval_sub); + mmap_read_lock(mm); + ret = hmm_range_fault(&range); + if (ret) { + mmap_read_unlock(mm); + if (ret == -EBUSY) + goto again; + return ret; + } + mmap_read_unlock(mm); + + take_lock(driver->update); + if (mmu_interval_read_retry(&ni, range.notifier_seq) { + release_lock(driver->update); + goto again; + } + + /* Use pfns array content to update device page table, + * under the update lock */ + + release_lock(driver->update); + return 0; + } + +driver->update 锁与驱动程序在其 invalidate() 回调中使用的锁相同。该锁必须在调用 +mmu_interval_read_retry() 之前保持，以避免与并发 CPU 页表更新发生任何竞争。 + +利用 default_flags 和 pfn_flags_mask +==================================== + +hmm_range 结构有 2 个字段，default_flags 和 pfn_flags_mask，它们指定整个范围 +的故障或快照策略，而不必为 pfns 数组中的每个条目设置它们。 + +例如，如果设备驱动程序需要至少具有读取权限的范围的页面，它会设置:: + + range->default_flags = HMM_PFN_REQ_FAULT; + range->pfn_flags_mask = 0; + +并如上所述调用 hmm_range_fault()。这将填充至少具有读取权限的范围内的所有页面。 + +现在假设驱动程序想要做同样的事情，除了它想要拥有写权限的范围内的一页。现在驱动程序设 +置:: + + range->default_flags = HMM_PFN_REQ_FAULT; + range->pfn_flags_mask = HMM_PFN_REQ_WRITE; + range->pfns[index_of_write] = HMM_PFN_REQ_WRITE; + +有了这个，HMM 将在至少读取（即有效）的所有页面中异常，并且对于地址 +== range->start + (index_of_write << PAGE_SHIFT) 它将异常写入权限，即，如果 +CPU pte 没有设置写权限，那么HMM将调用handle_mm_fault()。 + +hmm_range_fault 完成后，标志位被设置为页表的当前状态，即 HMM_PFN_VALID | 如果页 +面可写，将设置 HMM_PFN_WRITE。 + + +从核心内核的角度表示和管理设备内存 +================================== + +尝试了几种不同的设计来支持设备内存。第一个使用特定于设备的数据结构来保存有关迁移内存 +的信息，HMM 将自身挂接到 mm 代码的各个位置，以处理对设备内存支持的地址的任何访问。 +事实证明，这最终复制了 struct page 的大部分字段，并且还需要更新许多内核代码路径才 +能理解这种新的内存类型。 + +大多数内核代码路径从不尝试访问页面后面的内存，而只关心struct page的内容。正因为如此， +HMM 切换到直接使用 struct page 用于设备内存，这使得大多数内核代码路径不知道差异。 +我们只需要确保没有人试图从 CPU 端映射这些页面。 + +移入和移出设备内存 +================== + +由于 CPU 无法直接访问设备内存，因此设备驱动程序必须使用硬件 DMA 或设备特定的加载/存 +储指令来迁移数据。migrate_vma_setup()、migrate_vma_pages() 和 +migrate_vma_finalize() 函数旨在使驱动程序更易于编写并集中跨驱动程序的通用代码。 + +在将页面迁移到设备私有内存之前，需要创建特殊的设备私有 ``struct page`` 。这些将用 +作特殊的“交换”页表条目，以便 CPU 进程在尝试访问已迁移到设备专用内存的页面时会发生异常。 + +这些可以通过以下方式分配和释放:: + + struct resource *res; + struct dev_pagemap pagemap; + + res = request_free_mem_region(&iomem_resource, /* number of bytes */, + "name of driver resource"); + pagemap.type = MEMORY_DEVICE_PRIVATE; + pagemap.range.start = res->start; + pagemap.range.end = res->end; + pagemap.nr_range = 1; + pagemap.ops = &device_devmem_ops; + memremap_pages(&pagemap, numa_node_id()); + + memunmap_pages(&pagemap); + release_mem_region(pagemap.range.start, range_len(&pagemap.range)); + +还有devm_request_free_mem_region(), devm_memremap_pages(), +devm_memunmap_pages() 和 devm_release_mem_region() 当资源可以绑定到 ``struct device``. + +整体迁移步骤类似于在系统内存中迁移 NUMA 页面(see :ref:`Page migration `) ， +但这些步骤分为设备驱动程序特定代码和共享公共代码: + +1. ``mmap_read_lock()`` + + 设备驱动程序必须将 ``struct vm_area_struct`` 传递给migrate_vma_setup()， + 因此需要在迁移期间保留 mmap_read_lock() 或 mmap_write_lock()。 + +2. ``migrate_vma_setup(struct migrate_vma *args)`` + + 设备驱动初始化了 ``struct migrate_vma`` 的字段，并将该指针传递给 + migrate_vma_setup()。``args->flags`` 字段是用来过滤哪些源页面应该被迁移。 + 例如，设置 ``MIGRATE_VMA_SELECT_SYSTEM`` 将只迁移系统内存，设置 + ``MIGRATE_VMA_SELECT_DEVICE_PRIVATE`` 将只迁移驻留在设备私有内存中的页 + 面。如果后者被设置， ``args->pgmap_owner`` 字段被用来识别驱动所拥有的设备 + 私有页。这就避免了试图迁移驻留在其他设备中的设备私有页。目前，只有匿名的私有VMA + 范围可以被迁移到系统内存和设备私有内存。 + + migrate_vma_setup()所做的第一步是用 ``mmu_notifier_invalidate_range_start()`` + 和 ``mmu_notifier_invalidate_range_end()`` 调用来遍历设备周围的页表，使 + 其他设备的MMU无效，以便在 ``args->src`` 数组中填写要迁移的PFN。 + ``invalidate_range_start()`` 回调传递给一个``struct mmu_notifier_range`` ， + 其 ``event`` 字段设置为MMU_NOTIFY_MIGRATE， ``owner`` 字段设置为传递给 + migrate_vma_setup()的 ``args->pgmap_owner`` 字段。这允许设备驱动跳过无 + 效化回调，只无效化那些实际正在迁移的设备私有MMU映射。这一点将在下一节详细解释。 + + + 在遍历页表时，一个 ``pte_none()`` 或 ``is_zero_pfn()`` 条目导致一个有效 + 的 “zero” PFN 存储在 ``args->src`` 阵列中。这让驱动分配设备私有内存并清 + 除它，而不是复制一个零页。到系统内存或设备私有结构页的有效PTE条目将被 + ``lock_page()``锁定，与LRU隔离（如果系统内存和设备私有页不在LRU上），从进 + 程中取消映射，并插入一个特殊的迁移PTE来代替原来的PTE。 migrate_vma_setup() + 还清除了 ``args->dst`` 数组。 + +3. 设备驱动程序分配目标页面并将源页面复制到目标页面。 + + 驱动程序检查每个 ``src`` 条目以查看该 ``MIGRATE_PFN_MIGRATE`` 位是否已 + 设置并跳过未迁移的条目。设备驱动程序还可以通过不填充页面的 ``dst`` 数组来选 + 择跳过页面迁移。 + + 然后，驱动程序分配一个设备私有 struct page 或一个系统内存页，用 ``lock_page()`` + 锁定该页，并将 ``dst`` 数组条目填入:: + + dst[i] = migrate_pfn(page_to_pfn(dpage)); + + 现在驱动程序知道这个页面正在被迁移，它可以使设备私有 MMU 映射无效并将设备私有 + 内存复制到系统内存或另一个设备私有页面。由于核心 Linux 内核会处理 CPU 页表失 + 效，因此设备驱动程序只需使其自己的 MMU 映射失效。 + + 驱动程序可以使用 ``migrate_pfn_to_page(src[i])`` 来获取源设备的 + ``struct page`` 面，并将源页面复制到目标设备上，如果指针为 ``NULL`` ，意 + 味着源页面没有被填充到系统内存中，则清除目标设备的私有内存。 + +4. ``migrate_vma_pages()`` + + 这一步是实际“提交”迁移的地方。 + + 如果源页是 ``pte_none()`` 或 ``is_zero_pfn()`` 页，这时新分配的页会被插 + 入到CPU的页表中。如果一个CPU线程在同一页面上发生异常，这可能会失败。然而，页 + 表被锁定，只有一个新页会被插入。如果它失去了竞争，设备驱动将看到 + ``MIGRATE_PFN_MIGRATE`` 位被清除。 + + 如果源页被锁定、隔离等，源 ``struct page`` 信息现在被复制到目标 + ``struct page`` ，最终完成CPU端的迁移。 + +5. 设备驱动为仍在迁移的页面更新设备MMU页表，回滚未迁移的页面。 + + 如果 ``src`` 条目仍然有 ``MIGRATE_PFN_MIGRATE`` 位被设置，设备驱动可以 + 更新设备MMU，如果 ``MIGRATE_PFN_WRITE`` 位被设置，则设置写启用位。 + +6. ``migrate_vma_finalize()`` + + 这一步用新页的页表项替换特殊的迁移页表项，并释放对源和目的 ``struct page`` + 的引用。 + +7. ``mmap_read_unlock()`` + + 现在可以释放锁了。 + +独占访问存储器 +============== + +一些设备具有诸如原子PTE位的功能，可以用来实现对系统内存的原子访问。为了支持对一 +个共享的虚拟内存页的原子操作，这样的设备需要对该页的访问是排他的，而不是来自CPU +的任何用户空间访问。 ``make_device_exclusive_range()`` 函数可以用来使一 +个内存范围不能从用户空间访问。 + +这将用特殊的交换条目替换给定范围内的所有页的映射。任何试图访问交换条目的行为都会 +导致一个异常，该异常会通过用原始映射替换该条目而得到恢复。驱动程序会被通知映射已 +经被MMU通知器改变，之后它将不再有对该页的独占访问。独占访问被保证持续到驱动程序 +放弃页面锁和页面引用为止，这时页面上的任何CPU异常都可以按所述进行。 + +内存 cgroup (memcg) 和 rss 统计 +=============================== + +目前，设备内存被视为 rss 计数器中的任何常规页面（如果设备页面用于匿名，则为匿名， +如果设备页面用于文件支持页面，则为文件，如果设备页面用于共享内存，则为 shmem）。 +这是为了保持现有应用程序的故意选择，这些应用程序可能在不知情的情况下开始使用设备 +内存，运行不受影响。 + +一个缺点是 OOM 杀手可能会杀死使用大量设备内存而不是大量常规系统内存的应用程序， +因此不会释放太多系统内存。在决定以不同方式计算设备内存之前，我们希望收集更多关 +于应用程序和系统在存在设备内存的情况下在内存压力下如何反应的实际经验。 + +对内存 cgroup 做出了相同的决定。设备内存页面根据相同的内存 cgroup 计算，常规 +页面将被计算在内。这确实简化了进出设备内存的迁移。这也意味着从设备内存迁移回常规 +内存不会失败，因为它会超过内存 cgroup 限制。一旦我们对设备内存的使用方式及其对 +内存资源控制的影响有了更多的了解，我们可能会在后面重新考虑这个选择。 + +请注意，设备内存永远不能由设备驱动程序或通过 GUP 固定，因此此类内存在进程退出时 +总是被释放的。或者在共享内存或文件支持内存的情况下，当删除最后一个引用时。 diff --git a/Documentation/translations/zh_CN/mm/hugetlbfs_reserv.rst b/Documentation/translations/zh_CN/mm/hugetlbfs_reserv.rst new file mode 100644 index 000000000000..752e5696cd47 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/hugetlbfs_reserv.rst @@ -0,0 +1,436 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/hugetlbfs_reserv.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +============== +Hugetlbfs 预留 +============== + +概述 +==== + +:ref:`hugetlbpage` 中描述的巨页通常是预先分配给应用程序使用的。如果VMA指 +示要使用巨页，这些巨页会在缺页异常时被实例化到任务的地址空间。如果在缺页异常 +时没有巨页存在，任务就会被发送一个SIGBUS，并经常不高兴地死去。在加入巨页支 +持后不久，人们决定，在mmap()时检测巨页的短缺情况会更好。这个想法是，如果 +没有足够的巨页来覆盖映射，mmap()将失败。这首先是在mmap()时在代码中做一个 +简单的检查，以确定是否有足够的空闲巨页来覆盖映射。就像内核中的大多数东西一 +样，代码随着时间的推移而不断发展。然而，基本的想法是在mmap()时 “预留” +巨页，以确保巨页可以用于该映射中的缺页异常。下面的描述试图描述在v4.10内核 +中是如何进行巨页预留处理的。 + + +读者 +==== +这个描述主要是针对正在修改hugetlbfs代码的内核开发者。 + + +数据结构 +======== + +resv_huge_pages + 这是一个全局的（per-hstate）预留的巨页的计数。预留的巨页只对预留它们的任 + 务可用。因此，一般可用的巨页的数量被计算为(``free_huge_pages - resv_huge_pages``)。 +Reserve Map + 预留映射由以下结构体描述:: + + struct resv_map { + struct kref refs; + spinlock_t lock; + struct list_head regions; + long adds_in_progress; + struct list_head region_cache; + long region_cache_count; + }; + + 系统中每个巨页映射都有一个预留映射。resv_map中的regions列表描述了映射中的 + 区域。一个区域被描述为:: + + struct file_region { + struct list_head link; + long from; + long to; + }; + + file_region结构体的 ‘from’ 和 ‘to’ 字段是进入映射的巨页索引。根据映射的类型，在 + reserv_map 中的一个区域可能表示该范围存在预留，或预留不存在。 +Flags for MAP_PRIVATE Reservations + 这些被存储在预留的映射指针的底部。 + + ``#define HPAGE_RESV_OWNER (1UL << 0)`` + 表示该任务是与该映射相关的预留的所有者。 + ``#define HPAGE_RESV_UNMAPPED (1UL << 1)`` + 表示最初映射此范围（并创建储备）的任务由于COW失败而从该任务（子任务）中取消映 + 射了一个页面。 +Page Flags + PagePrivate页面标志是用来指示在释放巨页时必须恢复巨页的预留。更多细节将在 + “释放巨页” 一节中讨论。 + + +预留映射位置（私有或共享） +========================== + +一个巨页映射或段要么是私有的，要么是共享的。如果是私有的，它通常只对一个地址空间 +（任务）可用。如果是共享的，它可以被映射到多个地址空间（任务）。对于这两种类型的映射， +预留映射的位置和语义是明显不同的。位置的差异是： + +- 对于私有映射，预留映射挂在VMA结构体上。具体来说，就是vma->vm_private_data。这个保 + 留映射是在创建映射（mmap(MAP_PRIVATE)）时创建的。 +- 对于共享映射，预留映射挂在inode上。具体来说，就是inode->i_mapping->private_data。 + 由于共享映射总是由hugetlbfs文件系统中的文件支持，hugetlbfs代码确保每个节点包含一个预 + 留映射。因此，预留映射在创建节点时被分配。 + + +创建预留 +======== +当创建一个巨大的有页面支持的共享内存段（shmget(SHM_HUGETLB)）或通过mmap(MAP_HUGETLB) +创建一个映射时，就会创建预留。这些操作会导致对函数hugetlb_reserve_pages()的调用:: + + int hugetlb_reserve_pages(struct inode *inode, + long from, long to, + struct vm_area_struct *vma, + vm_flags_t vm_flags) + +hugetlb_reserve_pages()做的第一件事是检查在调用shmget()或mmap()时是否指定了NORESERVE +标志。如果指定了NORESERVE，那么这个函数立即返回，因为不需要预留。 + +参数'from'和'to'是映射或基础文件的巨页索引。对于shmget()，'from'总是0，'to'对应于段/映射 +的长度。对于mmap()，offset参数可以用来指定进入底层文件的偏移量。在这种情况下，'from'和'to' +参数已经被这个偏移量所调整。 + +PRIVATE和SHARED映射之间的一个很大的区别是预留在预留映射中的表示方式。 + +- 对于共享映射，预留映射中的条目表示对应页面的预留存在或曾经存在。当预留被消耗时，预留映射不被 + 修改。 +- 对于私有映射，预留映射中没有条目表示相应页面存在预留。随着预留被消耗，条目被添加到预留映射中。 + 因此，预留映射也可用于确定哪些预留已被消耗。 + +对于私有映射，hugetlb_reserve_pages()创建预留映射并将其挂在VMA结构体上。此外， +HPAGE_RESV_OWNER标志被设置，以表明该VMA拥有预留。 + +预留映射被查阅以确定当前映射/段需要多少巨页预留。对于私有映射，这始终是一个值（to - from）。 +然而，对于共享映射来说，一些预留可能已经存在于(to - from)的范围内。关于如何实现这一点的细节， +请参见 :ref:`预留映射的修改 ` 一节。 + +该映射可能与一个子池（subpool）相关联。如果是这样，将查询子池以确保有足够的空间用于映射。子池 +有可能已经预留了可用于映射的预留空间。更多细节请参见 :ref: `子池预留 ` +一节。 + +在咨询了预留映射和子池之后，就知道了需要的新预留数量。hugetlb_acct_memory()函数被调用以检查 +并获取所要求的预留数量。hugetlb_acct_memory()调用到可能分配和调整剩余页数的函数。然而，在这 +些函数中，代码只是检查以确保有足够的空闲的巨页来容纳预留。如果有的话，全局预留计数resv_huge_pages +会被调整，如下所示:: + + if (resv_needed <= (resv_huge_pages - free_huge_pages)) + resv_huge_pages += resv_needed; + +注意，在检查和调整这些计数器时，全局锁hugetlb_lock会被预留。 + +如果有足够的空闲的巨页，并且全局计数resv_huge_pages被调整，那么与映射相关的预留映射被修改以 +反映预留。在共享映射的情况下，将存在一个file_region，包括'from'-'to'范围。对于私有映射， +不对预留映射进行修改，因为没有条目表示存在预留。 + +如果hugetlb_reserve_pages()成功，全局预留数和与映射相关的预留映射将根据需要被修改，以确保 +在'from'-'to'范围内存在预留。 + +消耗预留/分配一个巨页 +=========================== + +当与预留相关的巨页在相应的映射中被分配和实例化时，预留就被消耗了。该分配是在函数alloc_huge_page() +中进行的:: + + struct page *alloc_huge_page(struct vm_area_struct *vma, + unsigned long addr, int avoid_reserve) + +alloc_huge_page被传递给一个VMA指针和一个虚拟地址，因此它可以查阅预留映射以确定是否存在预留。 +此外，alloc_huge_page需要一个参数avoid_reserve，该参数表示即使看起来已经为指定的地址预留了 +预留，也不应该使用预留。avoid_reserve参数最常被用于写时拷贝和页面迁移的情况下，即现有页面的额 +外拷贝被分配。 + + +调用辅助函数vma_needs_reservation()来确定是否存在对映射(vma)中地址的预留。关于这个函数的详 +细内容，请参见 :ref:`预留映射帮助函数 ` 一节。从 +vma_needs_reservation()返回的值通常为0或1。如果该地址存在预留，则为0，如果不存在预留，则为1。 +如果不存在预留，并且有一个与映射相关联的子池，则查询子池以确定它是否包含预留。如果子池包含预留， +则可将其中一个用于该分配。然而，在任何情况下，avoid_reserve参数都会优先考虑为分配使用预留。在 +确定预留是否存在并可用于分配后，调用dequeue_huge_page_vma()函数。这个函数需要两个与预留有关 +的参数： + +- avoid_reserve，这是传递给alloc_huge_page()的同一个值/参数。 +- chg，尽管这个参数的类型是long，但只有0或1的值被传递给dequeue_huge_page_vma。如果该值为0， + 则表明存在预留（关于可能的问题，请参见 “预留和内存策略” 一节）。如果值 + 为1，则表示不存在预留，如果可能的话，必须从全局空闲池中取出该页。 + +与VMA的内存策略相关的空闲列表被搜索到一个空闲页。如果找到了一个页面，当该页面从空闲列表中移除时， +free_huge_pages的值被递减。如果有一个与该页相关的预留，将进行以下调整:: + + SetPagePrivate(page); /* 表示分配这个页面消耗了一个预留， + * 如果遇到错误，以至于必须释放这个页面，预留将被 + * 恢复。 */ + resv_huge_pages--; /* 减少全局预留计数 */ + +注意，如果找不到满足VMA内存策略的巨页，将尝试使用伙伴分配器分配一个。这就带来了超出预留范围 +的剩余巨页和超额分配的问题。即使分配了一个多余的页面，也会进行与上面一样的基于预留的调整: +SetPagePrivate(page) 和 resv_huge_pages--. + +在获得一个新的巨页后，(page)->private被设置为与该页面相关的子池的值，如果它存在的话。当页 +面被释放时，这将被用于子池的计数。 + +然后调用函数vma_commit_reservation()，根据预留的消耗情况调整预留映射。一般来说，这涉及 +到确保页面在区域映射的file_region结构体中被表示。对于预留存在的共享映射，预留映射中的条目 +已经存在，所以不做任何改变。然而，如果共享映射中没有预留，或者这是一个私有映射，则必须创建一 +个新的条目。 + +注意，如果找不到满足VMA内存策略的巨页，将尝试使用伙伴分配器分配一个。这就带来了超出预留范围 +的剩余巨页和过度分配的问题。即使分配了一个多余的页面，也会进行与上面一样的基于预留的调整。 +SetPagePrivate(page)和resv_huge_pages-。 + +在获得一个新的巨页后，(page)->private被设置为与该页面相关的子池的值，如果它存在的话。当页 +面被释放时，这将被用于子池的计数。 + +然后调用函数vma_commit_reservation()，根据预留的消耗情况调整预留映射。一般来说，这涉及 +到确保页面在区域映射的file_region结构体中被表示。对于预留存在的共享映射，预留映射中的条目 +已经存在，所以不做任何改变。然而，如果共享映射中没有预留，或者这是一个私有映射，则必须创建 +一个新的条目。 + +在alloc_huge_page()开始调用vma_needs_reservation()和页面分配后调用 +vma_commit_reservation()之间，预留映射有可能被改变。如果hugetlb_reserve_pages在共 +享映射中为同一页面被调用，这将是可能的。在这种情况下，预留计数和子池空闲页计数会有一个偏差。 +这种罕见的情况可以通过比较vma_needs_reservation和vma_commit_reservation的返回值来 +识别。如果检测到这种竞争，子池和全局预留计数将被调整以进行补偿。关于这些函数的更多信息，请 +参见 :ref:`预留映射帮助函数 ` 一节。 + + +实例化巨页 +========== + +在巨页分配之后，页面通常被添加到分配任务的页表中。在此之前，共享映射中的页面被添加到页面缓 +存中，私有映射中的页面被添加到匿名反向映射中。在这两种情况下，PagePrivate标志被清除。因此， +当一个已经实例化的巨页被释放时，不会对全局预留计数（resv_huge_pages）进行调整。 + + +释放巨页 +======== + +巨页释放是由函数free_huge_page()执行的。这个函数是hugetlbfs复合页的析构器。因此，它只传 +递一个指向页面结构体的指针。当一个巨页被释放时，可能需要进行预留计算。如果该页与包含保 +留的子池相关联，或者该页在错误路径上被释放，必须恢复全局预留计数，就会出现这种情况。 + +page->private字段指向与该页相关的任何子池。如果PagePrivate标志被设置，它表明全局预留计数 +应该被调整（关于如何设置这些标志的信息，请参见 +:ref: `消耗预留/分配一个巨页 ` ）。 + + +该函数首先调用hugepage_subpool_put_pages()来处理该页。如果这个函数返回一个0的值（不等于 +传递的1的值），它表明预留与子池相关联，这个新释放的页面必须被用来保持子池预留的数量超过最小值。 +因此，在这种情况下，全局resv_huge_pages计数器被递增。 + +如果页面中设置了PagePrivate标志，那么全局resv_huge_pages计数器将永远被递增。 + +子池预留 +======== + +有一个结构体hstate与每个巨页尺寸相关联。hstate跟踪所有指定大小的巨页。一个子池代表一 +个hstate中的页面子集，它与一个已挂载的hugetlbfs文件系统相关 + +当一个hugetlbfs文件系统被挂载时，可以指定min_size选项，它表示文件系统所需的最小的巨页数量。 +如果指定了这个选项，与min_size相对应的巨页的数量将被预留给文件系统使用。这个数字在结构体 +hugepage_subpool的min_hpages字段中被跟踪。在挂载时，hugetlb_acct_memory(min_hpages) +被调用以预留指定数量的巨页。如果它们不能被预留，挂载就会失败。 + +当从子池中获取或释放页面时，会调用hugepage_subpool_get/put_pages()函数。 +hugepage_subpool_get/put_pages被传递给巨页数量，以此来调整子池的 “已用页面” 计数 +（get为下降，put为上升）。通常情况下，如果子池中没有足够的页面，它们会返回与传递的相同的值或 +一个错误。 + +然而，如果预留与子池相关联，可能会返回一个小于传递值的返回值。这个返回值表示必须进行的额外全局 +池调整的数量。例如，假设一个子池包含3个预留的巨页，有人要求5个。与子池相关的3个预留页可以用来 +满足部分请求。但是，必须从全局池中获得2个页面。为了向调用者转达这一信息，将返回值2。然后，调用 +者要负责从全局池中获取另外两个页面。 + + +COW和预留 +========== + +由于共享映射都指向并使用相同的底层页面，COW最大的预留问题是私有映射。在这种情况下，两个任务可 +以指向同一个先前分配的页面。一个任务试图写到该页，所以必须分配一个新的页，以便每个任务都指向它 +自己的页。 + +当该页最初被分配时，该页的预留被消耗了。当由于COW而试图分配一个新的页面时，有可能没有空闲的巨 +页，分配会失败。 + +当最初创建私有映射时，通过设置所有者的预留映射指针中的HPAGE_RESV_OWNER位来标记映射的所有者。 +由于所有者创建了映射，所有者拥有与映射相关的所有预留。因此，当一个写异常发生并且没有可用的页面 +时，对预留的所有者和非所有者采取不同的行动。 + +在发生异常的任务不是所有者的情况下，异常将失败，该任务通常会收到一个SIGBUS。 + +如果所有者是发生异常的任务，我们希望它能够成功，因为它拥有原始的预留。为了达到这个目的，该页被 +从非所有者任务中解映射出来。这样一来，唯一的引用就是来自拥有者的任务。此外，HPAGE_RESV_UNMAPPED +位被设置在非拥有任务的预留映射指针中。如果非拥有者任务后来在一个不存在的页面上发生异常，它可能 +会收到一个SIGBUS。但是，映射/预留的原始拥有者的行为将与预期一致。 + +预留映射的修改 +============== + +以下低级函数用于对预留映射进行修改。通常情况下，这些函数不会被直接调用。而是调用一个预留映射辅 +助函数，该函数调用这些低级函数中的一个。这些低级函数在源代码（mm/hugetlb.c）中得到了相当好的 +记录。这些函数是:: + + long region_chg(struct resv_map *resv, long f, long t); + long region_add(struct resv_map *resv, long f, long t); + void region_abort(struct resv_map *resv, long f, long t); + long region_count(struct resv_map *resv, long f, long t); + +在预留映射上的操作通常涉及两个操作: + +1) region_chg()被调用来检查预留映射，并确定在指定的范围[f, t]内有多少页目前没有被代表。 + + 调用代码执行全局检查和分配，以确定是否有足够的巨页使操作成功。 + +2) + a) 如果操作能够成功，regi_add()将被调用，以实际修改先前传递给regi_chg()的相同范围 + [f, t]的预留映射。 + b) 如果操作不能成功，region_abort被调用，在相同的范围[f, t]内中止操作。 + +注意，这是一个两步的过程， region_add()和 region_abort()在事先调用 region_chg()后保证 +成功。 region_chg()负责预先分配任何必要的数据结构以确保后续操作（特别是 region_add()）的 +成功。 + +如上所述，region_chg()确定该范围内当前没有在映射中表示的页面的数量。region_add()返回添加 +到映射中的范围内的页数。在大多数情况下， region_add() 的返回值与 region_chg() 的返回值相 +同。然而，在共享映射的情况下，有可能在调用 region_chg() 和 region_add() 之间对预留映射进 +行更改。在这种情况下，regi_add()的返回值将与regi_chg()的返回值不符。在这种情况下，全局计数 +和子池计数很可能是不正确的，需要调整。检查这种情况并进行适当的调整是调用者的责任。 + +函数region_del()被调用以从预留映射中移除区域。 +它通常在以下情况下被调用: + +- 当hugetlbfs文件系统中的一个文件被删除时，该节点将被释放，预留映射也被释放。在释放预留映射 + 之前，所有单独的file_region结构体必须被释放。在这种情况下，region_del的范围是[0, LONG_MAX]。 +- 当一个hugetlbfs文件正在被截断时。在这种情况下，所有在新文件大小之后分配的页面必须被释放。 + 此外，预留映射中任何超过新文件大小的file_region条目必须被删除。在这种情况下，region_del + 的范围是[new_end_of_file, LONG_MAX]。 +- 当在一个hugetlbfs文件中打洞时。在这种情况下，巨页被一次次从文件的中间移除。当这些页被移除 + 时，region_del()被调用以从预留映射中移除相应的条目。在这种情况下，region_del被传递的范 + 围是[page_idx, page_idx + 1]。 + +在任何情况下，region_del()都会返回从预留映射中删除的页面数量。在非常罕见的情况下，region_del() +会失败。这只能发生在打洞的情况下，即它必须分割一个现有的file_region条目，而不能分配一个新的 +结构体。在这种错误情况下，region_del()将返回-ENOMEM。这里的问题是，预留映射将显示对该页有 +预留。然而，子池和全局预留计数将不反映该预留。为了处理这种情况，调用函数hugetlb_fix_reserve_counts() +来调整计数器，使其与不能被删除的预留映射条目相对应。 + +region_count()在解除私有巨页映射时被调用。在私有映射中，预留映射中没有条目表明存在一个预留。 +因此，通过计算预留映射中的条目数，我们知道有多少预留被消耗了，有多少预留是未完成的 +（Outstanding = (end - start) - region_count（resv, start, end））。由于映射正在消 +失，子池和全局预留计数被未完成的预留数量所减去。 + +预留映射帮助函数 +================ + +有几个辅助函数可以查询和修改预留映射。这些函数只对特定的巨页的预留感兴趣，所以它们只是传入一个 +地址而不是一个范围。此外，它们还传入相关的VMA。从VMA中，可以确定映射的类型（私有或共享）和预留 +映射的位置（inode或VMA）。这些函数只是调用 “预留映射的修改” 一节中描述的基础函数。然而， +它们确实考虑到了私有和共享映射的预留映射条目的 “相反” 含义，并向调用者隐藏了这个细节:: + + long vma_needs_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +该函数为指定的页面调用 region_chg()。如果不存在预留，则返回1。如果存在预留，则返回0:: + + long vma_commit_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +这将调用 region_add()，用于指定的页面。与region_chg和region_add的情况一样，该函数应在 +先前调用的vma_needs_reservation后调用。它将为该页添加一个预留条目。如果预留被添加，它将 +返回1，如果没有则返回0。返回值应与之前调用vma_needs_reservation的返回值进行比较。如果出 +现意外的差异，说明在两次调用之间修改了预留映射:: + + void vma_end_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +这将调用指定页面的 region_abort()。与region_chg和region_abort的情况一样，该函数应在 +先前调用的vma_needs_reservation后被调用。它将中止/结束正在进行的预留添加操作:: + + long vma_add_reservation(struct hstate *h, + struct vm_area_struct *vma, + unsigned long addr) + +这是一个特殊的包装函数，有助于在错误路径上清理预留。它只从repare_reserve_on_error()函数 +中调用。该函数与vma_needs_reservation一起使用，试图将一个预留添加到预留映射中。它考虑到 +了私有和共享映射的不同预留映射语义。因此，region_add被调用用于共享映射（因为映射中的条目表 +示预留），而region_del被调用用于私有映射（因为映射中没有条目表示预留）。关于在错误路径上需 +要做什么的更多信息，请参见 “错误路径中的预留清理” 。 + + +错误路径中的预留清理 +==================== + +正如在:ref:`预留映射帮助函数` 一节中提到的，预留的修改分两步进行。首 +先，在分配页面之前调用vma_needs_reservation。如果分配成功，则调用vma_commit_reservation。 +如果不是，则调用vma_end_reservation。全局和子池的预留计数根据操作的成功或失败进行调整， +一切都很好。 + +此外，在一个巨页被实例化后，PagePrivate标志被清空，这样，当页面最终被释放时，计数是 +正确的。 + +然而，有几种情况是，在一个巨页被分配后，但在它被实例化之前，就遇到了错误。在这种情况下， +页面分配已经消耗了预留，并进行了适当的子池、预留映射和全局计数调整。如果页面在这个时候被释放 +（在实例化和清除PagePrivate之前），那么free_huge_page将增加全局预留计数。然而，预留映射 +显示报留被消耗了。这种不一致的状态将导致预留的巨页的 “泄漏” 。全局预留计数将比它原本的要高， +并阻止分配一个预先分配的页面。 + +函数 restore_reserve_on_error() 试图处理这种情况。它有相当完善的文档。这个函数的目的 +是将预留映射恢复到页面分配前的状态。通过这种方式，预留映射的状态将与页面释放后的全局预留计 +数相对应。 + +函数restore_reserve_on_error本身在试图恢复预留映射条目时可能会遇到错误。在这种情况下， +它将简单地清除该页的PagePrivate标志。这样一来，当页面被释放时，全局预留计数将不会被递增。 +然而，预留映射将继续看起来像预留被消耗了一样。一个页面仍然可以被分配到该地址，但它不会像最 +初设想的那样使用一个预留页。 + +有一些代码（最明显的是userfaultfd）不能调用restore_reserve_on_error。在这种情况下， +它简单地修改了PagePrivate，以便在释放巨页时不会泄露预留。 + + +预留和内存策略 +============== +当git第一次被用来管理Linux代码时，每个节点的巨页列表就存在于hstate结构中。预留的概念是 +在一段时间后加入的。当预留被添加时，没有尝试将内存策略考虑在内。虽然cpusets与内存策略不 +完全相同，但hugetlb_acct_memory中的这个注释总结了预留和cpusets/内存策略之间的相互作 +用:: + + + /* + * 当cpuset被配置时，它打破了严格的hugetlb页面预留，因为计数是在一个全局变量上完 + * 成的。在有cpuset的情况下，这样的预留完全是垃圾，因为预留没有根据当前cpuset的 + * 页面可用性来检查。在任务所在的cpuset中缺乏空闲的htlb页面时，应用程序仍然有可能 + * 被内核OOM'ed。试图用cpuset来执行严格的计数几乎是不可能的（或者说太难看了），因 + * 为cpuset太不稳定了，任务或内存节点可以在cpuset之间动态移动。与cpuset共享 + * hugetlb映射的语义变化是不可取的。然而，为了预留一些语义，我们退回到检查当前空闲 + * 页的可用性，作为一种最好的尝试，希望能将cpuset改变语义的影响降到最低。 + */ + +添加巨页预留是为了防止在缺页异常时出现意外的页面分配失败（OOM）。然而，如果一个应用 +程序使用cpusets或内存策略，就不能保证在所需的节点上有巨页可用。即使有足够数量的全局 +预留，也是如此。 + +Hugetlbfs回归测试 +================= + +最完整的hugetlb测试集在libhugetlbfs仓库。如果你修改了任何hugetlb相关的代码，请使用 +libhugetlbfs测试套件来检查回归情况。此外，如果你添加了任何新的hugetlb功能，请在 +libhugetlbfs中添加适当的测试。 + +-- +Mike Kravetz，2017年4月7日 diff --git a/Documentation/translations/zh_CN/mm/hwpoison.rst b/Documentation/translations/zh_CN/mm/hwpoison.rst new file mode 100644 index 000000000000..310862edc937 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/hwpoison.rst @@ -0,0 +1,166 @@ + +:Original: Documentation/mm/hwpoison.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +======== +hwpoison +======== + +什么是hwpoison? +=============== + + +即将推出的英特尔CPU支持从一些内存错误中恢复（ ``MCA恢复`` ）。这需要操作系统宣布 +一个页面"poisoned"，杀死与之相关的进程，并避免在未来使用它。 + +这个补丁包在虚拟机中实现了必要的(编程)框架。 + +引用概述中的评论:: + + 高级机器的检查与处理。处理方法是损坏的页面被硬件报告，通常是由于2位ECC内 + 存或高速缓存故障。 + + 这主要是针对在后台检测到的损坏的页面。当当前的CPU试图访问它时，当前运行的进程 + 可以直接被杀死。因为还没有访问损坏的页面, 如果错误由于某种原因不能被处理，就可 + 以安全地忽略它. 而不是用另外一个机器检查去处理它。 + + 处理不同状态的页面缓存页。这里棘手的部分是，相对于其他虚拟内存用户， 我们可以异 + 步访问任何页面。因为内存故障可能随时随地发生，可能违反了他们的一些假设。这就是 + 为什么这段代码必须非常小心。一般来说，它试图使用正常的锁规则，如获得标准锁，即使 + 这意味着错误处理可能需要很长的时间。 + + 这里的一些操作有点低效，并且具有非线性的算法复杂性，因为数据结构没有针对这种情 + 况进行优化。特别是从vma到进程的映射就是这种情况。由于这种情况大概率是罕见的，所 + 以我们希望我们可以摆脱这种情况。 + +该代码由mm/memory-failure.c中的高级处理程序、一个新的页面poison位和虚拟机中的 +各种检查组成，用来处理poison的页面。 + +现在主要目标是KVM客户机，但它适用于所有类型的应用程序。支持KVM需要最近的qemu-kvm +版本。 + +对于KVM的使用，需要一个新的信号类型，这样KVM就可以用适当的地址将机器检查注入到客户 +机中。这在理论上也允许其他应用程序处理内存故障。我们的期望是，所有的应用程序都不要这 +样做，但一些非常专业的应用程序可能会这样做。 + +故障恢复模式 +============ + +有两种（实际上是三种）模式的内存故障恢复可以在。 + +vm.memory_failure_recovery sysctl 置零: + 所有的内存故障都会导致panic。请不要尝试恢复。 + +早期处理 + (可以在全局和每个进程中控制) 一旦检测到错误，立即向应用程序发送SIGBUS这允许 + 应用程序以温和的方式处理内存错误（例如，放弃受影响的对象） 这是KVM qemu使用的 + 模式。 + +推迟处理 + 当应用程序运行到损坏的页面时，发送SIGBUS。这对不知道内存错误的应用程序来说是 + 最好的，默认情况下注意一些页面总是被当作late kill处理。 + +用户控制 +======== + +vm.memory_failure_recovery + 参阅 sysctl.txt + +vm.memory_failure_early_kill + 全局启用early kill + +PR_MCE_KILL + 设置early/late kill mode/revert 到系统默认值。 + + arg1: PR_MCE_KILL_CLEAR: + 恢复到系统默认值 + arg1: PR_MCE_KILL_SET: + arg2定义了线程特定模式 + + PR_MCE_KILL_EARLY: + Early kill + PR_MCE_KILL_LATE: + Late kill + PR_MCE_KILL_DEFAULT + 使用系统全局默认值 + + 注意，如果你想有一个专门的线程代表进程处理SIGBUS(BUS_MCEERR_AO)，你应该在 + 指定线程上调用prctl(PR_MCE_KILL_EARLY)。否则，SIGBUS将被发送到主线程。 + +PR_MCE_KILL_GET + 返回当前模式 + +测试 +==== + +* madvise(MADV_HWPOISON, ....) (as root) - 在测试过程中Poison一个页面 + +* 通过debugfs ``/sys/kernel/debug/hwpoison/`` hwpoison-inject模块 + + corrupt-pfn + 在PFN处注入hwpoison故障，并echoed到这个文件。这做了一些早期过滤，以避 + 免在测试套件中损坏非预期页面。 + unpoison-pfn + 在PFN的Software-unpoison页面对应到这个文件。这样，一个页面可以再次被 + 复用。这只对Linux注入的故障起作用，对真正的内存故障不起作用。 + + 注意这些注入接口并不稳定，可能会在不同的内核版本中发生变化 + + corrupt-filter-dev-major, corrupt-filter-dev-minor + 只处理与块设备major/minor定义的文件系统相关的页面的内存故障。-1U是通 + 配符值。这应该只用于人工注入的测试。 + + corrupt-filter-memcg + 限制注入到memgroup拥有的页面。由memcg的inode号指定。 + + Example:: + + mkdir /sys/fs/cgroup/mem/hwpoison + + usemem -m 100 -s 1000 & + echo `jobs -p` > /sys/fs/cgroup/mem/hwpoison/tasks + + memcg_ino=$(ls -id /sys/fs/cgroup/mem/hwpoison | cut -f1 -d' ') + echo $memcg_ino > /debug/hwpoison/corrupt-filter-memcg + + page-types -p `pidof init` --hwpoison # shall do nothing + page-types -p `pidof usemem` --hwpoison # poison its pages + + corrupt-filter-flags-mask, corrupt-filter-flags-value + 当指定时，只有在((page_flags & mask) == value)的情况下才会poison页面。 + 这允许对许多种类的页面进行压力测试。page_flags与/proc/kpageflags中的相 + 同。这些标志位在include/linux/kernel-page-flags.h中定义，并在 + Documentation/admin-guide/mm/pagemap.rst中记录。 + +* 架构特定的MCE注入器 + + x86 有 mce-inject, mce-test + + 在mce-test中的一些便携式hwpoison测试程序，见下文。 + +引用 +==== + +http://halobates.de/mce-lc09-2.pdf + 09年LinuxCon的概述演讲 + +git://git.kernel.org/pub/scm/utils/cpu/mce/mce-test.git + 测试套件（在tsrc中的hwpoison特定可移植测试）。 + +git://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git + x86特定的注入器 + + +限制 +==== +- 不是所有的页面类型都被支持，而且永远不会。大多数内核内部对象不能被恢 + 复，目前只有LRU页。 + +--- +Andi Kleen, 2009年10月 diff --git a/Documentation/translations/zh_CN/mm/index.rst b/Documentation/translations/zh_CN/mm/index.rst new file mode 100644 index 000000000000..4c8c6b7b72a3 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/index.rst @@ -0,0 +1,54 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/index.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +================= +Linux内存管理文档 +================= + +这是一个关于Linux内存管理（mm）子系统内部的文档集，其中有不同层次的细节，包括注释 +和邮件列表的回复，用于阐述数据结构和算法的基本情况。如果你正在寻找关于简单分配内存的建 +议，请参阅(Documentation/translations/zh_CN/core-api/memory-allocation.rst)。 +对于控制和调整指南，请参阅(Documentation/admin-guide/mm/index)。 +TODO:待引用文档集被翻译完毕后请及时修改此处） + +.. toctree:: + :maxdepth: 1 + + active_mm + balance + damon/index + free_page_reporting + highmem + ksm + frontswap + hmm + hwpoison + hugetlbfs_reserv + memory-model + mmu_notifier + numa + overcommit-accounting + page_frags + page_owner + page_table_check + remap_file_pages + split_page_table_lock + z3fold + zsmalloc + +TODOLIST: +* arch_pgtable_helpers +* free_page_reporting +* hugetlbfs_reserv +* page_migration +* slub +* transhuge +* unevictable-lru +* vmalloced-kernel-stacks diff --git a/Documentation/translations/zh_CN/mm/ksm.rst b/Documentation/translations/zh_CN/mm/ksm.rst new file mode 100644 index 000000000000..d1f82e857ad7 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/ksm.rst @@ -0,0 +1,70 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/mm/ksm.rst + +:翻译: + + 徐鑫 xu xin + +============ +内核同页合并 +============ + +KSM 是一种节省内存的数据去重功能，由CONFIG_KSM=y启用，并在2.6.32版本时被添加 +到Linux内核。详见 ``mm/ksm.c`` 的实现，以及http://lwn.net/Articles/306704和 +https://lwn.net/Articles/330589 + +KSM的用户空间的接口在Documentation/translations/zh_CN/admin-guide/mm/ksm.rst +文档中有描述。 + +设计 +==== + +概述 +---- + +概述内容请见mm/ksm.c文档中的“DOC: Overview” + +逆映射 +------ +KSM维护着稳定树中的KSM页的逆映射信息。 + +当KSM页面的共享数小于 ``max_page_sharing`` 的虚拟内存区域(VMAs)时，则代表了 +KSM页的稳定树其中的节点指向了一个rmap_item结构体类型的列表。同时，这个KSM页 +的 ``page->mapping`` 指向了该稳定树节点。 + +如果共享数超过了阈值，KSM将给稳定树添加第二个维度。稳定树就变成链接一个或多 +个稳定树"副本"的"链"。每个副本都保留KSM页的逆映射信息，其中 ``page->mapping`` +指向该"副本"。 + +每个链以及链接到该链中的所有"副本"强制不变的是，它们代表了相同的写保护内存 +内容，尽管任中一个"副本"是由同一片内存区的不同的KSM复制页所指向的。 + +这样一来，相比与无限的逆映射链表，稳定树的查找计算复杂性不受影响。但在稳定树 +本身中不能有重复的KSM页面内容仍然是强制要求。 + +由 ``max_page_sharing`` 强制决定的数据去重限制是必要的，以此来避免虚拟内存 +rmap链表变得过大。rmap的遍历具有O(N)的复杂度，其中N是共享页面的rmap_项（即 +虚拟映射）的数量，而这个共享页面的节点数量又被 ``max_page_sharing`` 所限制。 +因此，这有效地将线性O(N)计算复杂度从rmap遍历中分散到不同的KSM页面上。ksmd进 +程在稳定节点"链"上的遍历也是O(N)，但这个N是稳定树"副本"的数量，而不是rmap项 +的数量，因此它对ksmd性能没有显著影响。实际上，最佳稳定树"副本"的候选节点将 +保留在"副本"列表的开头。 + +``max_page_sharing`` 的值设置得高了会促使更快的内存合并（因为将有更少的稳定 +树副本排队进入稳定节点chain->hlist）和更高的数据去重系数，但代价是在交换、压 +缩、NUMA平衡和页面迁移过程中可能导致KSM页的最大rmap遍历速度较慢。 + +``stable_node_dups/stable_node_chains`` 的比值还受 ``max_page_sharing`` 调控 +的影响，高比值可能意味着稳定节点dup中存在碎片，这可以通过在ksmd中引入碎片算 +法来解决，该算法将rmap项从一个稳定节点dup重定位到另一个稳定节点dup，以便释放 +那些仅包含极少rmap项的稳定节点"dup"，但这可能会增加ksmd进程的CPU使用率，并可 +能会减慢应用程序在KSM页面上的只读计算。 + +KSM会定期扫描稳定节点"链"中链接的所有稳定树"副本"，以便删减过时了的稳定节点。 +这种扫描的频率由 ``stable_node_chains_prune_millisecs`` 这个sysfs 接口定义。 + +参考 +==== +内核代码请见mm/ksm.c。 +涉及的函数(mm_slot ksm_scan stable_node rmap_item)。 diff --git a/Documentation/translations/zh_CN/mm/memory-model.rst b/Documentation/translations/zh_CN/mm/memory-model.rst new file mode 100644 index 000000000000..77ec149a970c --- /dev/null +++ b/Documentation/translations/zh_CN/mm/memory-model.rst @@ -0,0 +1,135 @@ +.. SPDX-License-Identifier: GPL-2.0 + +:Original: Documentation/mm/memory-model.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +============ +物理内存模型 +============ + +系统中的物理内存可以用不同的方式进行寻址。最简单的情况是，物理内存从地址0开 +始，跨越一个连续的范围，直到最大的地址。然而，这个范围可能包含CPU无法访问的 +小孔隙。那么，在完全不同的地址可能有几个连续的范围。而且，别忘了NUMA，即不 +同的内存库连接到不同的CPU。 + +Linux使用两种内存模型中的一种对这种多样性进行抽象。FLATMEM和SPARSEM。每 +个架构都定义了它所支持的内存模型，默认的内存模型是什么，以及是否有可能手动 +覆盖该默认值。 + +所有的内存模型都使用排列在一个或多个数组中的 `struct page` 来跟踪物理页 +帧的状态。 + +无论选择哪种内存模型，物理页框号（PFN）和相应的 `struct page` 之间都存 +在一对一的映射关系。 + +每个内存模型都定义了 :c:func:`pfn_to_page` 和 :c:func:`page_to_pfn` +帮助函数，允许从PFN到 `struct page` 的转换，反之亦然。 + +FLATMEM +======= + +最简单的内存模型是FLATMEM。这个模型适用于非NUMA系统的连续或大部分连续的 +物理内存。 + +在FLATMEM内存模型中，有一个全局的 `mem_map` 数组来映射整个物理内存。对 +于大多数架构，孔隙在 `mem_map` 数组中都有条目。与孔洞相对应的 `struct page` +对象从未被完全初始化。 + +为了分配 `mem_map` 数组，架构特定的设置代码应该调用free_area_init()函数。 +然而，在调用memblock_free_all()函数之前，映射数组是不能使用的，该函数 +将所有的内存交给页分配器。 + +一个架构可能会释放 `mem_map` 数组中不包括实际物理页的部分。在这种情况下，特 +定架构的 :c:func:`pfn_valid` 实现应该考虑到 `mem_map` 中的孔隙。 + +使用FLATMEM，PFN和 `struct page` 之间的转换是直接的。 `PFN - ARCH_PFN_OFFSET` +是 `mem_map` 数组的一个索引。 + +`ARCH_PFN_OFFSET` 定义了物理内存起始地址不同于0的系统的第一个页框号。 + +SPARSEMEM +========= + +SPARSEMEM是Linux中最通用的内存模型，它是唯一支持若干高级功能的内存模型， +如物理内存的热插拔、非易失性内存设备的替代内存图和较大系统的内存图的延迟 +初始化。 + +SPARSEMEM模型将物理内存显示为一个部分的集合。一个区段用mem_section结构 +体表示，它包含 `section_mem_map` ，从逻辑上讲，它是一个指向 `struct page` +阵列的指针。然而，它被存储在一些其他的magic中，以帮助分区管理。区段的大小 +和最大区段数是使用 `SECTION_SIZE_BITS` 和 `MAX_PHYSMEM_BITS` 常量 +来指定的，这两个常量是由每个支持SPARSEMEM的架构定义的。 `MAX_PHYSMEM_BITS` +是一个架构所支持的物理地址的实际宽度，而 `SECTION_SIZE_BITS` 是一个任 +意的值。 + +最大的段数表示为 `NR_MEM_SECTIONS` ，定义为 + +.. math:: + + NR\_MEM\_SECTIONS = 2 ^ {(MAX\_PHYSMEM\_BITS - SECTION\_SIZE\_BITS)} + +`mem_section` 对象被安排在一个叫做 `mem_sections` 的二维数组中。这个数组的 +大小和位置取决于 `CONFIG_SPARSEM_EXTREME` 和可能的最大段数: + +* 当 `CONFIG_SPARSEMEM_EXTREME` 被禁用时， `mem_sections` 数组是静态的，有 + `NR_MEM_SECTIONS` 行。每一行持有一个 `mem_section` 对象。 +* 当 `CONFIG_SPARSEMEM_EXTREME` 被启用时， `mem_sections` 数组被动态分配。 + 每一行包含价值 `PAGE_SIZE` 的 `mem_section` 对象，行数的计算是为了适应所有的 + 内存区。 + +架构设置代码应该调用sparse_init()来初始化内存区和内存映射。 + +通过SPARSEMEM，有两种可能的方式将PFN转换为相应的 `struct page` --"classic sparse"和 + "sparse vmemmap"。选择是在构建时进行的，它由 `CONFIG_SPARSEMEM_VMEMMAP` 的 + 值决定。 + +Classic sparse在page->flags中编码了一个页面的段号，并使用PFN的高位来访问映射该页 +框的段。在一个区段内，PFN是指向页数组的索引。 + +Sparse vmemmapvmemmap使用虚拟映射的内存映射来优化pfn_to_page和page_to_pfn操 +作。有一个全局的 `struct page *vmemmap` 指针，指向一个虚拟连续的 `struct page` +对象阵列。PFN是该数组的一个索引，`struct page` 从 `vmemmap` 的偏移量是该页的PFN。 + +为了使用vmemmap，一个架构必须保留一个虚拟地址的范围，以映射包含内存映射的物理页，并 +确保 `vmemmap`指向该范围。此外，架构应该实现 :c:func:`vmemmap_populate` 方法， +它将分配物理内存并为虚拟内存映射创建页表。如果一个架构对vmemmap映射没有任何特殊要求， +它可以使用通用内存管理提供的默认 :c:func:`vmemmap_populate_basepages`。 + +虚拟映射的内存映射允许将持久性内存设备的 `struct page` 对象存储在这些设备上预先分 +配的存储中。这种存储用vmem_altmap结构表示，最终通过一长串的函数调用传递给 +vmemmap_populate()。vmemmap_populate()实现可以使用 `vmem_altmap` 和 +:c:func:`vmemmap_alloc_block_buf` 助手来分配持久性内存设备上的内存映射。 + +ZONE_DEVICE +=========== +`ZONE_DEVICE` 设施建立在 `SPARSEM_VMEMMAP` 之上，为设备驱动识别的物理地址范 +围提供 `struct page` `mem_map` 服务。 `ZONE_DEVICE` 的 "设备" 方面与以下 +事实有关：这些地址范围的页面对象从未被在线标记过，而且必须对设备进行引用，而不仅仅 +是页面，以保持内存被“锁定”以便使用。 `ZONE_DEVICE` ，通过 :c:func:`devm_memremap_pages` ， +为给定的pfns范围执行足够的内存热插拔来开启 :c:func:`pfn_to_page`， +:c:func:`page_to_pfn`, ，和 :c:func:`get_user_pages` 服务。由于页面引 +用计数永远不会低于1，所以页面永远不会被追踪为空闲内存，页面的 `struct list_head lru` +空间被重新利用，用于向映射该内存的主机设备/驱动程序进行反向引用。 + +虽然 `SPARSEMEM` 将内存作为一个区段的集合，可以选择收集并合成内存块，但 +`ZONE_DEVICE` 用户需要更小的颗粒度来填充 `mem_map` 。鉴于 `ZONE_DEVICE` +内存从未被在线标记，因此它的内存范围从未通过sysfs内存热插拔api暴露在内存块边界 +上。这个实现依赖于这种缺乏用户接口的约束，允许子段大小的内存范围被指定给 +:c:func:`arch_add_memory` ，即内存热插拔的上半部分。子段支持允许2MB作为 +:c:func:`devm_memremap_pages` 的跨架构通用对齐颗粒度。 + +`ZONE_DEVICE` 的用户是: + +* pmem: 通过DAX映射将平台持久性内存作为直接I/O目标使用。 + +* hmm: 用 `->page_fault()` 和 `->page_free()` 事件回调扩展 `ZONE_DEVICE` ， + 以允许设备驱动程序协调与设备内存相关的内存管理事件，通常是GPU内存。参见Documentation/mm/hmm.rst。 + +* p2pdma: 创建 `struct page` 对象，允许PCI/E拓扑结构中的peer设备协调它们之间的 + 直接DMA操作，即绕过主机内存。 diff --git a/Documentation/translations/zh_CN/mm/mmu_notifier.rst b/Documentation/translations/zh_CN/mm/mmu_notifier.rst new file mode 100644 index 000000000000..ce3664d1a410 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/mmu_notifier.rst @@ -0,0 +1,97 @@ +:Original: Documentation/mm/mmu_notifier.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + + +什么时候需要页表锁内通知？ +========================== + +当清除一个pte/pmd时，我们可以选择通过在页表锁下（通知版的\*_clear_flush调用 +mmu_notifier_invalidate_range）通知事件。但这种通知并不是在所有情况下都需要的。 + +对于二级TLB（非CPU TLB），如IOMMU TLB或设备TLB（当设备使用类似ATS/PASID的东西让 +IOMMU走CPU页表来访问进程的虚拟地址空间）。只有两种情况需要在清除pte/pmd时在持有页 +表锁的同时通知这些二级TLB： + + A) 在mmu_notifier_invalidate_range_end()之前，支持页的地址被释放。 + B) 一个页表项被更新以指向一个新的页面（COW，零页上的写异常，__replace_page()，...）。 + +情况A很明显，你不想冒风险让设备写到一个现在可能被一些完全不同的任务使用的页面。 + +情况B更加微妙。为了正确起见，它需要按照以下序列发生: + + - 上页表锁 + - 清除页表项并通知 ([pmd/pte]p_huge_clear_flush_notify()) + - 设置页表项以指向新页 + +如果在设置新的pte/pmd值之前，清除页表项之后没有进行通知，那么你就会破坏设备的C11或 +C++11等内存模型。 + +考虑以下情况（设备使用类似于ATS/PASID的功能）。 + +两个地址addrA和addrB，这样|addrA - addrB| >= PAGE_SIZE，我们假设它们是COW的 +写保护（B的其他情况也适用）。 + +:: + + [Time N] -------------------------------------------------------------------- + CPU-thread-0 {尝试写到addrA} + CPU-thread-1 {尝试写到addrB} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {读取addrA并填充设备TLB} + DEV-thread-2 {读取addrB并填充设备TLB} + [Time N+1] ------------------------------------------------------------------ + CPU-thread-0 {COW_step0: {mmu_notifier_invalidate_range_start(addrA)}} + CPU-thread-1 {COW_step0: {mmu_notifier_invalidate_range_start(addrB)}} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+2] ------------------------------------------------------------------ + CPU-thread-0 {COW_step1: {更新页表以指向addrA的新页}} + CPU-thread-1 {COW_step1: {更新页表以指向addrB的新页}} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+3] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {preempted} + CPU-thread-2 {写入addrA，这是对新页面的写入} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+3] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {preempted} + CPU-thread-2 {} + CPU-thread-3 {写入addrB，这是一个写入新页的过程} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+4] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {COW_step3: {mmu_notifier_invalidate_range_end(addrB)}} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {} + DEV-thread-2 {} + [Time N+5] ------------------------------------------------------------------ + CPU-thread-0 {preempted} + CPU-thread-1 {} + CPU-thread-2 {} + CPU-thread-3 {} + DEV-thread-0 {从旧页中读取addrA} + DEV-thread-2 {从新页面读取addrB} + +所以在这里，因为在N+2的时候，清空页表项没有和通知一起作废二级TLB，设备在看到addrA的新值之前 +就看到了addrB的新值。这就破坏了设备的总内存序。 + +当改变一个pte的写保护或指向一个新的具有相同内容的写保护页（KSM）时，将mmu_notifier_invalidate_range +调用延迟到页表锁外的mmu_notifier_invalidate_range_end()是可以的。即使做页表更新的线程 +在释放页表锁后但在调用mmu_notifier_invalidate_range_end()前被抢占，也是如此。 diff --git a/Documentation/translations/zh_CN/mm/numa.rst b/Documentation/translations/zh_CN/mm/numa.rst new file mode 100644 index 000000000000..b15cfeeb6dfb --- /dev/null +++ b/Documentation/translations/zh_CN/mm/numa.rst @@ -0,0 +1,101 @@ +:Original: Documentation/mm/numa.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +始于1999年11月，作者： + +========================== +何为非统一内存访问(NUMA)？ +========================== + +这个问题可以从几个视角来回答：硬件观点和Linux软件视角。 + +从硬件角度看，NUMA系统是一个由多个组件或装配组成的计算机平台，每个组件可能包含0个或更多的CPU、 +本地内存和/或IO总线。为了简洁起见，并将这些物理组件/装配的硬件视角与软件抽象区分开来，我们在 +本文中称这些组件/装配为“单元”。 + +每个“单元”都可以看作是系统的一个SMP[对称多处理器]子集——尽管独立的SMP系统所需的一些组件可能 +不会在任何给定的单元上填充。NUMA系统的单元通过某种系统互连连接在一起——例如，交叉开关或点对点 +链接是NUMA系统互连的常见类型。这两种类型的互连都可以聚合起来，以创建NUMA平台，其中的单元与其 +他单元有多个距离。 + +对于Linux，感兴趣的NUMA平台主要是所谓的缓存相干NUMA--简称ccNUMA系统系统。在ccNUMA系统中， +所有的内存都是可见的，并且可以从连接到任何单元的任何CPU中访问，缓存一致性是由处理器缓存和/或 +系统互连在硬件中处理。 + +内存访问时间和有效的内存带宽取决于包含CPU的单元或进行内存访问的IO总线距离包含目标内存的单元 +有多远。例如，连接到同一单元的CPU对内存的访问将比访问其他远程单元的内存经历更快的访问时间和 +更高的带宽。 NUMA平台可以在任何给定单元上访问多种远程距离的（其他）单元。 + +平台供应商建立NUMA系统并不只是为了让软件开发人员的生活变得有趣。相反，这种架构是提供可扩展 +内存带宽的一种手段。然而，为了实现可扩展的内存带宽，系统和应用软件必须安排大部分的内存引用 +[cache misses]到“本地”内存——同一单元的内存，如果有的话——或者到最近的有内存的单元。 + +这就自然而然有了Linux软件对NUMA系统的视角: + +Linux将系统的硬件资源划分为多个软件抽象，称为“节点”。Linux将节点映射到硬件平台的物理单元 +上，对一些架构的细节进行了抽象。与物理单元一样，软件节点可能包含0或更多的CPU、内存和/或IO +总线。同样，对“较近”节点的内存访问——映射到较近单元的节点——通常会比对较远单元的访问经历更快 +的访问时间和更高的有效带宽。 + +对于一些架构，如x86，Linux将“隐藏”任何代表没有内存连接的物理单元的节点，并将连接到该单元 +的任何CPU重新分配到代表有内存的单元的节点上。因此，在这些架构上，我们不能假设Linux将所有 +的CPU与一个给定的节点相关联，会看到相同的本地内存访问时间和带宽。 + +此外，对于某些架构，同样以x86为例，Linux支持对额外节点的仿真。对于NUMA仿真，Linux会将现 +有的节点或者非NUMA平台的系统内存分割成多个节点。每个模拟的节点将管理底层单元物理内存的一部 +分。NUMA仿真对于在非NUMA平台上测试NUMA内核和应用功能是非常有用的，当与cpusets一起使用时， +可以作为一种内存资源管理机制。[见 Documentation/admin-guide/cgroup-v1/cpusets.rst] + +对于每个有内存的节点，Linux构建了一个独立的内存管理子系统，有自己的空闲页列表、使用中页列表、 +使用统计和锁来调解访问。此外，Linux为每个内存区[DMA、DMA32、NORMAL、HIGH_MEMORY、MOVABLE +中的一个或多个]构建了一个有序的“区列表”。zonelist指定了当一个选定的区/节点不能满足分配请求 +时要访问的区/节点。当一个区没有可用的内存来满足请求时，这种情况被称为“overflow 溢出”或 +“fallback 回退”。 + +由于一些节点包含多个包含不同类型内存的区，Linux必须决定是否对区列表进行排序，使分配回退到不同 +节点上的相同区类型，或同一节点上的不同区类型。这是一个重要的考虑因素，因为有些区，如DMA或DMA32， +代表了相对稀缺的资源。Linux选择了一个默认的Node ordered zonelist。这意味着在使用按NUMA距 +离排序的远程节点之前，它会尝试回退到同一节点的其他分区。 + +默认情况下，Linux会尝试从执行请求的CPU被分配到的节点中满足内存分配请求。具体来说，Linux将试 +图从请求来源的节点的适当分区列表中的第一个节点进行分配。这被称为“本地分配”。如果“本地”节点不能 +满足请求，内核将检查所选分区列表中其他节点的区域，寻找列表中第一个能满足请求的区域。 + +本地分配将倾向于保持对分配的内存的后续访问 “本地”的底层物理资源和系统互连——只要内核代表其分配 +一些内存的任务后来不从该内存迁移。Linux调度器知道平台的NUMA拓扑结构——体现在“调度域”数据结构 +中[见 Documentation/scheduler/sched-domains.rst]——并且调度器试图尽量减少任务迁移到遥 +远的调度域中。然而，调度器并没有直接考虑到任务的NUMA足迹。因此，在充分不平衡的情况下，任务可 +以在节点之间迁移，远离其初始节点和内核数据结构。 + +系统管理员和应用程序设计者可以使用各种CPU亲和命令行接口，如taskset(1)和numactl(1)，以及程 +序接口，如sched_setaffinity(2)，来限制任务的迁移，以改善NUMA定位。此外，人们可以使用 +Linux NUMA内存策略修改内核的默认本地分配行为。 [见 +:ref:`Documentation/admin-guide/mm/numa_memory_policy.rst `]. + +系统管理员可以使用控制组和CPUsets限制非特权用户在调度或NUMA命令和功能中可以指定的CPU和节点 +的内存。 [见 Documentation/admin-guide/cgroup-v1/cpusets.rst] + +在不隐藏无内存节点的架构上，Linux会在分区列表中只包括有内存的区域[节点]。这意味着对于一个无 +内存的节点，“本地内存节点”——CPU节点的分区列表中的第一个区域的节点——将不是节点本身。相反，它 +将是内核在建立分区列表时选择的离它最近的有内存的节点。所以，默认情况下，本地分配将由内核提供 +最近的可用内存来完成。这是同一机制的结果，该机制允许这种分配在一个包含内存的节点溢出时回退到 +其他附近的节点。 + +一些内核分配不希望或不能容忍这种分配回退行为。相反，他们想确保他们从指定的节点获得内存，或者 +得到通知说该节点没有空闲内存。例如，当一个子系统分配每个CPU的内存资源时，通常是这种情况。 + +一个典型的分配模式是使用内核的numa_node_id()或CPU_to_node()函数获得“当前CPU”所在节点的 +节点ID，然后只从返回的节点ID请求内存。当这样的分配失败时，请求的子系统可以恢复到它自己的回退 +路径。板块内核内存分配器就是这样的一个例子。或者，子系统可以选择在分配失败时禁用或不启用自己。 +内核分析子系统就是这样的一个例子。 + +如果架构支持——不隐藏无内存节点，那么连接到无内存节点的CPU将总是产生回退路径的开销，或者一些 +子系统如果试图完全从无内存的节点分配内存，将无法初始化。为了透明地支持这种架构，内核子系统可 +以使用numa_mem_id()或cpu_to_mem()函数来定位调用或指定CPU的“本地内存节点”。同样，这是同 +一个节点，默认的本地页分配将从这个节点开始尝试。 diff --git a/Documentation/translations/zh_CN/mm/overcommit-accounting.rst b/Documentation/translations/zh_CN/mm/overcommit-accounting.rst new file mode 100644 index 000000000000..d8452d8b7fbb --- /dev/null +++ b/Documentation/translations/zh_CN/mm/overcommit-accounting.rst @@ -0,0 +1,86 @@ +:Original: Documentation/mm/overcommit-accounting.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + + +============== +超量使用审计 +============== + +Linux内核支持下列超量使用处理模式 + +0 + 启发式超量使用处理。拒绝明显的地址空间超量使用。用于一个典型的系统。 + 它确保严重的疯狂分配失败，同时允许超量使用以减少swap的使用。在这种模式下， + 允许root分配稍多的内存。这是默认的。 +1 + 总是超量使用。适用于一些科学应用。经典的例子是使用稀疏数组的代码，只是依赖 + 几乎完全由零页组成的虚拟内存 + +2 + 不超量使用。系统提交的总地址空间不允许超过swap+一个可配置的物理RAM的数量 + （默认为50%）。根据你使用的数量，在大多数情况下，这意味着一个进程在访问页面时 + 不会被杀死，但会在内存分配上收到相应的错误。 + + 对于那些想保证他们的内存分配在未来可用而又不需要初始化每一个页面的应用程序来说 + 是很有用的。 + +超量使用策略是通过sysctl `vm.overcommit_memory` 设置的。 + +可以通过 `vm.overcommit_ratio` （百分比）或 `vm.overcommit_kbytes` （绝对值） +来设置超限数量。这些只有在 `vm.overcommit_memory` 被设置为2时才有效果。 + +在 ``/proc/meminfo`` 中可以分别以CommitLimit和Committed_AS的形式查看当前 +的超量使用和提交量。 + +陷阱 +==== + +C语言的堆栈增长是一个隐含的mremap。如果你想得到绝对的保证，并在接近边缘的地方运行， +你 **必须** 为你认为你需要的最大尺寸的堆栈进行mmap。对于典型的堆栈使用来说，这并 +不重要，但如果你真的非常关心的话，这就是一个值得关注的案例。 + + +在模式2中，MAP_NORESERVE标志被忽略。 + + +它是如何工作的 +============== + +超量使用是基于以下规则 + +对于文件映射 + | SHARED or READ-only - 0 cost (该文件是映射而不是交换) + | PRIVATE WRITABLE - 每个实例的映射大小 + +对于匿名或者 ``/dev/zero`` 映射 + | SHARED - 映射的大小 + | PRIVATE READ-only - 0 cost (但作用不大) + | PRIVATE WRITABLE - 每个实例的映射大小 + +额外的计数 + | 通过mmap制作可写副本的页面 + | 从同一池中提取的shmfs内存 + +状态 +==== + +* 我们核算mmap内存映射 +* 我们核算mprotect在提交中的变化 +* 我们核算mremap的大小变化 +* 我们的审计 brk +* 审计munmap +* 我们在/proc中报告commit 状态 +* 核对并检查分叉的情况 +* 审查堆栈处理/执行中的构建 +* 叙述SHMfs的情况 +* 实现实际限制的执行 + +待续 +==== +* ptrace 页计数（这很难）。 diff --git a/Documentation/translations/zh_CN/mm/page_frags.rst b/Documentation/translations/zh_CN/mm/page_frags.rst new file mode 100644 index 000000000000..320952ca93af --- /dev/null +++ b/Documentation/translations/zh_CN/mm/page_frags.rst @@ -0,0 +1,38 @@ +:Original: Documentation/mm/page_frag.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +======== +页面片段 +======== + +一个页面片段是一个任意长度的任意偏移的内存区域，它位于一个0或更高阶的复合页面中。 +该页中的多个碎片在该页的引用计数器中被单独计算。 + +page_frag函数，page_frag_alloc和page_frag_free，为页面片段提供了一个简单 +的分配框架。这被网络堆栈和网络设备驱动使用，以提供一个内存的支持区域，作为 +sk_buff->head使用，或者用于skb_shared_info的 “frags” 部分。 + +为了使用页面片段API，需要一个支持页面片段的缓冲区。这为碎片分配提供了一个中心点， +并允许多个调用使用一个缓存的页面。这样做的好处是可以避免对get_page的多次调用， +这在分配时开销可能会很大。然而，由于这种缓存的性质，要求任何对缓存的调用都要受到每 +个CPU的限制，或者每个CPU的限制，并在执行碎片分配时强制禁止中断。 + +网络堆栈在每个CPU使用两个独立的缓存来处理碎片分配。netdev_alloc_cache被使用 +netdev_alloc_frag和__netdev_alloc_skb调用的调用者使用。napi_alloc_cache +被调用__napi_alloc_frag和__napi_alloc_skb的调用者使用。这两个调用的主要区别是 +它们可能被调用的环境。“netdev” 前缀的函数可以在任何上下文中使用，因为这些函数 +将禁用中断，而 ”napi“ 前缀的函数只可以在softirq上下文中使用。 + +许多网络设备驱动程序使用类似的方法来分配页面片段，但页面片段是在环或描述符级别上 +缓存的。为了实现这些情况，有必要提供一种拆解页面缓存的通用方法。出于这个原因， +__page_frag_cache_drain被实现了。它允许通过一次调用从一个页面释放多个引用。 +这样做的好处是，它允许清理被添加到一个页面的多个引用，以避免每次分配都调用 +get_page。 + +Alexander Duyck，2016年11月29日。 diff --git a/Documentation/translations/zh_CN/mm/page_owner.rst b/Documentation/translations/zh_CN/mm/page_owner.rst new file mode 100644 index 000000000000..03d9e613094a --- /dev/null +++ b/Documentation/translations/zh_CN/mm/page_owner.rst @@ -0,0 +1,116 @@ +:Original: Documentation/mm/page_owner.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +================================ +page owner: 跟踪谁分配的每个页面 +================================ + +概述 +==== + +page owner是用来追踪谁分配的每一个页面。它可以用来调试内存泄漏或找到内存占用者。 +当分配发生时，有关分配的信息，如调用堆栈和页面的顺序被存储到每个页面的特定存储中。 +当我们需要了解所有页面的状态时，我们可以获得并分析这些信息。 + +尽管我们已经有了追踪页面分配/释放的tracepoint，但用它来分析谁分配的每个页面是 +相当复杂的。我们需要扩大跟踪缓冲区，以防止在用户空间程序启动前出现重叠。而且，启 +动的程序会不断地将跟踪缓冲区转出，供以后分析，这将会改变系统的行为，会产生更多的 +可能性，而不是仅仅保留在内存中，所以不利于调试。 + +页面所有者也可以用于各种目的。例如，可以通过每个页面的gfp标志信息获得精确的碎片 +统计。如果启用了page owner，它就已经实现并激活了。我们非常欢迎其他用途。 + +page owner在默认情况下是禁用的。所以，如果你想使用它，你需要在你的启动cmdline +中加入"page_owner=on"。如果内核是用page owner构建的，并且由于没有启用启动 +选项而在运行时禁用page owner，那么运行时的开销是很小的。如果在运行时禁用，它不 +需要内存来存储所有者信息，所以没有运行时内存开销。而且，页面所有者在页面分配器的 +热路径中只插入了两个不可能的分支，如果不启用，那么分配就会像没有页面所有者的内核 +一样进行。这两个不可能的分支应该不会影响到分配的性能，特别是在静态键跳转标签修补 +功能可用的情况下。以下是由于这个功能而导致的内核代码大小的变化。 + +- 没有page owner:: + + text data bss dec hex filename + 48392 2333 644 51369 c8a9 mm/page_alloc.o + +- 有page owner:: + + text data bss dec hex filename + 48800 2445 644 51889 cab1 mm/page_alloc.o + 6662 108 29 6799 1a8f mm/page_owner.o + 1025 8 8 1041 411 mm/page_ext.o + +虽然总共增加了8KB的代码，但page_alloc.o增加了520字节，其中不到一半是在hotpath +中。构建带有page owner的内核，并在需要时打开它，将是调试内核内存问题的最佳选择。 + +有一个问题是由实现细节引起的。页所有者将信息存储到struct page扩展的内存中。这 +个内存的初始化时间比稀疏内存系统中的页面分配器启动的时间要晚一些，所以，在初始化 +之前，许多页面可以被分配，但它们没有所有者信息。为了解决这个问题，这些早期分配的 +页面在初始化阶段被调查并标记为分配。虽然这并不意味着它们有正确的所有者信息，但至 +少，我们可以更准确地判断该页是否被分配。在2GB内存的x86-64虚拟机上，有13343 +个早期分配的页面被捕捉和标记，尽管它们大部分是由结构页扩展功能分配的。总之，在这 +之后，没有任何页面处于未追踪状态。 + +使用方法 +======== + +1) 构建用户空间的帮助:: + + cd tools/vm + make page_owner_sort + +2) 启用page owner: 添加 "page_owner=on" 到 boot cmdline. + +3) 做你想调试的工作。 + +4) 分析来自页面所有者的信息:: + + cat /sys/kernel/debug/page_owner > page_owner_full.txt + ./page_owner_sort page_owner_full.txt sorted_page_owner.txt + + ``page_owner_full.txt`` 的一般输出情况如下(输出信息无翻译价值):: + + Page allocated via order XXX, ... + PFN XXX ... + // Detailed stack + + Page allocated via order XXX, ... + PFN XXX ... + // Detailed stack + + ``page_owner_sort`` 工具忽略了 ``PFN`` 行，将剩余的行放在buf中，使用regexp提 + 取页序值，计算buf的次数和页数，最后根据参数进行排序。 + + 在 ``sorted_page_owner.txt`` 中可以看到关于谁分配了每个页面的结果。一般输出:: + + XXX times, XXX pages: + Page allocated via order XXX, ... + // Detailed stack + + 默认情况下， ``page_owner_sort`` 是根据buf的时间来排序的。如果你想 + 按buf的页数排序，请使用-m参数。详细的参数是: + + 基本函数: + + Sort: + -a 按内存分配时间排序 + -m 按总内存排序 + -p 按pid排序。 + -P 按tgid排序。 + -r 按内存释放时间排序。 + -s 按堆栈跟踪排序。 + -t 按时间排序（默认）。 + + 其它函数: + + Cull: + -c 通过比较堆栈跟踪而不是总块来进行剔除。 + + Filter: + -f 过滤掉内存已被释放的块的信息。 diff --git a/Documentation/translations/zh_CN/mm/page_table_check.rst b/Documentation/translations/zh_CN/mm/page_table_check.rst new file mode 100644 index 000000000000..e8077310a76c --- /dev/null +++ b/Documentation/translations/zh_CN/mm/page_table_check.rst @@ -0,0 +1,56 @@ +.. SPDX-License-Identifier: GPL-2.0 + +:Original: Documentation/mm/page_table_check.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +======== +页表检查 +======== + +概述 +==== + +页表检查允许通过确保防止某些类型的内存损坏来强化内核。 + +当新的页面可以从用户空间访问时，页表检查通过将它们的页表项（PTEs PMD等）添加到页表中来执行额外 +的验证。 + +在检测到损坏的情况下，内核会被崩溃。页表检查有一个小的性能和内存开销。因此，它在默认情况下是禁用 +的，但是在额外的加固超过性能成本的系统上，可以选择启用。另外，由于页表检查是同步的，它可以帮助调 +试双映射内存损坏问题，在错误的映射发生时崩溃内核，而不是在内存损坏错误发生后内核崩溃。 + +双重映射检测逻辑 +================ + ++-------------------+-------------------+-------------------+------------------+ +| Current Mapping | New mapping | Permissions | Rule | ++===================+===================+===================+==================+ +| Anonymous | Anonymous | Read | Allow | ++-------------------+-------------------+-------------------+------------------+ +| Anonymous | Anonymous | Read / Write | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Anonymous | Named | Any | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Named | Anonymous | Any | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Named | Named | Any | Allow | ++-------------------+-------------------+-------------------+------------------+ + +启用页表检查 +============ + +用以下方法构建内核: + +- PAGE_TABLE_CHECK=y + 注意，它只能在ARCH_SUPPORTS_PAGE_TABLE_CHECK可用的平台上启用。 + +- 使用 "page_table_check=on" 内核参数启动。 + +可以选择用PAGE_TABLE_CHECK_ENFORCED来构建内核，以便在没有额外的内核参数的情况下获得页表 +支持。 diff --git a/Documentation/translations/zh_CN/mm/remap_file_pages.rst b/Documentation/translations/zh_CN/mm/remap_file_pages.rst new file mode 100644 index 000000000000..31e0c54dc36f --- /dev/null +++ b/Documentation/translations/zh_CN/mm/remap_file_pages.rst @@ -0,0 +1,32 @@ +:Original: Documentation/mm/remap_file_pages.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +============================== +remap_file_pages()系统调用 +============================== + +remap_file_pages()系统调用被用来创建一个非线性映射，也就是说，在这个映射中， +文件的页面被无序映射到内存中。使用remap_file_pages()比重复调用mmap(2)的好 +处是，前者不需要内核创建额外的VMA（虚拟内存区）数据结构。 + +支持非线性映射需要在内核虚拟内存子系统中编写大量的non-trivial的代码，包括热 +路径。另外，为了使非线性映射工作，内核需要一种方法来区分正常的页表项和带有文件 +偏移的项（pte_file）。内核为达到这个目的在PTE中保留了标志。PTE标志是稀缺资 +源，特别是在某些CPU架构上。如果能腾出这个标志用于其他用途就更好了。 + +幸运的是，在生活中并没有很多remap_file_pages()的用户。只知道有一个企业的RDBMS +实现在32位系统上使用这个系统调用来映射比32位虚拟地址空间线性尺寸更大的文件。 +由于64位系统的广泛使用，这种使用情况已经不重要了。 + +syscall被废弃了，现在用一个模拟来代替它。仿真会创建新的VMA，而不是非线性映射。 +对于remap_file_pages()的少数用户来说，它的工作速度会变慢，但ABI被保留了。 + +仿真的一个副作用（除了性能之外）是，由于额外的VMA，用户可以更容易达到 +vm.max_map_count的限制。关于限制的更多细节，请参见DEFAULT_MAX_MAP_COUNT +的注释。 diff --git a/Documentation/translations/zh_CN/mm/split_page_table_lock.rst b/Documentation/translations/zh_CN/mm/split_page_table_lock.rst new file mode 100644 index 000000000000..4fb7aa666037 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/split_page_table_lock.rst @@ -0,0 +1,96 @@ +:Original: Documentation/mm/split_page_table_lock.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +================================= +分页表锁（split page table lock） +================================= + +最初，mm->page_table_lock spinlock保护了mm_struct的所有页表。但是这种方 +法导致了多线程应用程序的缺页异常可扩展性差，因为对锁的争夺很激烈。为了提高可扩 +展性，我们引入了分页表锁。 + +有了分页表锁，我们就有了单独的每张表锁来顺序化对表的访问。目前，我们对PTE和 +PMD表使用分页锁。对高层表的访问由mm->page_table_lock保护。 + +有一些辅助工具来锁定/解锁一个表和其他访问器函数: + + - pte_offset_map_lock() + 映射pte并获取PTE表锁，返回所取锁的指针； + - pte_unmap_unlock() + 解锁和解映射PTE表； + - pte_alloc_map_lock() + 如果需要的话，分配PTE表并获取锁，如果分配失败，返回已获取的锁的指针 + 或NULL； + - pte_lockptr() + 返回指向PTE表锁的指针； + - pmd_lock() + 取得PMD表锁，返回所取锁的指针。 + - pmd_lockptr() + 返回指向PMD表锁的指针； + +如果CONFIG_SPLIT_PTLOCK_CPUS（通常为4）小于或等于NR_CPUS，则在编译 +时启用PTE表的分页表锁。如果分页锁被禁用，所有的表都由mm->page_table_lock +来保护。 + +如果PMD表启用了分页锁，并且架构支持它，那么PMD表的分页锁就会被启用（见 +下文）。 + +Hugetlb 和分页表锁 +================== + +Hugetlb可以支持多种页面大小。我们只对PMD级别使用分页锁，但不对PUD使用。 + +Hugetlb特定的辅助函数: + + - huge_pte_lock() + 对PMD_SIZE页面采取pmd分割锁，否则mm->page_table_lock； + - huge_pte_lockptr() + 返回指向表锁的指针。 + +架构对分页表锁的支持 +==================== + +没有必要特别启用PTE分页表锁：所有需要的东西都由pgtable_pte_page_ctor() +和pgtable_pte_page_dtor()完成，它们必须在PTE表分配/释放时被调用。 + +确保架构不使用slab分配器来分配页表：slab使用page->slab_cache来分配其页 +面。这个区域与page->ptl共享存储。 + +PMD分页锁只有在你有两个以上的页表级别时才有意义。 + +启用PMD分页锁需要在PMD表分配时调用pgtable_pmd_page_ctor()，在释放时调 +用pgtable_pmd_page_dtor()。 + +分配通常发生在pmd_alloc_one()中，释放发生在pmd_free()和pmd_free_tlb() +中，但要确保覆盖所有的PMD表分配/释放路径：即X86_PAE在pgd_alloc()中预先 +分配一些PMD。 + +一切就绪后，你可以设置CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK。 + +注意：pgtable_pte_page_ctor()和pgtable_pmd_page_ctor()可能失败--必 +须正确处理。 + +page->ptl +========= + +page->ptl用于访问分割页表锁，其中'page'是包含该表的页面struct page。它 +与page->private（以及union中的其他几个字段）共享存储。 + +为了避免增加struct page的大小并获得最佳性能，我们使用了一个技巧: + + - 如果spinlock_t适合于long，我们使用page->ptr作为spinlock，这样我们 + 就可以避免间接访问并节省一个缓存行。 + - 如果spinlock_t的大小大于long的大小，我们使用page->ptl作为spinlock_t + 的指针并动态分配它。这允许在启用DEBUG_SPINLOCK或DEBUG_LOCK_ALLOC的 + 情况下使用分页锁，但由于间接访问而多花了一个缓存行。 + +PTE表的spinlock_t分配在pgtable_pte_page_ctor()中，PMD表的spinlock_t +分配在pgtable_pmd_page_ctor()中。 + +请不要直接访问page->ptl - -使用适当的辅助函数。 diff --git a/Documentation/translations/zh_CN/mm/z3fold.rst b/Documentation/translations/zh_CN/mm/z3fold.rst new file mode 100644 index 000000000000..9569a6d88270 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/z3fold.rst @@ -0,0 +1,31 @@ +:Original: Documentation/mm/z3fold.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +====== +z3fold +====== + +z3fold是一个专门用于存储压缩页的分配器。它被设计为每个物理页最多可以存储三个压缩页。 +它是zbud的衍生物，允许更高的压缩率，保持其前辈的简单性和确定性。 + +z3fold和zbud的主要区别是: + +* 与zbud不同的是，z3fold允许最大的PAGE_SIZE分配。 +* z3fold在其页面中最多可以容纳3个压缩页面 +* z3fold本身没有输出任何API，因此打算通过zpool的API来使用 + +为了保持确定性和简单性，z3fold，就像zbud一样，总是在每页存储一个整数的压缩页，但是 +它最多可以存储3页，不像zbud最多可以存储2页。因此压缩率达到2.7倍左右，而zbud的压缩 +率是1.7倍左右。 + +不像zbud（但也像zsmalloc），z3fold_alloc()那样不返回一个可重复引用的指针。相反，它 +返回一个无符号长句柄，它编码了被分配对象的实际位置。 + +保持有效的压缩率接近于zsmalloc，z3fold不依赖于MMU的启用，并提供更可预测的回收行 +为，这使得它更适合于小型和反应迅速的系统。 diff --git a/Documentation/translations/zh_CN/mm/zsmalloc.rst b/Documentation/translations/zh_CN/mm/zsmalloc.rst new file mode 100644 index 000000000000..b5596ea08ae4 --- /dev/null +++ b/Documentation/translations/zh_CN/mm/zsmalloc.rst @@ -0,0 +1,78 @@ +:Original: Documentation/mm/zs_malloc.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +======== +zsmalloc +======== + +这个分配器是为与zram一起使用而设计的。因此，该分配器应该在低内存条件下工作良好。特别是， +它从未尝试过higher order页面的分配，这在内存压力下很可能会失败。另一方面，如果我们只 +是使用单（0-order）页，它将遭受非常高的碎片化 - 任何大小为PAGE_SIZE/2或更大的对象将 +占据整个页面。这是其前身（xvmalloc）的主要问题之一。 + +为了克服这些问题，zsmalloc分配了一堆0-order页面，并使用各种"struct page"字段将它 +们链接起来。这些链接的页面作为一个单一的higher order页面，即一个对象可以跨越0-order +页面的边界。代码将这些链接的页面作为一个实体，称为zspage。 + +为了简单起见，zsmalloc只能分配大小不超过PAGE_SIZE的对象，因为这满足了所有当前用户的 +要求（在最坏的情况下，页面是不可压缩的，因此以"原样"即未压缩的形式存储）。对于大于这 +个大小的分配请求，会返回失败（见zs_malloc）。 + +此外，zs_malloc()并不返回一个可重复引用的指针。相反，它返回一个不透明的句柄（无符号 +长），它编码了被分配对象的实际位置。这种间接性的原因是zsmalloc并不保持zspages的永久 +映射，因为这在32位系统上会导致问题，因为内核空间映射的VA区域非常小。因此，在使用分配 +的内存之前，对象必须使用zs_map_object()进行映射以获得一个可用的指针，随后使用 +zs_unmap_object()解除映射。 + +stat +==== + +通过CONFIG_ZSMALLOC_STAT，我们可以通过 ``/sys/kernel/debug/zsmalloc/`` +看到zsmalloc内部信息。下面是一个统计输出的例子。:: + + # cat /sys/kernel/debug/zsmalloc/zram0/classes + + class size almost_full almost_empty obj_allocated obj_used pages_used pages_per_zspage + ... + ... + 9 176 0 1 186 129 8 4 + 10 192 1 0 2880 2872 135 3 + 11 208 0 1 819 795 42 2 + 12 224 0 1 219 159 12 4 + ... + ... + + +class + 索引 +size + zspage存储对象大小 +almost_empty + ZS_ALMOST_EMPTY zspage的数量（见下文）。 +almost_full + ZS_ALMOST_FULL zspage的数量(见下图) +obj_allocated + 已分配对象的数量 +obj_used + 分配给用户的对象的数量 +pages_used + 为该类分配的页数 +pages_per_zspage + 组成一个zspage的0-order页面的数量 + +当n <= N / f时，我们将一个zspage分配给ZS_ALMOST_EMPTYfullness组，其中 + +* n = 已分配对象的数量 +* N = zspage可以存储的对象总数 +* f = fullness_threshold_frac(即，目前是4个) + +同样地，我们将zspage分配给: + +* ZS_ALMOST_FULL when n > N / f +* ZS_EMPTY when n == 0 +* ZS_FULL when n == N diff --git a/Documentation/translations/zh_CN/vm/active_mm.rst b/Documentation/translations/zh_CN/vm/active_mm.rst deleted file mode 100644 index 366609ea4f37..000000000000 --- a/Documentation/translations/zh_CN/vm/active_mm.rst +++ /dev/null @@ -1,85 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/active_mm.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -========= -Active MM -========= - -这是一封linux之父回复开发者的一封邮件，所以翻译时我尽量保持邮件格式的完整。 - -:: - - List: linux-kernel - Subject: Re: active_mm - From: Linus Torvalds - Date: 1999-07-30 21:36:24 - - 因为我并不经常写解释，所以已经抄送到linux-kernel邮件列表，而当我做这些， - 且更多的人在阅读它们时，我觉得棒极了。 - - 1999年7月30日 星期五， David Mosberger 写道： - > - > 是否有一个简短的描述，说明task_struct中的 - > "mm" 和 "active_mm"应该如何使用？ (如果 - > 这个问题在邮件列表中讨论过，我表示歉意--我刚 - > 刚度假回来，有一段时间没能关注linux-kernel了）。 - - 基本上，新的设定是： - - - 我们有“真实地址空间”和“匿名地址空间”。区别在于，匿名地址空间根本不关心用 - 户级页表，所以当我们做上下文切换到匿名地址空间时，我们只是让以前的地址空间 - 处于活动状态。 - - 一个“匿名地址空间”的明显用途是任何不需要任何用户映射的线程--所有的内核线 - 程基本上都属于这一类，但即使是“真正的”线程也可以暂时说在一定时间内它们不 - 会对用户空间感兴趣，调度器不妨试着避免在切换VM状态上浪费时间。目前只有老 - 式的bdflush sync能做到这一点。 - - - “tsk->mm” 指向 “真实地址空间”。对于一个匿名进程来说，tsk->mm将是NULL， - 其逻辑原因是匿名进程实际上根本就 “没有” 真正的地址空间。 - - - 然而，我们显然需要跟踪我们为这样的匿名用户“偷用”了哪个地址空间。为此，我们 - 有 “tsk->active_mm”，它显示了当前活动的地址空间是什么。 - - 规则是，对于一个有真实地址空间的进程（即tsk->mm是 non-NULL），active_mm - 显然必须与真实的mm相同。 - - 对于一个匿名进程，tsk->mm == NULL，而tsk->active_mm是匿名进程运行时 - “借用”的mm。当匿名进程被调度走时，借用的地址空间被返回并清除。 - - 为了支持所有这些，“struct mm_struct”现在有两个计数器：一个是 “mm_users” - 计数器，即有多少 “真正的地址空间用户”，另一个是 “mm_count”计数器，即 “lazy” - 用户（即匿名用户）的数量，如果有任何真正的用户，则加1。 - - 通常情况下，至少有一个真正的用户，但也可能是真正的用户在另一个CPU上退出，而 - 一个lazy的用户仍在活动，所以你实际上得到的情况是，你有一个地址空间 **只** - 被lazy的用户使用。这通常是一个短暂的生命周期状态，因为一旦这个线程被安排给一 - 个真正的线程，这个 “僵尸” mm就会被释放，因为 “mm_count”变成了零。 - - 另外，一个新的规则是，**没有人** 再把 “init_mm” 作为一个真正的MM了。 - “init_mm”应该被认为只是一个 “没有其他上下文时的lazy上下文”，事实上，它主 - 要是在启动时使用，当时还没有真正的VM被创建。因此，用来检查的代码 - - if (current->mm == &init_mm) - - 一般来说，应该用 - - if (!current->mm) - - 取代上面的写法（这更有意义--测试基本上是 “我们是否有一个用户环境”，并且通常 - 由缺页异常处理程序和类似的东西来完成）。 - - 总之，我刚才在ftp.kernel.org上放了一个pre-patch-2.3.13-1，因为它稍微改 - 变了接口以适配alpha（谁会想到呢，但alpha体系结构上下文切换代码实际上最终是 - 最丑陋的之一--不像其他架构的MM和寄存器状态是分开的，alpha的PALcode将两者 - 连接起来，你需要同时切换两者）。 - - (文档来源 http://marc.info/?l=linux-kernel&m=93337278602211&w=2) diff --git a/Documentation/translations/zh_CN/vm/balance.rst b/Documentation/translations/zh_CN/vm/balance.rst deleted file mode 100644 index e98a47ef24a8..000000000000 --- a/Documentation/translations/zh_CN/vm/balance.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/balance.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -======== -内存平衡 -======== - -2000年1月开始，作者：Kanoj Sarcar - -对于 !__GFP_HIGH 和 !__GFP_KSWAPD_RECLAIM 以及非 __GFP_IO 的分配，需要进行 -内存平衡。 - -调用者避免回收的第一个原因是调用者由于持有自旋锁或处于中断环境中而无法睡眠。第二个 -原因可能是，调用者愿意在不产生页面回收开销的情况下分配失败。这可能发生在有0阶回退 -选项的机会主义高阶分配请求中。在这种情况下，调用者可能也希望避免唤醒kswapd。 - -__GFP_IO分配请求是为了防止文件系统死锁。 - -在没有非睡眠分配请求的情况下，做平衡似乎是有害的。页面回收可以被懒散地启动，也就是 -说，只有在需要的时候（也就是区域的空闲内存为0），而不是让它成为一个主动的过程。 - -也就是说，内核应该尝试从直接映射池中满足对直接映射页的请求，而不是回退到dma池中， -这样就可以保持dma池为dma请求（不管是不是原子的）所填充。类似的争论也适用于高内存 -和直接映射的页面。相反，如果有很多空闲的dma页，最好是通过从dma池中分配一个来满足 -常规的内存请求，而不是产生常规区域平衡的开销。 - -在2.2中，只有当空闲页总数低于总内存的1/64时，才会启动内存平衡/页面回收。如果dma -和常规内存的比例合适，即使dma区完全空了，也很可能不会进行平衡。2.2已经在不同内存 -大小的生产机器上运行，即使有这个问题存在，似乎也做得不错。在2.3中，由于HIGHMEM的 -存在，这个问题变得更加严重。 - -在2.3中，区域平衡可以用两种方式之一来完成：根据区域的大小（可能是低级区域的大小）， -我们可以在初始化阶段决定在平衡任何区域时应该争取多少空闲页。好的方面是，在平衡的时 -候，我们不需要看低级区的大小，坏的方面是，我们可能会因为忽略低级区可能较低的使用率 -而做过于频繁的平衡。另外，只要对分配程序稍作修改，就有可能将memclass()宏简化为一 -个简单的等式。 - -另一个可能的解决方案是，我们只在一个区 **和** 其所有低级区的空闲内存低于该区及其 -低级区总内存的1/64时进行平衡。这就解决了2.2的平衡问题，并尽可能地保持了与2.2行为 -的接近。另外，平衡算法在各种架构上的工作方式也是一样的，这些架构有不同数量和类型的 -内存区。如果我们想变得更花哨一点，我们可以在未来为不同区域的自由页面分配不同的权重。 - -请注意，如果普通区的大小与dma区相比是巨大的，那么在决定是否平衡普通区的时候，考虑 -空闲的dma页就变得不那么重要了。那么第一个解决方案就变得更有吸引力。 - -所附的补丁实现了第二个解决方案。它还 “修复”了两个问题：首先，在低内存条件下，kswapd -被唤醒，就像2.2中的非睡眠分配。第二，HIGHMEM区也被平衡了，以便给replace_with_highmem() -一个争取获得HIGHMEM页的机会，同时确保HIGHMEM分配不会落回普通区。这也确保了HIGHMEM -页不会被泄露（例如，在一个HIGHMEM页在交换缓存中但没有被任何人使用的情况下）。 - -kswapd还需要知道它应该平衡哪些区。kswapd主要是在无法进行平衡的情况下需要的，可能 -是因为所有的分配请求都来自中断上下文，而所有的进程上下文都在睡眠。对于2.3， -kswapd并不真正需要平衡高内存区，因为中断上下文并不请求高内存页。kswapd看zone -结构体中的zone_wake_kswapd字段来决定一个区是否需要平衡。 - -如果从进程内存和shm中偷取页面可以减轻该页面节点中任何区的内存压力，而该区的内存压力 -已经低于其水位，则会进行偷取。 - -watemark[WMARK_MIN/WMARK_LOW/WMARK_HIGH]/low_on_memory/zone_wake_kswapd： -这些是每个区的字段，用于确定一个区何时需要平衡。当页面数低于水位[WMARK_MIN]时， -hysteric 的字段low_on_memory被设置。这个字段会一直被设置，直到空闲页数变成水位 -[WMARK_HIGH]。当low_on_memory被设置时，页面分配请求将尝试释放该区域的一些页面（如果 -请求中设置了GFP_WAIT）。与此相反的是，决定唤醒kswapd以释放一些区的页。这个决定不是基于 -hysteresis 的，而是当空闲页的数量低于watermark[WMARK_LOW]时就会进行；在这种情况下， -zone_wake_kswapd也被设置。 - - -我所听到的（超棒的）想法： - -1. 动态经历应该影响平衡：可以跟踪一个区的失败请求的数量，并反馈到平衡方案中（jalvo@mbay.net）。 - -2. 实现一个类似于replace_with_highmem()的replace_with_regular()，以保留dma页面。 - (lkd@tantalophile.demon.co.uk) diff --git a/Documentation/translations/zh_CN/vm/damon/api.rst b/Documentation/translations/zh_CN/vm/damon/api.rst deleted file mode 100644 index 21143eea4ebe..000000000000 --- a/Documentation/translations/zh_CN/vm/damon/api.rst +++ /dev/null @@ -1,32 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -:Original: Documentation/vm/damon/api.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -======= -API参考 -======= - -内核空间的程序可以使用下面的API来使用DAMON的每个功能。你所需要做的就是引用 ``damon.h`` ， -它位于源代码树的include/linux/。 - -结构体 -====== - -该API在以下内核代码中: - -include/linux/damon.h - - -函数 -==== - -该API在以下内核代码中: - -mm/damon/core.c diff --git a/Documentation/translations/zh_CN/vm/damon/design.rst b/Documentation/translations/zh_CN/vm/damon/design.rst deleted file mode 100644 index 46128b77c2b3..000000000000 --- a/Documentation/translations/zh_CN/vm/damon/design.rst +++ /dev/null @@ -1,140 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -:Original: Documentation/vm/damon/design.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -==== -设计 -==== - -可配置的层 -========== - -DAMON提供了数据访问监控功能，同时使其准确性和开销可控。基本的访问监控需要依赖于目标地址空间 -并为之优化的基元。另一方面，作为DAMON的核心，准确性和开销的权衡机制是在纯逻辑空间中。DAMON -将这两部分分离在不同的层中，并定义了它的接口，以允许各种低层次的基元实现与核心逻辑的配置。 - -由于这种分离的设计和可配置的接口，用户可以通过配置核心逻辑和适当的低级基元实现来扩展DAMON的 -任何地址空间。如果没有提供合适的，用户可以自己实现基元。 - -例如，物理内存、虚拟内存、交换空间、那些特定的进程、NUMA节点、文件和支持的内存设备将被支持。 -另外，如果某些架构或设备支持特殊的优化访问检查基元，这些基元将很容易被配置。 - - -特定地址空间基元的参考实现 -========================== - -基本访问监测的低级基元被定义为两部分。: - -1. 确定地址空间的监测目标地址范围 -2. 目标空间中特定地址范围的访问检查。 - -DAMON目前为物理和虚拟地址空间提供了基元的实现。下面两个小节描述了这些工作的方式。 - - -基于VMA的目标地址范围构造 -------------------------- - -这仅仅是针对虚拟地址空间基元的实现。对于物理地址空间，只是要求用户手动设置监控目标地址范围。 - -在进程的超级巨大的虚拟地址空间中，只有小部分被映射到物理内存并被访问。因此，跟踪未映射的地 -址区域只是一种浪费。然而，由于DAMON可以使用自适应区域调整机制来处理一定程度的噪声，所以严 -格来说，跟踪每一个映射并不是必须的，但在某些情况下甚至会产生很高的开销。也就是说，监测目标 -内部过于巨大的未映射区域应该被移除，以不占用自适应机制的时间。 - -出于这个原因，这个实现将复杂的映射转换为三个不同的区域，覆盖地址空间的每个映射区域。这三个 -区域之间的两个空隙是给定地址空间中两个最大的未映射区域。这两个最大的未映射区域是堆和最上面 -的mmap()区域之间的间隙，以及在大多数情况下最下面的mmap()区域和堆之间的间隙。因为这些间隙 -在通常的地址空间中是异常巨大的，排除这些间隙就足以做出合理的权衡。下面详细说明了这一点:: - - - - - (small mmap()-ed regions and munmap()-ed regions) - - - - - -基于PTE访问位的访问检查 ------------------------ - -物理和虚拟地址空间的实现都使用PTE Accessed-bit进行基本访问检查。唯一的区别在于从地址中 -找到相关的PTE访问位的方式。虚拟地址的实现是为该地址的目标任务查找页表，而物理地址的实现则 -是查找与该地址有映射关系的每一个页表。通过这种方式，实现者找到并清除下一个采样目标地址的位， -并检查该位是否在一个采样周期后再次设置。这可能会干扰其他使用访问位的内核子系统，即空闲页跟 -踪和回收逻辑。为了避免这种干扰，DAMON使其与空闲页面跟踪相互排斥，并使用 ``PG_idle`` 和 -``PG_young`` 页面标志来解决与回收逻辑的冲突，就像空闲页面跟踪那样。 - - -独立于地址空间的核心机制 -======================== - -下面四个部分分别描述了DAMON的核心机制和五个监测属性，即 ``采样间隔`` 、 ``聚集间隔`` 、 -``更新间隔`` 、 ``最小区域数`` 和 ``最大区域数`` 。 - - -访问频率监测 ------------- - -DAMON的输出显示了在给定的时间内哪些页面的访问频率是多少。访问频率的分辨率是通过设置 -``采样间隔`` 和 ``聚集间隔`` 来控制的。详细地说，DAMON检查每个 ``采样间隔`` 对每 -个页面的访问，并将结果汇总。换句话说，计算每个页面的访问次数。在每个 ``聚合间隔`` 过 -去后，DAMON调用先前由用户注册的回调函数，以便用户可以阅读聚合的结果，然后再清除这些结 -果。这可以用以下简单的伪代码来描述:: - - while monitoring_on: - for page in monitoring_target: - if accessed(page): - nr_accesses[page] += 1 - if time() % aggregation_interval == 0: - for callback in user_registered_callbacks: - callback(monitoring_target, nr_accesses) - for page in monitoring_target: - nr_accesses[page] = 0 - sleep(sampling interval) - -这种机制的监测开销将随着目标工作负载规模的增长而任意增加。 - - -基于区域的抽样调查 ------------------- - -为了避免开销的无限制增加，DAMON将假定具有相同访问频率的相邻页面归入一个区域。只要保持 -这个假设（一个区域内的页面具有相同的访问频率），该区域内就只需要检查一个页面。因此，对 -于每个 ``采样间隔`` ，DAMON在每个区域中随机挑选一个页面，等待一个 ``采样间隔`` ，检 -查该页面是否同时被访问，如果被访问则增加该区域的访问频率。因此，监测开销是可以通过设置 -区域的数量来控制的。DAMON允许用户设置最小和最大的区域数量来进行权衡。 - -然而，如果假设没有得到保证，这个方案就不能保持输出的质量。 - - -适应性区域调整 --------------- - -即使最初的监测目标区域被很好地构建以满足假设（同一区域内的页面具有相似的访问频率），数 -据访问模式也会被动态地改变。这将导致监测质量下降。为了尽可能地保持假设，DAMON根据每个 -区域的访问频率自适应地进行合并和拆分。 - -对于每个 ``聚集区间`` ，它比较相邻区域的访问频率，如果频率差异较小，就合并这些区域。 -然后，在它报告并清除每个区域的聚合接入频率后，如果区域总数不超过用户指定的最大区域数， -它将每个区域拆分为两个或三个区域。 - -通过这种方式，DAMON提供了其最佳的质量和最小的开销，同时保持了用户为其权衡设定的界限。 - - -动态目标空间更新处理 --------------------- - -监测目标地址范围可以动态改变。例如，虚拟内存可以动态地被映射和解映射。物理内存可以被 -热插拔。 - -由于在某些情况下变化可能相当频繁，DAMON允许监控操作检查动态变化，包括内存映射变化， -并仅在用户指定的时间间隔（ ``更新间隔`` ）中的每个时间段，将其应用于监控操作相关的 -数据结构，如抽象的监控目标内存区。 \ No newline at end of file diff --git a/Documentation/translations/zh_CN/vm/damon/faq.rst b/Documentation/translations/zh_CN/vm/damon/faq.rst deleted file mode 100644 index 07b4ac19407d..000000000000 --- a/Documentation/translations/zh_CN/vm/damon/faq.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -:Original: Documentation/vm/damon/faq.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -======== -常见问题 -======== - -为什么是一个新的子系统，而不是扩展perf或其他用户空间工具？ -========================================================== - -首先，因为它需要尽可能的轻量级，以便可以在线使用，所以应该避免任何不必要的开销，如内核-用户 -空间的上下文切换成本。第二，DAMON的目标是被包括内核在内的其他程序所使用。因此，对特定工具 -（如perf）的依赖性是不可取的。这就是DAMON在内核空间实现的两个最大的原因。 - - -“闲置页面跟踪” 或 “perf mem” 可以替代DAMON吗？ -============================================== - -闲置页跟踪是物理地址空间访问检查的一个低层次的原始方法。“perf mem”也是类似的，尽管它可以 -使用采样来减少开销。另一方面，DAMON是一个更高层次的框架，用于监控各种地址空间。它专注于内 -存管理优化，并提供复杂的精度/开销处理机制。因此，“空闲页面跟踪” 和 “perf mem” 可以提供 -DAMON输出的一个子集，但不能替代DAMON。 - - -DAMON是否只支持虚拟内存？ -========================= - -不，DAMON的核心是独立于地址空间的。用户可以在DAMON核心上实现和配置特定地址空间的低级原始 -部分，包括监测目标区域的构造和实际的访问检查。通过这种方式，DAMON用户可以用任何访问检查技 -术来监测任何地址空间。 - -尽管如此，DAMON默认为虚拟内存和物理内存提供了基于vma/rmap跟踪和PTE访问位检查的地址空间 -相关功能的实现，以供参考和方便使用。 - - -我可以简单地监测页面的粒度吗？ -============================== - -是的，你可以通过设置 ``min_nr_regions`` 属性高于工作集大小除以页面大小的值来实现。 -因为监视目标区域的大小被强制为 ``>=page size`` ，所以区域分割不会产生任何影响。 diff --git a/Documentation/translations/zh_CN/vm/damon/index.rst b/Documentation/translations/zh_CN/vm/damon/index.rst deleted file mode 100644 index 84d36d90c9b0..000000000000 --- a/Documentation/translations/zh_CN/vm/damon/index.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -:Original: Documentation/vm/damon/index.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -========================== -DAMON:数据访问监视器 -========================== - -DAMON是Linux内核的一个数据访问监控框架子系统。DAMON的核心机制使其成为 -（该核心机制详见(Documentation/translations/zh_CN/vm/damon/design.rst)） - - - *准确度* （监测输出对DRAM级别的内存管理足够有用；但可能不适合CPU Cache级别）， - - *轻量级* （监控开销低到可以在线应用），以及 - - *可扩展* （无论目标工作负载的大小，开销的上限值都在恒定范围内）。 - -因此，利用这个框架，内核的内存管理机制可以做出高级决策。会导致高数据访问监控开销的实 -验性内存管理优化工作可以再次进行。同时，在用户空间，有一些特殊工作负载的用户可以编写 -个性化的应用程序，以便更好地了解和优化他们的工作负载和系统。 - -.. toctree:: - :maxdepth: 2 - - faq - design - api - diff --git a/Documentation/translations/zh_CN/vm/free_page_reporting.rst b/Documentation/translations/zh_CN/vm/free_page_reporting.rst deleted file mode 100644 index 31d6c34b956b..000000000000 --- a/Documentation/translations/zh_CN/vm/free_page_reporting.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/_free_page_reporting.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - -========== -空闲页报告 -========== - -空闲页报告是一个API，设备可以通过它来注册接收系统当前未使用的页面列表。这在虚拟 -化的情况下是很有用的，客户机能够使用这些数据来通知管理器它不再使用内存中的某些页 -面。 - -对于驱动，通常是气球驱动要使用这个功能，它将分配和初始化一个page_reporting_dev_info -结构体。它要填充的结构体中的字段是用于处理散点列表的 "report" 函数指针。它还必 -须保证每次调用该函数时能处理至少相当于PAGE_REPORTING_CAPACITY的散点列表条目。 -假设没有其他页面报告设备已经注册， 对page_reporting_register的调用将向报告框 -架注册页面报告接口。 - -一旦注册，页面报告API将开始向驱动报告成批的页面。API将在接口被注册后2秒开始报告 -页面，并在任何足够高的页面被释放之后2秒继续报告。 - -报告的页面将被存储在传递给报告函数的散列表中，最后一个条目的结束位被设置在条目 -nent-1中。 当页面被报告函数处理时，分配器将无法访问它们。一旦报告函数完成，这些 -页将被返回到它们所获得的自由区域。 - -在移除使用空闲页报告的驱动之前，有必要调用page_reporting_unregister，以移除 -目前被空闲页报告使用的page_reporting_dev_info结构体。这样做将阻止进一步的报 -告通过该接口发出。如果另一个驱动或同一驱动被注册，它就有可能恢复前一个驱动在报告 -空闲页方面的工作。 - - -Alexander Duyck, 2019年12月04日 diff --git a/Documentation/translations/zh_CN/vm/frontswap.rst b/Documentation/translations/zh_CN/vm/frontswap.rst deleted file mode 100644 index 3eb07870e2ef..000000000000 --- a/Documentation/translations/zh_CN/vm/frontswap.rst +++ /dev/null @@ -1,196 +0,0 @@ -:Original: Documentation/vm/_free_page_reporting.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - -========= -Frontswap -========= - -Frontswap为交换页提供了一个 “transcendent memory” 的接口。在一些环境中，由 -于交换页被保存在RAM（或类似RAM的设备）中，而不是交换磁盘，因此可以获得巨大的性能 -节省（提高）。 - -.. _Transcendent memory in a nutshell: https://lwn.net/Articles/454795/ - -Frontswap之所以这么命名，是因为它可以被认为是与swap设备的“back”存储相反。存 -储器被认为是一个同步并发安全的面向页面的“伪RAM设备”，符合transcendent memory -（如Xen的“tmem”，或内核内压缩内存，又称“zcache”，或未来的类似RAM的设备）的要 -求；这个伪RAM设备不能被内核直接访问或寻址，其大小未知且可能随时间变化。驱动程序通过 -调用frontswap_register_ops将自己与frontswap链接起来，以适当地设置frontswap_ops -的功能，它提供的功能必须符合某些策略，如下所示: - -一个 “init” 将设备准备好接收与指定的交换设备编号（又称“类型”）相关的frontswap -交换页。一个 “store” 将把该页复制到transcendent memory，并与该页的类型和偏移 -量相关联。一个 “load” 将把该页，如果找到的话，从transcendent memory复制到内核 -内存，但不会从transcendent memory中删除该页。一个 “invalidate_page” 将从 -transcendent memory中删除该页，一个 “invalidate_area” 将删除所有与交换类型 -相关的页（例如，像swapoff）并通知 “device” 拒绝进一步存储该交换类型。 - -一旦一个页面被成功存储，在该页面上的匹配加载通常会成功。因此，当内核发现自己处于需 -要交换页面的情况时，它首先尝试使用frontswap。如果存储的结果是成功的，那么数据就已 -经成功的保存到了transcendent memory中，并且避免了磁盘写入，如果后来再读回数据， -也避免了磁盘读取。如果存储返回失败，transcendent memory已经拒绝了该数据，且该页 -可以像往常一样被写入交换空间。 - -请注意，如果一个页面被存储，而该页面已经存在于transcendent memory中（一个 “重复” -的存储），要么存储成功，数据被覆盖，要么存储失败，该页面被废止。这确保了旧的数据永远 -不会从frontswap中获得。 - -如果配置正确，对frontswap的监控是通过 `/sys/kernel/debug/frontswap` 目录下的 -debugfs完成的。frontswap的有效性可以通过以下方式测量（在所有交换设备中）: - -``failed_stores`` - 有多少次存储的尝试是失败的 - -``loads`` - 尝试了多少次加载（应该全部成功） - -``succ_stores`` - 有多少次存储的尝试是成功的 - -``invalidates`` - 尝试了多少次作废 - -后台实现可以提供额外的指标。 - -经常问到的问题 -============== - -* 价值在哪里? - -当一个工作负载开始交换时，性能就会下降。Frontswap通过提供一个干净的、动态的接口来 -读取和写入交换页到 “transcendent memory”，从而大大增加了许多这样的工作负载的性 -能，否则内核是无法直接寻址的。当数据被转换为不同的形式和大小（比如压缩）或者被秘密 -移动（对于一些类似RAM的设备来说，这可能对写平衡很有用）时，这个接口是理想的。交换 -页（和被驱逐的页面缓存页）是这种比RAM慢但比磁盘快得多的“伪RAM设备”的一大用途。 - -Frontswap对内核的影响相当小，为各种系统配置中更动态、更灵活的RAM利用提供了巨大的 -灵活性： - -在单一内核的情况下，又称“zcache”，页面被压缩并存储在本地内存中，从而增加了可以安 -全保存在RAM中的匿名页面总数。Zcache本质上是用压缩/解压缩的CPU周期换取更好的内存利 -用率。Benchmarks测试显示，当内存压力较低时，几乎没有影响，而在高内存压力下的一些 -工作负载上，则有明显的性能改善（25%以上）。 - -“RAMster” 在zcache的基础上增加了对集群系统的 “peer-to-peer” transcendent memory -的支持。Frontswap页面像zcache一样被本地压缩，但随后被“remotified” 到另一个系 -统的RAM。这使得RAM可以根据需要动态地来回负载平衡，也就是说，当系统A超载时，它可以 -交换到系统B，反之亦然。RAMster也可以被配置成一个内存服务器，因此集群中的许多服务器 -可以根据需要动态地交换到配置有大量内存的单一服务器上......而不需要预先配置每个客户 -有多少内存可用 - -在虚拟情况下，虚拟化的全部意义在于统计地将物理资源在多个虚拟机的不同需求之间进行复 -用。对于RAM来说，这真的很难做到，而且在不改变内核的情况下，要做好这一点的努力基本上 -是失败的（除了一些广为人知的特殊情况下的工作负载）。具体来说，Xen Transcendent Memory -后端允许管理器拥有的RAM “fallow”，不仅可以在多个虚拟机之间进行“time-shared”， -而且页面可以被压缩和重复利用，以优化RAM的利用率。当客户操作系统被诱导交出未充分利用 -的RAM时（如 “selfballooning”），突然出现的意外内存压力可能会导致交换；frontswap -允许这些页面被交换到管理器RAM中或从管理器RAM中交换（如果整体主机系统内存条件允许）， -从而减轻计划外交换可能带来的可怕的性能影响。 - -一个KVM的实现正在进行中，并且已经被RFC'ed到lkml。而且，利用frontswap，对NVM作为 -内存扩展技术的调查也在进行中。 - -* 当然，在某些情况下可能有性能上的优势，但frontswap的空间/时间开销是多少？ - -如果 CONFIG_FRONTSWAP 被禁用，每个 frontswap 钩子都会编译成空，唯一的开销是每 -个 swapon'ed swap 设备的几个额外字节。如果 CONFIG_FRONTSWAP 被启用，但没有 -frontswap的 “backend” 寄存器，每读或写一个交换页就会有一个额外的全局变量，而不 -是零。如果 CONFIG_FRONTSWAP 被启用，并且有一个frontswap的backend寄存器，并且 -后端每次 “store” 请求都失败（即尽管声称可能，但没有提供内存），CPU 的开销仍然可以 -忽略不计 - 因为每次frontswap失败都是在交换页写到磁盘之前，系统很可能是 I/O 绑定 -的，无论如何使用一小部分的 CPU 都是不相关的。 - -至于空间，如果CONFIG_FRONTSWAP被启用，并且有一个frontswap的backend注册，那么 -每个交换设备的每个交换页都会被分配一个比特。这是在内核已经为每个交换设备的每个交换 -页分配的8位（在2.6.34之前是16位）上增加的。(Hugh Dickins观察到，frontswap可能 -会偷取现有的8个比特，但是我们以后再来担心这个小的优化问题)。对于标准的4K页面大小的 -非常大的交换盘（这很罕见），这是每32GB交换盘1MB开销。 - -当交换页存储在transcendent memory中而不是写到磁盘上时，有一个副作用，即这可能会 -产生更多的内存压力，有可能超过其他的优点。一个backend，比如zcache，必须实现策略 -来仔细（但动态地）管理内存限制，以确保这种情况不会发生。 - -* 好吧，那就用内核骇客能理解的术语来快速概述一下这个frontswap补丁的作用如何？ - -我们假设在内核初始化过程中，一个frontswap 的 “backend” 已经注册了；这个注册表 -明这个frontswap 的 “backend” 可以访问一些不被内核直接访问的“内存”。它到底提 -供了多少内存是完全动态和随机的。 - -每当一个交换设备被交换时，就会调用frontswap_init()，把交换设备的编号（又称“类 -型”）作为一个参数传给它。这就通知了frontswap，以期待 “store” 与该号码相关的交 -换页的尝试。 - -每当交换子系统准备将一个页面写入交换设备时（参见swap_writepage()），就会调用 -frontswap_store。Frontswap与frontswap backend协商，如果backend说它没有空 -间，frontswap_store返回-1，内核就会照常把页换到交换设备上。注意，来自frontswap -backend的响应对内核来说是不可预测的；它可能选择从不接受一个页面，可能接受每九个 -页面，也可能接受每一个页面。但是如果backend确实接受了一个页面，那么这个页面的数 -据已经被复制并与类型和偏移量相关联了，而且backend保证了数据的持久性。在这种情况 -下，frontswap在交换设备的“frontswap_map” 中设置了一个位，对应于交换设备上的 -页面偏移量，否则它就会将数据写入该设备。 - -当交换子系统需要交换一个页面时（swap_readpage()），它首先调用frontswap_load()， -检查frontswap_map，看这个页面是否早先被frontswap backend接受。如果是，该页 -的数据就会从frontswap后端填充，换入就完成了。如果不是，正常的交换代码将被执行， -以便从真正的交换设备上获得这一页的数据。 - -所以每次frontswap backend接受一个页面时，交换设备的读取和（可能）交换设备的写 -入都被 “frontswap backend store” 和（可能）“frontswap backend loads” -所取代，这可能会快得多。 - -* frontswap不能被配置为一个 “特殊的” 交换设备，它的优先级要高于任何真正的交换 - 设备（例如像zswap，或者可能是swap-over-nbd/NFS）？ - -首先，现有的交换子系统不允许有任何种类的交换层次结构。也许它可以被重写以适应层次 -结构，但这将需要相当大的改变。即使它被重写，现有的交换子系统也使用了块I/O层，它 -假定交换设备是固定大小的，其中的任何页面都是可线性寻址的。Frontswap几乎没有触 -及现有的交换子系统，而是围绕着块I/O子系统的限制，提供了大量的灵活性和动态性。 - -例如，frontswap backend对任何交换页的接受是完全不可预测的。这对frontswap backend -的定义至关重要，因为它赋予了backend完全动态的决定权。在zcache中，人们无法预 -先知道一个页面的可压缩性如何。可压缩性 “差” 的页面会被拒绝，而 “差” 本身也可 -以根据当前的内存限制动态地定义。 - -此外，frontswap是完全同步的，而真正的交换设备，根据定义，是异步的，并且使用 -块I/O。块I/O层不仅是不必要的，而且可能进行 “优化”，这对面向RAM的设备来说是 -不合适的，包括将一些页面的写入延迟相当长的时间。同步是必须的，以确保后端的动 -态性，并避免棘手的竞争条件，这将不必要地大大增加frontswap和/或块I/O子系统的 -复杂性。也就是说，只有最初的 “store” 和 “load” 操作是需要同步的。一个独立 -的异步线程可以自由地操作由frontswap存储的页面。例如，RAMster中的 “remotification” -线程使用标准的异步内核套接字，将压缩的frontswap页面移动到远程机器。同样， -KVM的客户方实现可以进行客户内压缩，并使用 “batched” hypercalls。 - -在虚拟化环境中，动态性允许管理程序（或主机操作系统）做“intelligent overcommit”。 -例如，它可以选择只接受页面，直到主机交换可能即将发生，然后强迫客户机做他们 -自己的交换。 - -transcendent memory规格的frontswap有一个坏处。因为任何 “store” 都可 -能失败，所以必须在一个真正的交换设备上有一个真正的插槽来交换页面。因此， -frontswap必须作为每个交换设备的 “影子” 来实现，它有可能容纳交换设备可能 -容纳的每一个页面，也有可能根本不容纳任何页面。这意味着frontswap不能包含比 -swap设备总数更多的页面。例如，如果在某些安装上没有配置交换设备，frontswap -就没有用。无交换设备的便携式设备仍然可以使用frontswap，但是这种设备的 -backend必须配置某种 “ghost” 交换设备，并确保它永远不会被使用。 - - -* 为什么会有这种关于 “重复存储” 的奇怪定义？如果一个页面以前被成功地存储过， - 难道它不能总是被成功地覆盖吗？ - -几乎总是可以的，不，有时不能。考虑一个例子，数据被压缩了，原来的4K页面被压 -缩到了1K。现在，有人试图用不可压缩的数据覆盖该页，因此会占用整个4K。但是 -backend没有更多的空间了。在这种情况下，这个存储必须被拒绝。每当frontswap -拒绝一个会覆盖的存储时，它也必须使旧的数据作废，并确保它不再被访问。因为交 -换子系统会把新的数据写到读交换设备上，这是确保一致性的正确做法。 - -* 为什么frontswap补丁会创建新的头文件swapfile.h？ - -frontswap代码依赖于一些swap子系统内部的数据结构，这些数据结构多年来一直 -在静态和全局之间来回移动。这似乎是一个合理的妥协：将它们定义为全局，但在一 -个新的包含文件中声明它们，该文件不被包含swap.h的大量源文件所包含。 - -Dan Magenheimer，最后更新于2012年4月9日 diff --git a/Documentation/translations/zh_CN/vm/highmem.rst b/Documentation/translations/zh_CN/vm/highmem.rst deleted file mode 100644 index 018838e58c3e..000000000000 --- a/Documentation/translations/zh_CN/vm/highmem.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/highmem.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - -========== -高内存处理 -========== - -作者: Peter Zijlstra - -.. contents:: :local: - -高内存是什么？ -============== - -当物理内存的大小接近或超过虚拟内存的最大大小时，就会使用高内存（highmem）。在这一点上，内 -核不可能在任何时候都保持所有可用的物理内存的映射。这意味着内核需要开始使用它想访问的物理内 -存的临时映射。 - -没有被永久映射覆盖的那部分（物理）内存就是我们所说的 "高内存"。对于这个边界的确切位置，有 -各种架构上的限制。 - -例如，在i386架构中，我们选择将内核映射到每个进程的虚拟空间，这样我们就不必为内核的进入/退 -出付出全部的TLB作废代价。这意味着可用的虚拟内存空间（i386上为4GiB）必须在用户和内核空间之 -间进行划分。 - -使用这种方法的架构的传统分配方式是3:1，3GiB用于用户空间，顶部的1GiB用于内核空间。:: - - +--------+ 0xffffffff - | Kernel | - +--------+ 0xc0000000 - | | - | User | - | | - +--------+ 0x00000000 - -这意味着内核在任何时候最多可以映射1GiB的物理内存，但是由于我们需要虚拟地址空间来做其他事 -情--包括访问其余物理内存的临时映射--实际的直接映射通常会更少（通常在~896MiB左右）。 - -其他有mm上下文标签的TLB的架构可以有独立的内核和用户映射。然而，一些硬件（如一些ARM）在使 -用mm上下文标签时，其虚拟空间有限。 - - -临时虚拟映射 -============ - -内核包含几种创建临时映射的方法。: - -* vmap(). 这可以用来将多个物理页长期映射到一个连续的虚拟空间。它需要synchronization - 来解除映射。 - -* kmap(). 这允许对单个页面进行短期映射。它需要synchronization，但在一定程度上被摊销。 - 当以嵌套方式使用时，它也很容易出现死锁，因此不建议在新代码中使用它。 - -* kmap_atomic(). 这允许对单个页面进行非常短的时间映射。由于映射被限制在发布它的CPU上， - 它表现得很好，但发布任务因此被要求留在该CPU上直到它完成，以免其他任务取代它的映射。 - - kmap_atomic() 也可以由中断上下文使用，因为它不睡眠，而且调用者可能在调用kunmap_atomic() - 之后才睡眠。 - - 可以假设k[un]map_atomic()不会失败。 - - -使用kmap_atomic -=============== - -何时何地使用 kmap_atomic() 是很直接的。当代码想要访问一个可能从高内存（见__GFP_HIGHMEM） -分配的页面的内容时，例如在页缓存中的页面，就会使用它。该API有两个函数，它们的使用方式与 -下面类似:: - - /* 找到感兴趣的页面。 */ - struct page *page = find_get_page(mapping, offset); - - /* 获得对该页内容的访问权。 */ - void *vaddr = kmap_atomic(page); - - /* 对该页的内容做一些处理。 */ - memset(vaddr, 0, PAGE_SIZE); - - /* 解除该页面的映射。 */ - kunmap_atomic(vaddr); - -注意，kunmap_atomic()调用的是kmap_atomic()调用的结果而不是参数。 - -如果你需要映射两个页面，因为你想从一个页面复制到另一个页面，你需要保持kmap_atomic调用严 -格嵌套，如:: - - vaddr1 = kmap_atomic(page1); - vaddr2 = kmap_atomic(page2); - - memcpy(vaddr1, vaddr2, PAGE_SIZE); - - kunmap_atomic(vaddr2); - kunmap_atomic(vaddr1); - - -临时映射的成本 -============== - -创建临时映射的代价可能相当高。体系架构必须操作内核的页表、数据TLB和/或MMU的寄存器。 - -如果CONFIG_HIGHMEM没有被设置，那么内核会尝试用一点计算来创建映射，将页面结构地址转换成 -指向页面内容的指针，而不是去捣鼓映射。在这种情况下，解映射操作可能是一个空操作。 - -如果CONFIG_MMU没有被设置，那么就不可能有临时映射和高内存。在这种情况下，也将使用计算方法。 - - -i386 PAE -======== - -在某些情况下，i386 架构将允许你在 32 位机器上安装多达 64GiB 的内存。但这有一些后果: - -* Linux需要为系统中的每个页面建立一个页帧结构，而且页帧需要驻在永久映射中，这意味着： - -* 你最多可以有896M/sizeof(struct page)页帧；由于页结构体是32字节的，所以最终会有 - 112G的页；然而，内核需要在内存中存储更多的页帧...... - -* PAE使你的页表变大--这使系统变慢，因为更多的数据需要在TLB填充等方面被访问。一个好处 - 是，PAE有更多的PTE位，可以提供像NX和PAT这样的高级功能。 - -一般的建议是，你不要在32位机器上使用超过8GiB的空间--尽管更多的空间可能对你和你的工作 -量有用，但你几乎是靠你自己--不要指望内核开发者真的会很关心事情的进展情况。 diff --git a/Documentation/translations/zh_CN/vm/hmm.rst b/Documentation/translations/zh_CN/vm/hmm.rst deleted file mode 100644 index 2379df95aa58..000000000000 --- a/Documentation/translations/zh_CN/vm/hmm.rst +++ /dev/null @@ -1,361 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/hmm.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - -================== -异构内存管理 (HMM) -================== - -提供基础设施和帮助程序以将非常规内存（设备内存，如板上 GPU 内存）集成到常规内核路径中，其 -基石是此类内存的专用struct page（请参阅本文档的第 5 至 7 节）。 - -HMM 还为 SVM（共享虚拟内存）提供了可选的帮助程序，即允许设备透明地访问与 CPU 一致的程序 -地址，这意味着 CPU 上的任何有效指针也是该设备的有效指针。这对于简化高级异构计算的使用变得 -必不可少，其中 GPU、DSP 或 FPGA 用于代表进程执行各种计算。 - -本文档分为以下部分：在第一部分中，我揭示了与使用特定于设备的内存分配器相关的问题。在第二 -部分中，我揭示了许多平台固有的硬件限制。第三部分概述了 HMM 设计。第四部分解释了 CPU 页 -表镜像的工作原理以及 HMM 在这种情况下的目的。第五部分处理内核中如何表示设备内存。最后， -最后一节介绍了一个新的迁移助手，它允许利用设备 DMA 引擎。 - -.. contents:: :local: - -使用特定于设备的内存分配器的问题 -================================ - -具有大量板载内存（几 GB）的设备（如 GPU）历来通过专用驱动程序特定 API 管理其内存。这会 -造成设备驱动程序分配和管理的内存与常规应用程序内存（私有匿名、共享内存或常规文件支持内存） -之间的隔断。从这里开始，我将把这个方面称为分割的地址空间。我使用共享地址空间来指代相反的情况： -即，设备可以透明地使用任何应用程序内存区域。 - -分割的地址空间的发生是因为设备只能访问通过设备特定 API 分配的内存。这意味着从设备的角度来 -看，程序中的所有内存对象并不平等，这使得依赖于广泛的库的大型程序变得复杂。 - -具体来说，这意味着想要利用像 GPU 这样的设备的代码需要在通用分配的内存（malloc、mmap -私有、mmap 共享）和通过设备驱动程序 API 分配的内存之间复制对象（这仍然以 mmap 结束， -但是是设备文件）。 - -对于平面数据集（数组、网格、图像……），这并不难实现，但对于复杂数据集（列表、树……）， -很难做到正确。复制一个复杂的数据集需要重新映射其每个元素之间的所有指针关系。这很容易出错， -而且由于数据集和地址的重复，程序更难调试。 - -分割地址空间也意味着库不能透明地使用它们从核心程序或另一个库中获得的数据，因此每个库可能 -不得不使用设备特定的内存分配器来重复其输入数据集。大型项目会因此受到影响，并因为各种内存 -拷贝而浪费资源。 - -复制每个库的API以接受每个设备特定分配器分配的内存作为输入或输出，并不是一个可行的选择。 -这将导致库入口点的组合爆炸。 - -最后，随着高级语言结构（在 C++ 中，当然也在其他语言中）的进步，编译器现在有可能在没有程 -序员干预的情况下利用 GPU 和其他设备。某些编译器识别的模式仅适用于共享地址空间。对所有 -其他模式，使用共享地址空间也更合理。 - - -I/O 总线、设备内存特性 -====================== - -由于一些限制，I/O 总线削弱了共享地址空间。大多数 I/O 总线只允许从设备到主内存的基本 -内存访问；甚至缓存一致性通常是可选的。从 CPU 访问设备内存甚至更加有限。通常情况下，它 -不是缓存一致的。 - -如果我们只考虑 PCIE 总线，那么设备可以访问主内存（通常通过 IOMMU）并与 CPU 缓存一 -致。但是，它只允许设备对主存储器进行一组有限的原子操作。这在另一个方向上更糟：CPU -只能访问有限范围的设备内存，而不能对其执行原子操作。因此，从内核的角度来看，设备内存不 -能被视为与常规内存等同。 - -另一个严重的因素是带宽有限（约 32GBytes/s，PCIE 4.0 和 16 通道）。这比最快的 GPU -内存 (1 TBytes/s) 慢 33 倍。最后一个限制是延迟。从设备访问主内存的延迟比设备访问自 -己的内存时高一个数量级。 - -一些平台正在开发新的 I/O 总线或对 PCIE 的添加/修改以解决其中一些限制 -（OpenCAPI、CCIX）。它们主要允许 CPU 和设备之间的双向缓存一致性，并允许架构支持的所 -有原子操作。遗憾的是，并非所有平台都遵循这一趋势，并且一些主要架构没有针对这些问题的硬 -件解决方案。 - -因此，为了使共享地址空间有意义，我们不仅必须允许设备访问任何内存，而且还必须允许任何内 -存在设备使用时迁移到设备内存（在迁移时阻止 CPU 访问）。 - - -共享地址空间和迁移 -================== - -HMM 打算提供两个主要功能。第一个是通过复制cpu页表到设备页表中来共享地址空间，因此对 -于进程地址空间中的任何有效主内存地址，相同的地址指向相同的物理内存。 - -为了实现这一点，HMM 提供了一组帮助程序来填充设备页表，同时跟踪 CPU 页表更新。设备页表 -更新不像 CPU 页表更新那么容易。要更新设备页表，您必须分配一个缓冲区（或使用预先分配的 -缓冲区池）并在其中写入 GPU 特定命令以执行更新（取消映射、缓存失效和刷新等）。这不能通 -过所有设备的通用代码来完成。因此，为什么HMM提供了帮助器，在把硬件的具体细节留给设备驱 -动程序的同时，把一切可以考虑的因素都考虑进去了。 - -HMM 提供的第二种机制是一种新的 ZONE_DEVICE 内存，它允许为设备内存的每个页面分配一个 -struct page。这些页面很特殊，因为 CPU 无法映射它们。然而，它们允许使用现有的迁移机 -制将主内存迁移到设备内存，从 CPU 的角度来看，一切看起来都像是换出到磁盘的页面。使用 -struct page可以与现有的 mm 机制进行最简单、最干净的集成。再次，HMM 仅提供帮助程序， -首先为设备内存热插拔新的 ZONE_DEVICE 内存，然后执行迁移。迁移内容和时间的策略决定留 -给设备驱动程序。 - -请注意，任何 CPU 对设备页面的访问都会触发缺页异常并迁移回主内存。例如，当支持给定CPU -地址 A 的页面从主内存页面迁移到设备页面时，对地址 A 的任何 CPU 访问都会触发缺页异常 -并启动向主内存的迁移。 - -凭借这两个特性，HMM 不仅允许设备镜像进程地址空间并保持 CPU 和设备页表同步，而且还通 -过迁移设备正在使用的数据集部分来利用设备内存。 - - -地址空间镜像实现和API -===================== - -地址空间镜像的主要目标是允许将一定范围的 CPU 页表复制到一个设备页表中；HMM 有助于 -保持两者同步。想要镜像进程地址空间的设备驱动程序必须从注册 mmu_interval_notifier -开始:: - - int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub, - struct mm_struct *mm, unsigned long start, - unsigned long length, - const struct mmu_interval_notifier_ops *ops); - -在 ops->invalidate() 回调期间，设备驱动程序必须对范围执行更新操作（将范围标记为只 -读，或完全取消映射等）。设备必须在驱动程序回调返回之前完成更新。 - -当设备驱动程序想要填充一个虚拟地址范围时，它可以使用:: - - int hmm_range_fault(struct hmm_range *range); - -如果请求写访问，它将在丢失或只读条目上触发缺页异常（见下文）。缺页异常使用通用的 mm 缺 -页异常代码路径，就像 CPU 缺页异常一样。 - -这两个函数都将 CPU 页表条目复制到它们的 pfns 数组参数中。该数组中的每个条目对应于虚拟 -范围中的一个地址。HMM 提供了一组标志来帮助驱动程序识别特殊的 CPU 页表项。 - -在 sync_cpu_device_pagetables() 回调中锁定是驱动程序必须尊重的最重要的方面，以保 -持事物正确同步。使用模式是:: - - int driver_populate_range(...) - { - struct hmm_range range; - ... - - range.notifier = &interval_sub; - range.start = ...; - range.end = ...; - range.hmm_pfns = ...; - - if (!mmget_not_zero(interval_sub->notifier.mm)) - return -EFAULT; - - again: - range.notifier_seq = mmu_interval_read_begin(&interval_sub); - mmap_read_lock(mm); - ret = hmm_range_fault(&range); - if (ret) { - mmap_read_unlock(mm); - if (ret == -EBUSY) - goto again; - return ret; - } - mmap_read_unlock(mm); - - take_lock(driver->update); - if (mmu_interval_read_retry(&ni, range.notifier_seq) { - release_lock(driver->update); - goto again; - } - - /* Use pfns array content to update device page table, - * under the update lock */ - - release_lock(driver->update); - return 0; - } - -driver->update 锁与驱动程序在其 invalidate() 回调中使用的锁相同。该锁必须在调用 -mmu_interval_read_retry() 之前保持，以避免与并发 CPU 页表更新发生任何竞争。 - -利用 default_flags 和 pfn_flags_mask -==================================== - -hmm_range 结构有 2 个字段，default_flags 和 pfn_flags_mask，它们指定整个范围 -的故障或快照策略，而不必为 pfns 数组中的每个条目设置它们。 - -例如，如果设备驱动程序需要至少具有读取权限的范围的页面，它会设置:: - - range->default_flags = HMM_PFN_REQ_FAULT; - range->pfn_flags_mask = 0; - -并如上所述调用 hmm_range_fault()。这将填充至少具有读取权限的范围内的所有页面。 - -现在假设驱动程序想要做同样的事情，除了它想要拥有写权限的范围内的一页。现在驱动程序设 -置:: - - range->default_flags = HMM_PFN_REQ_FAULT; - range->pfn_flags_mask = HMM_PFN_REQ_WRITE; - range->pfns[index_of_write] = HMM_PFN_REQ_WRITE; - -有了这个，HMM 将在至少读取（即有效）的所有页面中异常，并且对于地址 -== range->start + (index_of_write << PAGE_SHIFT) 它将异常写入权限，即，如果 -CPU pte 没有设置写权限，那么HMM将调用handle_mm_fault()。 - -hmm_range_fault 完成后，标志位被设置为页表的当前状态，即 HMM_PFN_VALID | 如果页 -面可写，将设置 HMM_PFN_WRITE。 - - -从核心内核的角度表示和管理设备内存 -================================== - -尝试了几种不同的设计来支持设备内存。第一个使用特定于设备的数据结构来保存有关迁移内存 -的信息，HMM 将自身挂接到 mm 代码的各个位置，以处理对设备内存支持的地址的任何访问。 -事实证明，这最终复制了 struct page 的大部分字段，并且还需要更新许多内核代码路径才 -能理解这种新的内存类型。 - -大多数内核代码路径从不尝试访问页面后面的内存，而只关心struct page的内容。正因为如此， -HMM 切换到直接使用 struct page 用于设备内存，这使得大多数内核代码路径不知道差异。 -我们只需要确保没有人试图从 CPU 端映射这些页面。 - -移入和移出设备内存 -================== - -由于 CPU 无法直接访问设备内存，因此设备驱动程序必须使用硬件 DMA 或设备特定的加载/存 -储指令来迁移数据。migrate_vma_setup()、migrate_vma_pages() 和 -migrate_vma_finalize() 函数旨在使驱动程序更易于编写并集中跨驱动程序的通用代码。 - -在将页面迁移到设备私有内存之前，需要创建特殊的设备私有 ``struct page`` 。这些将用 -作特殊的“交换”页表条目，以便 CPU 进程在尝试访问已迁移到设备专用内存的页面时会发生异常。 - -这些可以通过以下方式分配和释放:: - - struct resource *res; - struct dev_pagemap pagemap; - - res = request_free_mem_region(&iomem_resource, /* number of bytes */, - "name of driver resource"); - pagemap.type = MEMORY_DEVICE_PRIVATE; - pagemap.range.start = res->start; - pagemap.range.end = res->end; - pagemap.nr_range = 1; - pagemap.ops = &device_devmem_ops; - memremap_pages(&pagemap, numa_node_id()); - - memunmap_pages(&pagemap); - release_mem_region(pagemap.range.start, range_len(&pagemap.range)); - -还有devm_request_free_mem_region(), devm_memremap_pages(), -devm_memunmap_pages() 和 devm_release_mem_region() 当资源可以绑定到 ``struct device``. - -整体迁移步骤类似于在系统内存中迁移 NUMA 页面(see :ref:`Page migration `) ， -但这些步骤分为设备驱动程序特定代码和共享公共代码: - -1. ``mmap_read_lock()`` - - 设备驱动程序必须将 ``struct vm_area_struct`` 传递给migrate_vma_setup()， - 因此需要在迁移期间保留 mmap_read_lock() 或 mmap_write_lock()。 - -2. ``migrate_vma_setup(struct migrate_vma *args)`` - - 设备驱动初始化了 ``struct migrate_vma`` 的字段，并将该指针传递给 - migrate_vma_setup()。``args->flags`` 字段是用来过滤哪些源页面应该被迁移。 - 例如，设置 ``MIGRATE_VMA_SELECT_SYSTEM`` 将只迁移系统内存，设置 - ``MIGRATE_VMA_SELECT_DEVICE_PRIVATE`` 将只迁移驻留在设备私有内存中的页 - 面。如果后者被设置， ``args->pgmap_owner`` 字段被用来识别驱动所拥有的设备 - 私有页。这就避免了试图迁移驻留在其他设备中的设备私有页。目前，只有匿名的私有VMA - 范围可以被迁移到系统内存和设备私有内存。 - - migrate_vma_setup()所做的第一步是用 ``mmu_notifier_invalidate_range_start()`` - 和 ``mmu_notifier_invalidate_range_end()`` 调用来遍历设备周围的页表，使 - 其他设备的MMU无效，以便在 ``args->src`` 数组中填写要迁移的PFN。 - ``invalidate_range_start()`` 回调传递给一个``struct mmu_notifier_range`` ， - 其 ``event`` 字段设置为MMU_NOTIFY_MIGRATE， ``owner`` 字段设置为传递给 - migrate_vma_setup()的 ``args->pgmap_owner`` 字段。这允许设备驱动跳过无 - 效化回调，只无效化那些实际正在迁移的设备私有MMU映射。这一点将在下一节详细解释。 - - - 在遍历页表时，一个 ``pte_none()`` 或 ``is_zero_pfn()`` 条目导致一个有效 - 的 “zero” PFN 存储在 ``args->src`` 阵列中。这让驱动分配设备私有内存并清 - 除它，而不是复制一个零页。到系统内存或设备私有结构页的有效PTE条目将被 - ``lock_page()``锁定，与LRU隔离（如果系统内存和设备私有页不在LRU上），从进 - 程中取消映射，并插入一个特殊的迁移PTE来代替原来的PTE。 migrate_vma_setup() - 还清除了 ``args->dst`` 数组。 - -3. 设备驱动程序分配目标页面并将源页面复制到目标页面。 - - 驱动程序检查每个 ``src`` 条目以查看该 ``MIGRATE_PFN_MIGRATE`` 位是否已 - 设置并跳过未迁移的条目。设备驱动程序还可以通过不填充页面的 ``dst`` 数组来选 - 择跳过页面迁移。 - - 然后，驱动程序分配一个设备私有 struct page 或一个系统内存页，用 ``lock_page()`` - 锁定该页，并将 ``dst`` 数组条目填入:: - - dst[i] = migrate_pfn(page_to_pfn(dpage)); - - 现在驱动程序知道这个页面正在被迁移，它可以使设备私有 MMU 映射无效并将设备私有 - 内存复制到系统内存或另一个设备私有页面。由于核心 Linux 内核会处理 CPU 页表失 - 效，因此设备驱动程序只需使其自己的 MMU 映射失效。 - - 驱动程序可以使用 ``migrate_pfn_to_page(src[i])`` 来获取源设备的 - ``struct page`` 面，并将源页面复制到目标设备上，如果指针为 ``NULL`` ，意 - 味着源页面没有被填充到系统内存中，则清除目标设备的私有内存。 - -4. ``migrate_vma_pages()`` - - 这一步是实际“提交”迁移的地方。 - - 如果源页是 ``pte_none()`` 或 ``is_zero_pfn()`` 页，这时新分配的页会被插 - 入到CPU的页表中。如果一个CPU线程在同一页面上发生异常，这可能会失败。然而，页 - 表被锁定，只有一个新页会被插入。如果它失去了竞争，设备驱动将看到 - ``MIGRATE_PFN_MIGRATE`` 位被清除。 - - 如果源页被锁定、隔离等，源 ``struct page`` 信息现在被复制到目标 - ``struct page`` ，最终完成CPU端的迁移。 - -5. 设备驱动为仍在迁移的页面更新设备MMU页表，回滚未迁移的页面。 - - 如果 ``src`` 条目仍然有 ``MIGRATE_PFN_MIGRATE`` 位被设置，设备驱动可以 - 更新设备MMU，如果 ``MIGRATE_PFN_WRITE`` 位被设置，则设置写启用位。 - -6. ``migrate_vma_finalize()`` - - 这一步用新页的页表项替换特殊的迁移页表项，并释放对源和目的 ``struct page`` - 的引用。 - -7. ``mmap_read_unlock()`` - - 现在可以释放锁了。 - -独占访问存储器 -============== - -一些设备具有诸如原子PTE位的功能，可以用来实现对系统内存的原子访问。为了支持对一 -个共享的虚拟内存页的原子操作，这样的设备需要对该页的访问是排他的，而不是来自CPU -的任何用户空间访问。 ``make_device_exclusive_range()`` 函数可以用来使一 -个内存范围不能从用户空间访问。 - -这将用特殊的交换条目替换给定范围内的所有页的映射。任何试图访问交换条目的行为都会 -导致一个异常，该异常会通过用原始映射替换该条目而得到恢复。驱动程序会被通知映射已 -经被MMU通知器改变，之后它将不再有对该页的独占访问。独占访问被保证持续到驱动程序 -放弃页面锁和页面引用为止，这时页面上的任何CPU异常都可以按所述进行。 - -内存 cgroup (memcg) 和 rss 统计 -=============================== - -目前，设备内存被视为 rss 计数器中的任何常规页面（如果设备页面用于匿名，则为匿名， -如果设备页面用于文件支持页面，则为文件，如果设备页面用于共享内存，则为 shmem）。 -这是为了保持现有应用程序的故意选择，这些应用程序可能在不知情的情况下开始使用设备 -内存，运行不受影响。 - -一个缺点是 OOM 杀手可能会杀死使用大量设备内存而不是大量常规系统内存的应用程序， -因此不会释放太多系统内存。在决定以不同方式计算设备内存之前，我们希望收集更多关 -于应用程序和系统在存在设备内存的情况下在内存压力下如何反应的实际经验。 - -对内存 cgroup 做出了相同的决定。设备内存页面根据相同的内存 cgroup 计算，常规 -页面将被计算在内。这确实简化了进出设备内存的迁移。这也意味着从设备内存迁移回常规 -内存不会失败，因为它会超过内存 cgroup 限制。一旦我们对设备内存的使用方式及其对 -内存资源控制的影响有了更多的了解，我们可能会在后面重新考虑这个选择。 - -请注意，设备内存永远不能由设备驱动程序或通过 GUP 固定，因此此类内存在进程退出时 -总是被释放的。或者在共享内存或文件支持内存的情况下，当删除最后一个引用时。 diff --git a/Documentation/translations/zh_CN/vm/hugetlbfs_reserv.rst b/Documentation/translations/zh_CN/vm/hugetlbfs_reserv.rst deleted file mode 100644 index c6d471ce2131..000000000000 --- a/Documentation/translations/zh_CN/vm/hugetlbfs_reserv.rst +++ /dev/null @@ -1,436 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/hugetlbfs_reserv.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - -============== -Hugetlbfs 预留 -============== - -概述 -==== - -:ref:`hugetlbpage` 中描述的巨页通常是预先分配给应用程序使用的。如果VMA指 -示要使用巨页，这些巨页会在缺页异常时被实例化到任务的地址空间。如果在缺页异常 -时没有巨页存在，任务就会被发送一个SIGBUS，并经常不高兴地死去。在加入巨页支 -持后不久，人们决定，在mmap()时检测巨页的短缺情况会更好。这个想法是，如果 -没有足够的巨页来覆盖映射，mmap()将失败。这首先是在mmap()时在代码中做一个 -简单的检查，以确定是否有足够的空闲巨页来覆盖映射。就像内核中的大多数东西一 -样，代码随着时间的推移而不断发展。然而，基本的想法是在mmap()时 “预留” -巨页，以确保巨页可以用于该映射中的缺页异常。下面的描述试图描述在v4.10内核 -中是如何进行巨页预留处理的。 - - -读者 -==== -这个描述主要是针对正在修改hugetlbfs代码的内核开发者。 - - -数据结构 -======== - -resv_huge_pages - 这是一个全局的（per-hstate）预留的巨页的计数。预留的巨页只对预留它们的任 - 务可用。因此，一般可用的巨页的数量被计算为(``free_huge_pages - resv_huge_pages``)。 -Reserve Map - 预留映射由以下结构体描述:: - - struct resv_map { - struct kref refs; - spinlock_t lock; - struct list_head regions; - long adds_in_progress; - struct list_head region_cache; - long region_cache_count; - }; - - 系统中每个巨页映射都有一个预留映射。resv_map中的regions列表描述了映射中的 - 区域。一个区域被描述为:: - - struct file_region { - struct list_head link; - long from; - long to; - }; - - file_region结构体的 ‘from’ 和 ‘to’ 字段是进入映射的巨页索引。根据映射的类型，在 - reserv_map 中的一个区域可能表示该范围存在预留，或预留不存在。 -Flags for MAP_PRIVATE Reservations - 这些被存储在预留的映射指针的底部。 - - ``#define HPAGE_RESV_OWNER (1UL << 0)`` - 表示该任务是与该映射相关的预留的所有者。 - ``#define HPAGE_RESV_UNMAPPED (1UL << 1)`` - 表示最初映射此范围（并创建储备）的任务由于COW失败而从该任务（子任务）中取消映 - 射了一个页面。 -Page Flags - PagePrivate页面标志是用来指示在释放巨页时必须恢复巨页的预留。更多细节将在 - “释放巨页” 一节中讨论。 - - -预留映射位置（私有或共享） -========================== - -一个巨页映射或段要么是私有的，要么是共享的。如果是私有的，它通常只对一个地址空间 -（任务）可用。如果是共享的，它可以被映射到多个地址空间（任务）。对于这两种类型的映射， -预留映射的位置和语义是明显不同的。位置的差异是： - -- 对于私有映射，预留映射挂在VMA结构体上。具体来说，就是vma->vm_private_data。这个保 - 留映射是在创建映射（mmap(MAP_PRIVATE)）时创建的。 -- 对于共享映射，预留映射挂在inode上。具体来说，就是inode->i_mapping->private_data。 - 由于共享映射总是由hugetlbfs文件系统中的文件支持，hugetlbfs代码确保每个节点包含一个预 - 留映射。因此，预留映射在创建节点时被分配。 - - -创建预留 -======== -当创建一个巨大的有页面支持的共享内存段（shmget(SHM_HUGETLB)）或通过mmap(MAP_HUGETLB) -创建一个映射时，就会创建预留。这些操作会导致对函数hugetlb_reserve_pages()的调用:: - - int hugetlb_reserve_pages(struct inode *inode, - long from, long to, - struct vm_area_struct *vma, - vm_flags_t vm_flags) - -hugetlb_reserve_pages()做的第一件事是检查在调用shmget()或mmap()时是否指定了NORESERVE -标志。如果指定了NORESERVE，那么这个函数立即返回，因为不需要预留。 - -参数'from'和'to'是映射或基础文件的巨页索引。对于shmget()，'from'总是0，'to'对应于段/映射 -的长度。对于mmap()，offset参数可以用来指定进入底层文件的偏移量。在这种情况下，'from'和'to' -参数已经被这个偏移量所调整。 - -PRIVATE和SHARED映射之间的一个很大的区别是预留在预留映射中的表示方式。 - -- 对于共享映射，预留映射中的条目表示对应页面的预留存在或曾经存在。当预留被消耗时，预留映射不被 - 修改。 -- 对于私有映射，预留映射中没有条目表示相应页面存在预留。随着预留被消耗，条目被添加到预留映射中。 - 因此，预留映射也可用于确定哪些预留已被消耗。 - -对于私有映射，hugetlb_reserve_pages()创建预留映射并将其挂在VMA结构体上。此外， -HPAGE_RESV_OWNER标志被设置，以表明该VMA拥有预留。 - -预留映射被查阅以确定当前映射/段需要多少巨页预留。对于私有映射，这始终是一个值（to - from）。 -然而，对于共享映射来说，一些预留可能已经存在于(to - from)的范围内。关于如何实现这一点的细节， -请参见 :ref:`预留映射的修改 ` 一节。 - -该映射可能与一个子池（subpool）相关联。如果是这样，将查询子池以确保有足够的空间用于映射。子池 -有可能已经预留了可用于映射的预留空间。更多细节请参见 :ref: `子池预留 ` -一节。 - -在咨询了预留映射和子池之后，就知道了需要的新预留数量。hugetlb_acct_memory()函数被调用以检查 -并获取所要求的预留数量。hugetlb_acct_memory()调用到可能分配和调整剩余页数的函数。然而，在这 -些函数中，代码只是检查以确保有足够的空闲的巨页来容纳预留。如果有的话，全局预留计数resv_huge_pages -会被调整，如下所示:: - - if (resv_needed <= (resv_huge_pages - free_huge_pages)) - resv_huge_pages += resv_needed; - -注意，在检查和调整这些计数器时，全局锁hugetlb_lock会被预留。 - -如果有足够的空闲的巨页，并且全局计数resv_huge_pages被调整，那么与映射相关的预留映射被修改以 -反映预留。在共享映射的情况下，将存在一个file_region，包括'from'-'to'范围。对于私有映射， -不对预留映射进行修改，因为没有条目表示存在预留。 - -如果hugetlb_reserve_pages()成功，全局预留数和与映射相关的预留映射将根据需要被修改，以确保 -在'from'-'to'范围内存在预留。 - -消耗预留/分配一个巨页 -=========================== - -当与预留相关的巨页在相应的映射中被分配和实例化时，预留就被消耗了。该分配是在函数alloc_huge_page() -中进行的:: - - struct page *alloc_huge_page(struct vm_area_struct *vma, - unsigned long addr, int avoid_reserve) - -alloc_huge_page被传递给一个VMA指针和一个虚拟地址，因此它可以查阅预留映射以确定是否存在预留。 -此外，alloc_huge_page需要一个参数avoid_reserve，该参数表示即使看起来已经为指定的地址预留了 -预留，也不应该使用预留。avoid_reserve参数最常被用于写时拷贝和页面迁移的情况下，即现有页面的额 -外拷贝被分配。 - - -调用辅助函数vma_needs_reservation()来确定是否存在对映射(vma)中地址的预留。关于这个函数的详 -细内容，请参见 :ref:`预留映射帮助函数 ` 一节。从 -vma_needs_reservation()返回的值通常为0或1。如果该地址存在预留，则为0，如果不存在预留，则为1。 -如果不存在预留，并且有一个与映射相关联的子池，则查询子池以确定它是否包含预留。如果子池包含预留， -则可将其中一个用于该分配。然而，在任何情况下，avoid_reserve参数都会优先考虑为分配使用预留。在 -确定预留是否存在并可用于分配后，调用dequeue_huge_page_vma()函数。这个函数需要两个与预留有关 -的参数： - -- avoid_reserve，这是传递给alloc_huge_page()的同一个值/参数。 -- chg，尽管这个参数的类型是long，但只有0或1的值被传递给dequeue_huge_page_vma。如果该值为0， - 则表明存在预留（关于可能的问题，请参见 “预留和内存策略” 一节）。如果值 - 为1，则表示不存在预留，如果可能的话，必须从全局空闲池中取出该页。 - -与VMA的内存策略相关的空闲列表被搜索到一个空闲页。如果找到了一个页面，当该页面从空闲列表中移除时， -free_huge_pages的值被递减。如果有一个与该页相关的预留，将进行以下调整:: - - SetPagePrivate(page); /* 表示分配这个页面消耗了一个预留， - * 如果遇到错误，以至于必须释放这个页面，预留将被 - * 恢复。 */ - resv_huge_pages--; /* 减少全局预留计数 */ - -注意，如果找不到满足VMA内存策略的巨页，将尝试使用伙伴分配器分配一个。这就带来了超出预留范围 -的剩余巨页和超额分配的问题。即使分配了一个多余的页面，也会进行与上面一样的基于预留的调整: -SetPagePrivate(page) 和 resv_huge_pages--. - -在获得一个新的巨页后，(page)->private被设置为与该页面相关的子池的值，如果它存在的话。当页 -面被释放时，这将被用于子池的计数。 - -然后调用函数vma_commit_reservation()，根据预留的消耗情况调整预留映射。一般来说，这涉及 -到确保页面在区域映射的file_region结构体中被表示。对于预留存在的共享映射，预留映射中的条目 -已经存在，所以不做任何改变。然而，如果共享映射中没有预留，或者这是一个私有映射，则必须创建一 -个新的条目。 - -注意，如果找不到满足VMA内存策略的巨页，将尝试使用伙伴分配器分配一个。这就带来了超出预留范围 -的剩余巨页和过度分配的问题。即使分配了一个多余的页面，也会进行与上面一样的基于预留的调整。 -SetPagePrivate(page)和resv_huge_pages-。 - -在获得一个新的巨页后，(page)->private被设置为与该页面相关的子池的值，如果它存在的话。当页 -面被释放时，这将被用于子池的计数。 - -然后调用函数vma_commit_reservation()，根据预留的消耗情况调整预留映射。一般来说，这涉及 -到确保页面在区域映射的file_region结构体中被表示。对于预留存在的共享映射，预留映射中的条目 -已经存在，所以不做任何改变。然而，如果共享映射中没有预留，或者这是一个私有映射，则必须创建 -一个新的条目。 - -在alloc_huge_page()开始调用vma_needs_reservation()和页面分配后调用 -vma_commit_reservation()之间，预留映射有可能被改变。如果hugetlb_reserve_pages在共 -享映射中为同一页面被调用，这将是可能的。在这种情况下，预留计数和子池空闲页计数会有一个偏差。 -这种罕见的情况可以通过比较vma_needs_reservation和vma_commit_reservation的返回值来 -识别。如果检测到这种竞争，子池和全局预留计数将被调整以进行补偿。关于这些函数的更多信息，请 -参见 :ref:`预留映射帮助函数 ` 一节。 - - -实例化巨页 -========== - -在巨页分配之后，页面通常被添加到分配任务的页表中。在此之前，共享映射中的页面被添加到页面缓 -存中，私有映射中的页面被添加到匿名反向映射中。在这两种情况下，PagePrivate标志被清除。因此， -当一个已经实例化的巨页被释放时，不会对全局预留计数（resv_huge_pages）进行调整。 - - -释放巨页 -======== - -巨页释放是由函数free_huge_page()执行的。这个函数是hugetlbfs复合页的析构器。因此，它只传 -递一个指向页面结构体的指针。当一个巨页被释放时，可能需要进行预留计算。如果该页与包含保 -留的子池相关联，或者该页在错误路径上被释放，必须恢复全局预留计数，就会出现这种情况。 - -page->private字段指向与该页相关的任何子池。如果PagePrivate标志被设置，它表明全局预留计数 -应该被调整（关于如何设置这些标志的信息，请参见 -:ref: `消耗预留/分配一个巨页 ` ）。 - - -该函数首先调用hugepage_subpool_put_pages()来处理该页。如果这个函数返回一个0的值（不等于 -传递的1的值），它表明预留与子池相关联，这个新释放的页面必须被用来保持子池预留的数量超过最小值。 -因此，在这种情况下，全局resv_huge_pages计数器被递增。 - -如果页面中设置了PagePrivate标志，那么全局resv_huge_pages计数器将永远被递增。 - -子池预留 -======== - -有一个结构体hstate与每个巨页尺寸相关联。hstate跟踪所有指定大小的巨页。一个子池代表一 -个hstate中的页面子集，它与一个已挂载的hugetlbfs文件系统相关 - -当一个hugetlbfs文件系统被挂载时，可以指定min_size选项，它表示文件系统所需的最小的巨页数量。 -如果指定了这个选项，与min_size相对应的巨页的数量将被预留给文件系统使用。这个数字在结构体 -hugepage_subpool的min_hpages字段中被跟踪。在挂载时，hugetlb_acct_memory(min_hpages) -被调用以预留指定数量的巨页。如果它们不能被预留，挂载就会失败。 - -当从子池中获取或释放页面时，会调用hugepage_subpool_get/put_pages()函数。 -hugepage_subpool_get/put_pages被传递给巨页数量，以此来调整子池的 “已用页面” 计数 -（get为下降，put为上升）。通常情况下，如果子池中没有足够的页面，它们会返回与传递的相同的值或 -一个错误。 - -然而，如果预留与子池相关联，可能会返回一个小于传递值的返回值。这个返回值表示必须进行的额外全局 -池调整的数量。例如，假设一个子池包含3个预留的巨页，有人要求5个。与子池相关的3个预留页可以用来 -满足部分请求。但是，必须从全局池中获得2个页面。为了向调用者转达这一信息，将返回值2。然后，调用 -者要负责从全局池中获取另外两个页面。 - - -COW和预留 -========== - -由于共享映射都指向并使用相同的底层页面，COW最大的预留问题是私有映射。在这种情况下，两个任务可 -以指向同一个先前分配的页面。一个任务试图写到该页，所以必须分配一个新的页，以便每个任务都指向它 -自己的页。 - -当该页最初被分配时，该页的预留被消耗了。当由于COW而试图分配一个新的页面时，有可能没有空闲的巨 -页，分配会失败。 - -当最初创建私有映射时，通过设置所有者的预留映射指针中的HPAGE_RESV_OWNER位来标记映射的所有者。 -由于所有者创建了映射，所有者拥有与映射相关的所有预留。因此，当一个写异常发生并且没有可用的页面 -时，对预留的所有者和非所有者采取不同的行动。 - -在发生异常的任务不是所有者的情况下，异常将失败，该任务通常会收到一个SIGBUS。 - -如果所有者是发生异常的任务，我们希望它能够成功，因为它拥有原始的预留。为了达到这个目的，该页被 -从非所有者任务中解映射出来。这样一来，唯一的引用就是来自拥有者的任务。此外，HPAGE_RESV_UNMAPPED -位被设置在非拥有任务的预留映射指针中。如果非拥有者任务后来在一个不存在的页面上发生异常，它可能 -会收到一个SIGBUS。但是，映射/预留的原始拥有者的行为将与预期一致。 - -预留映射的修改 -============== - -以下低级函数用于对预留映射进行修改。通常情况下，这些函数不会被直接调用。而是调用一个预留映射辅 -助函数，该函数调用这些低级函数中的一个。这些低级函数在源代码（mm/hugetlb.c）中得到了相当好的 -记录。这些函数是:: - - long region_chg(struct resv_map *resv, long f, long t); - long region_add(struct resv_map *resv, long f, long t); - void region_abort(struct resv_map *resv, long f, long t); - long region_count(struct resv_map *resv, long f, long t); - -在预留映射上的操作通常涉及两个操作: - -1) region_chg()被调用来检查预留映射，并确定在指定的范围[f, t]内有多少页目前没有被代表。 - - 调用代码执行全局检查和分配，以确定是否有足够的巨页使操作成功。 - -2) - a) 如果操作能够成功，regi_add()将被调用，以实际修改先前传递给regi_chg()的相同范围 - [f, t]的预留映射。 - b) 如果操作不能成功，region_abort被调用，在相同的范围[f, t]内中止操作。 - -注意，这是一个两步的过程， region_add()和 region_abort()在事先调用 region_chg()后保证 -成功。 region_chg()负责预先分配任何必要的数据结构以确保后续操作（特别是 region_add()）的 -成功。 - -如上所述，region_chg()确定该范围内当前没有在映射中表示的页面的数量。region_add()返回添加 -到映射中的范围内的页数。在大多数情况下， region_add() 的返回值与 region_chg() 的返回值相 -同。然而，在共享映射的情况下，有可能在调用 region_chg() 和 region_add() 之间对预留映射进 -行更改。在这种情况下，regi_add()的返回值将与regi_chg()的返回值不符。在这种情况下，全局计数 -和子池计数很可能是不正确的，需要调整。检查这种情况并进行适当的调整是调用者的责任。 - -函数region_del()被调用以从预留映射中移除区域。 -它通常在以下情况下被调用: - -- 当hugetlbfs文件系统中的一个文件被删除时，该节点将被释放，预留映射也被释放。在释放预留映射 - 之前，所有单独的file_region结构体必须被释放。在这种情况下，region_del的范围是[0, LONG_MAX]。 -- 当一个hugetlbfs文件正在被截断时。在这种情况下，所有在新文件大小之后分配的页面必须被释放。 - 此外，预留映射中任何超过新文件大小的file_region条目必须被删除。在这种情况下，region_del - 的范围是[new_end_of_file, LONG_MAX]。 -- 当在一个hugetlbfs文件中打洞时。在这种情况下，巨页被一次次从文件的中间移除。当这些页被移除 - 时，region_del()被调用以从预留映射中移除相应的条目。在这种情况下，region_del被传递的范 - 围是[page_idx, page_idx + 1]。 - -在任何情况下，region_del()都会返回从预留映射中删除的页面数量。在非常罕见的情况下，region_del() -会失败。这只能发生在打洞的情况下，即它必须分割一个现有的file_region条目，而不能分配一个新的 -结构体。在这种错误情况下，region_del()将返回-ENOMEM。这里的问题是，预留映射将显示对该页有 -预留。然而，子池和全局预留计数将不反映该预留。为了处理这种情况，调用函数hugetlb_fix_reserve_counts() -来调整计数器，使其与不能被删除的预留映射条目相对应。 - -region_count()在解除私有巨页映射时被调用。在私有映射中，预留映射中没有条目表明存在一个预留。 -因此，通过计算预留映射中的条目数，我们知道有多少预留被消耗了，有多少预留是未完成的 -（Outstanding = (end - start) - region_count（resv, start, end））。由于映射正在消 -失，子池和全局预留计数被未完成的预留数量所减去。 - -预留映射帮助函数 -================ - -有几个辅助函数可以查询和修改预留映射。这些函数只对特定的巨页的预留感兴趣，所以它们只是传入一个 -地址而不是一个范围。此外，它们还传入相关的VMA。从VMA中，可以确定映射的类型（私有或共享）和预留 -映射的位置（inode或VMA）。这些函数只是调用 “预留映射的修改” 一节中描述的基础函数。然而， -它们确实考虑到了私有和共享映射的预留映射条目的 “相反” 含义，并向调用者隐藏了这个细节:: - - long vma_needs_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -该函数为指定的页面调用 region_chg()。如果不存在预留，则返回1。如果存在预留，则返回0:: - - long vma_commit_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -这将调用 region_add()，用于指定的页面。与region_chg和region_add的情况一样，该函数应在 -先前调用的vma_needs_reservation后调用。它将为该页添加一个预留条目。如果预留被添加，它将 -返回1，如果没有则返回0。返回值应与之前调用vma_needs_reservation的返回值进行比较。如果出 -现意外的差异，说明在两次调用之间修改了预留映射:: - - void vma_end_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -这将调用指定页面的 region_abort()。与region_chg和region_abort的情况一样，该函数应在 -先前调用的vma_needs_reservation后被调用。它将中止/结束正在进行的预留添加操作:: - - long vma_add_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -这是一个特殊的包装函数，有助于在错误路径上清理预留。它只从repare_reserve_on_error()函数 -中调用。该函数与vma_needs_reservation一起使用，试图将一个预留添加到预留映射中。它考虑到 -了私有和共享映射的不同预留映射语义。因此，region_add被调用用于共享映射（因为映射中的条目表 -示预留），而region_del被调用用于私有映射（因为映射中没有条目表示预留）。关于在错误路径上需 -要做什么的更多信息，请参见 “错误路径中的预留清理” 。 - - -错误路径中的预留清理 -==================== - -正如在:ref:`预留映射帮助函数` 一节中提到的，预留的修改分两步进行。首 -先，在分配页面之前调用vma_needs_reservation。如果分配成功，则调用vma_commit_reservation。 -如果不是，则调用vma_end_reservation。全局和子池的预留计数根据操作的成功或失败进行调整， -一切都很好。 - -此外，在一个巨页被实例化后，PagePrivate标志被清空，这样，当页面最终被释放时，计数是 -正确的。 - -然而，有几种情况是，在一个巨页被分配后，但在它被实例化之前，就遇到了错误。在这种情况下， -页面分配已经消耗了预留，并进行了适当的子池、预留映射和全局计数调整。如果页面在这个时候被释放 -（在实例化和清除PagePrivate之前），那么free_huge_page将增加全局预留计数。然而，预留映射 -显示报留被消耗了。这种不一致的状态将导致预留的巨页的 “泄漏” 。全局预留计数将比它原本的要高， -并阻止分配一个预先分配的页面。 - -函数 restore_reserve_on_error() 试图处理这种情况。它有相当完善的文档。这个函数的目的 -是将预留映射恢复到页面分配前的状态。通过这种方式，预留映射的状态将与页面释放后的全局预留计 -数相对应。 - -函数restore_reserve_on_error本身在试图恢复预留映射条目时可能会遇到错误。在这种情况下， -它将简单地清除该页的PagePrivate标志。这样一来，当页面被释放时，全局预留计数将不会被递增。 -然而，预留映射将继续看起来像预留被消耗了一样。一个页面仍然可以被分配到该地址，但它不会像最 -初设想的那样使用一个预留页。 - -有一些代码（最明显的是userfaultfd）不能调用restore_reserve_on_error。在这种情况下， -它简单地修改了PagePrivate，以便在释放巨页时不会泄露预留。 - - -预留和内存策略 -============== -当git第一次被用来管理Linux代码时，每个节点的巨页列表就存在于hstate结构中。预留的概念是 -在一段时间后加入的。当预留被添加时，没有尝试将内存策略考虑在内。虽然cpusets与内存策略不 -完全相同，但hugetlb_acct_memory中的这个注释总结了预留和cpusets/内存策略之间的相互作 -用:: - - - /* - * 当cpuset被配置时，它打破了严格的hugetlb页面预留，因为计数是在一个全局变量上完 - * 成的。在有cpuset的情况下，这样的预留完全是垃圾，因为预留没有根据当前cpuset的 - * 页面可用性来检查。在任务所在的cpuset中缺乏空闲的htlb页面时，应用程序仍然有可能 - * 被内核OOM'ed。试图用cpuset来执行严格的计数几乎是不可能的（或者说太难看了），因 - * 为cpuset太不稳定了，任务或内存节点可以在cpuset之间动态移动。与cpuset共享 - * hugetlb映射的语义变化是不可取的。然而，为了预留一些语义，我们退回到检查当前空闲 - * 页的可用性，作为一种最好的尝试，希望能将cpuset改变语义的影响降到最低。 - */ - -添加巨页预留是为了防止在缺页异常时出现意外的页面分配失败（OOM）。然而，如果一个应用 -程序使用cpusets或内存策略，就不能保证在所需的节点上有巨页可用。即使有足够数量的全局 -预留，也是如此。 - -Hugetlbfs回归测试 -================= - -最完整的hugetlb测试集在libhugetlbfs仓库。如果你修改了任何hugetlb相关的代码，请使用 -libhugetlbfs测试套件来检查回归情况。此外，如果你添加了任何新的hugetlb功能，请在 -libhugetlbfs中添加适当的测试。 - --- -Mike Kravetz，2017年4月7日 diff --git a/Documentation/translations/zh_CN/vm/hwpoison.rst b/Documentation/translations/zh_CN/vm/hwpoison.rst deleted file mode 100644 index c6e1e7bdb05b..000000000000 --- a/Documentation/translations/zh_CN/vm/hwpoison.rst +++ /dev/null @@ -1,166 +0,0 @@ - -:Original: Documentation/vm/hwpoison.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -======== -hwpoison -======== - -什么是hwpoison? -=============== - - -即将推出的英特尔CPU支持从一些内存错误中恢复（ ``MCA恢复`` ）。这需要操作系统宣布 -一个页面"poisoned"，杀死与之相关的进程，并避免在未来使用它。 - -这个补丁包在虚拟机中实现了必要的(编程)框架。 - -引用概述中的评论:: - - 高级机器的检查与处理。处理方法是损坏的页面被硬件报告，通常是由于2位ECC内 - 存或高速缓存故障。 - - 这主要是针对在后台检测到的损坏的页面。当当前的CPU试图访问它时，当前运行的进程 - 可以直接被杀死。因为还没有访问损坏的页面, 如果错误由于某种原因不能被处理，就可 - 以安全地忽略它. 而不是用另外一个机器检查去处理它。 - - 处理不同状态的页面缓存页。这里棘手的部分是，相对于其他虚拟内存用户， 我们可以异 - 步访问任何页面。因为内存故障可能随时随地发生，可能违反了他们的一些假设。这就是 - 为什么这段代码必须非常小心。一般来说，它试图使用正常的锁规则，如获得标准锁，即使 - 这意味着错误处理可能需要很长的时间。 - - 这里的一些操作有点低效，并且具有非线性的算法复杂性，因为数据结构没有针对这种情 - 况进行优化。特别是从vma到进程的映射就是这种情况。由于这种情况大概率是罕见的，所 - 以我们希望我们可以摆脱这种情况。 - -该代码由mm/memory-failure.c中的高级处理程序、一个新的页面poison位和虚拟机中的 -各种检查组成，用来处理poison的页面。 - -现在主要目标是KVM客户机，但它适用于所有类型的应用程序。支持KVM需要最近的qemu-kvm -版本。 - -对于KVM的使用，需要一个新的信号类型，这样KVM就可以用适当的地址将机器检查注入到客户 -机中。这在理论上也允许其他应用程序处理内存故障。我们的期望是，所有的应用程序都不要这 -样做，但一些非常专业的应用程序可能会这样做。 - -故障恢复模式 -============ - -有两种（实际上是三种）模式的内存故障恢复可以在。 - -vm.memory_failure_recovery sysctl 置零: - 所有的内存故障都会导致panic。请不要尝试恢复。 - -早期处理 - (可以在全局和每个进程中控制) 一旦检测到错误，立即向应用程序发送SIGBUS这允许 - 应用程序以温和的方式处理内存错误（例如，放弃受影响的对象） 这是KVM qemu使用的 - 模式。 - -推迟处理 - 当应用程序运行到损坏的页面时，发送SIGBUS。这对不知道内存错误的应用程序来说是 - 最好的，默认情况下注意一些页面总是被当作late kill处理。 - -用户控制 -======== - -vm.memory_failure_recovery - 参阅 sysctl.txt - -vm.memory_failure_early_kill - 全局启用early kill - -PR_MCE_KILL - 设置early/late kill mode/revert 到系统默认值。 - - arg1: PR_MCE_KILL_CLEAR: - 恢复到系统默认值 - arg1: PR_MCE_KILL_SET: - arg2定义了线程特定模式 - - PR_MCE_KILL_EARLY: - Early kill - PR_MCE_KILL_LATE: - Late kill - PR_MCE_KILL_DEFAULT - 使用系统全局默认值 - - 注意，如果你想有一个专门的线程代表进程处理SIGBUS(BUS_MCEERR_AO)，你应该在 - 指定线程上调用prctl(PR_MCE_KILL_EARLY)。否则，SIGBUS将被发送到主线程。 - -PR_MCE_KILL_GET - 返回当前模式 - -测试 -==== - -* madvise(MADV_HWPOISON, ....) (as root) - 在测试过程中Poison一个页面 - -* 通过debugfs ``/sys/kernel/debug/hwpoison/`` hwpoison-inject模块 - - corrupt-pfn - 在PFN处注入hwpoison故障，并echoed到这个文件。这做了一些早期过滤，以避 - 免在测试套件中损坏非预期页面。 - unpoison-pfn - 在PFN的Software-unpoison页面对应到这个文件。这样，一个页面可以再次被 - 复用。这只对Linux注入的故障起作用，对真正的内存故障不起作用。 - - 注意这些注入接口并不稳定，可能会在不同的内核版本中发生变化 - - corrupt-filter-dev-major, corrupt-filter-dev-minor - 只处理与块设备major/minor定义的文件系统相关的页面的内存故障。-1U是通 - 配符值。这应该只用于人工注入的测试。 - - corrupt-filter-memcg - 限制注入到memgroup拥有的页面。由memcg的inode号指定。 - - Example:: - - mkdir /sys/fs/cgroup/mem/hwpoison - - usemem -m 100 -s 1000 & - echo `jobs -p` > /sys/fs/cgroup/mem/hwpoison/tasks - - memcg_ino=$(ls -id /sys/fs/cgroup/mem/hwpoison | cut -f1 -d' ') - echo $memcg_ino > /debug/hwpoison/corrupt-filter-memcg - - page-types -p `pidof init` --hwpoison # shall do nothing - page-types -p `pidof usemem` --hwpoison # poison its pages - - corrupt-filter-flags-mask, corrupt-filter-flags-value - 当指定时，只有在((page_flags & mask) == value)的情况下才会poison页面。 - 这允许对许多种类的页面进行压力测试。page_flags与/proc/kpageflags中的相 - 同。这些标志位在include/linux/kernel-page-flags.h中定义，并在 - Documentation/admin-guide/mm/pagemap.rst中记录。 - -* 架构特定的MCE注入器 - - x86 有 mce-inject, mce-test - - 在mce-test中的一些便携式hwpoison测试程序，见下文。 - -引用 -==== - -http://halobates.de/mce-lc09-2.pdf - 09年LinuxCon的概述演讲 - -git://git.kernel.org/pub/scm/utils/cpu/mce/mce-test.git - 测试套件（在tsrc中的hwpoison特定可移植测试）。 - -git://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git - x86特定的注入器 - - -限制 -==== -- 不是所有的页面类型都被支持，而且永远不会。大多数内核内部对象不能被恢 - 复，目前只有LRU页。 - ---- -Andi Kleen, 2009年10月 diff --git a/Documentation/translations/zh_CN/vm/index.rst b/Documentation/translations/zh_CN/vm/index.rst deleted file mode 100644 index a1c6d529b6ff..000000000000 --- a/Documentation/translations/zh_CN/vm/index.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/index.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - -================= -Linux内存管理文档 -================= - -这是一个关于Linux内存管理（mm）子系统内部的文档集，其中有不同层次的细节，包括注释 -和邮件列表的回复，用于阐述数据结构和算法的基本情况。如果你正在寻找关于简单分配内存的建 -议，请参阅(Documentation/translations/zh_CN/core-api/memory-allocation.rst)。 -对于控制和调整指南，请参阅(Documentation/admin-guide/mm/index)。 -TODO:待引用文档集被翻译完毕后请及时修改此处） - -.. toctree:: - :maxdepth: 1 - - active_mm - balance - damon/index - free_page_reporting - highmem - ksm - frontswap - hmm - hwpoison - hugetlbfs_reserv - memory-model - mmu_notifier - numa - overcommit-accounting - page_frags - page_owner - page_table_check - remap_file_pages - split_page_table_lock - z3fold - zsmalloc - -TODOLIST: -* arch_pgtable_helpers -* free_page_reporting -* hugetlbfs_reserv -* page_migration -* slub -* transhuge -* unevictable-lru -* vmalloced-kernel-stacks diff --git a/Documentation/translations/zh_CN/vm/ksm.rst b/Documentation/translations/zh_CN/vm/ksm.rst deleted file mode 100644 index 83b0c73984da..000000000000 --- a/Documentation/translations/zh_CN/vm/ksm.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/vm/ksm.rst - -:翻译: - - 徐鑫 xu xin - -============ -内核同页合并 -============ - -KSM 是一种节省内存的数据去重功能，由CONFIG_KSM=y启用，并在2.6.32版本时被添加 -到Linux内核。详见 ``mm/ksm.c`` 的实现，以及http://lwn.net/Articles/306704和 -https://lwn.net/Articles/330589 - -KSM的用户空间的接口在Documentation/translations/zh_CN/admin-guide/mm/ksm.rst -文档中有描述。 - -设计 -==== - -概述 ----- - -概述内容请见mm/ksm.c文档中的“DOC: Overview” - -逆映射 ------- -KSM维护着稳定树中的KSM页的逆映射信息。 - -当KSM页面的共享数小于 ``max_page_sharing`` 的虚拟内存区域(VMAs)时，则代表了 -KSM页的稳定树其中的节点指向了一个rmap_item结构体类型的列表。同时，这个KSM页 -的 ``page->mapping`` 指向了该稳定树节点。 - -如果共享数超过了阈值，KSM将给稳定树添加第二个维度。稳定树就变成链接一个或多 -个稳定树"副本"的"链"。每个副本都保留KSM页的逆映射信息，其中 ``page->mapping`` -指向该"副本"。 - -每个链以及链接到该链中的所有"副本"强制不变的是，它们代表了相同的写保护内存 -内容，尽管任中一个"副本"是由同一片内存区的不同的KSM复制页所指向的。 - -这样一来，相比与无限的逆映射链表，稳定树的查找计算复杂性不受影响。但在稳定树 -本身中不能有重复的KSM页面内容仍然是强制要求。 - -由 ``max_page_sharing`` 强制决定的数据去重限制是必要的，以此来避免虚拟内存 -rmap链表变得过大。rmap的遍历具有O(N)的复杂度，其中N是共享页面的rmap_项（即 -虚拟映射）的数量，而这个共享页面的节点数量又被 ``max_page_sharing`` 所限制。 -因此，这有效地将线性O(N)计算复杂度从rmap遍历中分散到不同的KSM页面上。ksmd进 -程在稳定节点"链"上的遍历也是O(N)，但这个N是稳定树"副本"的数量，而不是rmap项 -的数量，因此它对ksmd性能没有显著影响。实际上，最佳稳定树"副本"的候选节点将 -保留在"副本"列表的开头。 - -``max_page_sharing`` 的值设置得高了会促使更快的内存合并（因为将有更少的稳定 -树副本排队进入稳定节点chain->hlist）和更高的数据去重系数，但代价是在交换、压 -缩、NUMA平衡和页面迁移过程中可能导致KSM页的最大rmap遍历速度较慢。 - -``stable_node_dups/stable_node_chains`` 的比值还受 ``max_page_sharing`` 调控 -的影响，高比值可能意味着稳定节点dup中存在碎片，这可以通过在ksmd中引入碎片算 -法来解决，该算法将rmap项从一个稳定节点dup重定位到另一个稳定节点dup，以便释放 -那些仅包含极少rmap项的稳定节点"dup"，但这可能会增加ksmd进程的CPU使用率，并可 -能会减慢应用程序在KSM页面上的只读计算。 - -KSM会定期扫描稳定节点"链"中链接的所有稳定树"副本"，以便删减过时了的稳定节点。 -这种扫描的频率由 ``stable_node_chains_prune_millisecs`` 这个sysfs 接口定义。 - -参考 -==== -内核代码请见mm/ksm.c。 -涉及的函数(mm_slot ksm_scan stable_node rmap_item)。 diff --git a/Documentation/translations/zh_CN/vm/memory-model.rst b/Documentation/translations/zh_CN/vm/memory-model.rst deleted file mode 100644 index 013e30c88d72..000000000000 --- a/Documentation/translations/zh_CN/vm/memory-model.rst +++ /dev/null @@ -1,135 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -:Original: Documentation/vm/memory-model.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -============ -物理内存模型 -============ - -系统中的物理内存可以用不同的方式进行寻址。最简单的情况是，物理内存从地址0开 -始，跨越一个连续的范围，直到最大的地址。然而，这个范围可能包含CPU无法访问的 -小孔隙。那么，在完全不同的地址可能有几个连续的范围。而且，别忘了NUMA，即不 -同的内存库连接到不同的CPU。 - -Linux使用两种内存模型中的一种对这种多样性进行抽象。FLATMEM和SPARSEM。每 -个架构都定义了它所支持的内存模型，默认的内存模型是什么，以及是否有可能手动 -覆盖该默认值。 - -所有的内存模型都使用排列在一个或多个数组中的 `struct page` 来跟踪物理页 -帧的状态。 - -无论选择哪种内存模型，物理页框号（PFN）和相应的 `struct page` 之间都存 -在一对一的映射关系。 - -每个内存模型都定义了 :c:func:`pfn_to_page` 和 :c:func:`page_to_pfn` -帮助函数，允许从PFN到 `struct page` 的转换，反之亦然。 - -FLATMEM -======= - -最简单的内存模型是FLATMEM。这个模型适用于非NUMA系统的连续或大部分连续的 -物理内存。 - -在FLATMEM内存模型中，有一个全局的 `mem_map` 数组来映射整个物理内存。对 -于大多数架构，孔隙在 `mem_map` 数组中都有条目。与孔洞相对应的 `struct page` -对象从未被完全初始化。 - -为了分配 `mem_map` 数组，架构特定的设置代码应该调用free_area_init()函数。 -然而，在调用memblock_free_all()函数之前，映射数组是不能使用的，该函数 -将所有的内存交给页分配器。 - -一个架构可能会释放 `mem_map` 数组中不包括实际物理页的部分。在这种情况下，特 -定架构的 :c:func:`pfn_valid` 实现应该考虑到 `mem_map` 中的孔隙。 - -使用FLATMEM，PFN和 `struct page` 之间的转换是直接的。 `PFN - ARCH_PFN_OFFSET` -是 `mem_map` 数组的一个索引。 - -`ARCH_PFN_OFFSET` 定义了物理内存起始地址不同于0的系统的第一个页框号。 - -SPARSEMEM -========= - -SPARSEMEM是Linux中最通用的内存模型，它是唯一支持若干高级功能的内存模型， -如物理内存的热插拔、非易失性内存设备的替代内存图和较大系统的内存图的延迟 -初始化。 - -SPARSEMEM模型将物理内存显示为一个部分的集合。一个区段用mem_section结构 -体表示，它包含 `section_mem_map` ，从逻辑上讲，它是一个指向 `struct page` -阵列的指针。然而，它被存储在一些其他的magic中，以帮助分区管理。区段的大小 -和最大区段数是使用 `SECTION_SIZE_BITS` 和 `MAX_PHYSMEM_BITS` 常量 -来指定的，这两个常量是由每个支持SPARSEMEM的架构定义的。 `MAX_PHYSMEM_BITS` -是一个架构所支持的物理地址的实际宽度，而 `SECTION_SIZE_BITS` 是一个任 -意的值。 - -最大的段数表示为 `NR_MEM_SECTIONS` ，定义为 - -.. math:: - - NR\_MEM\_SECTIONS = 2 ^ {(MAX\_PHYSMEM\_BITS - SECTION\_SIZE\_BITS)} - -`mem_section` 对象被安排在一个叫做 `mem_sections` 的二维数组中。这个数组的 -大小和位置取决于 `CONFIG_SPARSEM_EXTREME` 和可能的最大段数: - -* 当 `CONFIG_SPARSEMEM_EXTREME` 被禁用时， `mem_sections` 数组是静态的，有 - `NR_MEM_SECTIONS` 行。每一行持有一个 `mem_section` 对象。 -* 当 `CONFIG_SPARSEMEM_EXTREME` 被启用时， `mem_sections` 数组被动态分配。 - 每一行包含价值 `PAGE_SIZE` 的 `mem_section` 对象，行数的计算是为了适应所有的 - 内存区。 - -架构设置代码应该调用sparse_init()来初始化内存区和内存映射。 - -通过SPARSEMEM，有两种可能的方式将PFN转换为相应的 `struct page` --"classic sparse"和 - "sparse vmemmap"。选择是在构建时进行的，它由 `CONFIG_SPARSEMEM_VMEMMAP` 的 - 值决定。 - -Classic sparse在page->flags中编码了一个页面的段号，并使用PFN的高位来访问映射该页 -框的段。在一个区段内，PFN是指向页数组的索引。 - -Sparse vmemmapvmemmap使用虚拟映射的内存映射来优化pfn_to_page和page_to_pfn操 -作。有一个全局的 `struct page *vmemmap` 指针，指向一个虚拟连续的 `struct page` -对象阵列。PFN是该数组的一个索引，`struct page` 从 `vmemmap` 的偏移量是该页的PFN。 - -为了使用vmemmap，一个架构必须保留一个虚拟地址的范围，以映射包含内存映射的物理页，并 -确保 `vmemmap`指向该范围。此外，架构应该实现 :c:func:`vmemmap_populate` 方法， -它将分配物理内存并为虚拟内存映射创建页表。如果一个架构对vmemmap映射没有任何特殊要求， -它可以使用通用内存管理提供的默认 :c:func:`vmemmap_populate_basepages`。 - -虚拟映射的内存映射允许将持久性内存设备的 `struct page` 对象存储在这些设备上预先分 -配的存储中。这种存储用vmem_altmap结构表示，最终通过一长串的函数调用传递给 -vmemmap_populate()。vmemmap_populate()实现可以使用 `vmem_altmap` 和 -:c:func:`vmemmap_alloc_block_buf` 助手来分配持久性内存设备上的内存映射。 - -ZONE_DEVICE -=========== -`ZONE_DEVICE` 设施建立在 `SPARSEM_VMEMMAP` 之上，为设备驱动识别的物理地址范 -围提供 `struct page` `mem_map` 服务。 `ZONE_DEVICE` 的 "设备" 方面与以下 -事实有关：这些地址范围的页面对象从未被在线标记过，而且必须对设备进行引用，而不仅仅 -是页面，以保持内存被“锁定”以便使用。 `ZONE_DEVICE` ，通过 :c:func:`devm_memremap_pages` ， -为给定的pfns范围执行足够的内存热插拔来开启 :c:func:`pfn_to_page`， -:c:func:`page_to_pfn`, ，和 :c:func:`get_user_pages` 服务。由于页面引 -用计数永远不会低于1，所以页面永远不会被追踪为空闲内存，页面的 `struct list_head lru` -空间被重新利用，用于向映射该内存的主机设备/驱动程序进行反向引用。 - -虽然 `SPARSEMEM` 将内存作为一个区段的集合，可以选择收集并合成内存块，但 -`ZONE_DEVICE` 用户需要更小的颗粒度来填充 `mem_map` 。鉴于 `ZONE_DEVICE` -内存从未被在线标记，因此它的内存范围从未通过sysfs内存热插拔api暴露在内存块边界 -上。这个实现依赖于这种缺乏用户接口的约束，允许子段大小的内存范围被指定给 -:c:func:`arch_add_memory` ，即内存热插拔的上半部分。子段支持允许2MB作为 -:c:func:`devm_memremap_pages` 的跨架构通用对齐颗粒度。 - -`ZONE_DEVICE` 的用户是: - -* pmem: 通过DAX映射将平台持久性内存作为直接I/O目标使用。 - -* hmm: 用 `->page_fault()` 和 `->page_free()` 事件回调扩展 `ZONE_DEVICE` ， - 以允许设备驱动程序协调与设备内存相关的内存管理事件，通常是GPU内存。参见/vm/hmm.rst。 - -* p2pdma: 创建 `struct page` 对象，允许PCI/E拓扑结构中的peer设备协调它们之间的 - 直接DMA操作，即绕过主机内存。 diff --git a/Documentation/translations/zh_CN/vm/mmu_notifier.rst b/Documentation/translations/zh_CN/vm/mmu_notifier.rst deleted file mode 100644 index b29a37b33628..000000000000 --- a/Documentation/translations/zh_CN/vm/mmu_notifier.rst +++ /dev/null @@ -1,97 +0,0 @@ -:Original: Documentation/vm/mmu_notifier.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - - -什么时候需要页表锁内通知？ -========================== - -当清除一个pte/pmd时，我们可以选择通过在页表锁下（通知版的\*_clear_flush调用 -mmu_notifier_invalidate_range）通知事件。但这种通知并不是在所有情况下都需要的。 - -对于二级TLB（非CPU TLB），如IOMMU TLB或设备TLB（当设备使用类似ATS/PASID的东西让 -IOMMU走CPU页表来访问进程的虚拟地址空间）。只有两种情况需要在清除pte/pmd时在持有页 -表锁的同时通知这些二级TLB： - - A) 在mmu_notifier_invalidate_range_end()之前，支持页的地址被释放。 - B) 一个页表项被更新以指向一个新的页面（COW，零页上的写异常，__replace_page()，...）。 - -情况A很明显，你不想冒风险让设备写到一个现在可能被一些完全不同的任务使用的页面。 - -情况B更加微妙。为了正确起见，它需要按照以下序列发生: - - - 上页表锁 - - 清除页表项并通知 ([pmd/pte]p_huge_clear_flush_notify()) - - 设置页表项以指向新页 - -如果在设置新的pte/pmd值之前，清除页表项之后没有进行通知，那么你就会破坏设备的C11或 -C++11等内存模型。 - -考虑以下情况（设备使用类似于ATS/PASID的功能）。 - -两个地址addrA和addrB，这样|addrA - addrB| >= PAGE_SIZE，我们假设它们是COW的 -写保护（B的其他情况也适用）。 - -:: - - [Time N] -------------------------------------------------------------------- - CPU-thread-0 {尝试写到addrA} - CPU-thread-1 {尝试写到addrB} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {读取addrA并填充设备TLB} - DEV-thread-2 {读取addrB并填充设备TLB} - [Time N+1] ------------------------------------------------------------------ - CPU-thread-0 {COW_step0: {mmu_notifier_invalidate_range_start(addrA)}} - CPU-thread-1 {COW_step0: {mmu_notifier_invalidate_range_start(addrB)}} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+2] ------------------------------------------------------------------ - CPU-thread-0 {COW_step1: {更新页表以指向addrA的新页}} - CPU-thread-1 {COW_step1: {更新页表以指向addrB的新页}} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+3] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {preempted} - CPU-thread-2 {写入addrA，这是对新页面的写入} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+3] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {preempted} - CPU-thread-2 {} - CPU-thread-3 {写入addrB，这是一个写入新页的过程} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+4] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {COW_step3: {mmu_notifier_invalidate_range_end(addrB)}} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+5] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {从旧页中读取addrA} - DEV-thread-2 {从新页面读取addrB} - -所以在这里，因为在N+2的时候，清空页表项没有和通知一起作废二级TLB，设备在看到addrA的新值之前 -就看到了addrB的新值。这就破坏了设备的总内存序。 - -当改变一个pte的写保护或指向一个新的具有相同内容的写保护页（KSM）时，将mmu_notifier_invalidate_range -调用延迟到页表锁外的mmu_notifier_invalidate_range_end()是可以的。即使做页表更新的线程 -在释放页表锁后但在调用mmu_notifier_invalidate_range_end()前被抢占，也是如此。 diff --git a/Documentation/translations/zh_CN/vm/numa.rst b/Documentation/translations/zh_CN/vm/numa.rst deleted file mode 100644 index 6af412b924ad..000000000000 --- a/Documentation/translations/zh_CN/vm/numa.rst +++ /dev/null @@ -1,101 +0,0 @@ -:Original: Documentation/vm/numa.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -始于1999年11月，作者： - -========================== -何为非统一内存访问(NUMA)？ -========================== - -这个问题可以从几个视角来回答：硬件观点和Linux软件视角。 - -从硬件角度看，NUMA系统是一个由多个组件或装配组成的计算机平台，每个组件可能包含0个或更多的CPU、 -本地内存和/或IO总线。为了简洁起见，并将这些物理组件/装配的硬件视角与软件抽象区分开来，我们在 -本文中称这些组件/装配为“单元”。 - -每个“单元”都可以看作是系统的一个SMP[对称多处理器]子集——尽管独立的SMP系统所需的一些组件可能 -不会在任何给定的单元上填充。NUMA系统的单元通过某种系统互连连接在一起——例如，交叉开关或点对点 -链接是NUMA系统互连的常见类型。这两种类型的互连都可以聚合起来，以创建NUMA平台，其中的单元与其 -他单元有多个距离。 - -对于Linux，感兴趣的NUMA平台主要是所谓的缓存相干NUMA--简称ccNUMA系统系统。在ccNUMA系统中， -所有的内存都是可见的，并且可以从连接到任何单元的任何CPU中访问，缓存一致性是由处理器缓存和/或 -系统互连在硬件中处理。 - -内存访问时间和有效的内存带宽取决于包含CPU的单元或进行内存访问的IO总线距离包含目标内存的单元 -有多远。例如，连接到同一单元的CPU对内存的访问将比访问其他远程单元的内存经历更快的访问时间和 -更高的带宽。 NUMA平台可以在任何给定单元上访问多种远程距离的（其他）单元。 - -平台供应商建立NUMA系统并不只是为了让软件开发人员的生活变得有趣。相反，这种架构是提供可扩展 -内存带宽的一种手段。然而，为了实现可扩展的内存带宽，系统和应用软件必须安排大部分的内存引用 -[cache misses]到“本地”内存——同一单元的内存，如果有的话——或者到最近的有内存的单元。 - -这就自然而然有了Linux软件对NUMA系统的视角: - -Linux将系统的硬件资源划分为多个软件抽象，称为“节点”。Linux将节点映射到硬件平台的物理单元 -上，对一些架构的细节进行了抽象。与物理单元一样，软件节点可能包含0或更多的CPU、内存和/或IO -总线。同样，对“较近”节点的内存访问——映射到较近单元的节点——通常会比对较远单元的访问经历更快 -的访问时间和更高的有效带宽。 - -对于一些架构，如x86，Linux将“隐藏”任何代表没有内存连接的物理单元的节点，并将连接到该单元 -的任何CPU重新分配到代表有内存的单元的节点上。因此，在这些架构上，我们不能假设Linux将所有 -的CPU与一个给定的节点相关联，会看到相同的本地内存访问时间和带宽。 - -此外，对于某些架构，同样以x86为例，Linux支持对额外节点的仿真。对于NUMA仿真，Linux会将现 -有的节点或者非NUMA平台的系统内存分割成多个节点。每个模拟的节点将管理底层单元物理内存的一部 -分。NUMA仿真对于在非NUMA平台上测试NUMA内核和应用功能是非常有用的，当与cpusets一起使用时， -可以作为一种内存资源管理机制。[见 Documentation/admin-guide/cgroup-v1/cpusets.rst] - -对于每个有内存的节点，Linux构建了一个独立的内存管理子系统，有自己的空闲页列表、使用中页列表、 -使用统计和锁来调解访问。此外，Linux为每个内存区[DMA、DMA32、NORMAL、HIGH_MEMORY、MOVABLE -中的一个或多个]构建了一个有序的“区列表”。zonelist指定了当一个选定的区/节点不能满足分配请求 -时要访问的区/节点。当一个区没有可用的内存来满足请求时，这种情况被称为“overflow 溢出”或 -“fallback 回退”。 - -由于一些节点包含多个包含不同类型内存的区，Linux必须决定是否对区列表进行排序，使分配回退到不同 -节点上的相同区类型，或同一节点上的不同区类型。这是一个重要的考虑因素，因为有些区，如DMA或DMA32， -代表了相对稀缺的资源。Linux选择了一个默认的Node ordered zonelist。这意味着在使用按NUMA距 -离排序的远程节点之前，它会尝试回退到同一节点的其他分区。 - -默认情况下，Linux会尝试从执行请求的CPU被分配到的节点中满足内存分配请求。具体来说，Linux将试 -图从请求来源的节点的适当分区列表中的第一个节点进行分配。这被称为“本地分配”。如果“本地”节点不能 -满足请求，内核将检查所选分区列表中其他节点的区域，寻找列表中第一个能满足请求的区域。 - -本地分配将倾向于保持对分配的内存的后续访问 “本地”的底层物理资源和系统互连——只要内核代表其分配 -一些内存的任务后来不从该内存迁移。Linux调度器知道平台的NUMA拓扑结构——体现在“调度域”数据结构 -中[见 Documentation/scheduler/sched-domains.rst]——并且调度器试图尽量减少任务迁移到遥 -远的调度域中。然而，调度器并没有直接考虑到任务的NUMA足迹。因此，在充分不平衡的情况下，任务可 -以在节点之间迁移，远离其初始节点和内核数据结构。 - -系统管理员和应用程序设计者可以使用各种CPU亲和命令行接口，如taskset(1)和numactl(1)，以及程 -序接口，如sched_setaffinity(2)，来限制任务的迁移，以改善NUMA定位。此外，人们可以使用 -Linux NUMA内存策略修改内核的默认本地分配行为。 [见 -:ref:`Documentation/admin-guide/mm/numa_memory_policy.rst `]. - -系统管理员可以使用控制组和CPUsets限制非特权用户在调度或NUMA命令和功能中可以指定的CPU和节点 -的内存。 [见 Documentation/admin-guide/cgroup-v1/cpusets.rst] - -在不隐藏无内存节点的架构上，Linux会在分区列表中只包括有内存的区域[节点]。这意味着对于一个无 -内存的节点，“本地内存节点”——CPU节点的分区列表中的第一个区域的节点——将不是节点本身。相反，它 -将是内核在建立分区列表时选择的离它最近的有内存的节点。所以，默认情况下，本地分配将由内核提供 -最近的可用内存来完成。这是同一机制的结果，该机制允许这种分配在一个包含内存的节点溢出时回退到 -其他附近的节点。 - -一些内核分配不希望或不能容忍这种分配回退行为。相反，他们想确保他们从指定的节点获得内存，或者 -得到通知说该节点没有空闲内存。例如，当一个子系统分配每个CPU的内存资源时，通常是这种情况。 - -一个典型的分配模式是使用内核的numa_node_id()或CPU_to_node()函数获得“当前CPU”所在节点的 -节点ID，然后只从返回的节点ID请求内存。当这样的分配失败时，请求的子系统可以恢复到它自己的回退 -路径。板块内核内存分配器就是这样的一个例子。或者，子系统可以选择在分配失败时禁用或不启用自己。 -内核分析子系统就是这样的一个例子。 - -如果架构支持——不隐藏无内存节点，那么连接到无内存节点的CPU将总是产生回退路径的开销，或者一些 -子系统如果试图完全从无内存的节点分配内存，将无法初始化。为了透明地支持这种架构，内核子系统可 -以使用numa_mem_id()或cpu_to_mem()函数来定位调用或指定CPU的“本地内存节点”。同样，这是同 -一个节点，默认的本地页分配将从这个节点开始尝试。 diff --git a/Documentation/translations/zh_CN/vm/overcommit-accounting.rst b/Documentation/translations/zh_CN/vm/overcommit-accounting.rst deleted file mode 100644 index 8765cb118f24..000000000000 --- a/Documentation/translations/zh_CN/vm/overcommit-accounting.rst +++ /dev/null @@ -1,86 +0,0 @@ -:Original: Documentation/vm/overcommit-accounting.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - - -============== -超量使用审计 -============== - -Linux内核支持下列超量使用处理模式 - -0 - 启发式超量使用处理。拒绝明显的地址空间超量使用。用于一个典型的系统。 - 它确保严重的疯狂分配失败，同时允许超量使用以减少swap的使用。在这种模式下， - 允许root分配稍多的内存。这是默认的。 -1 - 总是超量使用。适用于一些科学应用。经典的例子是使用稀疏数组的代码，只是依赖 - 几乎完全由零页组成的虚拟内存 - -2 - 不超量使用。系统提交的总地址空间不允许超过swap+一个可配置的物理RAM的数量 - （默认为50%）。根据你使用的数量，在大多数情况下，这意味着一个进程在访问页面时 - 不会被杀死，但会在内存分配上收到相应的错误。 - - 对于那些想保证他们的内存分配在未来可用而又不需要初始化每一个页面的应用程序来说 - 是很有用的。 - -超量使用策略是通过sysctl `vm.overcommit_memory` 设置的。 - -可以通过 `vm.overcommit_ratio` （百分比）或 `vm.overcommit_kbytes` （绝对值） -来设置超限数量。这些只有在 `vm.overcommit_memory` 被设置为2时才有效果。 - -在 ``/proc/meminfo`` 中可以分别以CommitLimit和Committed_AS的形式查看当前 -的超量使用和提交量。 - -陷阱 -==== - -C语言的堆栈增长是一个隐含的mremap。如果你想得到绝对的保证，并在接近边缘的地方运行， -你 **必须** 为你认为你需要的最大尺寸的堆栈进行mmap。对于典型的堆栈使用来说，这并 -不重要，但如果你真的非常关心的话，这就是一个值得关注的案例。 - - -在模式2中，MAP_NORESERVE标志被忽略。 - - -它是如何工作的 -============== - -超量使用是基于以下规则 - -对于文件映射 - | SHARED or READ-only - 0 cost (该文件是映射而不是交换) - | PRIVATE WRITABLE - 每个实例的映射大小 - -对于匿名或者 ``/dev/zero`` 映射 - | SHARED - 映射的大小 - | PRIVATE READ-only - 0 cost (但作用不大) - | PRIVATE WRITABLE - 每个实例的映射大小 - -额外的计数 - | 通过mmap制作可写副本的页面 - | 从同一池中提取的shmfs内存 - -状态 -==== - -* 我们核算mmap内存映射 -* 我们核算mprotect在提交中的变化 -* 我们核算mremap的大小变化 -* 我们的审计 brk -* 审计munmap -* 我们在/proc中报告commit 状态 -* 核对并检查分叉的情况 -* 审查堆栈处理/执行中的构建 -* 叙述SHMfs的情况 -* 实现实际限制的执行 - -待续 -==== -* ptrace 页计数（这很难）。 diff --git a/Documentation/translations/zh_CN/vm/page_frags.rst b/Documentation/translations/zh_CN/vm/page_frags.rst deleted file mode 100644 index ad27fed33634..000000000000 --- a/Documentation/translations/zh_CN/vm/page_frags.rst +++ /dev/null @@ -1,38 +0,0 @@ -:Original: Documentation/vm/page_frag.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -======== -页面片段 -======== - -一个页面片段是一个任意长度的任意偏移的内存区域，它位于一个0或更高阶的复合页面中。 -该页中的多个碎片在该页的引用计数器中被单独计算。 - -page_frag函数，page_frag_alloc和page_frag_free，为页面片段提供了一个简单 -的分配框架。这被网络堆栈和网络设备驱动使用，以提供一个内存的支持区域，作为 -sk_buff->head使用，或者用于skb_shared_info的 “frags” 部分。 - -为了使用页面片段API，需要一个支持页面片段的缓冲区。这为碎片分配提供了一个中心点， -并允许多个调用使用一个缓存的页面。这样做的好处是可以避免对get_page的多次调用， -这在分配时开销可能会很大。然而，由于这种缓存的性质，要求任何对缓存的调用都要受到每 -个CPU的限制，或者每个CPU的限制，并在执行碎片分配时强制禁止中断。 - -网络堆栈在每个CPU使用两个独立的缓存来处理碎片分配。netdev_alloc_cache被使用 -netdev_alloc_frag和__netdev_alloc_skb调用的调用者使用。napi_alloc_cache -被调用__napi_alloc_frag和__napi_alloc_skb的调用者使用。这两个调用的主要区别是 -它们可能被调用的环境。“netdev” 前缀的函数可以在任何上下文中使用，因为这些函数 -将禁用中断，而 ”napi“ 前缀的函数只可以在softirq上下文中使用。 - -许多网络设备驱动程序使用类似的方法来分配页面片段，但页面片段是在环或描述符级别上 -缓存的。为了实现这些情况，有必要提供一种拆解页面缓存的通用方法。出于这个原因， -__page_frag_cache_drain被实现了。它允许通过一次调用从一个页面释放多个引用。 -这样做的好处是，它允许清理被添加到一个页面的多个引用，以避免每次分配都调用 -get_page。 - -Alexander Duyck，2016年11月29日。 diff --git a/Documentation/translations/zh_CN/vm/page_owner.rst b/Documentation/translations/zh_CN/vm/page_owner.rst deleted file mode 100644 index 9e951fabba9d..000000000000 --- a/Documentation/translations/zh_CN/vm/page_owner.rst +++ /dev/null @@ -1,116 +0,0 @@ -:Original: Documentation/vm/page_owner.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -================================ -page owner: 跟踪谁分配的每个页面 -================================ - -概述 -==== - -page owner是用来追踪谁分配的每一个页面。它可以用来调试内存泄漏或找到内存占用者。 -当分配发生时，有关分配的信息，如调用堆栈和页面的顺序被存储到每个页面的特定存储中。 -当我们需要了解所有页面的状态时，我们可以获得并分析这些信息。 - -尽管我们已经有了追踪页面分配/释放的tracepoint，但用它来分析谁分配的每个页面是 -相当复杂的。我们需要扩大跟踪缓冲区，以防止在用户空间程序启动前出现重叠。而且，启 -动的程序会不断地将跟踪缓冲区转出，供以后分析，这将会改变系统的行为，会产生更多的 -可能性，而不是仅仅保留在内存中，所以不利于调试。 - -页面所有者也可以用于各种目的。例如，可以通过每个页面的gfp标志信息获得精确的碎片 -统计。如果启用了page owner，它就已经实现并激活了。我们非常欢迎其他用途。 - -page owner在默认情况下是禁用的。所以，如果你想使用它，你需要在你的启动cmdline -中加入"page_owner=on"。如果内核是用page owner构建的，并且由于没有启用启动 -选项而在运行时禁用page owner，那么运行时的开销是很小的。如果在运行时禁用，它不 -需要内存来存储所有者信息，所以没有运行时内存开销。而且，页面所有者在页面分配器的 -热路径中只插入了两个不可能的分支，如果不启用，那么分配就会像没有页面所有者的内核 -一样进行。这两个不可能的分支应该不会影响到分配的性能，特别是在静态键跳转标签修补 -功能可用的情况下。以下是由于这个功能而导致的内核代码大小的变化。 - -- 没有page owner:: - - text data bss dec hex filename - 48392 2333 644 51369 c8a9 mm/page_alloc.o - -- 有page owner:: - - text data bss dec hex filename - 48800 2445 644 51889 cab1 mm/page_alloc.o - 6662 108 29 6799 1a8f mm/page_owner.o - 1025 8 8 1041 411 mm/page_ext.o - -虽然总共增加了8KB的代码，但page_alloc.o增加了520字节，其中不到一半是在hotpath -中。构建带有page owner的内核，并在需要时打开它，将是调试内核内存问题的最佳选择。 - -有一个问题是由实现细节引起的。页所有者将信息存储到struct page扩展的内存中。这 -个内存的初始化时间比稀疏内存系统中的页面分配器启动的时间要晚一些，所以，在初始化 -之前，许多页面可以被分配，但它们没有所有者信息。为了解决这个问题，这些早期分配的 -页面在初始化阶段被调查并标记为分配。虽然这并不意味着它们有正确的所有者信息，但至 -少，我们可以更准确地判断该页是否被分配。在2GB内存的x86-64虚拟机上，有13343 -个早期分配的页面被捕捉和标记，尽管它们大部分是由结构页扩展功能分配的。总之，在这 -之后，没有任何页面处于未追踪状态。 - -使用方法 -======== - -1) 构建用户空间的帮助:: - - cd tools/vm - make page_owner_sort - -2) 启用page owner: 添加 "page_owner=on" 到 boot cmdline. - -3) 做你想调试的工作。 - -4) 分析来自页面所有者的信息:: - - cat /sys/kernel/debug/page_owner > page_owner_full.txt - ./page_owner_sort page_owner_full.txt sorted_page_owner.txt - - ``page_owner_full.txt`` 的一般输出情况如下(输出信息无翻译价值):: - - Page allocated via order XXX, ... - PFN XXX ... - // Detailed stack - - Page allocated via order XXX, ... - PFN XXX ... - // Detailed stack - - ``page_owner_sort`` 工具忽略了 ``PFN`` 行，将剩余的行放在buf中，使用regexp提 - 取页序值，计算buf的次数和页数，最后根据参数进行排序。 - - 在 ``sorted_page_owner.txt`` 中可以看到关于谁分配了每个页面的结果。一般输出:: - - XXX times, XXX pages: - Page allocated via order XXX, ... - // Detailed stack - - 默认情况下， ``page_owner_sort`` 是根据buf的时间来排序的。如果你想 - 按buf的页数排序，请使用-m参数。详细的参数是: - - 基本函数: - - Sort: - -a 按内存分配时间排序 - -m 按总内存排序 - -p 按pid排序。 - -P 按tgid排序。 - -r 按内存释放时间排序。 - -s 按堆栈跟踪排序。 - -t 按时间排序（默认）。 - - 其它函数: - - Cull: - -c 通过比较堆栈跟踪而不是总块来进行剔除。 - - Filter: - -f 过滤掉内存已被释放的块的信息。 diff --git a/Documentation/translations/zh_CN/vm/page_table_check.rst b/Documentation/translations/zh_CN/vm/page_table_check.rst deleted file mode 100644 index a29fc1b360e6..000000000000 --- a/Documentation/translations/zh_CN/vm/page_table_check.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -:Original: Documentation/vm/page_table_check.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -======== -页表检查 -======== - -概述 -==== - -页表检查允许通过确保防止某些类型的内存损坏来强化内核。 - -当新的页面可以从用户空间访问时，页表检查通过将它们的页表项（PTEs PMD等）添加到页表中来执行额外 -的验证。 - -在检测到损坏的情况下，内核会被崩溃。页表检查有一个小的性能和内存开销。因此，它在默认情况下是禁用 -的，但是在额外的加固超过性能成本的系统上，可以选择启用。另外，由于页表检查是同步的，它可以帮助调 -试双映射内存损坏问题，在错误的映射发生时崩溃内核，而不是在内存损坏错误发生后内核崩溃。 - -双重映射检测逻辑 -================ - -+-------------------+-------------------+-------------------+------------------+ -| Current Mapping | New mapping | Permissions | Rule | -+===================+===================+===================+==================+ -| Anonymous | Anonymous | Read | Allow | -+-------------------+-------------------+-------------------+------------------+ -| Anonymous | Anonymous | Read / Write | Prohibit | -+-------------------+-------------------+-------------------+------------------+ -| Anonymous | Named | Any | Prohibit | -+-------------------+-------------------+-------------------+------------------+ -| Named | Anonymous | Any | Prohibit | -+-------------------+-------------------+-------------------+------------------+ -| Named | Named | Any | Allow | -+-------------------+-------------------+-------------------+------------------+ - -启用页表检查 -============ - -用以下方法构建内核: - -- PAGE_TABLE_CHECK=y - 注意，它只能在ARCH_SUPPORTS_PAGE_TABLE_CHECK可用的平台上启用。 - -- 使用 "page_table_check=on" 内核参数启动。 - -可以选择用PAGE_TABLE_CHECK_ENFORCED来构建内核，以便在没有额外的内核参数的情况下获得页表 -支持。 diff --git a/Documentation/translations/zh_CN/vm/remap_file_pages.rst b/Documentation/translations/zh_CN/vm/remap_file_pages.rst deleted file mode 100644 index af6b7e28af23..000000000000 --- a/Documentation/translations/zh_CN/vm/remap_file_pages.rst +++ /dev/null @@ -1,32 +0,0 @@ -:Original: Documentation/vm/remap_file_pages.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -============================== -remap_file_pages()系统调用 -============================== - -remap_file_pages()系统调用被用来创建一个非线性映射，也就是说，在这个映射中， -文件的页面被无序映射到内存中。使用remap_file_pages()比重复调用mmap(2)的好 -处是，前者不需要内核创建额外的VMA（虚拟内存区）数据结构。 - -支持非线性映射需要在内核虚拟内存子系统中编写大量的non-trivial的代码，包括热 -路径。另外，为了使非线性映射工作，内核需要一种方法来区分正常的页表项和带有文件 -偏移的项（pte_file）。内核为达到这个目的在PTE中保留了标志。PTE标志是稀缺资 -源，特别是在某些CPU架构上。如果能腾出这个标志用于其他用途就更好了。 - -幸运的是，在生活中并没有很多remap_file_pages()的用户。只知道有一个企业的RDBMS -实现在32位系统上使用这个系统调用来映射比32位虚拟地址空间线性尺寸更大的文件。 -由于64位系统的广泛使用，这种使用情况已经不重要了。 - -syscall被废弃了，现在用一个模拟来代替它。仿真会创建新的VMA，而不是非线性映射。 -对于remap_file_pages()的少数用户来说，它的工作速度会变慢，但ABI被保留了。 - -仿真的一个副作用（除了性能之外）是，由于额外的VMA，用户可以更容易达到 -vm.max_map_count的限制。关于限制的更多细节，请参见DEFAULT_MAX_MAP_COUNT -的注释。 diff --git a/Documentation/translations/zh_CN/vm/split_page_table_lock.rst b/Documentation/translations/zh_CN/vm/split_page_table_lock.rst deleted file mode 100644 index 50694d97c426..000000000000 --- a/Documentation/translations/zh_CN/vm/split_page_table_lock.rst +++ /dev/null @@ -1,96 +0,0 @@ -:Original: Documentation/vm/split_page_table_lock.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -================================= -分页表锁（split page table lock） -================================= - -最初，mm->page_table_lock spinlock保护了mm_struct的所有页表。但是这种方 -法导致了多线程应用程序的缺页异常可扩展性差，因为对锁的争夺很激烈。为了提高可扩 -展性，我们引入了分页表锁。 - -有了分页表锁，我们就有了单独的每张表锁来顺序化对表的访问。目前，我们对PTE和 -PMD表使用分页锁。对高层表的访问由mm->page_table_lock保护。 - -有一些辅助工具来锁定/解锁一个表和其他访问器函数: - - - pte_offset_map_lock() - 映射pte并获取PTE表锁，返回所取锁的指针； - - pte_unmap_unlock() - 解锁和解映射PTE表； - - pte_alloc_map_lock() - 如果需要的话，分配PTE表并获取锁，如果分配失败，返回已获取的锁的指针 - 或NULL； - - pte_lockptr() - 返回指向PTE表锁的指针； - - pmd_lock() - 取得PMD表锁，返回所取锁的指针。 - - pmd_lockptr() - 返回指向PMD表锁的指针； - -如果CONFIG_SPLIT_PTLOCK_CPUS（通常为4）小于或等于NR_CPUS，则在编译 -时启用PTE表的分页表锁。如果分页锁被禁用，所有的表都由mm->page_table_lock -来保护。 - -如果PMD表启用了分页锁，并且架构支持它，那么PMD表的分页锁就会被启用（见 -下文）。 - -Hugetlb 和分页表锁 -================== - -Hugetlb可以支持多种页面大小。我们只对PMD级别使用分页锁，但不对PUD使用。 - -Hugetlb特定的辅助函数: - - - huge_pte_lock() - 对PMD_SIZE页面采取pmd分割锁，否则mm->page_table_lock； - - huge_pte_lockptr() - 返回指向表锁的指针。 - -架构对分页表锁的支持 -==================== - -没有必要特别启用PTE分页表锁：所有需要的东西都由pgtable_pte_page_ctor() -和pgtable_pte_page_dtor()完成，它们必须在PTE表分配/释放时被调用。 - -确保架构不使用slab分配器来分配页表：slab使用page->slab_cache来分配其页 -面。这个区域与page->ptl共享存储。 - -PMD分页锁只有在你有两个以上的页表级别时才有意义。 - -启用PMD分页锁需要在PMD表分配时调用pgtable_pmd_page_ctor()，在释放时调 -用pgtable_pmd_page_dtor()。 - -分配通常发生在pmd_alloc_one()中，释放发生在pmd_free()和pmd_free_tlb() -中，但要确保覆盖所有的PMD表分配/释放路径：即X86_PAE在pgd_alloc()中预先 -分配一些PMD。 - -一切就绪后，你可以设置CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK。 - -注意：pgtable_pte_page_ctor()和pgtable_pmd_page_ctor()可能失败--必 -须正确处理。 - -page->ptl -========= - -page->ptl用于访问分割页表锁，其中'page'是包含该表的页面struct page。它 -与page->private（以及union中的其他几个字段）共享存储。 - -为了避免增加struct page的大小并获得最佳性能，我们使用了一个技巧: - - - 如果spinlock_t适合于long，我们使用page->ptr作为spinlock，这样我们 - 就可以避免间接访问并节省一个缓存行。 - - 如果spinlock_t的大小大于long的大小，我们使用page->ptl作为spinlock_t - 的指针并动态分配它。这允许在启用DEBUG_SPINLOCK或DEBUG_LOCK_ALLOC的 - 情况下使用分页锁，但由于间接访问而多花了一个缓存行。 - -PTE表的spinlock_t分配在pgtable_pte_page_ctor()中，PMD表的spinlock_t -分配在pgtable_pmd_page_ctor()中。 - -请不要直接访问page->ptl - -使用适当的辅助函数。 diff --git a/Documentation/translations/zh_CN/vm/z3fold.rst b/Documentation/translations/zh_CN/vm/z3fold.rst deleted file mode 100644 index 57204aa08caa..000000000000 --- a/Documentation/translations/zh_CN/vm/z3fold.rst +++ /dev/null @@ -1,31 +0,0 @@ -:Original: Documentation/vm/z3fold.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - - -====== -z3fold -====== - -z3fold是一个专门用于存储压缩页的分配器。它被设计为每个物理页最多可以存储三个压缩页。 -它是zbud的衍生物，允许更高的压缩率，保持其前辈的简单性和确定性。 - -z3fold和zbud的主要区别是: - -* 与zbud不同的是，z3fold允许最大的PAGE_SIZE分配。 -* z3fold在其页面中最多可以容纳3个压缩页面 -* z3fold本身没有输出任何API，因此打算通过zpool的API来使用 - -为了保持确定性和简单性，z3fold，就像zbud一样，总是在每页存储一个整数的压缩页，但是 -它最多可以存储3页，不像zbud最多可以存储2页。因此压缩率达到2.7倍左右，而zbud的压缩 -率是1.7倍左右。 - -不像zbud（但也像zsmalloc），z3fold_alloc()那样不返回一个可重复引用的指针。相反，它 -返回一个无符号长句柄，它编码了被分配对象的实际位置。 - -保持有效的压缩率接近于zsmalloc，z3fold不依赖于MMU的启用，并提供更可预测的回收行 -为，这使得它更适合于小型和反应迅速的系统。 diff --git a/Documentation/translations/zh_CN/vm/zsmalloc.rst b/Documentation/translations/zh_CN/vm/zsmalloc.rst deleted file mode 100644 index 29e9c70a8eb6..000000000000 --- a/Documentation/translations/zh_CN/vm/zsmalloc.rst +++ /dev/null @@ -1,78 +0,0 @@ -:Original: Documentation/vm/zs_malloc.rst - -:翻译: - - 司延腾 Yanteng Si - -:校译: - -======== -zsmalloc -======== - -这个分配器是为与zram一起使用而设计的。因此，该分配器应该在低内存条件下工作良好。特别是， -它从未尝试过higher order页面的分配，这在内存压力下很可能会失败。另一方面，如果我们只 -是使用单（0-order）页，它将遭受非常高的碎片化 - 任何大小为PAGE_SIZE/2或更大的对象将 -占据整个页面。这是其前身（xvmalloc）的主要问题之一。 - -为了克服这些问题，zsmalloc分配了一堆0-order页面，并使用各种"struct page"字段将它 -们链接起来。这些链接的页面作为一个单一的higher order页面，即一个对象可以跨越0-order -页面的边界。代码将这些链接的页面作为一个实体，称为zspage。 - -为了简单起见，zsmalloc只能分配大小不超过PAGE_SIZE的对象，因为这满足了所有当前用户的 -要求（在最坏的情况下，页面是不可压缩的，因此以"原样"即未压缩的形式存储）。对于大于这 -个大小的分配请求，会返回失败（见zs_malloc）。 - -此外，zs_malloc()并不返回一个可重复引用的指针。相反，它返回一个不透明的句柄（无符号 -长），它编码了被分配对象的实际位置。这种间接性的原因是zsmalloc并不保持zspages的永久 -映射，因为这在32位系统上会导致问题，因为内核空间映射的VA区域非常小。因此，在使用分配 -的内存之前，对象必须使用zs_map_object()进行映射以获得一个可用的指针，随后使用 -zs_unmap_object()解除映射。 - -stat -==== - -通过CONFIG_ZSMALLOC_STAT，我们可以通过 ``/sys/kernel/debug/zsmalloc/`` -看到zsmalloc内部信息。下面是一个统计输出的例子。:: - - # cat /sys/kernel/debug/zsmalloc/zram0/classes - - class size almost_full almost_empty obj_allocated obj_used pages_used pages_per_zspage - ... - ... - 9 176 0 1 186 129 8 4 - 10 192 1 0 2880 2872 135 3 - 11 208 0 1 819 795 42 2 - 12 224 0 1 219 159 12 4 - ... - ... - - -class - 索引 -size - zspage存储对象大小 -almost_empty - ZS_ALMOST_EMPTY zspage的数量（见下文）。 -almost_full - ZS_ALMOST_FULL zspage的数量(见下图) -obj_allocated - 已分配对象的数量 -obj_used - 分配给用户的对象的数量 -pages_used - 为该类分配的页数 -pages_per_zspage - 组成一个zspage的0-order页面的数量 - -当n <= N / f时，我们将一个zspage分配给ZS_ALMOST_EMPTYfullness组，其中 - -* n = 已分配对象的数量 -* N = zspage可以存储的对象总数 -* f = fullness_threshold_frac(即，目前是4个) - -同样地，我们将zspage分配给: - -* ZS_ALMOST_FULL when n > N / f -* ZS_EMPTY when n == 0 -* ZS_FULL when n == N diff --git a/Documentation/translations/zh_TW/index.rst b/Documentation/translations/zh_TW/index.rst index e1ce9d8c06f8..e97d7d578751 100644 --- a/Documentation/translations/zh_TW/index.rst +++ b/Documentation/translations/zh_TW/index.rst @@ -128,7 +128,7 @@ TODOList: * security/index * sound/index * crypto/index -* vm/index +* mm/index * bpf/index * usb/index * PCI/index diff --git a/Documentation/vm/.gitignore b/Documentation/vm/.gitignore deleted file mode 100644 index bc74f5643008..000000000000 --- a/Documentation/vm/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -page-types -slabinfo diff --git a/Documentation/vm/active_mm.rst b/Documentation/vm/active_mm.rst deleted file mode 100644 index 6f8269c284ed..000000000000 --- a/Documentation/vm/active_mm.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. _active_mm: - -========= -Active MM -========= - -:: - - List: linux-kernel - Subject: Re: active_mm - From: Linus Torvalds - Date: 1999-07-30 21:36:24 - - Cc'd to linux-kernel, because I don't write explanations all that often, - and when I do I feel better about more people reading them. - - On Fri, 30 Jul 1999, David Mosberger wrote: - > - > Is there a brief description someplace on how "mm" vs. "active_mm" in - > the task_struct are supposed to be used? (My apologies if this was - > discussed on the mailing lists---I just returned from vacation and - > wasn't able to follow linux-kernel for a while). - - Basically, the new setup is: - - - we have "real address spaces" and "anonymous address spaces". The - difference is that an anonymous address space doesn't care about the - user-level page tables at all, so when we do a context switch into an - anonymous address space we just leave the previous address space - active. - - The obvious use for a "anonymous address space" is any thread that - doesn't need any user mappings - all kernel threads basically fall into - this category, but even "real" threads can temporarily say that for - some amount of time they are not going to be interested in user space, - and that the scheduler might as well try to avoid wasting time on - switching the VM state around. Currently only the old-style bdflush - sync does that. - - - "tsk->mm" points to the "real address space". For an anonymous process, - tsk->mm will be NULL, for the logical reason that an anonymous process - really doesn't _have_ a real address space at all. - - - however, we obviously need to keep track of which address space we - "stole" for such an anonymous user. For that, we have "tsk->active_mm", - which shows what the currently active address space is. - - The rule is that for a process with a real address space (ie tsk->mm is - non-NULL) the active_mm obviously always has to be the same as the real - one. - - For a anonymous process, tsk->mm == NULL, and tsk->active_mm is the - "borrowed" mm while the anonymous process is running. When the - anonymous process gets scheduled away, the borrowed address space is - returned and cleared. - - To support all that, the "struct mm_struct" now has two counters: a - "mm_users" counter that is how many "real address space users" there are, - and a "mm_count" counter that is the number of "lazy" users (ie anonymous - users) plus one if there are any real users. - - Usually there is at least one real user, but it could be that the real - user exited on another CPU while a lazy user was still active, so you do - actually get cases where you have a address space that is _only_ used by - lazy users. That is often a short-lived state, because once that thread - gets scheduled away in favour of a real thread, the "zombie" mm gets - released because "mm_count" becomes zero. - - Also, a new rule is that _nobody_ ever has "init_mm" as a real MM any - more. "init_mm" should be considered just a "lazy context when no other - context is available", and in fact it is mainly used just at bootup when - no real VM has yet been created. So code that used to check - - if (current->mm == &init_mm) - - should generally just do - - if (!current->mm) - - instead (which makes more sense anyway - the test is basically one of "do - we have a user context", and is generally done by the page fault handler - and things like that). - - Anyway, I put a pre-patch-2.3.13-1 on ftp.kernel.org just a moment ago, - because it slightly changes the interfaces to accommodate the alpha (who - would have thought it, but the alpha actually ends up having one of the - ugliest context switch codes - unlike the other architectures where the MM - and register state is separate, the alpha PALcode joins the two, and you - need to switch both together). - - (From http://marc.info/?l=linux-kernel&m=93337278602211&w=2) diff --git a/Documentation/vm/arch_pgtable_helpers.rst b/Documentation/vm/arch_pgtable_helpers.rst deleted file mode 100644 index cbaee9e59241..000000000000 --- a/Documentation/vm/arch_pgtable_helpers.rst +++ /dev/null @@ -1,260 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -.. _arch_page_table_helpers: - -=============================== -Architecture Page Table Helpers -=============================== - -Generic MM expects architectures (with MMU) to provide helpers to create, access -and modify page table entries at various level for different memory functions. -These page table helpers need to conform to a common semantics across platforms. -Following tables describe the expected semantics which can also be tested during -boot via CONFIG_DEBUG_VM_PGTABLE option. All future changes in here or the debug -test need to be in sync. - - -PTE Page Table Helpers -====================== - -+---------------------------+--------------------------------------------------+ -| pte_same | Tests whether both PTE entries are the same | -+---------------------------+--------------------------------------------------+ -| pte_bad | Tests a non-table mapped PTE | -+---------------------------+--------------------------------------------------+ -| pte_present | Tests a valid mapped PTE | -+---------------------------+--------------------------------------------------+ -| pte_young | Tests a young PTE | -+---------------------------+--------------------------------------------------+ -| pte_dirty | Tests a dirty PTE | -+---------------------------+--------------------------------------------------+ -| pte_write | Tests a writable PTE | -+---------------------------+--------------------------------------------------+ -| pte_special | Tests a special PTE | -+---------------------------+--------------------------------------------------+ -| pte_protnone | Tests a PROT_NONE PTE | -+---------------------------+--------------------------------------------------+ -| pte_devmap | Tests a ZONE_DEVICE mapped PTE | -+---------------------------+--------------------------------------------------+ -| pte_soft_dirty | Tests a soft dirty PTE | -+---------------------------+--------------------------------------------------+ -| pte_swp_soft_dirty | Tests a soft dirty swapped PTE | -+---------------------------+--------------------------------------------------+ -| pte_mkyoung | Creates a young PTE | -+---------------------------+--------------------------------------------------+ -| pte_mkold | Creates an old PTE | -+---------------------------+--------------------------------------------------+ -| pte_mkdirty | Creates a dirty PTE | -+---------------------------+--------------------------------------------------+ -| pte_mkclean | Creates a clean PTE | -+---------------------------+--------------------------------------------------+ -| pte_mkwrite | Creates a writable PTE | -+---------------------------+--------------------------------------------------+ -| pte_wrprotect | Creates a write protected PTE | -+---------------------------+--------------------------------------------------+ -| pte_mkspecial | Creates a special PTE | -+---------------------------+--------------------------------------------------+ -| pte_mkdevmap | Creates a ZONE_DEVICE mapped PTE | -+---------------------------+--------------------------------------------------+ -| pte_mksoft_dirty | Creates a soft dirty PTE | -+---------------------------+--------------------------------------------------+ -| pte_clear_soft_dirty | Clears a soft dirty PTE | -+---------------------------+--------------------------------------------------+ -| pte_swp_mksoft_dirty | Creates a soft dirty swapped PTE | -+---------------------------+--------------------------------------------------+ -| pte_swp_clear_soft_dirty | Clears a soft dirty swapped PTE | -+---------------------------+--------------------------------------------------+ -| pte_mknotpresent | Invalidates a mapped PTE | -+---------------------------+--------------------------------------------------+ -| ptep_clear | Clears a PTE | -+---------------------------+--------------------------------------------------+ -| ptep_get_and_clear | Clears and returns PTE | -+---------------------------+--------------------------------------------------+ -| ptep_get_and_clear_full | Clears and returns PTE (batched PTE unmap) | -+---------------------------+--------------------------------------------------+ -| ptep_test_and_clear_young | Clears young from a PTE | -+---------------------------+--------------------------------------------------+ -| ptep_set_wrprotect | Converts into a write protected PTE | -+---------------------------+--------------------------------------------------+ -| ptep_set_access_flags | Converts into a more permissive PTE | -+---------------------------+--------------------------------------------------+ - - -PMD Page Table Helpers -====================== - -+---------------------------+--------------------------------------------------+ -| pmd_same | Tests whether both PMD entries are the same | -+---------------------------+--------------------------------------------------+ -| pmd_bad | Tests a non-table mapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_leaf | Tests a leaf mapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_huge | Tests a HugeTLB mapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_trans_huge | Tests a Transparent Huge Page (THP) at PMD | -+---------------------------+--------------------------------------------------+ -| pmd_present | Tests a valid mapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_young | Tests a young PMD | -+---------------------------+--------------------------------------------------+ -| pmd_dirty | Tests a dirty PMD | -+---------------------------+--------------------------------------------------+ -| pmd_write | Tests a writable PMD | -+---------------------------+--------------------------------------------------+ -| pmd_special | Tests a special PMD | -+---------------------------+--------------------------------------------------+ -| pmd_protnone | Tests a PROT_NONE PMD | -+---------------------------+--------------------------------------------------+ -| pmd_devmap | Tests a ZONE_DEVICE mapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_soft_dirty | Tests a soft dirty PMD | -+---------------------------+--------------------------------------------------+ -| pmd_swp_soft_dirty | Tests a soft dirty swapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkyoung | Creates a young PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkold | Creates an old PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkdirty | Creates a dirty PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkclean | Creates a clean PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkwrite | Creates a writable PMD | -+---------------------------+--------------------------------------------------+ -| pmd_wrprotect | Creates a write protected PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkspecial | Creates a special PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkdevmap | Creates a ZONE_DEVICE mapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mksoft_dirty | Creates a soft dirty PMD | -+---------------------------+--------------------------------------------------+ -| pmd_clear_soft_dirty | Clears a soft dirty PMD | -+---------------------------+--------------------------------------------------+ -| pmd_swp_mksoft_dirty | Creates a soft dirty swapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_swp_clear_soft_dirty | Clears a soft dirty swapped PMD | -+---------------------------+--------------------------------------------------+ -| pmd_mkinvalid | Invalidates a mapped PMD [1] | -+---------------------------+--------------------------------------------------+ -| pmd_set_huge | Creates a PMD huge mapping | -+---------------------------+--------------------------------------------------+ -| pmd_clear_huge | Clears a PMD huge mapping | -+---------------------------+--------------------------------------------------+ -| pmdp_get_and_clear | Clears a PMD | -+---------------------------+--------------------------------------------------+ -| pmdp_get_and_clear_full | Clears a PMD | -+---------------------------+--------------------------------------------------+ -| pmdp_test_and_clear_young | Clears young from a PMD | -+---------------------------+--------------------------------------------------+ -| pmdp_set_wrprotect | Converts into a write protected PMD | -+---------------------------+--------------------------------------------------+ -| pmdp_set_access_flags | Converts into a more permissive PMD | -+---------------------------+--------------------------------------------------+ - - -PUD Page Table Helpers -====================== - -+---------------------------+--------------------------------------------------+ -| pud_same | Tests whether both PUD entries are the same | -+---------------------------+--------------------------------------------------+ -| pud_bad | Tests a non-table mapped PUD | -+---------------------------+--------------------------------------------------+ -| pud_leaf | Tests a leaf mapped PUD | -+---------------------------+--------------------------------------------------+ -| pud_huge | Tests a HugeTLB mapped PUD | -+---------------------------+--------------------------------------------------+ -| pud_trans_huge | Tests a Transparent Huge Page (THP) at PUD | -+---------------------------+--------------------------------------------------+ -| pud_present | Tests a valid mapped PUD | -+---------------------------+--------------------------------------------------+ -| pud_young | Tests a young PUD | -+---------------------------+--------------------------------------------------+ -| pud_dirty | Tests a dirty PUD | -+---------------------------+--------------------------------------------------+ -| pud_write | Tests a writable PUD | -+---------------------------+--------------------------------------------------+ -| pud_devmap | Tests a ZONE_DEVICE mapped PUD | -+---------------------------+--------------------------------------------------+ -| pud_mkyoung | Creates a young PUD | -+---------------------------+--------------------------------------------------+ -| pud_mkold | Creates an old PUD | -+---------------------------+--------------------------------------------------+ -| pud_mkdirty | Creates a dirty PUD | -+---------------------------+--------------------------------------------------+ -| pud_mkclean | Creates a clean PUD | -+---------------------------+--------------------------------------------------+ -| pud_mkwrite | Creates a writable PUD | -+---------------------------+--------------------------------------------------+ -| pud_wrprotect | Creates a write protected PUD | -+---------------------------+--------------------------------------------------+ -| pud_mkdevmap | Creates a ZONE_DEVICE mapped PUD | -+---------------------------+--------------------------------------------------+ -| pud_mkinvalid | Invalidates a mapped PUD [1] | -+---------------------------+--------------------------------------------------+ -| pud_set_huge | Creates a PUD huge mapping | -+---------------------------+--------------------------------------------------+ -| pud_clear_huge | Clears a PUD huge mapping | -+---------------------------+--------------------------------------------------+ -| pudp_get_and_clear | Clears a PUD | -+---------------------------+--------------------------------------------------+ -| pudp_get_and_clear_full | Clears a PUD | -+---------------------------+--------------------------------------------------+ -| pudp_test_and_clear_young | Clears young from a PUD | -+---------------------------+--------------------------------------------------+ -| pudp_set_wrprotect | Converts into a write protected PUD | -+---------------------------+--------------------------------------------------+ -| pudp_set_access_flags | Converts into a more permissive PUD | -+---------------------------+--------------------------------------------------+ - - -HugeTLB Page Table Helpers -========================== - -+---------------------------+--------------------------------------------------+ -| pte_huge | Tests a HugeTLB | -+---------------------------+--------------------------------------------------+ -| pte_mkhuge | Creates a HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_pte_dirty | Tests a dirty HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_pte_write | Tests a writable HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_pte_mkdirty | Creates a dirty HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_pte_mkwrite | Creates a writable HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_pte_wrprotect | Creates a write protected HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_ptep_get_and_clear | Clears a HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_ptep_set_wrprotect | Converts into a write protected HugeTLB | -+---------------------------+--------------------------------------------------+ -| huge_ptep_set_access_flags | Converts into a more permissive HugeTLB | -+---------------------------+--------------------------------------------------+ - - -SWAP Page Table Helpers -======================== - -+---------------------------+--------------------------------------------------+ -| __pte_to_swp_entry | Creates a swapped entry (arch) from a mapped PTE | -+---------------------------+--------------------------------------------------+ -| __swp_to_pte_entry | Creates a mapped PTE from a swapped entry (arch) | -+---------------------------+--------------------------------------------------+ -| __pmd_to_swp_entry | Creates a swapped entry (arch) from a mapped PMD | -+---------------------------+--------------------------------------------------+ -| __swp_to_pmd_entry | Creates a mapped PMD from a swapped entry (arch) | -+---------------------------+--------------------------------------------------+ -| is_migration_entry | Tests a migration (read or write) swapped entry | -+-------------------------------+----------------------------------------------+ -| is_writable_migration_entry | Tests a write migration swapped entry | -+-------------------------------+----------------------------------------------+ -| make_readable_migration_entry | Creates a read migration swapped entry | -+-------------------------------+----------------------------------------------+ -| make_writable_migration_entry | Creates a write migration swapped entry | -+-------------------------------+----------------------------------------------+ - -[1] https://lore.kernel.org/linux-mm/20181017020930.GN30832@redhat.com/ diff --git a/Documentation/vm/balance.rst b/Documentation/vm/balance.rst deleted file mode 100644 index 6a1fadf3e173..000000000000 --- a/Documentation/vm/balance.rst +++ /dev/null @@ -1,102 +0,0 @@ -.. _balance: - -================ -Memory Balancing -================ - -Started Jan 2000 by Kanoj Sarcar - -Memory balancing is needed for !__GFP_ATOMIC and !__GFP_KSWAPD_RECLAIM as -well as for non __GFP_IO allocations. - -The first reason why a caller may avoid reclaim is that the caller can not -sleep due to holding a spinlock or is in interrupt context. The second may -be that the caller is willing to fail the allocation without incurring the -overhead of page reclaim. This may happen for opportunistic high-order -allocation requests that have order-0 fallback options. In such cases, -the caller may also wish to avoid waking kswapd. - -__GFP_IO allocation requests are made to prevent file system deadlocks. - -In the absence of non sleepable allocation requests, it seems detrimental -to be doing balancing. Page reclamation can be kicked off lazily, that -is, only when needed (aka zone free memory is 0), instead of making it -a proactive process. - -That being said, the kernel should try to fulfill requests for direct -mapped pages from the direct mapped pool, instead of falling back on -the dma pool, so as to keep the dma pool filled for dma requests (atomic -or not). A similar argument applies to highmem and direct mapped pages. -OTOH, if there is a lot of free dma pages, it is preferable to satisfy -regular memory requests by allocating one from the dma pool, instead -of incurring the overhead of regular zone balancing. - -In 2.2, memory balancing/page reclamation would kick off only when the -_total_ number of free pages fell below 1/64 th of total memory. With the -right ratio of dma and regular memory, it is quite possible that balancing -would not be done even when the dma zone was completely empty. 2.2 has -been running production machines of varying memory sizes, and seems to be -doing fine even with the presence of this problem. In 2.3, due to -HIGHMEM, this problem is aggravated. - -In 2.3, zone balancing can be done in one of two ways: depending on the -zone size (and possibly of the size of lower class zones), we can decide -at init time how many free pages we should aim for while balancing any -zone. The good part is, while balancing, we do not need to look at sizes -of lower class zones, the bad part is, we might do too frequent balancing -due to ignoring possibly lower usage in the lower class zones. Also, -with a slight change in the allocation routine, it is possible to reduce -the memclass() macro to be a simple equality. - -Another possible solution is that we balance only when the free memory -of a zone _and_ all its lower class zones falls below 1/64th of the -total memory in the zone and its lower class zones. This fixes the 2.2 -balancing problem, and stays as close to 2.2 behavior as possible. Also, -the balancing algorithm works the same way on the various architectures, -which have different numbers and types of zones. If we wanted to get -fancy, we could assign different weights to free pages in different -zones in the future. - -Note that if the size of the regular zone is huge compared to dma zone, -it becomes less significant to consider the free dma pages while -deciding whether to balance the regular zone. The first solution -becomes more attractive then. - -The appended patch implements the second solution. It also "fixes" two -problems: first, kswapd is woken up as in 2.2 on low memory conditions -for non-sleepable allocations. Second, the HIGHMEM zone is also balanced, -so as to give a fighting chance for replace_with_highmem() to get a -HIGHMEM page, as well as to ensure that HIGHMEM allocations do not -fall back into regular zone. This also makes sure that HIGHMEM pages -are not leaked (for example, in situations where a HIGHMEM page is in -the swapcache but is not being used by anyone) - -kswapd also needs to know about the zones it should balance. kswapd is -primarily needed in a situation where balancing can not be done, -probably because all allocation requests are coming from intr context -and all process contexts are sleeping. For 2.3, kswapd does not really -need to balance the highmem zone, since intr context does not request -highmem pages. kswapd looks at the zone_wake_kswapd field in the zone -structure to decide whether a zone needs balancing. - -Page stealing from process memory and shm is done if stealing the page would -alleviate memory pressure on any zone in the page's node that has fallen below -its watermark. - -watemark[WMARK_MIN/WMARK_LOW/WMARK_HIGH]/low_on_memory/zone_wake_kswapd: These -are per-zone fields, used to determine when a zone needs to be balanced. When -the number of pages falls below watermark[WMARK_MIN], the hysteric field -low_on_memory gets set. This stays set till the number of free pages becomes -watermark[WMARK_HIGH]. When low_on_memory is set, page allocation requests will -try to free some pages in the zone (providing GFP_WAIT is set in the request). -Orthogonal to this, is the decision to poke kswapd to free some zone pages. -That decision is not hysteresis based, and is done when the number of free -pages is below watermark[WMARK_LOW]; in which case zone_wake_kswapd is also set. - - -(Good) Ideas that I have heard: - -1. Dynamic experience should influence balancing: number of failed requests - for a zone can be tracked and fed into the balancing scheme (jalvo@mbay.net) -2. Implement a replace_with_highmem()-like replace_with_regular() to preserve - dma pages. (lkd@tantalophile.demon.co.uk) diff --git a/Documentation/vm/bootmem.rst b/Documentation/vm/bootmem.rst deleted file mode 100644 index eb2b31eedfa1..000000000000 --- a/Documentation/vm/bootmem.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -=========== -Boot Memory -=========== diff --git a/Documentation/vm/damon/api.rst b/Documentation/vm/damon/api.rst deleted file mode 100644 index 08f34df45523..000000000000 --- a/Documentation/vm/damon/api.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -============= -API Reference -============= - -Kernel space programs can use every feature of DAMON using below APIs. All you -need to do is including ``damon.h``, which is located in ``include/linux/`` of -the source tree. - -Structures -========== - -.. kernel-doc:: include/linux/damon.h - - -Functions -========= - -.. kernel-doc:: mm/damon/core.c diff --git a/Documentation/vm/damon/design.rst b/Documentation/vm/damon/design.rst deleted file mode 100644 index 0cff6fac6b7e..000000000000 --- a/Documentation/vm/damon/design.rst +++ /dev/null @@ -1,176 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -====== -Design -====== - -Configurable Layers -=================== - -DAMON provides data access monitoring functionality while making the accuracy -and the overhead controllable. The fundamental access monitorings require -primitives that dependent on and optimized for the target address space. On -the other hand, the accuracy and overhead tradeoff mechanism, which is the core -of DAMON, is in the pure logic space. DAMON separates the two parts in -different layers and defines its interface to allow various low level -primitives implementations configurable with the core logic. We call the low -level primitives implementations monitoring operations. - -Due to this separated design and the configurable interface, users can extend -DAMON for any address space by configuring the core logics with appropriate -monitoring operations. If appropriate one is not provided, users can implement -the operations on their own. - -For example, physical memory, virtual memory, swap space, those for specific -processes, NUMA nodes, files, and backing memory devices would be supportable. -Also, if some architectures or devices support special optimized access check -primitives, those will be easily configurable. - - -Reference Implementations of Address Space Specific Monitoring Operations -========================================================================= - -The monitoring operations are defined in two parts: - -1. Identification of the monitoring target address range for the address space. -2. Access check of specific address range in the target space. - -DAMON currently provides the implementations of the operations for the physical -and virtual address spaces. Below two subsections describe how those work. - - -VMA-based Target Address Range Construction -------------------------------------------- - -This is only for the virtual address space monitoring operations -implementation. That for the physical address space simply asks users to -manually set the monitoring target address ranges. - -Only small parts in the super-huge virtual address space of the processes are -mapped to the physical memory and accessed. Thus, tracking the unmapped -address regions is just wasteful. However, because DAMON can deal with some -level of noise using the adaptive regions adjustment mechanism, tracking every -mapping is not strictly required but could even incur a high overhead in some -cases. That said, too huge unmapped areas inside the monitoring target should -be removed to not take the time for the adaptive mechanism. - -For the reason, this implementation converts the complex mappings to three -distinct regions that cover every mapped area of the address space. The two -gaps between the three regions are the two biggest unmapped areas in the given -address space. The two biggest unmapped areas would be the gap between the -heap and the uppermost mmap()-ed region, and the gap between the lowermost -mmap()-ed region and the stack in most of the cases. Because these gaps are -exceptionally huge in usual address spaces, excluding these will be sufficient -to make a reasonable trade-off. Below shows this in detail:: - - - - - (small mmap()-ed regions and munmap()-ed regions) - - - - - -PTE Accessed-bit Based Access Check ------------------------------------ - -Both of the implementations for physical and virtual address spaces use PTE -Accessed-bit for basic access checks. Only one difference is the way of -finding the relevant PTE Accessed bit(s) from the address. While the -implementation for the virtual address walks the page table for the target task -of the address, the implementation for the physical address walks every page -table having a mapping to the address. In this way, the implementations find -and clear the bit(s) for next sampling target address and checks whether the -bit(s) set again after one sampling period. This could disturb other kernel -subsystems using the Accessed bits, namely Idle page tracking and the reclaim -logic. DAMON does nothing to avoid disturbing Idle page tracking, so handling -the interference is the responsibility of sysadmins. However, it solves the -conflict with the reclaim logic using ``PG_idle`` and ``PG_young`` page flags, -as Idle page tracking does. - - -Address Space Independent Core Mechanisms -========================================= - -Below four sections describe each of the DAMON core mechanisms and the five -monitoring attributes, ``sampling interval``, ``aggregation interval``, -``update interval``, ``minimum number of regions``, and ``maximum number of -regions``. - - -Access Frequency Monitoring ---------------------------- - -The output of DAMON says what pages are how frequently accessed for a given -duration. The resolution of the access frequency is controlled by setting -``sampling interval`` and ``aggregation interval``. In detail, DAMON checks -access to each page per ``sampling interval`` and aggregates the results. In -other words, counts the number of the accesses to each page. After each -``aggregation interval`` passes, DAMON calls callback functions that previously -registered by users so that users can read the aggregated results and then -clears the results. This can be described in below simple pseudo-code:: - - while monitoring_on: - for page in monitoring_target: - if accessed(page): - nr_accesses[page] += 1 - if time() % aggregation_interval == 0: - for callback in user_registered_callbacks: - callback(monitoring_target, nr_accesses) - for page in monitoring_target: - nr_accesses[page] = 0 - sleep(sampling interval) - -The monitoring overhead of this mechanism will arbitrarily increase as the -size of the target workload grows. - - -Region Based Sampling ---------------------- - -To avoid the unbounded increase of the overhead, DAMON groups adjacent pages -that assumed to have the same access frequencies into a region. As long as the -assumption (pages in a region have the same access frequencies) is kept, only -one page in the region is required to be checked. Thus, for each ``sampling -interval``, DAMON randomly picks one page in each region, waits for one -``sampling interval``, checks whether the page is accessed meanwhile, and -increases the access frequency of the region if so. Therefore, the monitoring -overhead is controllable by setting the number of regions. DAMON allows users -to set the minimum and the maximum number of regions for the trade-off. - -This scheme, however, cannot preserve the quality of the output if the -assumption is not guaranteed. - - -Adaptive Regions Adjustment ---------------------------- - -Even somehow the initial monitoring target regions are well constructed to -fulfill the assumption (pages in same region have similar access frequencies), -the data access pattern can be dynamically changed. This will result in low -monitoring quality. To keep the assumption as much as possible, DAMON -adaptively merges and splits each region based on their access frequency. - -For each ``aggregation interval``, it compares the access frequencies of -adjacent regions and merges those if the frequency difference is small. Then, -after it reports and clears the aggregated access frequency of each region, it -splits each region into two or three regions if the total number of regions -will not exceed the user-specified maximum number of regions after the split. - -In this way, DAMON provides its best-effort quality and minimal overhead while -keeping the bounds users set for their trade-off. - - -Dynamic Target Space Updates Handling -------------------------------------- - -The monitoring target address range could dynamically changed. For example, -virtual memory could be dynamically mapped and unmapped. Physical memory could -be hot-plugged. - -As the changes could be quite frequent in some cases, DAMON allows the -monitoring operations to check dynamic changes including memory mapping changes -and applies it to monitoring operations-related data structures such as the -abstracted monitoring target memory area only for each of a user-specified time -interval (``update interval``). diff --git a/Documentation/vm/damon/faq.rst b/Documentation/vm/damon/faq.rst deleted file mode 100644 index dde7e2414ee6..000000000000 --- a/Documentation/vm/damon/faq.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -========================== -Frequently Asked Questions -========================== - -Why a new subsystem, instead of extending perf or other user space tools? -========================================================================= - -First, because it needs to be lightweight as much as possible so that it can be -used online, any unnecessary overhead such as kernel - user space context -switching cost should be avoided. Second, DAMON aims to be used by other -programs including the kernel. Therefore, having a dependency on specific -tools like perf is not desirable. These are the two biggest reasons why DAMON -is implemented in the kernel space. - - -Can 'idle pages tracking' or 'perf mem' substitute DAMON? -========================================================= - -Idle page tracking is a low level primitive for access check of the physical -address space. 'perf mem' is similar, though it can use sampling to minimize -the overhead. On the other hand, DAMON is a higher-level framework for the -monitoring of various address spaces. It is focused on memory management -optimization and provides sophisticated accuracy/overhead handling mechanisms. -Therefore, 'idle pages tracking' and 'perf mem' could provide a subset of -DAMON's output, but cannot substitute DAMON. - - -Does DAMON support virtual memory only? -======================================= - -No. The core of the DAMON is address space independent. The address space -specific monitoring operations including monitoring target regions -constructions and actual access checks can be implemented and configured on the -DAMON core by the users. In this way, DAMON users can monitor any address -space with any access check technique. - -Nonetheless, DAMON provides vma/rmap tracking and PTE Accessed bit check based -implementations of the address space dependent functions for the virtual memory -and the physical memory by default, for a reference and convenient use. - - -Can I simply monitor page granularity? -====================================== - -Yes. You can do so by setting the ``min_nr_regions`` attribute higher than the -working set size divided by the page size. Because the monitoring target -regions size is forced to be ``>=page size``, the region split will make no -effect. diff --git a/Documentation/vm/damon/index.rst b/Documentation/vm/damon/index.rst deleted file mode 100644 index 48c0bbff98b2..000000000000 --- a/Documentation/vm/damon/index.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -========================== -DAMON: Data Access MONitor -========================== - -DAMON is a data access monitoring framework subsystem for the Linux kernel. -The core mechanisms of DAMON (refer to :doc:`design` for the detail) make it - - - *accurate* (the monitoring output is useful enough for DRAM level memory - management; It might not appropriate for CPU Cache levels, though), - - *light-weight* (the monitoring overhead is low enough to be applied online), - and - - *scalable* (the upper-bound of the overhead is in constant range regardless - of the size of target workloads). - -Using this framework, therefore, the kernel's memory management mechanisms can -make advanced decisions. Experimental memory management optimization works -that incurring high data accesses monitoring overhead could implemented again. -In user space, meanwhile, users who have some special workloads can write -personalized applications for better understanding and optimizations of their -workloads and systems. - -.. toctree:: - :maxdepth: 2 - - faq - design - api diff --git a/Documentation/vm/free_page_reporting.rst b/Documentation/vm/free_page_reporting.rst deleted file mode 100644 index 8c05e62d8b2b..000000000000 --- a/Documentation/vm/free_page_reporting.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. _free_page_reporting: - -===================== -Free Page Reporting -===================== - -Free page reporting is an API by which a device can register to receive -lists of pages that are currently unused by the system. This is useful in -the case of virtualization where a guest is then able to use this data to -notify the hypervisor that it is no longer using certain pages in memory. - -For the driver, typically a balloon driver, to use of this functionality -it will allocate and initialize a page_reporting_dev_info structure. The -field within the structure it will populate is the "report" function -pointer used to process the scatterlist. It must also guarantee that it can -handle at least PAGE_REPORTING_CAPACITY worth of scatterlist entries per -call to the function. A call to page_reporting_register will register the -page reporting interface with the reporting framework assuming no other -page reporting devices are already registered. - -Once registered the page reporting API will begin reporting batches of -pages to the driver. The API will start reporting pages 2 seconds after -the interface is registered and will continue to do so 2 seconds after any -page of a sufficiently high order is freed. - -Pages reported will be stored in the scatterlist passed to the reporting -function with the final entry having the end bit set in entry nent - 1. -While pages are being processed by the report function they will not be -accessible to the allocator. Once the report function has been completed -the pages will be returned to the free area from which they were obtained. - -Prior to removing a driver that is making use of free page reporting it -is necessary to call page_reporting_unregister to have the -page_reporting_dev_info structure that is currently in use by free page -reporting removed. Doing this will prevent further reports from being -issued via the interface. If another driver or the same driver is -registered it is possible for it to resume where the previous driver had -left off in terms of reporting free pages. - -Alexander Duyck, Dec 04, 2019 diff --git a/Documentation/vm/frontswap.rst b/Documentation/vm/frontswap.rst deleted file mode 100644 index feecc5e24477..000000000000 --- a/Documentation/vm/frontswap.rst +++ /dev/null @@ -1,266 +0,0 @@ -.. _frontswap: - -========= -Frontswap -========= - -Frontswap provides a "transcendent memory" interface for swap pages. -In some environments, dramatic performance savings may be obtained because -swapped pages are saved in RAM (or a RAM-like device) instead of a swap disk. - -.. _Transcendent memory in a nutshell: https://lwn.net/Articles/454795/ - -Frontswap is so named because it can be thought of as the opposite of -a "backing" store for a swap device. The storage is assumed to be -a synchronous concurrency-safe page-oriented "pseudo-RAM device" conforming -to the requirements of transcendent memory (such as Xen's "tmem", or -in-kernel compressed memory, aka "zcache", or future RAM-like devices); -this pseudo-RAM device is not directly accessible or addressable by the -kernel and is of unknown and possibly time-varying size. The driver -links itself to frontswap by calling frontswap_register_ops to set the -frontswap_ops funcs appropriately and the functions it provides must -conform to certain policies as follows: - -An "init" prepares the device to receive frontswap pages associated -with the specified swap device number (aka "type"). A "store" will -copy the page to transcendent memory and associate it with the type and -offset associated with the page. A "load" will copy the page, if found, -from transcendent memory into kernel memory, but will NOT remove the page -from transcendent memory. An "invalidate_page" will remove the page -from transcendent memory and an "invalidate_area" will remove ALL pages -associated with the swap type (e.g., like swapoff) and notify the "device" -to refuse further stores with that swap type. - -Once a page is successfully stored, a matching load on the page will normally -succeed. So when the kernel finds itself in a situation where it needs -to swap out a page, it first attempts to use frontswap. If the store returns -success, the data has been successfully saved to transcendent memory and -a disk write and, if the data is later read back, a disk read are avoided. -If a store returns failure, transcendent memory has rejected the data, and the -page can be written to swap as usual. - -Note that if a page is stored and the page already exists in transcendent memory -(a "duplicate" store), either the store succeeds and the data is overwritten, -or the store fails AND the page is invalidated. This ensures stale data may -never be obtained from frontswap. - -If properly configured, monitoring of frontswap is done via debugfs in -the `/sys/kernel/debug/frontswap` directory. The effectiveness of -frontswap can be measured (across all swap devices) with: - -``failed_stores`` - how many store attempts have failed - -``loads`` - how many loads were attempted (all should succeed) - -``succ_stores`` - how many store attempts have succeeded - -``invalidates`` - how many invalidates were attempted - -A backend implementation may provide additional metrics. - -FAQ -=== - -* Where's the value? - -When a workload starts swapping, performance falls through the floor. -Frontswap significantly increases performance in many such workloads by -providing a clean, dynamic interface to read and write swap pages to -"transcendent memory" that is otherwise not directly addressable to the kernel. -This interface is ideal when data is transformed to a different form -and size (such as with compression) or secretly moved (as might be -useful for write-balancing for some RAM-like devices). Swap pages (and -evicted page-cache pages) are a great use for this kind of slower-than-RAM- -but-much-faster-than-disk "pseudo-RAM device". - -Frontswap with a fairly small impact on the kernel, -provides a huge amount of flexibility for more dynamic, flexible RAM -utilization in various system configurations: - -In the single kernel case, aka "zcache", pages are compressed and -stored in local memory, thus increasing the total anonymous pages -that can be safely kept in RAM. Zcache essentially trades off CPU -cycles used in compression/decompression for better memory utilization. -Benchmarks have shown little or no impact when memory pressure is -low while providing a significant performance improvement (25%+) -on some workloads under high memory pressure. - -"RAMster" builds on zcache by adding "peer-to-peer" transcendent memory -support for clustered systems. Frontswap pages are locally compressed -as in zcache, but then "remotified" to another system's RAM. This -allows RAM to be dynamically load-balanced back-and-forth as needed, -i.e. when system A is overcommitted, it can swap to system B, and -vice versa. RAMster can also be configured as a memory server so -many servers in a cluster can swap, dynamically as needed, to a single -server configured with a large amount of RAM... without pre-configuring -how much of the RAM is available for each of the clients! - -In the virtual case, the whole point of virtualization is to statistically -multiplex physical resources across the varying demands of multiple -virtual machines. This is really hard to do with RAM and efforts to do -it well with no kernel changes have essentially failed (except in some -well-publicized special-case workloads). -Specifically, the Xen Transcendent Memory backend allows otherwise -"fallow" hypervisor-owned RAM to not only be "time-shared" between multiple -virtual machines, but the pages can be compressed and deduplicated to -optimize RAM utilization. And when guest OS's are induced to surrender -underutilized RAM (e.g. with "selfballooning"), sudden unexpected -memory pressure may result in swapping; frontswap allows those pages -to be swapped to and from hypervisor RAM (if overall host system memory -conditions allow), thus mitigating the potentially awful performance impact -of unplanned swapping. - -A KVM implementation is underway and has been RFC'ed to lkml. And, -using frontswap, investigation is also underway on the use of NVM as -a memory extension technology. - -* Sure there may be performance advantages in some situations, but - what's the space/time overhead of frontswap? - -If CONFIG_FRONTSWAP is disabled, every frontswap hook compiles into -nothingness and the only overhead is a few extra bytes per swapon'ed -swap device. If CONFIG_FRONTSWAP is enabled but no frontswap "backend" -registers, there is one extra global variable compared to zero for -every swap page read or written. If CONFIG_FRONTSWAP is enabled -AND a frontswap backend registers AND the backend fails every "store" -request (i.e. provides no memory despite claiming it might), -CPU overhead is still negligible -- and since every frontswap fail -precedes a swap page write-to-disk, the system is highly likely -to be I/O bound and using a small fraction of a percent of a CPU -will be irrelevant anyway. - -As for space, if CONFIG_FRONTSWAP is enabled AND a frontswap backend -registers, one bit is allocated for every swap page for every swap -device that is swapon'd. This is added to the EIGHT bits (which -was sixteen until about 2.6.34) that the kernel already allocates -for every swap page for every swap device that is swapon'd. (Hugh -Dickins has observed that frontswap could probably steal one of -the existing eight bits, but let's worry about that minor optimization -later.) For very large swap disks (which are rare) on a standard -4K pagesize, this is 1MB per 32GB swap. - -When swap pages are stored in transcendent memory instead of written -out to disk, there is a side effect that this may create more memory -pressure that can potentially outweigh the other advantages. A -backend, such as zcache, must implement policies to carefully (but -dynamically) manage memory limits to ensure this doesn't happen. - -* OK, how about a quick overview of what this frontswap patch does - in terms that a kernel hacker can grok? - -Let's assume that a frontswap "backend" has registered during -kernel initialization; this registration indicates that this -frontswap backend has access to some "memory" that is not directly -accessible by the kernel. Exactly how much memory it provides is -entirely dynamic and random. - -Whenever a swap-device is swapon'd frontswap_init() is called, -passing the swap device number (aka "type") as a parameter. -This notifies frontswap to expect attempts to "store" swap pages -associated with that number. - -Whenever the swap subsystem is readying a page to write to a swap -device (c.f swap_writepage()), frontswap_store is called. Frontswap -consults with the frontswap backend and if the backend says it does NOT -have room, frontswap_store returns -1 and the kernel swaps the page -to the swap device as normal. Note that the response from the frontswap -backend is unpredictable to the kernel; it may choose to never accept a -page, it could accept every ninth page, or it might accept every -page. But if the backend does accept a page, the data from the page -has already been copied and associated with the type and offset, -and the backend guarantees the persistence of the data. In this case, -frontswap sets a bit in the "frontswap_map" for the swap device -corresponding to the page offset on the swap device to which it would -otherwise have written the data. - -When the swap subsystem needs to swap-in a page (swap_readpage()), -it first calls frontswap_load() which checks the frontswap_map to -see if the page was earlier accepted by the frontswap backend. If -it was, the page of data is filled from the frontswap backend and -the swap-in is complete. If not, the normal swap-in code is -executed to obtain the page of data from the real swap device. - -So every time the frontswap backend accepts a page, a swap device read -and (potentially) a swap device write are replaced by a "frontswap backend -store" and (possibly) a "frontswap backend loads", which are presumably much -faster. - -* Can't frontswap be configured as a "special" swap device that is - just higher priority than any real swap device (e.g. like zswap, - or maybe swap-over-nbd/NFS)? - -No. First, the existing swap subsystem doesn't allow for any kind of -swap hierarchy. Perhaps it could be rewritten to accommodate a hierarchy, -but this would require fairly drastic changes. Even if it were -rewritten, the existing swap subsystem uses the block I/O layer which -assumes a swap device is fixed size and any page in it is linearly -addressable. Frontswap barely touches the existing swap subsystem, -and works around the constraints of the block I/O subsystem to provide -a great deal of flexibility and dynamicity. - -For example, the acceptance of any swap page by the frontswap backend is -entirely unpredictable. This is critical to the definition of frontswap -backends because it grants completely dynamic discretion to the -backend. In zcache, one cannot know a priori how compressible a page is. -"Poorly" compressible pages can be rejected, and "poorly" can itself be -defined dynamically depending on current memory constraints. - -Further, frontswap is entirely synchronous whereas a real swap -device is, by definition, asynchronous and uses block I/O. The -block I/O layer is not only unnecessary, but may perform "optimizations" -that are inappropriate for a RAM-oriented device including delaying -the write of some pages for a significant amount of time. Synchrony is -required to ensure the dynamicity of the backend and to avoid thorny race -conditions that would unnecessarily and greatly complicate frontswap -and/or the block I/O subsystem. That said, only the initial "store" -and "load" operations need be synchronous. A separate asynchronous thread -is free to manipulate the pages stored by frontswap. For example, -the "remotification" thread in RAMster uses standard asynchronous -kernel sockets to move compressed frontswap pages to a remote machine. -Similarly, a KVM guest-side implementation could do in-guest compression -and use "batched" hypercalls. - -In a virtualized environment, the dynamicity allows the hypervisor -(or host OS) to do "intelligent overcommit". For example, it can -choose to accept pages only until host-swapping might be imminent, -then force guests to do their own swapping. - -There is a downside to the transcendent memory specifications for -frontswap: Since any "store" might fail, there must always be a real -slot on a real swap device to swap the page. Thus frontswap must be -implemented as a "shadow" to every swapon'd device with the potential -capability of holding every page that the swap device might have held -and the possibility that it might hold no pages at all. This means -that frontswap cannot contain more pages than the total of swapon'd -swap devices. For example, if NO swap device is configured on some -installation, frontswap is useless. Swapless portable devices -can still use frontswap but a backend for such devices must configure -some kind of "ghost" swap device and ensure that it is never used. - -* Why this weird definition about "duplicate stores"? If a page - has been previously successfully stored, can't it always be - successfully overwritten? - -Nearly always it can, but no, sometimes it cannot. Consider an example -where data is compressed and the original 4K page has been compressed -to 1K. Now an attempt is made to overwrite the page with data that -is non-compressible and so would take the entire 4K. But the backend -has no more space. In this case, the store must be rejected. Whenever -frontswap rejects a store that would overwrite, it also must invalidate -the old data and ensure that it is no longer accessible. Since the -swap subsystem then writes the new data to the read swap device, -this is the correct course of action to ensure coherency. - -* Why does the frontswap patch create the new include file swapfile.h? - -The frontswap code depends on some swap-subsystem-internal data -structures that have, over the years, moved back and forth between -static and global. This seemed a reasonable compromise: Define -them as global but declare them in a new include file that isn't -included by the large number of source files that include swap.h. - -Dan Magenheimer, last updated April 9, 2012 diff --git a/Documentation/vm/highmem.rst b/Documentation/vm/highmem.rst deleted file mode 100644 index c9887f241c6c..000000000000 --- a/Documentation/vm/highmem.rst +++ /dev/null @@ -1,167 +0,0 @@ -.. _highmem: - -==================== -High Memory Handling -==================== - -By: Peter Zijlstra - -.. contents:: :local: - -What Is High Memory? -==================== - -High memory (highmem) is used when the size of physical memory approaches or -exceeds the maximum size of virtual memory. At that point it becomes -impossible for the kernel to keep all of the available physical memory mapped -at all times. This means the kernel needs to start using temporary mappings of -the pieces of physical memory that it wants to access. - -The part of (physical) memory not covered by a permanent mapping is what we -refer to as 'highmem'. There are various architecture dependent constraints on -where exactly that border lies. - -In the i386 arch, for example, we choose to map the kernel into every process's -VM space so that we don't have to pay the full TLB invalidation costs for -kernel entry/exit. This means the available virtual memory space (4GiB on -i386) has to be divided between user and kernel space. - -The traditional split for architectures using this approach is 3:1, 3GiB for -userspace and the top 1GiB for kernel space:: - - +--------+ 0xffffffff - | Kernel | - +--------+ 0xc0000000 - | | - | User | - | | - +--------+ 0x00000000 - -This means that the kernel can at most map 1GiB of physical memory at any one -time, but because we need virtual address space for other things - including -temporary maps to access the rest of the physical memory - the actual direct -map will typically be less (usually around ~896MiB). - -Other architectures that have mm context tagged TLBs can have separate kernel -and user maps. Some hardware (like some ARMs), however, have limited virtual -space when they use mm context tags. - - -Temporary Virtual Mappings -========================== - -The kernel contains several ways of creating temporary mappings. The following -list shows them in order of preference of use. - -* kmap_local_page(). This function is used to require short term mappings. - It can be invoked from any context (including interrupts) but the mappings - can only be used in the context which acquired them. - - This function should be preferred, where feasible, over all the others. - - These mappings are thread-local and CPU-local, meaning that the mapping - can only be accessed from within this thread and the thread is bound the - CPU while the mapping is active. Even if the thread is preempted (since - preemption is never disabled by the function) the CPU can not be - unplugged from the system via CPU-hotplug until the mapping is disposed. - - It's valid to take pagefaults in a local kmap region, unless the context - in which the local mapping is acquired does not allow it for other reasons. - - kmap_local_page() always returns a valid virtual address and it is assumed - that kunmap_local() will never fail. - - Nesting kmap_local_page() and kmap_atomic() mappings is allowed to a certain - extent (up to KMAP_TYPE_NR) but their invocations have to be strictly ordered - because the map implementation is stack based. See kmap_local_page() kdocs - (included in the "Functions" section) for details on how to manage nested - mappings. - -* kmap_atomic(). This permits a very short duration mapping of a single - page. Since the mapping is restricted to the CPU that issued it, it - performs well, but the issuing task is therefore required to stay on that - CPU until it has finished, lest some other task displace its mappings. - - kmap_atomic() may also be used by interrupt contexts, since it does not - sleep and the callers too may not sleep until after kunmap_atomic() is - called. - - Each call of kmap_atomic() in the kernel creates a non-preemptible section - and disable pagefaults. This could be a source of unwanted latency. Therefore - users should prefer kmap_local_page() instead of kmap_atomic(). - - It is assumed that k[un]map_atomic() won't fail. - -* kmap(). This should be used to make short duration mapping of a single - page with no restrictions on preemption or migration. It comes with an - overhead as mapping space is restricted and protected by a global lock - for synchronization. When mapping is no longer needed, the address that - the page was mapped to must be released with kunmap(). - - Mapping changes must be propagated across all the CPUs. kmap() also - requires global TLB invalidation when the kmap's pool wraps and it might - block when the mapping space is fully utilized until a slot becomes - available. Therefore, kmap() is only callable from preemptible context. - - All the above work is necessary if a mapping must last for a relatively - long time but the bulk of high-memory mappings in the kernel are - short-lived and only used in one place. This means that the cost of - kmap() is mostly wasted in such cases. kmap() was not intended for long - term mappings but it has morphed in that direction and its use is - strongly discouraged in newer code and the set of the preceding functions - should be preferred. - - On 64-bit systems, calls to kmap_local_page(), kmap_atomic() and kmap() have - no real work to do because a 64-bit address space is more than sufficient to - address all the physical memory whose pages are permanently mapped. - -* vmap(). This can be used to make a long duration mapping of multiple - physical pages into a contiguous virtual space. It needs global - synchronization to unmap. - - -Cost of Temporary Mappings -========================== - -The cost of creating temporary mappings can be quite high. The arch has to -manipulate the kernel's page tables, the data TLB and/or the MMU's registers. - -If CONFIG_HIGHMEM is not set, then the kernel will try and create a mapping -simply with a bit of arithmetic that will convert the page struct address into -a pointer to the page contents rather than juggling mappings about. In such a -case, the unmap operation may be a null operation. - -If CONFIG_MMU is not set, then there can be no temporary mappings and no -highmem. In such a case, the arithmetic approach will also be used. - - -i386 PAE -======== - -The i386 arch, under some circumstances, will permit you to stick up to 64GiB -of RAM into your 32-bit machine. This has a number of consequences: - -* Linux needs a page-frame structure for each page in the system and the - pageframes need to live in the permanent mapping, which means: - -* you can have 896M/sizeof(struct page) page-frames at most; with struct - page being 32-bytes that would end up being something in the order of 112G - worth of pages; the kernel, however, needs to store more than just - page-frames in that memory... - -* PAE makes your page tables larger - which slows the system down as more - data has to be accessed to traverse in TLB fills and the like. One - advantage is that PAE has more PTE bits and can provide advanced features - like NX and PAT. - -The general recommendation is that you don't use more than 8GiB on a 32-bit -machine - although more might work for you and your workload, you're pretty -much on your own - don't expect kernel developers to really care much if things -come apart. - - -Functions -========= - -.. kernel-doc:: include/linux/highmem.h -.. kernel-doc:: include/linux/highmem-internal.h diff --git a/Documentation/vm/hmm.rst b/Documentation/vm/hmm.rst deleted file mode 100644 index f2a59ed82ed3..000000000000 --- a/Documentation/vm/hmm.rst +++ /dev/null @@ -1,452 +0,0 @@ -.. _hmm: - -===================================== -Heterogeneous Memory Management (HMM) -===================================== - -Provide infrastructure and helpers to integrate non-conventional memory (device -memory like GPU on board memory) into regular kernel path, with the cornerstone -of this being specialized struct page for such memory (see sections 5 to 7 of -this document). - -HMM also provides optional helpers for SVM (Share Virtual Memory), i.e., -allowing a device to transparently access program addresses coherently with -the CPU meaning that any valid pointer on the CPU is also a valid pointer -for the device. This is becoming mandatory to simplify the use of advanced -heterogeneous computing where GPU, DSP, or FPGA are used to perform various -computations on behalf of a process. - -This document is divided as follows: in the first section I expose the problems -related to using device specific memory allocators. In the second section, I -expose the hardware limitations that are inherent to many platforms. The third -section gives an overview of the HMM design. The fourth section explains how -CPU page-table mirroring works and the purpose of HMM in this context. The -fifth section deals with how device memory is represented inside the kernel. -Finally, the last section presents a new migration helper that allows -leveraging the device DMA engine. - -.. contents:: :local: - -Problems of using a device specific memory allocator -==================================================== - -Devices with a large amount of on board memory (several gigabytes) like GPUs -have historically managed their memory through dedicated driver specific APIs. -This creates a disconnect between memory allocated and managed by a device -driver and regular application memory (private anonymous, shared memory, or -regular file backed memory). From here on I will refer to this aspect as split -address space. I use shared address space to refer to the opposite situation: -i.e., one in which any application memory region can be used by a device -transparently. - -Split address space happens because devices can only access memory allocated -through a device specific API. This implies that all memory objects in a program -are not equal from the device point of view which complicates large programs -that rely on a wide set of libraries. - -Concretely, this means that code that wants to leverage devices like GPUs needs -to copy objects between generically allocated memory (malloc, mmap private, mmap -share) and memory allocated through the device driver API (this still ends up -with an mmap but of the device file). - -For flat data sets (array, grid, image, ...) this isn't too hard to achieve but -for complex data sets (list, tree, ...) it's hard to get right. Duplicating a -complex data set needs to re-map all the pointer relations between each of its -elements. This is error prone and programs get harder to debug because of the -duplicate data set and addresses. - -Split address space also means that libraries cannot transparently use data -they are getting from the core program or another library and thus each library -might have to duplicate its input data set using the device specific memory -allocator. Large projects suffer from this and waste resources because of the -various memory copies. - -Duplicating each library API to accept as input or output memory allocated by -each device specific allocator is not a viable option. It would lead to a -combinatorial explosion in the library entry points. - -Finally, with the advance of high level language constructs (in C++ but in -other languages too) it is now possible for the compiler to leverage GPUs and -other devices without programmer knowledge. Some compiler identified patterns -are only do-able with a shared address space. It is also more reasonable to use -a shared address space for all other patterns. - - -I/O bus, device memory characteristics -====================================== - -I/O buses cripple shared address spaces due to a few limitations. Most I/O -buses only allow basic memory access from device to main memory; even cache -coherency is often optional. Access to device memory from a CPU is even more -limited. More often than not, it is not cache coherent. - -If we only consider the PCIE bus, then a device can access main memory (often -through an IOMMU) and be cache coherent with the CPUs. However, it only allows -a limited set of atomic operations from the device on main memory. This is worse -in the other direction: the CPU can only access a limited range of the device -memory and cannot perform atomic operations on it. Thus device memory cannot -be considered the same as regular memory from the kernel point of view. - -Another crippling factor is the limited bandwidth (~32GBytes/s with PCIE 4.0 -and 16 lanes). This is 33 times less than the fastest GPU memory (1 TBytes/s). -The final limitation is latency. Access to main memory from the device has an -order of magnitude higher latency than when the device accesses its own memory. - -Some platforms are developing new I/O buses or additions/modifications to PCIE -to address some of these limitations (OpenCAPI, CCIX). They mainly allow -two-way cache coherency between CPU and device and allow all atomic operations the -architecture supports. Sadly, not all platforms are following this trend and -some major architectures are left without hardware solutions to these problems. - -So for shared address space to make sense, not only must we allow devices to -access any memory but we must also permit any memory to be migrated to device -memory while the device is using it (blocking CPU access while it happens). - - -Shared address space and migration -================================== - -HMM intends to provide two main features. The first one is to share the address -space by duplicating the CPU page table in the device page table so the same -address points to the same physical memory for any valid main memory address in -the process address space. - -To achieve this, HMM offers a set of helpers to populate the device page table -while keeping track of CPU page table updates. Device page table updates are -not as easy as CPU page table updates. To update the device page table, you must -allocate a buffer (or use a pool of pre-allocated buffers) and write GPU -specific commands in it to perform the update (unmap, cache invalidations, and -flush, ...). This cannot be done through common code for all devices. Hence -why HMM provides helpers to factor out everything that can be while leaving the -hardware specific details to the device driver. - -The second mechanism HMM provides is a new kind of ZONE_DEVICE memory that -allows allocating a struct page for each page of device memory. Those pages -are special because the CPU cannot map them. However, they allow migrating -main memory to device memory using existing migration mechanisms and everything -looks like a page that is swapped out to disk from the CPU point of view. Using a -struct page gives the easiest and cleanest integration with existing mm -mechanisms. Here again, HMM only provides helpers, first to hotplug new ZONE_DEVICE -memory for the device memory and second to perform migration. Policy decisions -of what and when to migrate is left to the device driver. - -Note that any CPU access to a device page triggers a page fault and a migration -back to main memory. For example, when a page backing a given CPU address A is -migrated from a main memory page to a device page, then any CPU access to -address A triggers a page fault and initiates a migration back to main memory. - -With these two features, HMM not only allows a device to mirror process address -space and keeps both CPU and device page tables synchronized, but also -leverages device memory by migrating the part of the data set that is actively being -used by the device. - - -Address space mirroring implementation and API -============================================== - -Address space mirroring's main objective is to allow duplication of a range of -CPU page table into a device page table; HMM helps keep both synchronized. A -device driver that wants to mirror a process address space must start with the -registration of a mmu_interval_notifier:: - - int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub, - struct mm_struct *mm, unsigned long start, - unsigned long length, - const struct mmu_interval_notifier_ops *ops); - -During the ops->invalidate() callback the device driver must perform the -update action to the range (mark range read only, or fully unmap, etc.). The -device must complete the update before the driver callback returns. - -When the device driver wants to populate a range of virtual addresses, it can -use:: - - int hmm_range_fault(struct hmm_range *range); - -It will trigger a page fault on missing or read-only entries if write access is -requested (see below). Page faults use the generic mm page fault code path just -like a CPU page fault. - -Both functions copy CPU page table entries into their pfns array argument. Each -entry in that array corresponds to an address in the virtual range. HMM -provides a set of flags to help the driver identify special CPU page table -entries. - -Locking within the sync_cpu_device_pagetables() callback is the most important -aspect the driver must respect in order to keep things properly synchronized. -The usage pattern is:: - - int driver_populate_range(...) - { - struct hmm_range range; - ... - - range.notifier = &interval_sub; - range.start = ...; - range.end = ...; - range.hmm_pfns = ...; - - if (!mmget_not_zero(interval_sub->notifier.mm)) - return -EFAULT; - - again: - range.notifier_seq = mmu_interval_read_begin(&interval_sub); - mmap_read_lock(mm); - ret = hmm_range_fault(&range); - if (ret) { - mmap_read_unlock(mm); - if (ret == -EBUSY) - goto again; - return ret; - } - mmap_read_unlock(mm); - - take_lock(driver->update); - if (mmu_interval_read_retry(&ni, range.notifier_seq) { - release_lock(driver->update); - goto again; - } - - /* Use pfns array content to update device page table, - * under the update lock */ - - release_lock(driver->update); - return 0; - } - -The driver->update lock is the same lock that the driver takes inside its -invalidate() callback. That lock must be held before calling -mmu_interval_read_retry() to avoid any race with a concurrent CPU page table -update. - -Leverage default_flags and pfn_flags_mask -========================================= - -The hmm_range struct has 2 fields, default_flags and pfn_flags_mask, that specify -fault or snapshot policy for the whole range instead of having to set them -for each entry in the pfns array. - -For instance if the device driver wants pages for a range with at least read -permission, it sets:: - - range->default_flags = HMM_PFN_REQ_FAULT; - range->pfn_flags_mask = 0; - -and calls hmm_range_fault() as described above. This will fill fault all pages -in the range with at least read permission. - -Now let's say the driver wants to do the same except for one page in the range for -which it wants to have write permission. Now driver set:: - - range->default_flags = HMM_PFN_REQ_FAULT; - range->pfn_flags_mask = HMM_PFN_REQ_WRITE; - range->pfns[index_of_write] = HMM_PFN_REQ_WRITE; - -With this, HMM will fault in all pages with at least read (i.e., valid) and for the -address == range->start + (index_of_write << PAGE_SHIFT) it will fault with -write permission i.e., if the CPU pte does not have write permission set then HMM -will call handle_mm_fault(). - -After hmm_range_fault completes the flag bits are set to the current state of -the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be set if the page is -writable. - - -Represent and manage device memory from core kernel point of view -================================================================= - -Several different designs were tried to support device memory. The first one -used a device specific data structure to keep information about migrated memory -and HMM hooked itself in various places of mm code to handle any access to -addresses that were backed by device memory. It turns out that this ended up -replicating most of the fields of struct page and also needed many kernel code -paths to be updated to understand this new kind of memory. - -Most kernel code paths never try to access the memory behind a page -but only care about struct page contents. Because of this, HMM switched to -directly using struct page for device memory which left most kernel code paths -unaware of the difference. We only need to make sure that no one ever tries to -map those pages from the CPU side. - -Migration to and from device memory -=================================== - -Because the CPU cannot access device memory directly, the device driver must -use hardware DMA or device specific load/store instructions to migrate data. -The migrate_vma_setup(), migrate_vma_pages(), and migrate_vma_finalize() -functions are designed to make drivers easier to write and to centralize common -code across drivers. - -Before migrating pages to device private memory, special device private -``struct page`` need to be created. These will be used as special "swap" -page table entries so that a CPU process will fault if it tries to access -a page that has been migrated to device private memory. - -These can be allocated and freed with:: - - struct resource *res; - struct dev_pagemap pagemap; - - res = request_free_mem_region(&iomem_resource, /* number of bytes */, - "name of driver resource"); - pagemap.type = MEMORY_DEVICE_PRIVATE; - pagemap.range.start = res->start; - pagemap.range.end = res->end; - pagemap.nr_range = 1; - pagemap.ops = &device_devmem_ops; - memremap_pages(&pagemap, numa_node_id()); - - memunmap_pages(&pagemap); - release_mem_region(pagemap.range.start, range_len(&pagemap.range)); - -There are also devm_request_free_mem_region(), devm_memremap_pages(), -devm_memunmap_pages(), and devm_release_mem_region() when the resources can -be tied to a ``struct device``. - -The overall migration steps are similar to migrating NUMA pages within system -memory (see :ref:`Page migration `) but the steps are split -between device driver specific code and shared common code: - -1. ``mmap_read_lock()`` - - The device driver has to pass a ``struct vm_area_struct`` to - migrate_vma_setup() so the mmap_read_lock() or mmap_write_lock() needs to - be held for the duration of the migration. - -2. ``migrate_vma_setup(struct migrate_vma *args)`` - - The device driver initializes the ``struct migrate_vma`` fields and passes - the pointer to migrate_vma_setup(). The ``args->flags`` field is used to - filter which source pages should be migrated. For example, setting - ``MIGRATE_VMA_SELECT_SYSTEM`` will only migrate system memory and - ``MIGRATE_VMA_SELECT_DEVICE_PRIVATE`` will only migrate pages residing in - device private memory. If the latter flag is set, the ``args->pgmap_owner`` - field is used to identify device private pages owned by the driver. This - avoids trying to migrate device private pages residing in other devices. - Currently only anonymous private VMA ranges can be migrated to or from - system memory and device private memory. - - One of the first steps migrate_vma_setup() does is to invalidate other - device's MMUs with the ``mmu_notifier_invalidate_range_start(()`` and - ``mmu_notifier_invalidate_range_end()`` calls around the page table - walks to fill in the ``args->src`` array with PFNs to be migrated. - The ``invalidate_range_start()`` callback is passed a - ``struct mmu_notifier_range`` with the ``event`` field set to - ``MMU_NOTIFY_MIGRATE`` and the ``owner`` field set to - the ``args->pgmap_owner`` field passed to migrate_vma_setup(). This is - allows the device driver to skip the invalidation callback and only - invalidate device private MMU mappings that are actually migrating. - This is explained more in the next section. - - While walking the page tables, a ``pte_none()`` or ``is_zero_pfn()`` - entry results in a valid "zero" PFN stored in the ``args->src`` array. - This lets the driver allocate device private memory and clear it instead - of copying a page of zeros. Valid PTE entries to system memory or - device private struct pages will be locked with ``lock_page()``, isolated - from the LRU (if system memory since device private pages are not on - the LRU), unmapped from the process, and a special migration PTE is - inserted in place of the original PTE. - migrate_vma_setup() also clears the ``args->dst`` array. - -3. The device driver allocates destination pages and copies source pages to - destination pages. - - The driver checks each ``src`` entry to see if the ``MIGRATE_PFN_MIGRATE`` - bit is set and skips entries that are not migrating. The device driver - can also choose to skip migrating a page by not filling in the ``dst`` - array for that page. - - The driver then allocates either a device private struct page or a - system memory page, locks the page with ``lock_page()``, and fills in the - ``dst`` array entry with:: - - dst[i] = migrate_pfn(page_to_pfn(dpage)); - - Now that the driver knows that this page is being migrated, it can - invalidate device private MMU mappings and copy device private memory - to system memory or another device private page. The core Linux kernel - handles CPU page table invalidations so the device driver only has to - invalidate its own MMU mappings. - - The driver can use ``migrate_pfn_to_page(src[i])`` to get the - ``struct page`` of the source and either copy the source page to the - destination or clear the destination device private memory if the pointer - is ``NULL`` meaning the source page was not populated in system memory. - -4. ``migrate_vma_pages()`` - - This step is where the migration is actually "committed". - - If the source page was a ``pte_none()`` or ``is_zero_pfn()`` page, this - is where the newly allocated page is inserted into the CPU's page table. - This can fail if a CPU thread faults on the same page. However, the page - table is locked and only one of the new pages will be inserted. - The device driver will see that the ``MIGRATE_PFN_MIGRATE`` bit is cleared - if it loses the race. - - If the source page was locked, isolated, etc. the source ``struct page`` - information is now copied to destination ``struct page`` finalizing the - migration on the CPU side. - -5. Device driver updates device MMU page tables for pages still migrating, - rolling back pages not migrating. - - If the ``src`` entry still has ``MIGRATE_PFN_MIGRATE`` bit set, the device - driver can update the device MMU and set the write enable bit if the - ``MIGRATE_PFN_WRITE`` bit is set. - -6. ``migrate_vma_finalize()`` - - This step replaces the special migration page table entry with the new - page's page table entry and releases the reference to the source and - destination ``struct page``. - -7. ``mmap_read_unlock()`` - - The lock can now be released. - -Exclusive access memory -======================= - -Some devices have features such as atomic PTE bits that can be used to implement -atomic access to system memory. To support atomic operations to a shared virtual -memory page such a device needs access to that page which is exclusive of any -userspace access from the CPU. The ``make_device_exclusive_range()`` function -can be used to make a memory range inaccessible from userspace. - -This replaces all mappings for pages in the given range with special swap -entries. Any attempt to access the swap entry results in a fault which is -resovled by replacing the entry with the original mapping. A driver gets -notified that the mapping has been changed by MMU notifiers, after which point -it will no longer have exclusive access to the page. Exclusive access is -guranteed to last until the driver drops the page lock and page reference, at -which point any CPU faults on the page may proceed as described. - -Memory cgroup (memcg) and rss accounting -======================================== - -For now, device memory is accounted as any regular page in rss counters (either -anonymous if device page is used for anonymous, file if device page is used for -file backed page, or shmem if device page is used for shared memory). This is a -deliberate choice to keep existing applications, that might start using device -memory without knowing about it, running unimpacted. - -A drawback is that the OOM killer might kill an application using a lot of -device memory and not a lot of regular system memory and thus not freeing much -system memory. We want to gather more real world experience on how applications -and system react under memory pressure in the presence of device memory before -deciding to account device memory differently. - - -Same decision was made for memory cgroup. Device memory pages are accounted -against same memory cgroup a regular page would be accounted to. This does -simplify migration to and from device memory. This also means that migration -back from device memory to regular memory cannot fail because it would -go above memory cgroup limit. We might revisit this choice latter on once we -get more experience in how device memory is used and its impact on memory -resource control. - - -Note that device memory can never be pinned by a device driver nor through GUP -and thus such memory is always free upon process exit. Or when last reference -is dropped in case of shared memory or file backed memory. diff --git a/Documentation/vm/hugetlbfs_reserv.rst b/Documentation/vm/hugetlbfs_reserv.rst deleted file mode 100644 index f143954e0d05..000000000000 --- a/Documentation/vm/hugetlbfs_reserv.rst +++ /dev/null @@ -1,596 +0,0 @@ -.. _hugetlbfs_reserve: - -===================== -Hugetlbfs Reservation -===================== - -Overview -======== - -Huge pages as described at :ref:`hugetlbpage` are typically -preallocated for application use. These huge pages are instantiated in a -task's address space at page fault time if the VMA indicates huge pages are -to be used. If no huge page exists at page fault time, the task is sent -a SIGBUS and often dies an unhappy death. Shortly after huge page support -was added, it was determined that it would be better to detect a shortage -of huge pages at mmap() time. The idea is that if there were not enough -huge pages to cover the mapping, the mmap() would fail. This was first -done with a simple check in the code at mmap() time to determine if there -were enough free huge pages to cover the mapping. Like most things in the -kernel, the code has evolved over time. However, the basic idea was to -'reserve' huge pages at mmap() time to ensure that huge pages would be -available for page faults in that mapping. The description below attempts to -describe how huge page reserve processing is done in the v4.10 kernel. - - -Audience -======== -This description is primarily targeted at kernel developers who are modifying -hugetlbfs code. - - -The Data Structures -=================== - -resv_huge_pages - This is a global (per-hstate) count of reserved huge pages. Reserved - huge pages are only available to the task which reserved them. - Therefore, the number of huge pages generally available is computed - as (``free_huge_pages - resv_huge_pages``). -Reserve Map - A reserve map is described by the structure:: - - struct resv_map { - struct kref refs; - spinlock_t lock; - struct list_head regions; - long adds_in_progress; - struct list_head region_cache; - long region_cache_count; - }; - - There is one reserve map for each huge page mapping in the system. - The regions list within the resv_map describes the regions within - the mapping. A region is described as:: - - struct file_region { - struct list_head link; - long from; - long to; - }; - - The 'from' and 'to' fields of the file region structure are huge page - indices into the mapping. Depending on the type of mapping, a - region in the reserv_map may indicate reservations exist for the - range, or reservations do not exist. -Flags for MAP_PRIVATE Reservations - These are stored in the bottom bits of the reservation map pointer. - - ``#define HPAGE_RESV_OWNER (1UL << 0)`` - Indicates this task is the owner of the reservations - associated with the mapping. - ``#define HPAGE_RESV_UNMAPPED (1UL << 1)`` - Indicates task originally mapping this range (and creating - reserves) has unmapped a page from this task (the child) - due to a failed COW. -Page Flags - The PagePrivate page flag is used to indicate that a huge page - reservation must be restored when the huge page is freed. More - details will be discussed in the "Freeing huge pages" section. - - -Reservation Map Location (Private or Shared) -============================================ - -A huge page mapping or segment is either private or shared. If private, -it is typically only available to a single address space (task). If shared, -it can be mapped into multiple address spaces (tasks). The location and -semantics of the reservation map is significantly different for the two types -of mappings. Location differences are: - -- For private mappings, the reservation map hangs off the VMA structure. - Specifically, vma->vm_private_data. This reserve map is created at the - time the mapping (mmap(MAP_PRIVATE)) is created. -- For shared mappings, the reservation map hangs off the inode. Specifically, - inode->i_mapping->private_data. Since shared mappings are always backed - by files in the hugetlbfs filesystem, the hugetlbfs code ensures each inode - contains a reservation map. As a result, the reservation map is allocated - when the inode is created. - - -Creating Reservations -===================== -Reservations are created when a huge page backed shared memory segment is -created (shmget(SHM_HUGETLB)) or a mapping is created via mmap(MAP_HUGETLB). -These operations result in a call to the routine hugetlb_reserve_pages():: - - int hugetlb_reserve_pages(struct inode *inode, - long from, long to, - struct vm_area_struct *vma, - vm_flags_t vm_flags) - -The first thing hugetlb_reserve_pages() does is check if the NORESERVE -flag was specified in either the shmget() or mmap() call. If NORESERVE -was specified, then this routine returns immediately as no reservations -are desired. - -The arguments 'from' and 'to' are huge page indices into the mapping or -underlying file. For shmget(), 'from' is always 0 and 'to' corresponds to -the length of the segment/mapping. For mmap(), the offset argument could -be used to specify the offset into the underlying file. In such a case, -the 'from' and 'to' arguments have been adjusted by this offset. - -One of the big differences between PRIVATE and SHARED mappings is the way -in which reservations are represented in the reservation map. - -- For shared mappings, an entry in the reservation map indicates a reservation - exists or did exist for the corresponding page. As reservations are - consumed, the reservation map is not modified. -- For private mappings, the lack of an entry in the reservation map indicates - a reservation exists for the corresponding page. As reservations are - consumed, entries are added to the reservation map. Therefore, the - reservation map can also be used to determine which reservations have - been consumed. - -For private mappings, hugetlb_reserve_pages() creates the reservation map and -hangs it off the VMA structure. In addition, the HPAGE_RESV_OWNER flag is set -to indicate this VMA owns the reservations. - -The reservation map is consulted to determine how many huge page reservations -are needed for the current mapping/segment. For private mappings, this is -always the value (to - from). However, for shared mappings it is possible that -some reservations may already exist within the range (to - from). See the -section :ref:`Reservation Map Modifications ` -for details on how this is accomplished. - -The mapping may be associated with a subpool. If so, the subpool is consulted -to ensure there is sufficient space for the mapping. It is possible that the -subpool has set aside reservations that can be used for the mapping. See the -section :ref:`Subpool Reservations ` for more details. - -After consulting the reservation map and subpool, the number of needed new -reservations is known. The routine hugetlb_acct_memory() is called to check -for and take the requested number of reservations. hugetlb_acct_memory() -calls into routines that potentially allocate and adjust surplus page counts. -However, within those routines the code is simply checking to ensure there -are enough free huge pages to accommodate the reservation. If there are, -the global reservation count resv_huge_pages is adjusted something like the -following:: - - if (resv_needed <= (resv_huge_pages - free_huge_pages)) - resv_huge_pages += resv_needed; - -Note that the global lock hugetlb_lock is held when checking and adjusting -these counters. - -If there were enough free huge pages and the global count resv_huge_pages -was adjusted, then the reservation map associated with the mapping is -modified to reflect the reservations. In the case of a shared mapping, a -file_region will exist that includes the range 'from' - 'to'. For private -mappings, no modifications are made to the reservation map as lack of an -entry indicates a reservation exists. - -If hugetlb_reserve_pages() was successful, the global reservation count and -reservation map associated with the mapping will be modified as required to -ensure reservations exist for the range 'from' - 'to'. - -.. _consume_resv: - -Consuming Reservations/Allocating a Huge Page -============================================= - -Reservations are consumed when huge pages associated with the reservations -are allocated and instantiated in the corresponding mapping. The allocation -is performed within the routine alloc_huge_page():: - - struct page *alloc_huge_page(struct vm_area_struct *vma, - unsigned long addr, int avoid_reserve) - -alloc_huge_page is passed a VMA pointer and a virtual address, so it can -consult the reservation map to determine if a reservation exists. In addition, -alloc_huge_page takes the argument avoid_reserve which indicates reserves -should not be used even if it appears they have been set aside for the -specified address. The avoid_reserve argument is most often used in the case -of Copy on Write and Page Migration where additional copies of an existing -page are being allocated. - -The helper routine vma_needs_reservation() is called to determine if a -reservation exists for the address within the mapping(vma). See the section -:ref:`Reservation Map Helper Routines ` for detailed -information on what this routine does. -The value returned from vma_needs_reservation() is generally -0 or 1. 0 if a reservation exists for the address, 1 if no reservation exists. -If a reservation does not exist, and there is a subpool associated with the -mapping the subpool is consulted to determine if it contains reservations. -If the subpool contains reservations, one can be used for this allocation. -However, in every case the avoid_reserve argument overrides the use of -a reservation for the allocation. After determining whether a reservation -exists and can be used for the allocation, the routine dequeue_huge_page_vma() -is called. This routine takes two arguments related to reservations: - -- avoid_reserve, this is the same value/argument passed to alloc_huge_page() -- chg, even though this argument is of type long only the values 0 or 1 are - passed to dequeue_huge_page_vma. If the value is 0, it indicates a - reservation exists (see the section "Memory Policy and Reservations" for - possible issues). If the value is 1, it indicates a reservation does not - exist and the page must be taken from the global free pool if possible. - -The free lists associated with the memory policy of the VMA are searched for -a free page. If a page is found, the value free_huge_pages is decremented -when the page is removed from the free list. If there was a reservation -associated with the page, the following adjustments are made:: - - SetPagePrivate(page); /* Indicates allocating this page consumed - * a reservation, and if an error is - * encountered such that the page must be - * freed, the reservation will be restored. */ - resv_huge_pages--; /* Decrement the global reservation count */ - -Note, if no huge page can be found that satisfies the VMA's memory policy -an attempt will be made to allocate one using the buddy allocator. This -brings up the issue of surplus huge pages and overcommit which is beyond -the scope reservations. Even if a surplus page is allocated, the same -reservation based adjustments as above will be made: SetPagePrivate(page) and -resv_huge_pages--. - -After obtaining a new huge page, (page)->private is set to the value of -the subpool associated with the page if it exists. This will be used for -subpool accounting when the page is freed. - -The routine vma_commit_reservation() is then called to adjust the reserve -map based on the consumption of the reservation. In general, this involves -ensuring the page is represented within a file_region structure of the region -map. For shared mappings where the reservation was present, an entry -in the reserve map already existed so no change is made. However, if there -was no reservation in a shared mapping or this was a private mapping a new -entry must be created. - -It is possible that the reserve map could have been changed between the call -to vma_needs_reservation() at the beginning of alloc_huge_page() and the -call to vma_commit_reservation() after the page was allocated. This would -be possible if hugetlb_reserve_pages was called for the same page in a shared -mapping. In such cases, the reservation count and subpool free page count -will be off by one. This rare condition can be identified by comparing the -return value from vma_needs_reservation and vma_commit_reservation. If such -a race is detected, the subpool and global reserve counts are adjusted to -compensate. See the section -:ref:`Reservation Map Helper Routines ` for more -information on these routines. - - -Instantiate Huge Pages -====================== - -After huge page allocation, the page is typically added to the page tables -of the allocating task. Before this, pages in a shared mapping are added -to the page cache and pages in private mappings are added to an anonymous -reverse mapping. In both cases, the PagePrivate flag is cleared. Therefore, -when a huge page that has been instantiated is freed no adjustment is made -to the global reservation count (resv_huge_pages). - - -Freeing Huge Pages -================== - -Huge page freeing is performed by the routine free_huge_page(). This routine -is the destructor for hugetlbfs compound pages. As a result, it is only -passed a pointer to the page struct. When a huge page is freed, reservation -accounting may need to be performed. This would be the case if the page was -associated with a subpool that contained reserves, or the page is being freed -on an error path where a global reserve count must be restored. - -The page->private field points to any subpool associated with the page. -If the PagePrivate flag is set, it indicates the global reserve count should -be adjusted (see the section -:ref:`Consuming Reservations/Allocating a Huge Page ` -for information on how these are set). - -The routine first calls hugepage_subpool_put_pages() for the page. If this -routine returns a value of 0 (which does not equal the value passed 1) it -indicates reserves are associated with the subpool, and this newly free page -must be used to keep the number of subpool reserves above the minimum size. -Therefore, the global resv_huge_pages counter is incremented in this case. - -If the PagePrivate flag was set in the page, the global resv_huge_pages counter -will always be incremented. - -.. _sub_pool_resv: - -Subpool Reservations -==================== - -There is a struct hstate associated with each huge page size. The hstate -tracks all huge pages of the specified size. A subpool represents a subset -of pages within a hstate that is associated with a mounted hugetlbfs -filesystem. - -When a hugetlbfs filesystem is mounted a min_size option can be specified -which indicates the minimum number of huge pages required by the filesystem. -If this option is specified, the number of huge pages corresponding to -min_size are reserved for use by the filesystem. This number is tracked in -the min_hpages field of a struct hugepage_subpool. At mount time, -hugetlb_acct_memory(min_hpages) is called to reserve the specified number of -huge pages. If they can not be reserved, the mount fails. - -The routines hugepage_subpool_get/put_pages() are called when pages are -obtained from or released back to a subpool. They perform all subpool -accounting, and track any reservations associated with the subpool. -hugepage_subpool_get/put_pages are passed the number of huge pages by which -to adjust the subpool 'used page' count (down for get, up for put). Normally, -they return the same value that was passed or an error if not enough pages -exist in the subpool. - -However, if reserves are associated with the subpool a return value less -than the passed value may be returned. This return value indicates the -number of additional global pool adjustments which must be made. For example, -suppose a subpool contains 3 reserved huge pages and someone asks for 5. -The 3 reserved pages associated with the subpool can be used to satisfy part -of the request. But, 2 pages must be obtained from the global pools. To -relay this information to the caller, the value 2 is returned. The caller -is then responsible for attempting to obtain the additional two pages from -the global pools. - - -COW and Reservations -==================== - -Since shared mappings all point to and use the same underlying pages, the -biggest reservation concern for COW is private mappings. In this case, -two tasks can be pointing at the same previously allocated page. One task -attempts to write to the page, so a new page must be allocated so that each -task points to its own page. - -When the page was originally allocated, the reservation for that page was -consumed. When an attempt to allocate a new page is made as a result of -COW, it is possible that no free huge pages are free and the allocation -will fail. - -When the private mapping was originally created, the owner of the mapping -was noted by setting the HPAGE_RESV_OWNER bit in the pointer to the reservation -map of the owner. Since the owner created the mapping, the owner owns all -the reservations associated with the mapping. Therefore, when a write fault -occurs and there is no page available, different action is taken for the owner -and non-owner of the reservation. - -In the case where the faulting task is not the owner, the fault will fail and -the task will typically receive a SIGBUS. - -If the owner is the faulting task, we want it to succeed since it owned the -original reservation. To accomplish this, the page is unmapped from the -non-owning task. In this way, the only reference is from the owning task. -In addition, the HPAGE_RESV_UNMAPPED bit is set in the reservation map pointer -of the non-owning task. The non-owning task may receive a SIGBUS if it later -faults on a non-present page. But, the original owner of the -mapping/reservation will behave as expected. - - -.. _resv_map_modifications: - -Reservation Map Modifications -============================= - -The following low level routines are used to make modifications to a -reservation map. Typically, these routines are not called directly. Rather, -a reservation map helper routine is called which calls one of these low level -routines. These low level routines are fairly well documented in the source -code (mm/hugetlb.c). These routines are:: - - long region_chg(struct resv_map *resv, long f, long t); - long region_add(struct resv_map *resv, long f, long t); - void region_abort(struct resv_map *resv, long f, long t); - long region_count(struct resv_map *resv, long f, long t); - -Operations on the reservation map typically involve two operations: - -1) region_chg() is called to examine the reserve map and determine how - many pages in the specified range [f, t) are NOT currently represented. - - The calling code performs global checks and allocations to determine if - there are enough huge pages for the operation to succeed. - -2) - a) If the operation can succeed, region_add() is called to actually modify - the reservation map for the same range [f, t) previously passed to - region_chg(). - b) If the operation can not succeed, region_abort is called for the same - range [f, t) to abort the operation. - -Note that this is a two step process where region_add() and region_abort() -are guaranteed to succeed after a prior call to region_chg() for the same -range. region_chg() is responsible for pre-allocating any data structures -necessary to ensure the subsequent operations (specifically region_add())) -will succeed. - -As mentioned above, region_chg() determines the number of pages in the range -which are NOT currently represented in the map. This number is returned to -the caller. region_add() returns the number of pages in the range added to -the map. In most cases, the return value of region_add() is the same as the -return value of region_chg(). However, in the case of shared mappings it is -possible for changes to the reservation map to be made between the calls to -region_chg() and region_add(). In this case, the return value of region_add() -will not match the return value of region_chg(). It is likely that in such -cases global counts and subpool accounting will be incorrect and in need of -adjustment. It is the responsibility of the caller to check for this condition -and make the appropriate adjustments. - -The routine region_del() is called to remove regions from a reservation map. -It is typically called in the following situations: - -- When a file in the hugetlbfs filesystem is being removed, the inode will - be released and the reservation map freed. Before freeing the reservation - map, all the individual file_region structures must be freed. In this case - region_del is passed the range [0, LONG_MAX). -- When a hugetlbfs file is being truncated. In this case, all allocated pages - after the new file size must be freed. In addition, any file_region entries - in the reservation map past the new end of file must be deleted. In this - case, region_del is passed the range [new_end_of_file, LONG_MAX). -- When a hole is being punched in a hugetlbfs file. In this case, huge pages - are removed from the middle of the file one at a time. As the pages are - removed, region_del() is called to remove the corresponding entry from the - reservation map. In this case, region_del is passed the range - [page_idx, page_idx + 1). - -In every case, region_del() will return the number of pages removed from the -reservation map. In VERY rare cases, region_del() can fail. This can only -happen in the hole punch case where it has to split an existing file_region -entry and can not allocate a new structure. In this error case, region_del() -will return -ENOMEM. The problem here is that the reservation map will -indicate that there is a reservation for the page. However, the subpool and -global reservation counts will not reflect the reservation. To handle this -situation, the routine hugetlb_fix_reserve_counts() is called to adjust the -counters so that they correspond with the reservation map entry that could -not be deleted. - -region_count() is called when unmapping a private huge page mapping. In -private mappings, the lack of a entry in the reservation map indicates that -a reservation exists. Therefore, by counting the number of entries in the -reservation map we know how many reservations were consumed and how many are -outstanding (outstanding = (end - start) - region_count(resv, start, end)). -Since the mapping is going away, the subpool and global reservation counts -are decremented by the number of outstanding reservations. - -.. _resv_map_helpers: - -Reservation Map Helper Routines -=============================== - -Several helper routines exist to query and modify the reservation maps. -These routines are only interested with reservations for a specific huge -page, so they just pass in an address instead of a range. In addition, -they pass in the associated VMA. From the VMA, the type of mapping (private -or shared) and the location of the reservation map (inode or VMA) can be -determined. These routines simply call the underlying routines described -in the section "Reservation Map Modifications". However, they do take into -account the 'opposite' meaning of reservation map entries for private and -shared mappings and hide this detail from the caller:: - - long vma_needs_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -This routine calls region_chg() for the specified page. If no reservation -exists, 1 is returned. If a reservation exists, 0 is returned:: - - long vma_commit_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -This calls region_add() for the specified page. As in the case of region_chg -and region_add, this routine is to be called after a previous call to -vma_needs_reservation. It will add a reservation entry for the page. It -returns 1 if the reservation was added and 0 if not. The return value should -be compared with the return value of the previous call to -vma_needs_reservation. An unexpected difference indicates the reservation -map was modified between calls:: - - void vma_end_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -This calls region_abort() for the specified page. As in the case of region_chg -and region_abort, this routine is to be called after a previous call to -vma_needs_reservation. It will abort/end the in progress reservation add -operation:: - - long vma_add_reservation(struct hstate *h, - struct vm_area_struct *vma, - unsigned long addr) - -This is a special wrapper routine to help facilitate reservation cleanup -on error paths. It is only called from the routine restore_reserve_on_error(). -This routine is used in conjunction with vma_needs_reservation in an attempt -to add a reservation to the reservation map. It takes into account the -different reservation map semantics for private and shared mappings. Hence, -region_add is called for shared mappings (as an entry present in the map -indicates a reservation), and region_del is called for private mappings (as -the absence of an entry in the map indicates a reservation). See the section -"Reservation cleanup in error paths" for more information on what needs to -be done on error paths. - - -Reservation Cleanup in Error Paths -================================== - -As mentioned in the section -:ref:`Reservation Map Helper Routines `, reservation -map modifications are performed in two steps. First vma_needs_reservation -is called before a page is allocated. If the allocation is successful, -then vma_commit_reservation is called. If not, vma_end_reservation is called. -Global and subpool reservation counts are adjusted based on success or failure -of the operation and all is well. - -Additionally, after a huge page is instantiated the PagePrivate flag is -cleared so that accounting when the page is ultimately freed is correct. - -However, there are several instances where errors are encountered after a huge -page is allocated but before it is instantiated. In this case, the page -allocation has consumed the reservation and made the appropriate subpool, -reservation map and global count adjustments. If the page is freed at this -time (before instantiation and clearing of PagePrivate), then free_huge_page -will increment the global reservation count. However, the reservation map -indicates the reservation was consumed. This resulting inconsistent state -will cause the 'leak' of a reserved huge page. The global reserve count will -be higher than it should and prevent allocation of a pre-allocated page. - -The routine restore_reserve_on_error() attempts to handle this situation. It -is fairly well documented. The intention of this routine is to restore -the reservation map to the way it was before the page allocation. In this -way, the state of the reservation map will correspond to the global reservation -count after the page is freed. - -The routine restore_reserve_on_error itself may encounter errors while -attempting to restore the reservation map entry. In this case, it will -simply clear the PagePrivate flag of the page. In this way, the global -reserve count will not be incremented when the page is freed. However, the -reservation map will continue to look as though the reservation was consumed. -A page can still be allocated for the address, but it will not use a reserved -page as originally intended. - -There is some code (most notably userfaultfd) which can not call -restore_reserve_on_error. In this case, it simply modifies the PagePrivate -so that a reservation will not be leaked when the huge page is freed. - - -Reservations and Memory Policy -============================== -Per-node huge page lists existed in struct hstate when git was first used -to manage Linux code. The concept of reservations was added some time later. -When reservations were added, no attempt was made to take memory policy -into account. While cpusets are not exactly the same as memory policy, this -comment in hugetlb_acct_memory sums up the interaction between reservations -and cpusets/memory policy:: - - /* - * When cpuset is configured, it breaks the strict hugetlb page - * reservation as the accounting is done on a global variable. Such - * reservation is completely rubbish in the presence of cpuset because - * the reservation is not checked against page availability for the - * current cpuset. Application can still potentially OOM'ed by kernel - * with lack of free htlb page in cpuset that the task is in. - * Attempt to enforce strict accounting with cpuset is almost - * impossible (or too ugly) because cpuset is too fluid that - * task or memory node can be dynamically moved between cpusets. - * - * The change of semantics for shared hugetlb mapping with cpuset is - * undesirable. However, in order to preserve some of the semantics, - * we fall back to check against current free page availability as - * a best attempt and hopefully to minimize the impact of changing - * semantics that cpuset has. - */ - -Huge page reservations were added to prevent unexpected page allocation -failures (OOM) at page fault time. However, if an application makes use -of cpusets or memory policy there is no guarantee that huge pages will be -available on the required nodes. This is true even if there are a sufficient -number of global reservations. - -Hugetlbfs regression testing -============================ - -The most complete set of hugetlb tests are in the libhugetlbfs repository. -If you modify any hugetlb related code, use the libhugetlbfs test suite -to check for regressions. In addition, if you add any new hugetlb -functionality, please add appropriate tests to libhugetlbfs. - --- -Mike Kravetz, 7 April 2017 diff --git a/Documentation/vm/hwpoison.rst b/Documentation/vm/hwpoison.rst deleted file mode 100644 index b9d5253c1305..000000000000 --- a/Documentation/vm/hwpoison.rst +++ /dev/null @@ -1,184 +0,0 @@ -.. hwpoison: - -======== -hwpoison -======== - -What is hwpoison? -================= - -Upcoming Intel CPUs have support for recovering from some memory errors -(``MCA recovery``). This requires the OS to declare a page "poisoned", -kill the processes associated with it and avoid using it in the future. - -This patchkit implements the necessary infrastructure in the VM. - -To quote the overview comment:: - - High level machine check handler. Handles pages reported by the - hardware as being corrupted usually due to a 2bit ECC memory or cache - failure. - - This focusses on pages detected as corrupted in the background. - When the current CPU tries to consume corruption the currently - running process can just be killed directly instead. This implies - that if the error cannot be handled for some reason it's safe to - just ignore it because no corruption has been consumed yet. Instead - when that happens another machine check will happen. - - Handles page cache pages in various states. The tricky part - here is that we can access any page asynchronous to other VM - users, because memory failures could happen anytime and anywhere, - possibly violating some of their assumptions. This is why this code - has to be extremely careful. Generally it tries to use normal locking - rules, as in get the standard locks, even if that means the - error handling takes potentially a long time. - - Some of the operations here are somewhat inefficient and have non - linear algorithmic complexity, because the data structures have not - been optimized for this case. This is in particular the case - for the mapping from a vma to a process. Since this case is expected - to be rare we hope we can get away with this. - -The code consists of a the high level handler in mm/memory-failure.c, -a new page poison bit and various checks in the VM to handle poisoned -pages. - -The main target right now is KVM guests, but it works for all kinds -of applications. KVM support requires a recent qemu-kvm release. - -For the KVM use there was need for a new signal type so that -KVM can inject the machine check into the guest with the proper -address. This in theory allows other applications to handle -memory failures too. The expection is that near all applications -won't do that, but some very specialized ones might. - -Failure recovery modes -====================== - -There are two (actually three) modes memory failure recovery can be in: - -vm.memory_failure_recovery sysctl set to zero: - All memory failures cause a panic. Do not attempt recovery. - -early kill - (can be controlled globally and per process) - Send SIGBUS to the application as soon as the error is detected - This allows applications who can process memory errors in a gentle - way (e.g. drop affected object) - This is the mode used by KVM qemu. - -late kill - Send SIGBUS when the application runs into the corrupted page. - This is best for memory error unaware applications and default - Note some pages are always handled as late kill. - -User control -============ - -vm.memory_failure_recovery - See sysctl.txt - -vm.memory_failure_early_kill - Enable early kill mode globally - -PR_MCE_KILL - Set early/late kill mode/revert to system default - - arg1: PR_MCE_KILL_CLEAR: - Revert to system default - arg1: PR_MCE_KILL_SET: - arg2 defines thread specific mode - - PR_MCE_KILL_EARLY: - Early kill - PR_MCE_KILL_LATE: - Late kill - PR_MCE_KILL_DEFAULT - Use system global default - - Note that if you want to have a dedicated thread which handles - the SIGBUS(BUS_MCEERR_AO) on behalf of the process, you should - call prctl(PR_MCE_KILL_EARLY) on the designated thread. Otherwise, - the SIGBUS is sent to the main thread. - -PR_MCE_KILL_GET - return current mode - -Testing -======= - -* madvise(MADV_HWPOISON, ....) (as root) - Poison a page in the - process for testing - -* hwpoison-inject module through debugfs ``/sys/kernel/debug/hwpoison/`` - - corrupt-pfn - Inject hwpoison fault at PFN echoed into this file. This does - some early filtering to avoid corrupted unintended pages in test suites. - - unpoison-pfn - Software-unpoison page at PFN echoed into this file. This way - a page can be reused again. This only works for Linux - injected failures, not for real memory failures. Once any hardware - memory failure happens, this feature is disabled. - - Note these injection interfaces are not stable and might change between - kernel versions - - corrupt-filter-dev-major, corrupt-filter-dev-minor - Only handle memory failures to pages associated with the file - system defined by block device major/minor. -1U is the - wildcard value. This should be only used for testing with - artificial injection. - - corrupt-filter-memcg - Limit injection to pages owned by memgroup. Specified by inode - number of the memcg. - - Example:: - - mkdir /sys/fs/cgroup/mem/hwpoison - - usemem -m 100 -s 1000 & - echo `jobs -p` > /sys/fs/cgroup/mem/hwpoison/tasks - - memcg_ino=$(ls -id /sys/fs/cgroup/mem/hwpoison | cut -f1 -d' ') - echo $memcg_ino > /debug/hwpoison/corrupt-filter-memcg - - page-types -p `pidof init` --hwpoison # shall do nothing - page-types -p `pidof usemem` --hwpoison # poison its pages - - corrupt-filter-flags-mask, corrupt-filter-flags-value - When specified, only poison pages if ((page_flags & mask) == - value). This allows stress testing of many kinds of - pages. The page_flags are the same as in /proc/kpageflags. The - flag bits are defined in include/linux/kernel-page-flags.h and - documented in Documentation/admin-guide/mm/pagemap.rst - -* Architecture specific MCE injector - - x86 has mce-inject, mce-test - - Some portable hwpoison test programs in mce-test, see below. - -References -========== - -http://halobates.de/mce-lc09-2.pdf - Overview presentation from LinuxCon 09 - -git://git.kernel.org/pub/scm/utils/cpu/mce/mce-test.git - Test suite (hwpoison specific portable tests in tsrc) - -git://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git - x86 specific injector - - -Limitations -=========== -- Not all page types are supported and never will. Most kernel internal - objects cannot be recovered, only LRU pages for now. - ---- -Andi Kleen, Oct 2009 diff --git a/Documentation/vm/index.rst b/Documentation/vm/index.rst deleted file mode 100644 index 575ccd40e30c..000000000000 --- a/Documentation/vm/index.rst +++ /dev/null @@ -1,68 +0,0 @@ -===================================== -Linux Memory Management Documentation -===================================== - -Memory Management Guide -======================= - -This is a guide to understanding the memory management subsystem -of Linux. If you are looking for advice on simply allocating memory, -see the :ref:`memory_allocation`. For controlling and tuning guides, -see the :doc:`admin guide <../admin-guide/mm/index>`. - -.. toctree:: - :maxdepth: 1 - - physical_memory - page_tables - process_addrs - bootmem - page_allocation - vmalloc - slab - highmem - page_reclaim - swap - page_cache - shmfs - oom - -Legacy Documentation -==================== - -This is a collection of older documents about the Linux memory management -(MM) subsystem internals with different level of details ranging from -notes and mailing list responses for elaborating descriptions of data -structures and algorithms. It should all be integrated nicely into the -above structured documentation, or deleted if it has served its purpose. - -.. toctree:: - :maxdepth: 1 - - active_mm - arch_pgtable_helpers - balance - damon/index - free_page_reporting - frontswap - hmm - hwpoison - hugetlbfs_reserv - ksm - memory-model - mmu_notifier - numa - overcommit-accounting - page_migration - page_frags - page_owner - page_table_check - remap_file_pages - slub - split_page_table_lock - transhuge - unevictable-lru - vmalloced-kernel-stacks - vmemmap_dedup - z3fold - zsmalloc diff --git a/Documentation/vm/ksm.rst b/Documentation/vm/ksm.rst deleted file mode 100644 index 9e37add068e6..000000000000 --- a/Documentation/vm/ksm.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. _ksm: - -======================= -Kernel Samepage Merging -======================= - -KSM is a memory-saving de-duplication feature, enabled by CONFIG_KSM=y, -added to the Linux kernel in 2.6.32. See ``mm/ksm.c`` for its implementation, -and http://lwn.net/Articles/306704/ and https://lwn.net/Articles/330589/ - -The userspace interface of KSM is described in :ref:`Documentation/admin-guide/mm/ksm.rst ` - -Design -====== - -Overview --------- - -.. kernel-doc:: mm/ksm.c - :DOC: Overview - -Reverse mapping ---------------- -KSM maintains reverse mapping information for KSM pages in the stable -tree. - -If a KSM page is shared between less than ``max_page_sharing`` VMAs, -the node of the stable tree that represents such KSM page points to a -list of struct rmap_item and the ``page->mapping`` of the -KSM page points to the stable tree node. - -When the sharing passes this threshold, KSM adds a second dimension to -the stable tree. The tree node becomes a "chain" that links one or -more "dups". Each "dup" keeps reverse mapping information for a KSM -page with ``page->mapping`` pointing to that "dup". - -Every "chain" and all "dups" linked into a "chain" enforce the -invariant that they represent the same write protected memory content, -even if each "dup" will be pointed by a different KSM page copy of -that content. - -This way the stable tree lookup computational complexity is unaffected -if compared to an unlimited list of reverse mappings. It is still -enforced that there cannot be KSM page content duplicates in the -stable tree itself. - -The deduplication limit enforced by ``max_page_sharing`` is required -to avoid the virtual memory rmap lists to grow too large. The rmap -walk has O(N) complexity where N is the number of rmap_items -(i.e. virtual mappings) that are sharing the page, which is in turn -capped by ``max_page_sharing``. So this effectively spreads the linear -O(N) computational complexity from rmap walk context over different -KSM pages. The ksmd walk over the stable_node "chains" is also O(N), -but N is the number of stable_node "dups", not the number of -rmap_items, so it has not a significant impact on ksmd performance. In -practice the best stable_node "dup" candidate will be kept and found -at the head of the "dups" list. - -High values of ``max_page_sharing`` result in faster memory merging -(because there will be fewer stable_node dups queued into the -stable_node chain->hlist to check for pruning) and higher -deduplication factor at the expense of slower worst case for rmap -walks for any KSM page which can happen during swapping, compaction, -NUMA balancing and page migration. - -The ``stable_node_dups/stable_node_chains`` ratio is also affected by the -``max_page_sharing`` tunable, and an high ratio may indicate fragmentation -in the stable_node dups, which could be solved by introducing -fragmentation algorithms in ksmd which would refile rmap_items from -one stable_node dup to another stable_node dup, in order to free up -stable_node "dups" with few rmap_items in them, but that may increase -the ksmd CPU usage and possibly slowdown the readonly computations on -the KSM pages of the applications. - -The whole list of stable_node "dups" linked in the stable_node -"chains" is scanned periodically in order to prune stale stable_nodes. -The frequency of such scans is defined by -``stable_node_chains_prune_millisecs`` sysfs tunable. - -Reference ---------- -.. kernel-doc:: mm/ksm.c - :functions: mm_slot ksm_scan stable_node rmap_item - --- -Izik Eidus, -Hugh Dickins, 17 Nov 2009 diff --git a/Documentation/vm/memory-model.rst b/Documentation/vm/memory-model.rst deleted file mode 100644 index 30e8fbed6914..000000000000 --- a/Documentation/vm/memory-model.rst +++ /dev/null @@ -1,177 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -.. _physical_memory_model: - -===================== -Physical Memory Model -===================== - -Physical memory in a system may be addressed in different ways. The -simplest case is when the physical memory starts at address 0 and -spans a contiguous range up to the maximal address. It could be, -however, that this range contains small holes that are not accessible -for the CPU. Then there could be several contiguous ranges at -completely distinct addresses. And, don't forget about NUMA, where -different memory banks are attached to different CPUs. - -Linux abstracts this diversity using one of the two memory models: -FLATMEM and SPARSEMEM. Each architecture defines what -memory models it supports, what the default memory model is and -whether it is possible to manually override that default. - -All the memory models track the status of physical page frames using -struct page arranged in one or more arrays. - -Regardless of the selected memory model, there exists one-to-one -mapping between the physical page frame number (PFN) and the -corresponding `struct page`. - -Each memory model defines :c:func:`pfn_to_page` and :c:func:`page_to_pfn` -helpers that allow the conversion from PFN to `struct page` and vice -versa. - -FLATMEM -======= - -The simplest memory model is FLATMEM. This model is suitable for -non-NUMA systems with contiguous, or mostly contiguous, physical -memory. - -In the FLATMEM memory model, there is a global `mem_map` array that -maps the entire physical memory. For most architectures, the holes -have entries in the `mem_map` array. The `struct page` objects -corresponding to the holes are never fully initialized. - -To allocate the `mem_map` array, architecture specific setup code should -call :c:func:`free_area_init` function. Yet, the mappings array is not -usable until the call to :c:func:`memblock_free_all` that hands all the -memory to the page allocator. - -An architecture may free parts of the `mem_map` array that do not cover the -actual physical pages. In such case, the architecture specific -:c:func:`pfn_valid` implementation should take the holes in the -`mem_map` into account. - -With FLATMEM, the conversion between a PFN and the `struct page` is -straightforward: `PFN - ARCH_PFN_OFFSET` is an index to the -`mem_map` array. - -The `ARCH_PFN_OFFSET` defines the first page frame number for -systems with physical memory starting at address different from 0. - -SPARSEMEM -========= - -SPARSEMEM is the most versatile memory model available in Linux and it -is the only memory model that supports several advanced features such -as hot-plug and hot-remove of the physical memory, alternative memory -maps for non-volatile memory devices and deferred initialization of -the memory map for larger systems. - -The SPARSEMEM model presents the physical memory as a collection of -sections. A section is represented with struct mem_section -that contains `section_mem_map` that is, logically, a pointer to an -array of struct pages. However, it is stored with some other magic -that aids the sections management. The section size and maximal number -of section is specified using `SECTION_SIZE_BITS` and -`MAX_PHYSMEM_BITS` constants defined by each architecture that -supports SPARSEMEM. While `MAX_PHYSMEM_BITS` is an actual width of a -physical address that an architecture supports, the -`SECTION_SIZE_BITS` is an arbitrary value. - -The maximal number of sections is denoted `NR_MEM_SECTIONS` and -defined as - -.. math:: - - NR\_MEM\_SECTIONS = 2 ^ {(MAX\_PHYSMEM\_BITS - SECTION\_SIZE\_BITS)} - -The `mem_section` objects are arranged in a two-dimensional array -called `mem_sections`. The size and placement of this array depend -on `CONFIG_SPARSEMEM_EXTREME` and the maximal possible number of -sections: - -* When `CONFIG_SPARSEMEM_EXTREME` is disabled, the `mem_sections` - array is static and has `NR_MEM_SECTIONS` rows. Each row holds a - single `mem_section` object. -* When `CONFIG_SPARSEMEM_EXTREME` is enabled, the `mem_sections` - array is dynamically allocated. Each row contains PAGE_SIZE worth of - `mem_section` objects and the number of rows is calculated to fit - all the memory sections. - -The architecture setup code should call sparse_init() to -initialize the memory sections and the memory maps. - -With SPARSEMEM there are two possible ways to convert a PFN to the -corresponding `struct page` - a "classic sparse" and "sparse -vmemmap". The selection is made at build time and it is determined by -the value of `CONFIG_SPARSEMEM_VMEMMAP`. - -The classic sparse encodes the section number of a page in page->flags -and uses high bits of a PFN to access the section that maps that page -frame. Inside a section, the PFN is the index to the array of pages. - -The sparse vmemmap uses a virtually mapped memory map to optimize -pfn_to_page and page_to_pfn operations. There is a global `struct -page *vmemmap` pointer that points to a virtually contiguous array of -`struct page` objects. A PFN is an index to that array and the -offset of the `struct page` from `vmemmap` is the PFN of that -page. - -To use vmemmap, an architecture has to reserve a range of virtual -addresses that will map the physical pages containing the memory -map and make sure that `vmemmap` points to that range. In addition, -the architecture should implement :c:func:`vmemmap_populate` method -that will allocate the physical memory and create page tables for the -virtual memory map. If an architecture does not have any special -requirements for the vmemmap mappings, it can use default -:c:func:`vmemmap_populate_basepages` provided by the generic memory -management. - -The virtually mapped memory map allows storing `struct page` objects -for persistent memory devices in pre-allocated storage on those -devices. This storage is represented with struct vmem_altmap -that is eventually passed to vmemmap_populate() through a long chain -of function calls. The vmemmap_populate() implementation may use the -`vmem_altmap` along with :c:func:`vmemmap_alloc_block_buf` helper to -allocate memory map on the persistent memory device. - -ZONE_DEVICE -=========== -The `ZONE_DEVICE` facility builds upon `SPARSEMEM_VMEMMAP` to offer -`struct page` `mem_map` services for device driver identified physical -address ranges. The "device" aspect of `ZONE_DEVICE` relates to the fact -that the page objects for these address ranges are never marked online, -and that a reference must be taken against the device, not just the page -to keep the memory pinned for active use. `ZONE_DEVICE`, via -:c:func:`devm_memremap_pages`, performs just enough memory hotplug to -turn on :c:func:`pfn_to_page`, :c:func:`page_to_pfn`, and -:c:func:`get_user_pages` service for the given range of pfns. Since the -page reference count never drops below 1 the page is never tracked as -free memory and the page's `struct list_head lru` space is repurposed -for back referencing to the host device / driver that mapped the memory. - -While `SPARSEMEM` presents memory as a collection of sections, -optionally collected into memory blocks, `ZONE_DEVICE` users have a need -for smaller granularity of populating the `mem_map`. Given that -`ZONE_DEVICE` memory is never marked online it is subsequently never -subject to its memory ranges being exposed through the sysfs memory -hotplug api on memory block boundaries. The implementation relies on -this lack of user-api constraint to allow sub-section sized memory -ranges to be specified to :c:func:`arch_add_memory`, the top-half of -memory hotplug. Sub-section support allows for 2MB as the cross-arch -common alignment granularity for :c:func:`devm_memremap_pages`. - -The users of `ZONE_DEVICE` are: - -* pmem: Map platform persistent memory to be used as a direct-I/O target - via DAX mappings. - -* hmm: Extend `ZONE_DEVICE` with `->page_fault()` and `->page_free()` - event callbacks to allow a device-driver to coordinate memory management - events related to device-memory, typically GPU memory. See - Documentation/vm/hmm.rst. - -* p2pdma: Create `struct page` objects to allow peer devices in a - PCI/-E topology to coordinate direct-DMA operations between themselves, - i.e. bypass host memory. diff --git a/Documentation/vm/mmu_notifier.rst b/Documentation/vm/mmu_notifier.rst deleted file mode 100644 index df5d7777fc6b..000000000000 --- a/Documentation/vm/mmu_notifier.rst +++ /dev/null @@ -1,99 +0,0 @@ -.. _mmu_notifier: - -When do you need to notify inside page table lock ? -=================================================== - -When clearing a pte/pmd we are given a choice to notify the event through -(notify version of \*_clear_flush call mmu_notifier_invalidate_range) under -the page table lock. But that notification is not necessary in all cases. - -For secondary TLB (non CPU TLB) like IOMMU TLB or device TLB (when device use -thing like ATS/PASID to get the IOMMU to walk the CPU page table to access a -process virtual address space). There is only 2 cases when you need to notify -those secondary TLB while holding page table lock when clearing a pte/pmd: - - A) page backing address is free before mmu_notifier_invalidate_range_end() - B) a page table entry is updated to point to a new page (COW, write fault - on zero page, __replace_page(), ...) - -Case A is obvious you do not want to take the risk for the device to write to -a page that might now be used by some completely different task. - -Case B is more subtle. For correctness it requires the following sequence to -happen: - - - take page table lock - - clear page table entry and notify ([pmd/pte]p_huge_clear_flush_notify()) - - set page table entry to point to new page - -If clearing the page table entry is not followed by a notify before setting -the new pte/pmd value then you can break memory model like C11 or C++11 for -the device. - -Consider the following scenario (device use a feature similar to ATS/PASID): - -Two address addrA and addrB such that \|addrA - addrB\| >= PAGE_SIZE we assume -they are write protected for COW (other case of B apply too). - -:: - - [Time N] -------------------------------------------------------------------- - CPU-thread-0 {try to write to addrA} - CPU-thread-1 {try to write to addrB} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {read addrA and populate device TLB} - DEV-thread-2 {read addrB and populate device TLB} - [Time N+1] ------------------------------------------------------------------ - CPU-thread-0 {COW_step0: {mmu_notifier_invalidate_range_start(addrA)}} - CPU-thread-1 {COW_step0: {mmu_notifier_invalidate_range_start(addrB)}} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+2] ------------------------------------------------------------------ - CPU-thread-0 {COW_step1: {update page table to point to new page for addrA}} - CPU-thread-1 {COW_step1: {update page table to point to new page for addrB}} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+3] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {preempted} - CPU-thread-2 {write to addrA which is a write to new page} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+3] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {preempted} - CPU-thread-2 {} - CPU-thread-3 {write to addrB which is a write to new page} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+4] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {COW_step3: {mmu_notifier_invalidate_range_end(addrB)}} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {} - DEV-thread-2 {} - [Time N+5] ------------------------------------------------------------------ - CPU-thread-0 {preempted} - CPU-thread-1 {} - CPU-thread-2 {} - CPU-thread-3 {} - DEV-thread-0 {read addrA from old page} - DEV-thread-2 {read addrB from new page} - -So here because at time N+2 the clear page table entry was not pair with a -notification to invalidate the secondary TLB, the device see the new value for -addrB before seeing the new value for addrA. This break total memory ordering -for the device. - -When changing a pte to write protect or to point to a new write protected page -with same content (KSM) it is fine to delay the mmu_notifier_invalidate_range -call to mmu_notifier_invalidate_range_end() outside the page table lock. This -is true even if the thread doing the page table update is preempted right after -releasing page table lock but before call mmu_notifier_invalidate_range_end(). diff --git a/Documentation/vm/numa.rst b/Documentation/vm/numa.rst deleted file mode 100644 index 99fdeca917ca..000000000000 --- a/Documentation/vm/numa.rst +++ /dev/null @@ -1,150 +0,0 @@ -.. _numa: - -Started Nov 1999 by Kanoj Sarcar - -============= -What is NUMA? -============= - -This question can be answered from a couple of perspectives: the -hardware view and the Linux software view. - -From the hardware perspective, a NUMA system is a computer platform that -comprises multiple components or assemblies each of which may contain 0 -or more CPUs, local memory, and/or IO buses. For brevity and to -disambiguate the hardware view of these physical components/assemblies -from the software abstraction thereof, we'll call the components/assemblies -'cells' in this document. - -Each of the 'cells' may be viewed as an SMP [symmetric multi-processor] subset -of the system--although some components necessary for a stand-alone SMP system -may not be populated on any given cell. The cells of the NUMA system are -connected together with some sort of system interconnect--e.g., a crossbar or -point-to-point link are common types of NUMA system interconnects. Both of -these types of interconnects can be aggregated to create NUMA platforms with -cells at multiple distances from other cells. - -For Linux, the NUMA platforms of interest are primarily what is known as Cache -Coherent NUMA or ccNUMA systems. With ccNUMA systems, all memory is visible -to and accessible from any CPU attached to any cell and cache coherency -is handled in hardware by the processor caches and/or the system interconnect. - -Memory access time and effective memory bandwidth varies depending on how far -away the cell containing the CPU or IO bus making the memory access is from the -cell containing the target memory. For example, access to memory by CPUs -attached to the same cell will experience faster access times and higher -bandwidths than accesses to memory on other, remote cells. NUMA platforms -can have cells at multiple remote distances from any given cell. - -Platform vendors don't build NUMA systems just to make software developers' -lives interesting. Rather, this architecture is a means to provide scalable -memory bandwidth. However, to achieve scalable memory bandwidth, system and -application software must arrange for a large majority of the memory references -[cache misses] to be to "local" memory--memory on the same cell, if any--or -to the closest cell with memory. - -This leads to the Linux software view of a NUMA system: - -Linux divides the system's hardware resources into multiple software -abstractions called "nodes". Linux maps the nodes onto the physical cells -of the hardware platform, abstracting away some of the details for some -architectures. As with physical cells, software nodes may contain 0 or more -CPUs, memory and/or IO buses. And, again, memory accesses to memory on -"closer" nodes--nodes that map to closer cells--will generally experience -faster access times and higher effective bandwidth than accesses to more -remote cells. - -For some architectures, such as x86, Linux will "hide" any node representing a -physical cell that has no memory attached, and reassign any CPUs attached to -that cell to a node representing a cell that does have memory. Thus, on -these architectures, one cannot assume that all CPUs that Linux associates with -a given node will see the same local memory access times and bandwidth. - -In addition, for some architectures, again x86 is an example, Linux supports -the emulation of additional nodes. For NUMA emulation, linux will carve up -the existing nodes--or the system memory for non-NUMA platforms--into multiple -nodes. Each emulated node will manage a fraction of the underlying cells' -physical memory. NUMA emluation is useful for testing NUMA kernel and -application features on non-NUMA platforms, and as a sort of memory resource -management mechanism when used together with cpusets. -[see Documentation/admin-guide/cgroup-v1/cpusets.rst] - -For each node with memory, Linux constructs an independent memory management -subsystem, complete with its own free page lists, in-use page lists, usage -statistics and locks to mediate access. In addition, Linux constructs for -each memory zone [one or more of DMA, DMA32, NORMAL, HIGH_MEMORY, MOVABLE], -an ordered "zonelist". A zonelist specifies the zones/nodes to visit when a -selected zone/node cannot satisfy the allocation request. This situation, -when a zone has no available memory to satisfy a request, is called -"overflow" or "fallback". - -Because some nodes contain multiple zones containing different types of -memory, Linux must decide whether to order the zonelists such that allocations -fall back to the same zone type on a different node, or to a different zone -type on the same node. This is an important consideration because some zones, -such as DMA or DMA32, represent relatively scarce resources. Linux chooses -a default Node ordered zonelist. This means it tries to fallback to other zones -from the same node before using remote nodes which are ordered by NUMA distance. - -By default, Linux will attempt to satisfy memory allocation requests from the -node to which the CPU that executes the request is assigned. Specifically, -Linux will attempt to allocate from the first node in the appropriate zonelist -for the node where the request originates. This is called "local allocation." -If the "local" node cannot satisfy the request, the kernel will examine other -nodes' zones in the selected zonelist looking for the first zone in the list -that can satisfy the request. - -Local allocation will tend to keep subsequent access to the allocated memory -"local" to the underlying physical resources and off the system interconnect-- -as long as the task on whose behalf the kernel allocated some memory does not -later migrate away from that memory. The Linux scheduler is aware of the -NUMA topology of the platform--embodied in the "scheduling domains" data -structures [see Documentation/scheduler/sched-domains.rst]--and the scheduler -attempts to minimize task migration to distant scheduling domains. However, -the scheduler does not take a task's NUMA footprint into account directly. -Thus, under sufficient imbalance, tasks can migrate between nodes, remote -from their initial node and kernel data structures. - -System administrators and application designers can restrict a task's migration -to improve NUMA locality using various CPU affinity command line interfaces, -such as taskset(1) and numactl(1), and program interfaces such as -sched_setaffinity(2). Further, one can modify the kernel's default local -allocation behavior using Linux NUMA memory policy. [see -:ref:`Documentation/admin-guide/mm/numa_memory_policy.rst `]. - -System administrators can restrict the CPUs and nodes' memories that a non- -privileged user can specify in the scheduling or NUMA commands and functions -using control groups and CPUsets. [see Documentation/admin-guide/cgroup-v1/cpusets.rst] - -On architectures that do not hide memoryless nodes, Linux will include only -zones [nodes] with memory in the zonelists. This means that for a memoryless -node the "local memory node"--the node of the first zone in CPU's node's -zonelist--will not be the node itself. Rather, it will be the node that the -kernel selected as the nearest node with memory when it built the zonelists. -So, default, local allocations will succeed with the kernel supplying the -closest available memory. This is a consequence of the same mechanism that -allows such allocations to fallback to other nearby nodes when a node that -does contain memory overflows. - -Some kernel allocations do not want or cannot tolerate this allocation fallback -behavior. Rather they want to be sure they get memory from the specified node -or get notified that the node has no free memory. This is usually the case when -a subsystem allocates per CPU memory resources, for example. - -A typical model for making such an allocation is to obtain the node id of the -node to which the "current CPU" is attached using one of the kernel's -numa_node_id() or CPU_to_node() functions and then request memory from only -the node id returned. When such an allocation fails, the requesting subsystem -may revert to its own fallback path. The slab kernel memory allocator is an -example of this. Or, the subsystem may choose to disable or not to enable -itself on allocation failure. The kernel profiling subsystem is an example of -this. - -If the architecture supports--does not hide--memoryless nodes, then CPUs -attached to memoryless nodes would always incur the fallback path overhead -or some subsystems would fail to initialize if they attempted to allocated -memory exclusively from a node without memory. To support such -architectures transparently, kernel subsystems can use the numa_mem_id() -or cpu_to_mem() function to locate the "local memory node" for the calling or -specified CPU. Again, this is the same node from which default, local page -allocations will be attempted. diff --git a/Documentation/vm/oom.rst b/Documentation/vm/oom.rst deleted file mode 100644 index 18e9e40c1ec1..000000000000 --- a/Documentation/vm/oom.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -====================== -Out Of Memory Handling -====================== diff --git a/Documentation/vm/overcommit-accounting.rst b/Documentation/vm/overcommit-accounting.rst deleted file mode 100644 index 1addb0c374a4..000000000000 --- a/Documentation/vm/overcommit-accounting.rst +++ /dev/null @@ -1,88 +0,0 @@ -.. _overcommit_accounting: - -===================== -Overcommit Accounting -===================== - -The Linux kernel supports the following overcommit handling modes - -0 - Heuristic overcommit handling. Obvious overcommits of address - space are refused. Used for a typical system. It ensures a - seriously wild allocation fails while allowing overcommit to - reduce swap usage. root is allowed to allocate slightly more - memory in this mode. This is the default. - -1 - Always overcommit. Appropriate for some scientific - applications. Classic example is code using sparse arrays and - just relying on the virtual memory consisting almost entirely - of zero pages. - -2 - Don't overcommit. The total address space commit for the - system is not permitted to exceed swap + a configurable amount - (default is 50%) of physical RAM. Depending on the amount you - use, in most situations this means a process will not be - killed while accessing pages but will receive errors on memory - allocation as appropriate. - - Useful for applications that want to guarantee their memory - allocations will be available in the future without having to - initialize every page. - -The overcommit policy is set via the sysctl ``vm.overcommit_memory``. - -The overcommit amount can be set via ``vm.overcommit_ratio`` (percentage) -or ``vm.overcommit_kbytes`` (absolute value). These only have an effect -when ``vm.overcommit_memory`` is set to 2. - -The current overcommit limit and amount committed are viewable in -``/proc/meminfo`` as CommitLimit and Committed_AS respectively. - -Gotchas -======= - -The C language stack growth does an implicit mremap. If you want absolute -guarantees and run close to the edge you MUST mmap your stack for the -largest size you think you will need. For typical stack usage this does -not matter much but it's a corner case if you really really care - -In mode 2 the MAP_NORESERVE flag is ignored. - - -How It Works -============ - -The overcommit is based on the following rules - -For a file backed map - | SHARED or READ-only - 0 cost (the file is the map not swap) - | PRIVATE WRITABLE - size of mapping per instance - -For an anonymous or ``/dev/zero`` map - | SHARED - size of mapping - | PRIVATE READ-only - 0 cost (but of little use) - | PRIVATE WRITABLE - size of mapping per instance - -Additional accounting - | Pages made writable copies by mmap - | shmfs memory drawn from the same pool - -Status -====== - -* We account mmap memory mappings -* We account mprotect changes in commit -* We account mremap changes in size -* We account brk -* We account munmap -* We report the commit status in /proc -* Account and check on fork -* Review stack handling/building on exec -* SHMfs accounting -* Implement actual limit enforcement - -To Do -===== -* Account ptrace pages (this is hard) diff --git a/Documentation/vm/page_allocation.rst b/Documentation/vm/page_allocation.rst deleted file mode 100644 index d9b4495561f1..000000000000 --- a/Documentation/vm/page_allocation.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -=============== -Page Allocation -=============== diff --git a/Documentation/vm/page_cache.rst b/Documentation/vm/page_cache.rst deleted file mode 100644 index 75eba7c431b2..000000000000 --- a/Documentation/vm/page_cache.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -========== -Page Cache -========== diff --git a/Documentation/vm/page_frags.rst b/Documentation/vm/page_frags.rst deleted file mode 100644 index 7d6f9385d129..000000000000 --- a/Documentation/vm/page_frags.rst +++ /dev/null @@ -1,45 +0,0 @@ -.. _page_frags: - -============== -Page fragments -============== - -A page fragment is an arbitrary-length arbitrary-offset area of memory -which resides within a 0 or higher order compound page. Multiple -fragments within that page are individually refcounted, in the page's -reference counter. - -The page_frag functions, page_frag_alloc and page_frag_free, provide a -simple allocation framework for page fragments. This is used by the -network stack and network device drivers to provide a backing region of -memory for use as either an sk_buff->head, or to be used in the "frags" -portion of skb_shared_info. - -In order to make use of the page fragment APIs a backing page fragment -cache is needed. This provides a central point for the fragment allocation -and tracks allows multiple calls to make use of a cached page. The -advantage to doing this is that multiple calls to get_page can be avoided -which can be expensive at allocation time. However due to the nature of -this caching it is required that any calls to the cache be protected by -either a per-cpu limitation, or a per-cpu limitation and forcing interrupts -to be disabled when executing the fragment allocation. - -The network stack uses two separate caches per CPU to handle fragment -allocation. The netdev_alloc_cache is used by callers making use of the -netdev_alloc_frag and __netdev_alloc_skb calls. The napi_alloc_cache is -used by callers of the __napi_alloc_frag and __napi_alloc_skb calls. The -main difference between these two calls is the context in which they may be -called. The "netdev" prefixed functions are usable in any context as these -functions will disable interrupts, while the "napi" prefixed functions are -only usable within the softirq context. - -Many network device drivers use a similar methodology for allocating page -fragments, but the page fragments are cached at the ring or descriptor -level. In order to enable these cases it is necessary to provide a generic -way of tearing down a page cache. For this reason __page_frag_cache_drain -was implemented. It allows for freeing multiple references from a single -page via a single call. The advantage to doing this is that it allows for -cleaning up the multiple references that were added to a page in order to -avoid calling get_page per allocation. - -Alexander Duyck, Nov 29, 2016. diff --git a/Documentation/vm/page_migration.rst b/Documentation/vm/page_migration.rst deleted file mode 100644 index 8c5cb8147e55..000000000000 --- a/Documentation/vm/page_migration.rst +++ /dev/null @@ -1,288 +0,0 @@ -.. _page_migration: - -============== -Page migration -============== - -Page migration allows moving the physical location of pages between -nodes in a NUMA system while the process is running. This means that the -virtual addresses that the process sees do not change. However, the -system rearranges the physical location of those pages. - -Also see :ref:`Heterogeneous Memory Management (HMM) ` -for migrating pages to or from device private memory. - -The main intent of page migration is to reduce the latency of memory accesses -by moving pages near to the processor where the process accessing that memory -is running. - -Page migration allows a process to manually relocate the node on which its -pages are located through the MF_MOVE and MF_MOVE_ALL options while setting -a new memory policy via mbind(). The pages of a process can also be relocated -from another process using the sys_migrate_pages() function call. The -migrate_pages() function call takes two sets of nodes and moves pages of a -process that are located on the from nodes to the destination nodes. -Page migration functions are provided by the numactl package by Andi Kleen -(a version later than 0.9.3 is required. Get it from -https://github.com/numactl/numactl.git). numactl provides libnuma -which provides an interface similar to other NUMA functionality for page -migration. cat ``/proc//numa_maps`` allows an easy review of where the -pages of a process are located. See also the numa_maps documentation in the -proc(5) man page. - -Manual migration is useful if for example the scheduler has relocated -a process to a processor on a distant node. A batch scheduler or an -administrator may detect the situation and move the pages of the process -nearer to the new processor. The kernel itself only provides -manual page migration support. Automatic page migration may be implemented -through user space processes that move pages. A special function call -"move_pages" allows the moving of individual pages within a process. -For example, A NUMA profiler may obtain a log showing frequent off-node -accesses and may use the result to move pages to more advantageous -locations. - -Larger installations usually partition the system using cpusets into -sections of nodes. Paul Jackson has equipped cpusets with the ability to -move pages when a task is moved to another cpuset (See -:ref:`CPUSETS `). -Cpusets allow the automation of process locality. If a task is moved to -a new cpuset then also all its pages are moved with it so that the -performance of the process does not sink dramatically. Also the pages -of processes in a cpuset are moved if the allowed memory nodes of a -cpuset are changed. - -Page migration allows the preservation of the relative location of pages -within a group of nodes for all migration techniques which will preserve a -particular memory allocation pattern generated even after migrating a -process. This is necessary in order to preserve the memory latencies. -Processes will run with similar performance after migration. - -Page migration occurs in several steps. First a high level -description for those trying to use migrate_pages() from the kernel -(for userspace usage see the Andi Kleen's numactl package mentioned above) -and then a low level description of how the low level details work. - -In kernel use of migrate_pages() -================================ - -1. Remove pages from the LRU. - - Lists of pages to be migrated are generated by scanning over - pages and moving them into lists. This is done by - calling isolate_lru_page(). - Calling isolate_lru_page() increases the references to the page - so that it cannot vanish while the page migration occurs. - It also prevents the swapper or other scans from encountering - the page. - -2. We need to have a function of type new_page_t that can be - passed to migrate_pages(). This function should figure out - how to allocate the correct new page given the old page. - -3. The migrate_pages() function is called which attempts - to do the migration. It will call the function to allocate - the new page for each page that is considered for - moving. - -How migrate_pages() works -========================= - -migrate_pages() does several passes over its list of pages. A page is moved -if all references to a page are removable at the time. The page has -already been removed from the LRU via isolate_lru_page() and the refcount -is increased so that the page cannot be freed while page migration occurs. - -Steps: - -1. Lock the page to be migrated. - -2. Ensure that writeback is complete. - -3. Lock the new page that we want to move to. It is locked so that accesses to - this (not yet up-to-date) page immediately block while the move is in progress. - -4. All the page table references to the page are converted to migration - entries. This decreases the mapcount of a page. If the resulting - mapcount is not zero then we do not migrate the page. All user space - processes that attempt to access the page will now wait on the page lock - or wait for the migration page table entry to be removed. - -5. The i_pages lock is taken. This will cause all processes trying - to access the page via the mapping to block on the spinlock. - -6. The refcount of the page is examined and we back out if references remain. - Otherwise, we know that we are the only one referencing this page. - -7. The radix tree is checked and if it does not contain the pointer to this - page then we back out because someone else modified the radix tree. - -8. The new page is prepped with some settings from the old page so that - accesses to the new page will discover a page with the correct settings. - -9. The radix tree is changed to point to the new page. - -10. The reference count of the old page is dropped because the address space - reference is gone. A reference to the new page is established because - the new page is referenced by the address space. - -11. The i_pages lock is dropped. With that lookups in the mapping - become possible again. Processes will move from spinning on the lock - to sleeping on the locked new page. - -12. The page contents are copied to the new page. - -13. The remaining page flags are copied to the new page. - -14. The old page flags are cleared to indicate that the page does - not provide any information anymore. - -15. Queued up writeback on the new page is triggered. - -16. If migration entries were inserted into the page table, then replace them - with real ptes. Doing so will enable access for user space processes not - already waiting for the page lock. - -17. The page locks are dropped from the old and new page. - Processes waiting on the page lock will redo their page faults - and will reach the new page. - -18. The new page is moved to the LRU and can be scanned by the swapper, - etc. again. - -Non-LRU page migration -====================== - -Although migration originally aimed for reducing the latency of memory accesses -for NUMA, compaction also uses migration to create high-order pages. - -Current problem of the implementation is that it is designed to migrate only -*LRU* pages. However, there are potential non-LRU pages which can be migrated -in drivers, for example, zsmalloc, virtio-balloon pages. - -For virtio-balloon pages, some parts of migration code path have been hooked -up and added virtio-balloon specific functions to intercept migration logics. -It's too specific to a driver so other drivers who want to make their pages -movable would have to add their own specific hooks in the migration path. - -To overcome the problem, VM supports non-LRU page migration which provides -generic functions for non-LRU movable pages without driver specific hooks -in the migration path. - -If a driver wants to make its pages movable, it should define three functions -which are function pointers of struct address_space_operations. - -1. ``bool (*isolate_page) (struct page *page, isolate_mode_t mode);`` - - What VM expects from isolate_page() function of driver is to return *true* - if driver isolates the page successfully. On returning true, VM marks the page - as PG_isolated so concurrent isolation in several CPUs skip the page - for isolation. If a driver cannot isolate the page, it should return *false*. - - Once page is successfully isolated, VM uses page.lru fields so driver - shouldn't expect to preserve values in those fields. - -2. ``int (*migratepage) (struct address_space *mapping,`` -| ``struct page *newpage, struct page *oldpage, enum migrate_mode);`` - - After isolation, VM calls migratepage() of driver with the isolated page. - The function of migratepage() is to move the contents of the old page to the - new page - and set up fields of struct page newpage. Keep in mind that you should - indicate to the VM the oldpage is no longer movable via __ClearPageMovable() - under page_lock if you migrated the oldpage successfully and returned - MIGRATEPAGE_SUCCESS. If driver cannot migrate the page at the moment, driver - can return -EAGAIN. On -EAGAIN, VM will retry page migration in a short time - because VM interprets -EAGAIN as "temporary migration failure". On returning - any error except -EAGAIN, VM will give up the page migration without - retrying. - - Driver shouldn't touch the page.lru field while in the migratepage() function. - -3. ``void (*putback_page)(struct page *);`` - - If migration fails on the isolated page, VM should return the isolated page - to the driver so VM calls the driver's putback_page() with the isolated page. - In this function, the driver should put the isolated page back into its own data - structure. - -Non-LRU movable page flags - - There are two page flags for supporting non-LRU movable page. - - * PG_movable - - Driver should use the function below to make page movable under page_lock:: - - void __SetPageMovable(struct page *page, struct address_space *mapping) - - It needs argument of address_space for registering migration - family functions which will be called by VM. Exactly speaking, - PG_movable is not a real flag of struct page. Rather, VM - reuses the page->mapping's lower bits to represent it:: - - #define PAGE_MAPPING_MOVABLE 0x2 - page->mapping = page->mapping | PAGE_MAPPING_MOVABLE; - - so driver shouldn't access page->mapping directly. Instead, driver should - use page_mapping() which masks off the low two bits of page->mapping under - page lock so it can get the right struct address_space. - - For testing of non-LRU movable pages, VM supports __PageMovable() function. - However, it doesn't guarantee to identify non-LRU movable pages because - the page->mapping field is unified with other variables in struct page. - If the driver releases the page after isolation by VM, page->mapping - doesn't have a stable value although it has PAGE_MAPPING_MOVABLE set - (look at __ClearPageMovable). But __PageMovable() is cheap to call whether - page is LRU or non-LRU movable once the page has been isolated because LRU - pages can never have PAGE_MAPPING_MOVABLE set in page->mapping. It is also - good for just peeking to test non-LRU movable pages before more expensive - checking with lock_page() in pfn scanning to select a victim. - - For guaranteeing non-LRU movable page, VM provides PageMovable() function. - Unlike __PageMovable(), PageMovable() validates page->mapping and - mapping->a_ops->isolate_page under lock_page(). The lock_page() prevents - sudden destroying of page->mapping. - - Drivers using __SetPageMovable() should clear the flag via - __ClearMovablePage() under page_lock() before the releasing the page. - - * PG_isolated - - To prevent concurrent isolation among several CPUs, VM marks isolated page - as PG_isolated under lock_page(). So if a CPU encounters PG_isolated - non-LRU movable page, it can skip it. Driver doesn't need to manipulate the - flag because VM will set/clear it automatically. Keep in mind that if the - driver sees a PG_isolated page, it means the page has been isolated by the - VM so it shouldn't touch the page.lru field. - The PG_isolated flag is aliased with the PG_reclaim flag so drivers - shouldn't use PG_isolated for its own purposes. - -Monitoring Migration -===================== - -The following events (counters) can be used to monitor page migration. - -1. PGMIGRATE_SUCCESS: Normal page migration success. Each count means that a - page was migrated. If the page was a non-THP and non-hugetlb page, then - this counter is increased by one. If the page was a THP or hugetlb, then - this counter is increased by the number of THP or hugetlb subpages. - For example, migration of a single 2MB THP that has 4KB-size base pages - (subpages) will cause this counter to increase by 512. - -2. PGMIGRATE_FAIL: Normal page migration failure. Same counting rules as for - PGMIGRATE_SUCCESS, above: this will be increased by the number of subpages, - if it was a THP or hugetlb. - -3. THP_MIGRATION_SUCCESS: A THP was migrated without being split. - -4. THP_MIGRATION_FAIL: A THP could not be migrated nor it could be split. - -5. THP_MIGRATION_SPLIT: A THP was migrated, but not as such: first, the THP had - to be split. After splitting, a migration retry was used for it's sub-pages. - -THP_MIGRATION_* events also update the appropriate PGMIGRATE_SUCCESS or -PGMIGRATE_FAIL events. For example, a THP migration failure will cause both -THP_MIGRATION_FAIL and PGMIGRATE_FAIL to increase. - -Christoph Lameter, May 8, 2006. -Minchan Kim, Mar 28, 2016. diff --git a/Documentation/vm/page_owner.rst b/Documentation/vm/page_owner.rst deleted file mode 100644 index f5c954afe97c..000000000000 --- a/Documentation/vm/page_owner.rst +++ /dev/null @@ -1,196 +0,0 @@ -.. _page_owner: - -================================================== -page owner: Tracking about who allocated each page -================================================== - -Introduction -============ - -page owner is for the tracking about who allocated each page. -It can be used to debug memory leak or to find a memory hogger. -When allocation happens, information about allocation such as call stack -and order of pages is stored into certain storage for each page. -When we need to know about status of all pages, we can get and analyze -this information. - -Although we already have tracepoint for tracing page allocation/free, -using it for analyzing who allocate each page is rather complex. We need -to enlarge the trace buffer for preventing overlapping until userspace -program launched. And, launched program continually dump out the trace -buffer for later analysis and it would change system behaviour with more -possibility rather than just keeping it in memory, so bad for debugging. - -page owner can also be used for various purposes. For example, accurate -fragmentation statistics can be obtained through gfp flag information of -each page. It is already implemented and activated if page owner is -enabled. Other usages are more than welcome. - -page owner is disabled by default. So, if you'd like to use it, you need -to add "page_owner=on" to your boot cmdline. If the kernel is built -with page owner and page owner is disabled in runtime due to not enabling -boot option, runtime overhead is marginal. If disabled in runtime, it -doesn't require memory to store owner information, so there is no runtime -memory overhead. And, page owner inserts just two unlikely branches into -the page allocator hotpath and if not enabled, then allocation is done -like as the kernel without page owner. These two unlikely branches should -not affect to allocation performance, especially if the static keys jump -label patching functionality is available. Following is the kernel's code -size change due to this facility. - -- Without page owner:: - - text data bss dec hex filename - 48392 2333 644 51369 c8a9 mm/page_alloc.o - -- With page owner:: - - text data bss dec hex filename - 48800 2445 644 51889 cab1 mm/page_alloc.o - 6662 108 29 6799 1a8f mm/page_owner.o - 1025 8 8 1041 411 mm/page_ext.o - -Although, roughly, 8 KB code is added in total, page_alloc.o increase by -520 bytes and less than half of it is in hotpath. Building the kernel with -page owner and turning it on if needed would be great option to debug -kernel memory problem. - -There is one notice that is caused by implementation detail. page owner -stores information into the memory from struct page extension. This memory -is initialized some time later than that page allocator starts in sparse -memory system, so, until initialization, many pages can be allocated and -they would have no owner information. To fix it up, these early allocated -pages are investigated and marked as allocated in initialization phase. -Although it doesn't mean that they have the right owner information, -at least, we can tell whether the page is allocated or not, -more accurately. On 2GB memory x86-64 VM box, 13343 early allocated pages -are catched and marked, although they are mostly allocated from struct -page extension feature. Anyway, after that, no page is left in -un-tracking state. - -Usage -===== - -1) Build user-space helper:: - - cd tools/vm - make page_owner_sort - -2) Enable page owner: add "page_owner=on" to boot cmdline. - -3) Do the job that you want to debug. - -4) Analyze information from page owner:: - - cat /sys/kernel/debug/page_owner > page_owner_full.txt - ./page_owner_sort page_owner_full.txt sorted_page_owner.txt - - The general output of ``page_owner_full.txt`` is as follows:: - - Page allocated via order XXX, ... - PFN XXX ... - // Detailed stack - - Page allocated via order XXX, ... - PFN XXX ... - // Detailed stack - - The ``page_owner_sort`` tool ignores ``PFN`` rows, puts the remaining rows - in buf, uses regexp to extract the page order value, counts the times - and pages of buf, and finally sorts them according to the parameter(s). - - See the result about who allocated each page - in the ``sorted_page_owner.txt``. General output:: - - XXX times, XXX pages: - Page allocated via order XXX, ... - // Detailed stack - - By default, ``page_owner_sort`` is sorted according to the times of buf. - If you want to sort by the page nums of buf, use the ``-m`` parameter. - The detailed parameters are: - - fundamental function:: - - Sort: - -a Sort by memory allocation time. - -m Sort by total memory. - -p Sort by pid. - -P Sort by tgid. - -n Sort by task command name. - -r Sort by memory release time. - -s Sort by stack trace. - -t Sort by times (default). - --sort Specify sorting order. Sorting syntax is [+|-]key[,[+|-]key[,...]]. - Choose a key from the **STANDARD FORMAT SPECIFIERS** section. The "+" is - optional since default direction is increasing numerical or lexicographic - order. Mixed use of abbreviated and complete-form of keys is allowed. - - Examples: - ./page_owner_sort --sort=n,+pid,-tgid - ./page_owner_sort --sort=at - - additional function:: - - Cull: - --cull - Specify culling rules.Culling syntax is key[,key[,...]].Choose a - multi-letter key from the **STANDARD FORMAT SPECIFIERS** section. - - is a single argument in the form of a comma-separated list, - which offers a way to specify individual culling rules. The recognized - keywords are described in the **STANDARD FORMAT SPECIFIERS** section below. - can be specified by the sequence of keys k1,k2, ..., as described in - the STANDARD SORT KEYS section below. Mixed use of abbreviated and - complete-form of keys is allowed. - - Examples: - ./page_owner_sort --cull=stacktrace - ./page_owner_sort --cull=st,pid,name - ./page_owner_sort --cull=n,f - - Filter: - -f Filter out the information of blocks whose memory has been released. - - Select: - --pid Select by pid. This selects the blocks whose process ID - numbers appear in . - --tgid Select by tgid. This selects the blocks whose thread - group ID numbers appear in . - --name Select by task command name. This selects the blocks whose - task command name appear in . - - , , are single arguments in the form of a comma-separated list, - which offers a way to specify individual selecting rules. - - - Examples: - ./page_owner_sort --pid=1 - ./page_owner_sort --tgid=1,2,3 - ./page_owner_sort --name name1,name2 - -STANDARD FORMAT SPECIFIERS -========================== -:: - - For --sort option: - - KEY LONG DESCRIPTION - p pid process ID - tg tgid thread group ID - n name task command name - st stacktrace stack trace of the page allocation - T txt full text of block - ft free_ts timestamp of the page when it was released - at alloc_ts timestamp of the page when it was allocated - ator allocator memory allocator for pages - - For --curl option: - - KEY LONG DESCRIPTION - p pid process ID - tg tgid thread group ID - n name task command name - f free whether the page has been released or not - st stacktrace stack trace of the page allocation - ator allocator memory allocator for pages diff --git a/Documentation/vm/page_reclaim.rst b/Documentation/vm/page_reclaim.rst deleted file mode 100644 index 50a30b7f8ac3..000000000000 --- a/Documentation/vm/page_reclaim.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -============ -Page Reclaim -============ diff --git a/Documentation/vm/page_table_check.rst b/Documentation/vm/page_table_check.rst deleted file mode 100644 index 1a09472f10a3..000000000000 --- a/Documentation/vm/page_table_check.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -.. _page_table_check: - -================ -Page Table Check -================ - -Introduction -============ - -Page table check allows to harden the kernel by ensuring that some types of -the memory corruptions are prevented. - -Page table check performs extra verifications at the time when new pages become -accessible from the userspace by getting their page table entries (PTEs PMDs -etc.) added into the table. - -In case of detected corruption, the kernel is crashed. There is a small -performance and memory overhead associated with the page table check. Therefore, -it is disabled by default, but can be optionally enabled on systems where the -extra hardening outweighs the performance costs. Also, because page table check -is synchronous, it can help with debugging double map memory corruption issues, -by crashing kernel at the time wrong mapping occurs instead of later which is -often the case with memory corruptions bugs. - -Double mapping detection logic -============================== - -+-------------------+-------------------+-------------------+------------------+ -| Current Mapping | New mapping | Permissions | Rule | -+===================+===================+===================+==================+ -| Anonymous | Anonymous | Read | Allow | -+-------------------+-------------------+-------------------+------------------+ -| Anonymous | Anonymous | Read / Write | Prohibit | -+-------------------+-------------------+-------------------+------------------+ -| Anonymous | Named | Any | Prohibit | -+-------------------+-------------------+-------------------+------------------+ -| Named | Anonymous | Any | Prohibit | -+-------------------+-------------------+-------------------+------------------+ -| Named | Named | Any | Allow | -+-------------------+-------------------+-------------------+------------------+ - -Enabling Page Table Check -========================= - -Build kernel with: - -- PAGE_TABLE_CHECK=y - Note, it can only be enabled on platforms where ARCH_SUPPORTS_PAGE_TABLE_CHECK - is available. - -- Boot with 'page_table_check=on' kernel parameter. - -Optionally, build kernel with PAGE_TABLE_CHECK_ENFORCED in order to have page -table support without extra kernel parameter. diff --git a/Documentation/vm/page_tables.rst b/Documentation/vm/page_tables.rst deleted file mode 100644 index 96939571d7bc..000000000000 --- a/Documentation/vm/page_tables.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -=========== -Page Tables -=========== diff --git a/Documentation/vm/physical_memory.rst b/Documentation/vm/physical_memory.rst deleted file mode 100644 index 2ab7b8c1c863..000000000000 --- a/Documentation/vm/physical_memory.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -=============== -Physical Memory -=============== diff --git a/Documentation/vm/process_addrs.rst b/Documentation/vm/process_addrs.rst deleted file mode 100644 index e8618fbc62c9..000000000000 --- a/Documentation/vm/process_addrs.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -================= -Process Addresses -================= diff --git a/Documentation/vm/remap_file_pages.rst b/Documentation/vm/remap_file_pages.rst deleted file mode 100644 index 7bef6718e3a9..000000000000 --- a/Documentation/vm/remap_file_pages.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. _remap_file_pages: - -============================== -remap_file_pages() system call -============================== - -The remap_file_pages() system call is used to create a nonlinear mapping, -that is, a mapping in which the pages of the file are mapped into a -nonsequential order in memory. The advantage of using remap_file_pages() -over using repeated calls to mmap(2) is that the former approach does not -require the kernel to create additional VMA (Virtual Memory Area) data -structures. - -Supporting of nonlinear mapping requires significant amount of non-trivial -code in kernel virtual memory subsystem including hot paths. Also to get -nonlinear mapping work kernel need a way to distinguish normal page table -entries from entries with file offset (pte_file). Kernel reserves flag in -PTE for this purpose. PTE flags are scarce resource especially on some CPU -architectures. It would be nice to free up the flag for other usage. - -Fortunately, there are not many users of remap_file_pages() in the wild. -It's only known that one enterprise RDBMS implementation uses the syscall -on 32-bit systems to map files bigger than can linearly fit into 32-bit -virtual address space. This use-case is not critical anymore since 64-bit -systems are widely available. - -The syscall is deprecated and replaced it with an emulation now. The -emulation creates new VMAs instead of nonlinear mappings. It's going to -work slower for rare users of remap_file_pages() but ABI is preserved. - -One side effect of emulation (apart from performance) is that user can hit -vm.max_map_count limit more easily due to additional VMAs. See comment for -DEFAULT_MAX_MAP_COUNT for more details on the limit. diff --git a/Documentation/vm/shmfs.rst b/Documentation/vm/shmfs.rst deleted file mode 100644 index 8b01ebb4c30e..000000000000 --- a/Documentation/vm/shmfs.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -======================== -Shared Memory Filesystem -======================== diff --git a/Documentation/vm/slab.rst b/Documentation/vm/slab.rst deleted file mode 100644 index 87d5a5bb172f..000000000000 --- a/Documentation/vm/slab.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -=============== -Slab Allocation -=============== diff --git a/Documentation/vm/slub.rst b/Documentation/vm/slub.rst deleted file mode 100644 index 43063ade737a..000000000000 --- a/Documentation/vm/slub.rst +++ /dev/null @@ -1,452 +0,0 @@ -.. _slub: - -========================== -Short users guide for SLUB -========================== - -The basic philosophy of SLUB is very different from SLAB. SLAB -requires rebuilding the kernel to activate debug options for all -slab caches. SLUB always includes full debugging but it is off by default. -SLUB can enable debugging only for selected slabs in order to avoid -an impact on overall system performance which may make a bug more -difficult to find. - -In order to switch debugging on one can add an option ``slub_debug`` -to the kernel command line. That will enable full debugging for -all slabs. - -Typically one would then use the ``slabinfo`` command to get statistical -data and perform operation on the slabs. By default ``slabinfo`` only lists -slabs that have data in them. See "slabinfo -h" for more options when -running the command. ``slabinfo`` can be compiled with -:: - - gcc -o slabinfo tools/vm/slabinfo.c - -Some of the modes of operation of ``slabinfo`` require that slub debugging -be enabled on the command line. F.e. no tracking information will be -available without debugging on and validation can only partially -be performed if debugging was not switched on. - -Some more sophisticated uses of slub_debug: -------------------------------------------- - -Parameters may be given to ``slub_debug``. If none is specified then full -debugging is enabled. Format: - -slub_debug= - Enable options for all slabs - -slub_debug=,,,... - Enable options only for select slabs (no spaces - after a comma) - -Multiple blocks of options for all slabs or selected slabs can be given, with -blocks of options delimited by ';'. The last of "all slabs" blocks is applied -to all slabs except those that match one of the "select slabs" block. Options -of the first "select slabs" blocks that matches the slab's name are applied. - -Possible debug options are:: - - F Sanity checks on (enables SLAB_DEBUG_CONSISTENCY_CHECKS - Sorry SLAB legacy issues) - Z Red zoning - P Poisoning (object and padding) - U User tracking (free and alloc) - T Trace (please only use on single slabs) - A Enable failslab filter mark for the cache - O Switch debugging off for caches that would have - caused higher minimum slab orders - - Switch all debugging off (useful if the kernel is - configured with CONFIG_SLUB_DEBUG_ON) - -F.e. in order to boot just with sanity checks and red zoning one would specify:: - - slub_debug=FZ - -Trying to find an issue in the dentry cache? Try:: - - slub_debug=,dentry - -to only enable debugging on the dentry cache. You may use an asterisk at the -end of the slab name, in order to cover all slabs with the same prefix. For -example, here's how you can poison the dentry cache as well as all kmalloc -slabs:: - - slub_debug=P,kmalloc-*,dentry - -Red zoning and tracking may realign the slab. We can just apply sanity checks -to the dentry cache with:: - - slub_debug=F,dentry - -Debugging options may require the minimum possible slab order to increase as -a result of storing the metadata (for example, caches with PAGE_SIZE object -sizes). This has a higher liklihood of resulting in slab allocation errors -in low memory situations or if there's high fragmentation of memory. To -switch off debugging for such caches by default, use:: - - slub_debug=O - -You can apply different options to different list of slab names, using blocks -of options. This will enable red zoning for dentry and user tracking for -kmalloc. All other slabs will not get any debugging enabled:: - - slub_debug=Z,dentry;U,kmalloc-* - -You can also enable options (e.g. sanity checks and poisoning) for all caches -except some that are deemed too performance critical and don't need to be -debugged by specifying global debug options followed by a list of slab names -with "-" as options:: - - slub_debug=FZ;-,zs_handle,zspage - -The state of each debug option for a slab can be found in the respective files -under:: - - /sys/kernel/slab// - -If the file contains 1, the option is enabled, 0 means disabled. The debug -options from the ``slub_debug`` parameter translate to the following files:: - - F sanity_checks - Z red_zone - P poison - U store_user - T trace - A failslab - -Careful with tracing: It may spew out lots of information and never stop if -used on the wrong slab. - -Slab merging -============ - -If no debug options are specified then SLUB may merge similar slabs together -in order to reduce overhead and increase cache hotness of objects. -``slabinfo -a`` displays which slabs were merged together. - -Slab validation -=============== - -SLUB can validate all object if the kernel was booted with slub_debug. In -order to do so you must have the ``slabinfo`` tool. Then you can do -:: - - slabinfo -v - -which will test all objects. Output will be generated to the syslog. - -This also works in a more limited way if boot was without slab debug. -In that case ``slabinfo -v`` simply tests all reachable objects. Usually -these are in the cpu slabs and the partial slabs. Full slabs are not -tracked by SLUB in a non debug situation. - -Getting more performance -======================== - -To some degree SLUB's performance is limited by the need to take the -list_lock once in a while to deal with partial slabs. That overhead is -governed by the order of the allocation for each slab. The allocations -can be influenced by kernel parameters: - -.. slub_min_objects=x (default 4) -.. slub_min_order=x (default 0) -.. slub_max_order=x (default 3 (PAGE_ALLOC_COSTLY_ORDER)) - -``slub_min_objects`` - allows to specify how many objects must at least fit into one - slab in order for the allocation order to be acceptable. In - general slub will be able to perform this number of - allocations on a slab without consulting centralized resources - (list_lock) where contention may occur. - -``slub_min_order`` - specifies a minimum order of slabs. A similar effect like - ``slub_min_objects``. - -``slub_max_order`` - specified the order at which ``slub_min_objects`` should no - longer be checked. This is useful to avoid SLUB trying to - generate super large order pages to fit ``slub_min_objects`` - of a slab cache with large object sizes into one high order - page. Setting command line parameter - ``debug_guardpage_minorder=N`` (N > 0), forces setting - ``slub_max_order`` to 0, what cause minimum possible order of - slabs allocation. - -SLUB Debug output -================= - -Here is a sample of slub debug output:: - - ==================================================================== - BUG kmalloc-8: Right Redzone overwritten - -------------------------------------------------------------------- - - INFO: 0xc90f6d28-0xc90f6d2b. First byte 0x00 instead of 0xcc - INFO: Slab 0xc528c530 flags=0x400000c3 inuse=61 fp=0xc90f6d58 - INFO: Object 0xc90f6d20 @offset=3360 fp=0xc90f6d58 - INFO: Allocated in get_modalias+0x61/0xf5 age=53 cpu=1 pid=554 - - Bytes b4 (0xc90f6d10): 00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ - Object (0xc90f6d20): 31 30 31 39 2e 30 30 35 1019.005 - Redzone (0xc90f6d28): 00 cc cc cc . - Padding (0xc90f6d50): 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZ - - [] dump_trace+0x63/0x1eb - [] show_trace_log_lvl+0x1a/0x2f - [] show_trace+0x12/0x14 - [] dump_stack+0x16/0x18 - [] object_err+0x143/0x14b - [] check_object+0x66/0x234 - [] __slab_free+0x239/0x384 - [] kfree+0xa6/0xc6 - [] get_modalias+0xb9/0xf5 - [] dmi_dev_uevent+0x27/0x3c - [] dev_uevent+0x1ad/0x1da - [] kobject_uevent_env+0x20a/0x45b - [] kobject_uevent+0xa/0xf - [] store_uevent+0x4f/0x58 - [] dev_attr_store+0x29/0x2f - [] sysfs_write_file+0x16e/0x19c - [] vfs_write+0xd1/0x15a - [] sys_write+0x3d/0x72 - [] sysenter_past_esp+0x5f/0x99 - [] 0xb7f7b410 - ======================= - - FIX kmalloc-8: Restoring Redzone 0xc90f6d28-0xc90f6d2b=0xcc - -If SLUB encounters a corrupted object (full detection requires the kernel -to be booted with slub_debug) then the following output will be dumped -into the syslog: - -1. Description of the problem encountered - - This will be a message in the system log starting with:: - - =============================================== - BUG : - ----------------------------------------------- - - INFO: - - INFO: Slab
- INFO: Object
- INFO: Allocated in age= cpu= pid= - INFO: Freed in age= cpu= - pid= - - (Object allocation / free information is only available if SLAB_STORE_USER is - set for the slab. slub_debug sets that option) - -2. The object contents if an object was involved. - - Various types of lines can follow the BUG SLUB line: - - Bytes b4
: - Shows a few bytes before the object where the problem was detected. - Can be useful if the corruption does not stop with the start of the - object. - - Object
: - The bytes of the object. If the object is inactive then the bytes - typically contain poison values. Any non-poison value shows a - corruption by a write after free. - - Redzone
: - The Redzone following the object. The Redzone is used to detect - writes after the object. All bytes should always have the same - value. If there is any deviation then it is due to a write after - the object boundary. - - (Redzone information is only available if SLAB_RED_ZONE is set. - slub_debug sets that option) - - Padding
: - Unused data to fill up the space in order to get the next object - properly aligned. In the debug case we make sure that there are - at least 4 bytes of padding. This allows the detection of writes - before the object. - -3. A stackdump - - The stackdump describes the location where the error was detected. The cause - of the corruption is may be more likely found by looking at the function that - allocated or freed the object. - -4. Report on how the problem was dealt with in order to ensure the continued - operation of the system. - - These are messages in the system log beginning with:: - - FIX : - - In the above sample SLUB found that the Redzone of an active object has - been overwritten. Here a string of 8 characters was written into a slab that - has the length of 8 characters. However, a 8 character string needs a - terminating 0. That zero has overwritten the first byte of the Redzone field. - After reporting the details of the issue encountered the FIX SLUB message - tells us that SLUB has restored the Redzone to its proper value and then - system operations continue. - -Emergency operations -==================== - -Minimal debugging (sanity checks alone) can be enabled by booting with:: - - slub_debug=F - -This will be generally be enough to enable the resiliency features of slub -which will keep the system running even if a bad kernel component will -keep corrupting objects. This may be important for production systems. -Performance will be impacted by the sanity checks and there will be a -continual stream of error messages to the syslog but no additional memory -will be used (unlike full debugging). - -No guarantees. The kernel component still needs to be fixed. Performance -may be optimized further by locating the slab that experiences corruption -and enabling debugging only for that cache - -I.e.:: - - slub_debug=F,dentry - -If the corruption occurs by writing after the end of the object then it -may be advisable to enable a Redzone to avoid corrupting the beginning -of other objects:: - - slub_debug=FZ,dentry - -Extended slabinfo mode and plotting -=================================== - -The ``slabinfo`` tool has a special 'extended' ('-X') mode that includes: - - Slabcache Totals - - Slabs sorted by size (up to -N slabs, default 1) - - Slabs sorted by loss (up to -N slabs, default 1) - -Additionally, in this mode ``slabinfo`` does not dynamically scale -sizes (G/M/K) and reports everything in bytes (this functionality is -also available to other slabinfo modes via '-B' option) which makes -reporting more precise and accurate. Moreover, in some sense the `-X' -mode also simplifies the analysis of slabs' behaviour, because its -output can be plotted using the ``slabinfo-gnuplot.sh`` script. So it -pushes the analysis from looking through the numbers (tons of numbers) -to something easier -- visual analysis. - -To generate plots: - -a) collect slabinfo extended records, for example:: - - while [ 1 ]; do slabinfo -X >> FOO_STATS; sleep 1; done - -b) pass stats file(-s) to ``slabinfo-gnuplot.sh`` script:: - - slabinfo-gnuplot.sh FOO_STATS [FOO_STATS2 .. FOO_STATSN] - - The ``slabinfo-gnuplot.sh`` script will pre-processes the collected records - and generates 3 png files (and 3 pre-processing cache files) per STATS - file: - - Slabcache Totals: FOO_STATS-totals.png - - Slabs sorted by size: FOO_STATS-slabs-by-size.png - - Slabs sorted by loss: FOO_STATS-slabs-by-loss.png - -Another use case, when ``slabinfo-gnuplot.sh`` can be useful, is when you -need to compare slabs' behaviour "prior to" and "after" some code -modification. To help you out there, ``slabinfo-gnuplot.sh`` script -can 'merge' the `Slabcache Totals` sections from different -measurements. To visually compare N plots: - -a) Collect as many STATS1, STATS2, .. STATSN files as you need:: - - while [ 1 ]; do slabinfo -X >> STATS; sleep 1; done - -b) Pre-process those STATS files:: - - slabinfo-gnuplot.sh STATS1 STATS2 .. STATSN - -c) Execute ``slabinfo-gnuplot.sh`` in '-t' mode, passing all of the - generated pre-processed \*-totals:: - - slabinfo-gnuplot.sh -t STATS1-totals STATS2-totals .. STATSN-totals - - This will produce a single plot (png file). - - Plots, expectedly, can be large so some fluctuations or small spikes - can go unnoticed. To deal with that, ``slabinfo-gnuplot.sh`` has two - options to 'zoom-in'/'zoom-out': - - a) ``-s %d,%d`` -- overwrites the default image width and height - b) ``-r %d,%d`` -- specifies a range of samples to use (for example, - in ``slabinfo -X >> FOO_STATS; sleep 1;`` case, using a ``-r - 40,60`` range will plot only samples collected between 40th and - 60th seconds). - - -DebugFS files for SLUB -====================== - -For more information about current state of SLUB caches with the user tracking -debug option enabled, debugfs files are available, typically under -/sys/kernel/debug/slab// (created only for caches with enabled user -tracking). There are 2 types of these files with the following debug -information: - -1. alloc_traces:: - - Prints information about unique allocation traces of the currently - allocated objects. The output is sorted by frequency of each trace. - - Information in the output: - Number of objects, allocating function, minimal/average/maximal jiffies since alloc, - pid range of the allocating processes, cpu mask of allocating cpus, and stack trace. - - Example::: - - 1085 populate_error_injection_list+0x97/0x110 age=166678/166680/166682 pid=1 cpus=1:: - __slab_alloc+0x6d/0x90 - kmem_cache_alloc_trace+0x2eb/0x300 - populate_error_injection_list+0x97/0x110 - init_error_injection+0x1b/0x71 - do_one_initcall+0x5f/0x2d0 - kernel_init_freeable+0x26f/0x2d7 - kernel_init+0xe/0x118 - ret_from_fork+0x22/0x30 - - -2. free_traces:: - - Prints information about unique freeing traces of the currently allocated - objects. The freeing traces thus come from the previous life-cycle of the - objects and are reported as not available for objects allocated for the first - time. The output is sorted by frequency of each trace. - - Information in the output: - Number of objects, freeing function, minimal/average/maximal jiffies since free, - pid range of the freeing processes, cpu mask of freeing cpus, and stack trace. - - Example::: - - 1980 age=4294912290 pid=0 cpus=0 - 51 acpi_ut_update_ref_count+0x6a6/0x782 age=236886/237027/237772 pid=1 cpus=1 - kfree+0x2db/0x420 - acpi_ut_update_ref_count+0x6a6/0x782 - acpi_ut_update_object_reference+0x1ad/0x234 - acpi_ut_remove_reference+0x7d/0x84 - acpi_rs_get_prt_method_data+0x97/0xd6 - acpi_get_irq_routing_table+0x82/0xc4 - acpi_pci_irq_find_prt_entry+0x8e/0x2e0 - acpi_pci_irq_lookup+0x3a/0x1e0 - acpi_pci_irq_enable+0x77/0x240 - pcibios_enable_device+0x39/0x40 - do_pci_enable_device.part.0+0x5d/0xe0 - pci_enable_device_flags+0xfc/0x120 - pci_enable_device+0x13/0x20 - virtio_pci_probe+0x9e/0x170 - local_pci_probe+0x48/0x80 - pci_device_probe+0x105/0x1c0 - -Christoph Lameter, May 30, 2007 -Sergey Senozhatsky, October 23, 2015 diff --git a/Documentation/vm/split_page_table_lock.rst b/Documentation/vm/split_page_table_lock.rst deleted file mode 100644 index c08919662704..000000000000 --- a/Documentation/vm/split_page_table_lock.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. _split_page_table_lock: - -===================== -Split page table lock -===================== - -Originally, mm->page_table_lock spinlock protected all page tables of the -mm_struct. But this approach leads to poor page fault scalability of -multi-threaded applications due high contention on the lock. To improve -scalability, split page table lock was introduced. - -With split page table lock we have separate per-table lock to serialize -access to the table. At the moment we use split lock for PTE and PMD -tables. Access to higher level tables protected by mm->page_table_lock. - -There are helpers to lock/unlock a table and other accessor functions: - - - pte_offset_map_lock() - maps pte and takes PTE table lock, returns pointer to the taken - lock; - - pte_unmap_unlock() - unlocks and unmaps PTE table; - - pte_alloc_map_lock() - allocates PTE table if needed and take the lock, returns pointer - to taken lock or NULL if allocation failed; - - pte_lockptr() - returns pointer to PTE table lock; - - pmd_lock() - takes PMD table lock, returns pointer to taken lock; - - pmd_lockptr() - returns pointer to PMD table lock; - -Split page table lock for PTE tables is enabled compile-time if -CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS. -If split lock is disabled, all tables are guarded by mm->page_table_lock. - -Split page table lock for PMD tables is enabled, if it's enabled for PTE -tables and the architecture supports it (see below). - -Hugetlb and split page table lock -================================= - -Hugetlb can support several page sizes. We use split lock only for PMD -level, but not for PUD. - -Hugetlb-specific helpers: - - - huge_pte_lock() - takes pmd split lock for PMD_SIZE page, mm->page_table_lock - otherwise; - - huge_pte_lockptr() - returns pointer to table lock; - -Support of split page table lock by an architecture -=================================================== - -There's no need in special enabling of PTE split page table lock: everything -required is done by pgtable_pte_page_ctor() and pgtable_pte_page_dtor(), which -must be called on PTE table allocation / freeing. - -Make sure the architecture doesn't use slab allocator for page table -allocation: slab uses page->slab_cache for its pages. -This field shares storage with page->ptl. - -PMD split lock only makes sense if you have more than two page table -levels. - -PMD split lock enabling requires pgtable_pmd_page_ctor() call on PMD table -allocation and pgtable_pmd_page_dtor() on freeing. - -Allocation usually happens in pmd_alloc_one(), freeing in pmd_free() and -pmd_free_tlb(), but make sure you cover all PMD table allocation / freeing -paths: i.e X86_PAE preallocate few PMDs on pgd_alloc(). - -With everything in place you can set CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK. - -NOTE: pgtable_pte_page_ctor() and pgtable_pmd_page_ctor() can fail -- it must -be handled properly. - -page->ptl -========= - -page->ptl is used to access split page table lock, where 'page' is struct -page of page containing the table. It shares storage with page->private -(and few other fields in union). - -To avoid increasing size of struct page and have best performance, we use a -trick: - - - if spinlock_t fits into long, we use page->ptr as spinlock, so we - can avoid indirect access and save a cache line. - - if size of spinlock_t is bigger then size of long, we use page->ptl as - pointer to spinlock_t and allocate it dynamically. This allows to use - split lock with enabled DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC, but costs - one more cache line for indirect access; - -The spinlock_t allocated in pgtable_pte_page_ctor() for PTE table and in -pgtable_pmd_page_ctor() for PMD table. - -Please, never access page->ptl directly -- use appropriate helper. diff --git a/Documentation/vm/swap.rst b/Documentation/vm/swap.rst deleted file mode 100644 index 78819bd4d745..000000000000 --- a/Documentation/vm/swap.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -==== -Swap -==== diff --git a/Documentation/vm/transhuge.rst b/Documentation/vm/transhuge.rst deleted file mode 100644 index 216db1d67d04..000000000000 --- a/Documentation/vm/transhuge.rst +++ /dev/null @@ -1,187 +0,0 @@ -.. _transhuge: - -============================ -Transparent Hugepage Support -============================ - -This document describes design principles for Transparent Hugepage (THP) -support and its interaction with other parts of the memory management -system. - -Design principles -================= - -- "graceful fallback": mm components which don't have transparent hugepage - knowledge fall back to breaking huge pmd mapping into table of ptes and, - if necessary, split a transparent hugepage. Therefore these components - can continue working on the regular pages or regular pte mappings. - -- if a hugepage allocation fails because of memory fragmentation, - regular pages should be gracefully allocated instead and mixed in - the same vma without any failure or significant delay and without - userland noticing - -- if some task quits and more hugepages become available (either - immediately in the buddy or through the VM), guest physical memory - backed by regular pages should be relocated on hugepages - automatically (with khugepaged) - -- it doesn't require memory reservation and in turn it uses hugepages - whenever possible (the only possible reservation here is kernelcore= - to avoid unmovable pages to fragment all the memory but such a tweak - is not specific to transparent hugepage support and it's a generic - feature that applies to all dynamic high order allocations in the - kernel) - -get_user_pages and follow_page -============================== - -get_user_pages and follow_page if run on a hugepage, will return the -head or tail pages as usual (exactly as they would do on -hugetlbfs). Most GUP users will only care about the actual physical -address of the page and its temporary pinning to release after the I/O -is complete, so they won't ever notice the fact the page is huge. But -if any driver is going to mangle over the page structure of the tail -page (like for checking page->mapping or other bits that are relevant -for the head page and not the tail page), it should be updated to jump -to check head page instead. Taking a reference on any head/tail page would -prevent the page from being split by anyone. - -.. note:: - these aren't new constraints to the GUP API, and they match the - same constraints that apply to hugetlbfs too, so any driver capable - of handling GUP on hugetlbfs will also work fine on transparent - hugepage backed mappings. - -Graceful fallback -================= - -Code walking pagetables but unaware about huge pmds can simply call -split_huge_pmd(vma, pmd, addr) where the pmd is the one returned by -pmd_offset. It's trivial to make the code transparent hugepage aware -by just grepping for "pmd_offset" and adding split_huge_pmd where -missing after pmd_offset returns the pmd. Thanks to the graceful -fallback design, with a one liner change, you can avoid to write -hundreds if not thousands of lines of complex code to make your code -hugepage aware. - -If you're not walking pagetables but you run into a physical hugepage -that you can't handle natively in your code, you can split it by -calling split_huge_page(page). This is what the Linux VM does before -it tries to swapout the hugepage for example. split_huge_page() can fail -if the page is pinned and you must handle this correctly. - -Example to make mremap.c transparent hugepage aware with a one liner -change:: - - diff --git a/mm/mremap.c b/mm/mremap.c - --- a/mm/mremap.c - +++ b/mm/mremap.c - @@ -41,6 +41,7 @@ static pmd_t *get_old_pmd(struct mm_stru - return NULL; - - pmd = pmd_offset(pud, addr); - + split_huge_pmd(vma, pmd, addr); - if (pmd_none_or_clear_bad(pmd)) - return NULL; - -Locking in hugepage aware code -============================== - -We want as much code as possible hugepage aware, as calling -split_huge_page() or split_huge_pmd() has a cost. - -To make pagetable walks huge pmd aware, all you need to do is to call -pmd_trans_huge() on the pmd returned by pmd_offset. You must hold the -mmap_lock in read (or write) mode to be sure a huge pmd cannot be -created from under you by khugepaged (khugepaged collapse_huge_page -takes the mmap_lock in write mode in addition to the anon_vma lock). If -pmd_trans_huge returns false, you just fallback in the old code -paths. If instead pmd_trans_huge returns true, you have to take the -page table lock (pmd_lock()) and re-run pmd_trans_huge. Taking the -page table lock will prevent the huge pmd being converted into a -regular pmd from under you (split_huge_pmd can run in parallel to the -pagetable walk). If the second pmd_trans_huge returns false, you -should just drop the page table lock and fallback to the old code as -before. Otherwise, you can proceed to process the huge pmd and the -hugepage natively. Once finished, you can drop the page table lock. - -Refcounts and transparent huge pages -==================================== - -Refcounting on THP is mostly consistent with refcounting on other compound -pages: - - - get_page()/put_page() and GUP operate on head page's ->_refcount. - - - ->_refcount in tail pages is always zero: get_page_unless_zero() never - succeeds on tail pages. - - - map/unmap of the pages with PTE entry increment/decrement ->_mapcount - on relevant sub-page of the compound page. - - - map/unmap of the whole compound page is accounted for in compound_mapcount - (stored in first tail page). For file huge pages, we also increment - ->_mapcount of all sub-pages in order to have race-free detection of - last unmap of subpages. - -PageDoubleMap() indicates that the page is *possibly* mapped with PTEs. - -For anonymous pages, PageDoubleMap() also indicates ->_mapcount in all -subpages is offset up by one. This additional reference is required to -get race-free detection of unmap of subpages when we have them mapped with -both PMDs and PTEs. - -This optimization is required to lower the overhead of per-subpage mapcount -tracking. The alternative is to alter ->_mapcount in all subpages on each -map/unmap of the whole compound page. - -For anonymous pages, we set PG_double_map when a PMD of the page is split -for the first time, but still have a PMD mapping. The additional references -go away with the last compound_mapcount. - -File pages get PG_double_map set on the first map of the page with PTE and -goes away when the page gets evicted from the page cache. - -split_huge_page internally has to distribute the refcounts in the head -page to the tail pages before clearing all PG_head/tail bits from the page -structures. It can be done easily for refcounts taken by page table -entries, but we don't have enough information on how to distribute any -additional pins (i.e. from get_user_pages). split_huge_page() fails any -requests to split pinned huge pages: it expects page count to be equal to -the sum of mapcount of all sub-pages plus one (split_huge_page caller must -have a reference to the head page). - -split_huge_page uses migration entries to stabilize page->_refcount and -page->_mapcount of anonymous pages. File pages just get unmapped. - -We are safe against physical memory scanners too: the only legitimate way -a scanner can get a reference to a page is get_page_unless_zero(). - -All tail pages have zero ->_refcount until atomic_add(). This prevents the -scanner from getting a reference to the tail page up to that point. After the -atomic_add() we don't care about the ->_refcount value. We already know how -many references should be uncharged from the head page. - -For head page get_page_unless_zero() will succeed and we don't mind. It's -clear where references should go after split: it will stay on the head page. - -Note that split_huge_pmd() doesn't have any limitations on refcounting: -pmd can be split at any point and never fails. - -Partial unmap and deferred_split_huge_page() -============================================ - -Unmapping part of THP (with munmap() or other way) is not going to free -memory immediately. Instead, we detect that a subpage of THP is not in use -in page_remove_rmap() and queue the THP for splitting if memory pressure -comes. Splitting will free up unused subpages. - -Splitting the page right away is not an option due to locking context in -the place where we can detect partial unmap. It also might be -counterproductive since in many cases partial unmap happens during exit(2) if -a THP crosses a VMA boundary. - -The function deferred_split_huge_page() is used to queue a page for splitting. -The splitting itself will happen when we get memory pressure via shrinker -interface. diff --git a/Documentation/vm/unevictable-lru.rst b/Documentation/vm/unevictable-lru.rst deleted file mode 100644 index b280367d6a44..000000000000 --- a/Documentation/vm/unevictable-lru.rst +++ /dev/null @@ -1,554 +0,0 @@ -.. _unevictable_lru: - -============================== -Unevictable LRU Infrastructure -============================== - -.. contents:: :local: - - -Introduction -============ - -This document describes the Linux memory manager's "Unevictable LRU" -infrastructure and the use of this to manage several types of "unevictable" -pages. - -The document attempts to provide the overall rationale behind this mechanism -and the rationale for some of the design decisions that drove the -implementation. The latter design rationale is discussed in the context of an -implementation description. Admittedly, one can obtain the implementation -details - the "what does it do?" - by reading the code. One hopes that the -descriptions below add value by provide the answer to "why does it do that?". - - - -The Unevictable LRU -=================== - -The Unevictable LRU facility adds an additional LRU list to track unevictable -pages and to hide these pages from vmscan. This mechanism is based on a patch -by Larry Woodman of Red Hat to address several scalability problems with page -reclaim in Linux. The problems have been observed at customer sites on large -memory x86_64 systems. - -To illustrate this with an example, a non-NUMA x86_64 platform with 128GB of -main memory will have over 32 million 4k pages in a single node. When a large -fraction of these pages are not evictable for any reason [see below], vmscan -will spend a lot of time scanning the LRU lists looking for the small fraction -of pages that are evictable. This can result in a situation where all CPUs are -spending 100% of their time in vmscan for hours or days on end, with the system -completely unresponsive. - -The unevictable list addresses the following classes of unevictable pages: - - * Those owned by ramfs. - - * Those mapped into SHM_LOCK'd shared memory regions. - - * Those mapped into VM_LOCKED [mlock()ed] VMAs. - -The infrastructure may also be able to handle other conditions that make pages -unevictable, either by definition or by circumstance, in the future. - - -The Unevictable LRU Page List ------------------------------ - -The Unevictable LRU page list is a lie. It was never an LRU-ordered list, but a -companion to the LRU-ordered anonymous and file, active and inactive page lists; -and now it is not even a page list. But following familiar convention, here in -this document and in the source, we often imagine it as a fifth LRU page list. - -The Unevictable LRU infrastructure consists of an additional, per-node, LRU list -called the "unevictable" list and an associated page flag, PG_unevictable, to -indicate that the page is being managed on the unevictable list. - -The PG_unevictable flag is analogous to, and mutually exclusive with, the -PG_active flag in that it indicates on which LRU list a page resides when -PG_lru is set. - -The Unevictable LRU infrastructure maintains unevictable pages as if they were -on an additional LRU list for a few reasons: - - (1) We get to "treat unevictable pages just like we treat other pages in the - system - which means we get to use the same code to manipulate them, the - same code to isolate them (for migrate, etc.), the same code to keep track - of the statistics, etc..." [Rik van Riel] - - (2) We want to be able to migrate unevictable pages between nodes for memory - defragmentation, workload management and memory hotplug. The Linux kernel - can only migrate pages that it can successfully isolate from the LRU - lists (or "Movable" pages: outside of consideration here). If we were to - maintain pages elsewhere than on an LRU-like list, where they can be - detected by isolate_lru_page(), we would prevent their migration. - -The unevictable list does not differentiate between file-backed and anonymous, -swap-backed pages. This differentiation is only important while the pages are, -in fact, evictable. - -The unevictable list benefits from the "arrayification" of the per-node LRU -lists and statistics originally proposed and posted by Christoph Lameter. - - -Memory Control Group Interaction --------------------------------- - -The unevictable LRU facility interacts with the memory control group [aka -memory controller; see Documentation/admin-guide/cgroup-v1/memory.rst] by -extending the lru_list enum. - -The memory controller data structure automatically gets a per-node unevictable -list as a result of the "arrayification" of the per-node LRU lists (one per -lru_list enum element). The memory controller tracks the movement of pages to -and from the unevictable list. - -When a memory control group comes under memory pressure, the controller will -not attempt to reclaim pages on the unevictable list. This has a couple of -effects: - - (1) Because the pages are "hidden" from reclaim on the unevictable list, the - reclaim process can be more efficient, dealing only with pages that have a - chance of being reclaimed. - - (2) On the other hand, if too many of the pages charged to the control group - are unevictable, the evictable portion of the working set of the tasks in - the control group may not fit into the available memory. This can cause - the control group to thrash or to OOM-kill tasks. - - -.. _mark_addr_space_unevict: - -Marking Address Spaces Unevictable ----------------------------------- - -For facilities such as ramfs none of the pages attached to the address space -may be evicted. To prevent eviction of any such pages, the AS_UNEVICTABLE -address space flag is provided, and this can be manipulated by a filesystem -using a number of wrapper functions: - - * ``void mapping_set_unevictable(struct address_space *mapping);`` - - Mark the address space as being completely unevictable. - - * ``void mapping_clear_unevictable(struct address_space *mapping);`` - - Mark the address space as being evictable. - - * ``int mapping_unevictable(struct address_space *mapping);`` - - Query the address space, and return true if it is completely - unevictable. - -These are currently used in three places in the kernel: - - (1) By ramfs to mark the address spaces of its inodes when they are created, - and this mark remains for the life of the inode. - - (2) By SYSV SHM to mark SHM_LOCK'd address spaces until SHM_UNLOCK is called. - Note that SHM_LOCK is not required to page in the locked pages if they're - swapped out; the application must touch the pages manually if it wants to - ensure they're in memory. - - (3) By the i915 driver to mark pinned address space until it's unpinned. The - amount of unevictable memory marked by i915 driver is roughly the bounded - object size in debugfs/dri/0/i915_gem_objects. - - -Detecting Unevictable Pages ---------------------------- - -The function page_evictable() in mm/internal.h determines whether a page is -evictable or not using the query function outlined above [see section -:ref:`Marking address spaces unevictable `] -to check the AS_UNEVICTABLE flag. - -For address spaces that are so marked after being populated (as SHM regions -might be), the lock action (e.g. SHM_LOCK) can be lazy, and need not populate -the page tables for the region as does, for example, mlock(), nor need it make -any special effort to push any pages in the SHM_LOCK'd area to the unevictable -list. Instead, vmscan will do this if and when it encounters the pages during -a reclamation scan. - -On an unlock action (such as SHM_UNLOCK), the unlocker (e.g. shmctl()) must scan -the pages in the region and "rescue" them from the unevictable list if no other -condition is keeping them unevictable. If an unevictable region is destroyed, -the pages are also "rescued" from the unevictable list in the process of -freeing them. - -page_evictable() also checks for mlocked pages by testing an additional page -flag, PG_mlocked (as wrapped by PageMlocked()), which is set when a page is -faulted into a VM_LOCKED VMA, or found in a VMA being VM_LOCKED. - - -Vmscan's Handling of Unevictable Pages --------------------------------------- - -If unevictable pages are culled in the fault path, or moved to the unevictable -list at mlock() or mmap() time, vmscan will not encounter the pages until they -have become evictable again (via munlock() for example) and have been "rescued" -from the unevictable list. However, there may be situations where we decide, -for the sake of expediency, to leave an unevictable page on one of the regular -active/inactive LRU lists for vmscan to deal with. vmscan checks for such -pages in all of the shrink_{active|inactive|page}_list() functions and will -"cull" such pages that it encounters: that is, it diverts those pages to the -unevictable list for the memory cgroup and node being scanned. - -There may be situations where a page is mapped into a VM_LOCKED VMA, but the -page is not marked as PG_mlocked. Such pages will make it all the way to -shrink_active_list() or shrink_page_list() where they will be detected when -vmscan walks the reverse map in page_referenced() or try_to_unmap(). The page -is culled to the unevictable list when it is released by the shrinker. - -To "cull" an unevictable page, vmscan simply puts the page back on the LRU list -using putback_lru_page() - the inverse operation to isolate_lru_page() - after -dropping the page lock. Because the condition which makes the page unevictable -may change once the page is unlocked, __pagevec_lru_add_fn() will recheck the -unevictable state of a page before placing it on the unevictable list. - - -MLOCKED Pages -============= - -The unevictable page list is also useful for mlock(), in addition to ramfs and -SYSV SHM. Note that mlock() is only available in CONFIG_MMU=y situations; in -NOMMU situations, all mappings are effectively mlocked. - - -History -------- - -The "Unevictable mlocked Pages" infrastructure is based on work originally -posted by Nick Piggin in an RFC patch entitled "mm: mlocked pages off LRU". -Nick posted his patch as an alternative to a patch posted by Christoph Lameter -to achieve the same objective: hiding mlocked pages from vmscan. - -In Nick's patch, he used one of the struct page LRU list link fields as a count -of VM_LOCKED VMAs that map the page (Rik van Riel had the same idea three years -earlier). But this use of the link field for a count prevented the management -of the pages on an LRU list, and thus mlocked pages were not migratable as -isolate_lru_page() could not detect them, and the LRU list link field was not -available to the migration subsystem. - -Nick resolved this by putting mlocked pages back on the LRU list before -attempting to isolate them, thus abandoning the count of VM_LOCKED VMAs. When -Nick's patch was integrated with the Unevictable LRU work, the count was -replaced by walking the reverse map when munlocking, to determine whether any -other VM_LOCKED VMAs still mapped the page. - -However, walking the reverse map for each page when munlocking was ugly and -inefficient, and could lead to catastrophic contention on a file's rmap lock, -when many processes which had it mlocked were trying to exit. In 5.18, the -idea of keeping mlock_count in Unevictable LRU list link field was revived and -put to work, without preventing the migration of mlocked pages. This is why -the "Unevictable LRU list" cannot be a linked list of pages now; but there was -no use for that linked list anyway - though its size is maintained for meminfo. - - -Basic Management ----------------- - -mlocked pages - pages mapped into a VM_LOCKED VMA - are a class of unevictable -pages. When such a page has been "noticed" by the memory management subsystem, -the page is marked with the PG_mlocked flag. This can be manipulated using the -PageMlocked() functions. - -A PG_mlocked page will be placed on the unevictable list when it is added to -the LRU. Such pages can be "noticed" by memory management in several places: - - (1) in the mlock()/mlock2()/mlockall() system call handlers; - - (2) in the mmap() system call handler when mmapping a region with the - MAP_LOCKED flag; - - (3) mmapping a region in a task that has called mlockall() with the MCL_FUTURE - flag; - - (4) in the fault path and when a VM_LOCKED stack segment is expanded; or - - (5) as mentioned above, in vmscan:shrink_page_list() when attempting to - reclaim a page in a VM_LOCKED VMA by page_referenced() or try_to_unmap(). - -mlocked pages become unlocked and rescued from the unevictable list when: - - (1) mapped in a range unlocked via the munlock()/munlockall() system calls; - - (2) munmap()'d out of the last VM_LOCKED VMA that maps the page, including - unmapping at task exit; - - (3) when the page is truncated from the last VM_LOCKED VMA of an mmapped file; - or - - (4) before a page is COW'd in a VM_LOCKED VMA. - - -mlock()/mlock2()/mlockall() System Call Handling ------------------------------------------------- - -mlock(), mlock2() and mlockall() system call handlers proceed to mlock_fixup() -for each VMA in the range specified by the call. In the case of mlockall(), -this is the entire active address space of the task. Note that mlock_fixup() -is used for both mlocking and munlocking a range of memory. A call to mlock() -an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED, is -treated as a no-op and mlock_fixup() simply returns. - -If the VMA passes some filtering as described in "Filtering Special VMAs" -below, mlock_fixup() will attempt to merge the VMA with its neighbors or split -off a subset of the VMA if the range does not cover the entire VMA. Any pages -already present in the VMA are then marked as mlocked by mlock_page() via -mlock_pte_range() via walk_page_range() via mlock_vma_pages_range(). - -Before returning from the system call, do_mlock() or mlockall() will call -__mm_populate() to fault in the remaining pages via get_user_pages() and to -mark those pages as mlocked as they are faulted. - -Note that the VMA being mlocked might be mapped with PROT_NONE. In this case, -get_user_pages() will be unable to fault in the pages. That's okay. If pages -do end up getting faulted into this VM_LOCKED VMA, they will be handled in the -fault path - which is also how mlock2()'s MLOCK_ONFAULT areas are handled. - -For each PTE (or PMD) being faulted into a VMA, the page add rmap function -calls mlock_vma_page(), which calls mlock_page() when the VMA is VM_LOCKED -(unless it is a PTE mapping of a part of a transparent huge page). Or when -it is a newly allocated anonymous page, lru_cache_add_inactive_or_unevictable() -calls mlock_new_page() instead: similar to mlock_page(), but can make better -judgments, since this page is held exclusively and known not to be on LRU yet. - -mlock_page() sets PageMlocked immediately, then places the page on the CPU's -mlock pagevec, to batch up the rest of the work to be done under lru_lock by -__mlock_page(). __mlock_page() sets PageUnevictable, initializes mlock_count -and moves the page to unevictable state ("the unevictable LRU", but with -mlock_count in place of LRU threading). Or if the page was already PageLRU -and PageUnevictable and PageMlocked, it simply increments the mlock_count. - -But in practice that may not work ideally: the page may not yet be on an LRU, or -it may have been temporarily isolated from LRU. In such cases the mlock_count -field cannot be touched, but will be set to 0 later when __pagevec_lru_add_fn() -returns the page to "LRU". Races prohibit mlock_count from being set to 1 then: -rather than risk stranding a page indefinitely as unevictable, always err with -mlock_count on the low side, so that when munlocked the page will be rescued to -an evictable LRU, then perhaps be mlocked again later if vmscan finds it in a -VM_LOCKED VMA. - - -Filtering Special VMAs ----------------------- - -mlock_fixup() filters several classes of "special" VMAs: - -1) VMAs with VM_IO or VM_PFNMAP set are skipped entirely. The pages behind - these mappings are inherently pinned, so we don't need to mark them as - mlocked. In any case, most of the pages have no struct page in which to so - mark the page. Because of this, get_user_pages() will fail for these VMAs, - so there is no sense in attempting to visit them. - -2) VMAs mapping hugetlbfs page are already effectively pinned into memory. We - neither need nor want to mlock() these pages. But __mm_populate() includes - hugetlbfs ranges, allocating the huge pages and populating the PTEs. - -3) VMAs with VM_DONTEXPAND are generally userspace mappings of kernel pages, - such as the VDSO page, relay channel pages, etc. These pages are inherently - unevictable and are not managed on the LRU lists. __mm_populate() includes - these ranges, populating the PTEs if not already populated. - -4) VMAs with VM_MIXEDMAP set are not marked VM_LOCKED, but __mm_populate() - includes these ranges, populating the PTEs if not already populated. - -Note that for all of these special VMAs, mlock_fixup() does not set the -VM_LOCKED flag. Therefore, we won't have to deal with them later during -munlock(), munmap() or task exit. Neither does mlock_fixup() account these -VMAs against the task's "locked_vm". - - -munlock()/munlockall() System Call Handling -------------------------------------------- - -The munlock() and munlockall() system calls are handled by the same -mlock_fixup() function as mlock(), mlock2() and mlockall() system calls are. -If called to munlock an already munlocked VMA, mlock_fixup() simply returns. -Because of the VMA filtering discussed above, VM_LOCKED will not be set in -any "special" VMAs. So, those VMAs will be ignored for munlock. - -If the VMA is VM_LOCKED, mlock_fixup() again attempts to merge or split off the -specified range. All pages in the VMA are then munlocked by munlock_page() via -mlock_pte_range() via walk_page_range() via mlock_vma_pages_range() - the same -function used when mlocking a VMA range, with new flags for the VMA indicating -that it is munlock() being performed. - -munlock_page() uses the mlock pagevec to batch up work to be done under -lru_lock by __munlock_page(). __munlock_page() decrements the page's -mlock_count, and when that reaches 0 it clears PageMlocked and clears -PageUnevictable, moving the page from unevictable state to inactive LRU. - -But in practice that may not work ideally: the page may not yet have reached -"the unevictable LRU", or it may have been temporarily isolated from it. In -those cases its mlock_count field is unusable and must be assumed to be 0: so -that the page will be rescued to an evictable LRU, then perhaps be mlocked -again later if vmscan finds it in a VM_LOCKED VMA. - - -Migrating MLOCKED Pages ------------------------ - -A page that is being migrated has been isolated from the LRU lists and is held -locked across unmapping of the page, updating the page's address space entry -and copying the contents and state, until the page table entry has been -replaced with an entry that refers to the new page. Linux supports migration -of mlocked pages and other unevictable pages. PG_mlocked is cleared from the -the old page when it is unmapped from the last VM_LOCKED VMA, and set when the -new page is mapped in place of migration entry in a VM_LOCKED VMA. If the page -was unevictable because mlocked, PG_unevictable follows PG_mlocked; but if the -page was unevictable for other reasons, PG_unevictable is copied explicitly. - -Note that page migration can race with mlocking or munlocking of the same page. -There is mostly no problem since page migration requires unmapping all PTEs of -the old page (including munlock where VM_LOCKED), then mapping in the new page -(including mlock where VM_LOCKED). The page table locks provide sufficient -synchronization. - -However, since mlock_vma_pages_range() starts by setting VM_LOCKED on a VMA, -before mlocking any pages already present, if one of those pages were migrated -before mlock_pte_range() reached it, it would get counted twice in mlock_count. -To prevent that, mlock_vma_pages_range() temporarily marks the VMA as VM_IO, -so that mlock_vma_page() will skip it. - -To complete page migration, we place the old and new pages back onto the LRU -afterwards. The "unneeded" page - old page on success, new page on failure - -is freed when the reference count held by the migration process is released. - - -Compacting MLOCKED Pages ------------------------- - -The memory map can be scanned for compactable regions and the default behavior -is to let unevictable pages be moved. /proc/sys/vm/compact_unevictable_allowed -controls this behavior (see Documentation/admin-guide/sysctl/vm.rst). The work -of compaction is mostly handled by the page migration code and the same work -flow as described in Migrating MLOCKED Pages will apply. - - -MLOCKING Transparent Huge Pages -------------------------------- - -A transparent huge page is represented by a single entry on an LRU list. -Therefore, we can only make unevictable an entire compound page, not -individual subpages. - -If a user tries to mlock() part of a huge page, and no user mlock()s the -whole of the huge page, we want the rest of the page to be reclaimable. - -We cannot just split the page on partial mlock() as split_huge_page() can -fail and a new intermittent failure mode for the syscall is undesirable. - -We handle this by keeping PTE-mlocked huge pages on evictable LRU lists: -the PMD on the border of a VM_LOCKED VMA will be split into a PTE table. - -This way the huge page is accessible for vmscan. Under memory pressure the -page will be split, subpages which belong to VM_LOCKED VMAs will be moved -to the unevictable LRU and the rest can be reclaimed. - -/proc/meminfo's Unevictable and Mlocked amounts do not include those parts -of a transparent huge page which are mapped only by PTEs in VM_LOCKED VMAs. - - -mmap(MAP_LOCKED) System Call Handling -------------------------------------- - -In addition to the mlock(), mlock2() and mlockall() system calls, an application -can request that a region of memory be mlocked by supplying the MAP_LOCKED flag -to the mmap() call. There is one important and subtle difference here, though. -mmap() + mlock() will fail if the range cannot be faulted in (e.g. because -mm_populate fails) and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. -The mmaped area will still have properties of the locked area - pages will not -get swapped out - but major page faults to fault memory in might still happen. - -Furthermore, any mmap() call or brk() call that expands the heap by a task -that has previously called mlockall() with the MCL_FUTURE flag will result -in the newly mapped memory being mlocked. Before the unevictable/mlock -changes, the kernel simply called make_pages_present() to allocate pages -and populate the page table. - -To mlock a range of memory under the unevictable/mlock infrastructure, -the mmap() handler and task address space expansion functions call -populate_vma_page_range() specifying the vma and the address range to mlock. - - -munmap()/exit()/exec() System Call Handling -------------------------------------------- - -When unmapping an mlocked region of memory, whether by an explicit call to -munmap() or via an internal unmap from exit() or exec() processing, we must -munlock the pages if we're removing the last VM_LOCKED VMA that maps the pages. -Before the unevictable/mlock changes, mlocking did not mark the pages in any -way, so unmapping them required no processing. - -For each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls -munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED -(unless it was a PTE mapping of a part of a transparent huge page). - -munlock_page() uses the mlock pagevec to batch up work to be done under -lru_lock by __munlock_page(). __munlock_page() decrements the page's -mlock_count, and when that reaches 0 it clears PageMlocked and clears -PageUnevictable, moving the page from unevictable state to inactive LRU. - -But in practice that may not work ideally: the page may not yet have reached -"the unevictable LRU", or it may have been temporarily isolated from it. In -those cases its mlock_count field is unusable and must be assumed to be 0: so -that the page will be rescued to an evictable LRU, then perhaps be mlocked -again later if vmscan finds it in a VM_LOCKED VMA. - - -Truncating MLOCKED Pages ------------------------- - -File truncation or hole punching forcibly unmaps the deleted pages from -userspace; truncation even unmaps and deletes any private anonymous pages -which had been Copied-On-Write from the file pages now being truncated. - -Mlocked pages can be munlocked and deleted in this way: like with munmap(), -for each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls -munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED -(unless it was a PTE mapping of a part of a transparent huge page). - -However, if there is a racing munlock(), since mlock_vma_pages_range() starts -munlocking by clearing VM_LOCKED from a VMA, before munlocking all the pages -present, if one of those pages were unmapped by truncation or hole punch before -mlock_pte_range() reached it, it would not be recognized as mlocked by this VMA, -and would not be counted out of mlock_count. In this rare case, a page may -still appear as PageMlocked after it has been fully unmapped: and it is left to -release_pages() (or __page_cache_release()) to clear it and update statistics -before freeing (this event is counted in /proc/vmstat unevictable_pgs_cleared, -which is usually 0). - - -Page Reclaim in shrink_*_list() -------------------------------- - -vmscan's shrink_active_list() culls any obviously unevictable pages - -i.e. !page_evictable(page) pages - diverting those to the unevictable list. -However, shrink_active_list() only sees unevictable pages that made it onto the -active/inactive LRU lists. Note that these pages do not have PageUnevictable -set - otherwise they would be on the unevictable list and shrink_active_list() -would never see them. - -Some examples of these unevictable pages on the LRU lists are: - - (1) ramfs pages that have been placed on the LRU lists when first allocated. - - (2) SHM_LOCK'd shared memory pages. shmctl(SHM_LOCK) does not attempt to - allocate or fault in the pages in the shared memory region. This happens - when an application accesses the page the first time after SHM_LOCK'ing - the segment. - - (3) pages still mapped into VM_LOCKED VMAs, which should be marked mlocked, - but events left mlock_count too low, so they were munlocked too early. - -vmscan's shrink_inactive_list() and shrink_page_list() also divert obviously -unevictable pages found on the inactive lists to the appropriate memory cgroup -and node unevictable list. - -rmap's page_referenced_one(), called via vmscan's shrink_active_list() or -shrink_page_list(), and rmap's try_to_unmap_one() called via shrink_page_list(), -check for (3) pages still mapped into VM_LOCKED VMAs, and call mlock_vma_page() -to correct them. Such pages are culled to the unevictable list when released -by the shrinker. diff --git a/Documentation/vm/vmalloc.rst b/Documentation/vm/vmalloc.rst deleted file mode 100644 index 363fe20d6b9f..000000000000 --- a/Documentation/vm/vmalloc.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -====================================== -Virtually Contiguous Memory Allocation -====================================== diff --git a/Documentation/vm/vmalloced-kernel-stacks.rst b/Documentation/vm/vmalloced-kernel-stacks.rst deleted file mode 100644 index fc8c67833af6..000000000000 --- a/Documentation/vm/vmalloced-kernel-stacks.rst +++ /dev/null @@ -1,153 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -===================================== -Virtually Mapped Kernel Stack Support -===================================== - -:Author: Shuah Khan - -.. contents:: :local: - -Overview --------- - -This is a compilation of information from the code and original patch -series that introduced the `Virtually Mapped Kernel Stacks feature -` - -Introduction ------------- - -Kernel stack overflows are often hard to debug and make the kernel -susceptible to exploits. Problems could show up at a later time making -it difficult to isolate and root-cause. - -Virtually-mapped kernel stacks with guard pages causes kernel stack -overflows to be caught immediately rather than causing difficult to -diagnose corruptions. - -HAVE_ARCH_VMAP_STACK and VMAP_STACK configuration options enable -support for virtually mapped stacks with guard pages. This feature -causes reliable faults when the stack overflows. The usability of -the stack trace after overflow and response to the overflow itself -is architecture dependent. - -.. note:: - As of this writing, arm64, powerpc, riscv, s390, um, and x86 have - support for VMAP_STACK. - -HAVE_ARCH_VMAP_STACK --------------------- - -Architectures that can support Virtually Mapped Kernel Stacks should -enable this bool configuration option. The requirements are: - -- vmalloc space must be large enough to hold many kernel stacks. This - may rule out many 32-bit architectures. -- Stacks in vmalloc space need to work reliably. For example, if - vmap page tables are created on demand, either this mechanism - needs to work while the stack points to a virtual address with - unpopulated page tables or arch code (switch_to() and switch_mm(), - most likely) needs to ensure that the stack's page table entries - are populated before running on a possibly unpopulated stack. -- If the stack overflows into a guard page, something reasonable - should happen. The definition of "reasonable" is flexible, but - instantly rebooting without logging anything would be unfriendly. - -VMAP_STACK ----------- - -VMAP_STACK bool configuration option when enabled allocates virtually -mapped task stacks. This option depends on HAVE_ARCH_VMAP_STACK. - -- Enable this if you want the use virtually-mapped kernel stacks - with guard pages. This causes kernel stack overflows to be caught - immediately rather than causing difficult-to-diagnose corruption. - -.. note:: - - Using this feature with KASAN requires architecture support - for backing virtual mappings with real shadow memory, and - KASAN_VMALLOC must be enabled. - -.. note:: - - VMAP_STACK is enabled, it is not possible to run DMA on stack - allocated data. - -Kernel configuration options and dependencies keep changing. Refer to -the latest code base: - -`Kconfig ` - -Allocation ------------ - -When a new kernel thread is created, thread stack is allocated from -virtually contiguous memory pages from the page level allocator. These -pages are mapped into contiguous kernel virtual space with PAGE_KERNEL -protections. - -alloc_thread_stack_node() calls __vmalloc_node_range() to allocate stack -with PAGE_KERNEL protections. - -- Allocated stacks are cached and later reused by new threads, so memcg - accounting is performed manually on assigning/releasing stacks to tasks. - Hence, __vmalloc_node_range is called without __GFP_ACCOUNT. -- vm_struct is cached to be able to find when thread free is initiated - in interrupt context. free_thread_stack() can be called in interrupt - context. -- On arm64, all VMAP's stacks need to have the same alignment to ensure - that VMAP'd stack overflow detection works correctly. Arch specific - vmap stack allocator takes care of this detail. -- This does not address interrupt stacks - according to the original patch - -Thread stack allocation is initiated from clone(), fork(), vfork(), -kernel_thread() via kernel_clone(). Leaving a few hints for searching -the code base to understand when and how thread stack is allocated. - -Bulk of the code is in: -`kernel/fork.c `. - -stack_vm_area pointer in task_struct keeps track of the virtually allocated -stack and a non-null stack_vm_area pointer serves as a indication that the -virtually mapped kernel stacks are enabled. - -:: - - struct vm_struct *stack_vm_area; - -Stack overflow handling ------------------------ - -Leading and trailing guard pages help detect stack overflows. When stack -overflows into the guard pages, handlers have to be careful not overflow -the stack again. When handlers are called, it is likely that very little -stack space is left. - -On x86, this is done by handling the page fault indicating the kernel -stack overflow on the double-fault stack. - -Testing VMAP allocation with guard pages ----------------------------------------- - -How do we ensure that VMAP_STACK is actually allocating with a leading -and trailing guard page? The following lkdtm tests can help detect any -regressions. - -:: - - void lkdtm_STACK_GUARD_PAGE_LEADING() - void lkdtm_STACK_GUARD_PAGE_TRAILING() - -Conclusions ------------ - -- A percpu cache of vmalloced stacks appears to be a bit faster than a - high-order stack allocation, at least when the cache hits. -- THREAD_INFO_IN_TASK gets rid of arch-specific thread_info entirely and - simply embed the thread_info (containing only flags) and 'int cpu' into - task_struct. -- The thread stack can be free'ed as soon as the task is dead (without - waiting for RCU) and then, if vmapped stacks are in use, cache the - entire stack for reuse on the same cpu. diff --git a/Documentation/vm/vmemmap_dedup.rst b/Documentation/vm/vmemmap_dedup.rst deleted file mode 100644 index c9c495f62d12..000000000000 --- a/Documentation/vm/vmemmap_dedup.rst +++ /dev/null @@ -1,223 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -========================================= -A vmemmap diet for HugeTLB and Device DAX -========================================= - -HugeTLB -======= - -The struct page structures (page structs) are used to describe a physical -page frame. By default, there is a one-to-one mapping from a page frame to -it's corresponding page struct. - -HugeTLB pages consist of multiple base page size pages and is supported by many -architectures. See Documentation/admin-guide/mm/hugetlbpage.rst for more -details. On the x86-64 architecture, HugeTLB pages of size 2MB and 1GB are -currently supported. Since the base page size on x86 is 4KB, a 2MB HugeTLB page -consists of 512 base pages and a 1GB HugeTLB page consists of 4096 base pages. -For each base page, there is a corresponding page struct. - -Within the HugeTLB subsystem, only the first 4 page structs are used to -contain unique information about a HugeTLB page. __NR_USED_SUBPAGE provides -this upper limit. The only 'useful' information in the remaining page structs -is the compound_head field, and this field is the same for all tail pages. - -By removing redundant page structs for HugeTLB pages, memory can be returned -to the buddy allocator for other uses. - -Different architectures support different HugeTLB pages. For example, the -following table is the HugeTLB page size supported by x86 and arm64 -architectures. Because arm64 supports 4k, 16k, and 64k base pages and -supports contiguous entries, so it supports many kinds of sizes of HugeTLB -page. - -+--------------+-----------+-----------------------------------------------+ -| Architecture | Page Size | HugeTLB Page Size | -+--------------+-----------+-----------+-----------+-----------+-----------+ -| x86-64 | 4KB | 2MB | 1GB | | | -+--------------+-----------+-----------+-----------+-----------+-----------+ -| | 4KB | 64KB | 2MB | 32MB | 1GB | -| +-----------+-----------+-----------+-----------+-----------+ -| arm64 | 16KB | 2MB | 32MB | 1GB | | -| +-----------+-----------+-----------+-----------+-----------+ -| | 64KB | 2MB | 512MB | 16GB | | -+--------------+-----------+-----------+-----------+-----------+-----------+ - -When the system boot up, every HugeTLB page has more than one struct page -structs which size is (unit: pages):: - - struct_size = HugeTLB_Size / PAGE_SIZE * sizeof(struct page) / PAGE_SIZE - -Where HugeTLB_Size is the size of the HugeTLB page. We know that the size -of the HugeTLB page is always n times PAGE_SIZE. So we can get the following -relationship:: - - HugeTLB_Size = n * PAGE_SIZE - -Then:: - - struct_size = n * PAGE_SIZE / PAGE_SIZE * sizeof(struct page) / PAGE_SIZE - = n * sizeof(struct page) / PAGE_SIZE - -We can use huge mapping at the pud/pmd level for the HugeTLB page. - -For the HugeTLB page of the pmd level mapping, then:: - - struct_size = n * sizeof(struct page) / PAGE_SIZE - = PAGE_SIZE / sizeof(pte_t) * sizeof(struct page) / PAGE_SIZE - = sizeof(struct page) / sizeof(pte_t) - = 64 / 8 - = 8 (pages) - -Where n is how many pte entries which one page can contains. So the value of -n is (PAGE_SIZE / sizeof(pte_t)). - -This optimization only supports 64-bit system, so the value of sizeof(pte_t) -is 8. And this optimization also applicable only when the size of struct page -is a power of two. In most cases, the size of struct page is 64 bytes (e.g. -x86-64 and arm64). So if we use pmd level mapping for a HugeTLB page, the -size of struct page structs of it is 8 page frames which size depends on the -size of the base page. - -For the HugeTLB page of the pud level mapping, then:: - - struct_size = PAGE_SIZE / sizeof(pmd_t) * struct_size(pmd) - = PAGE_SIZE / 8 * 8 (pages) - = PAGE_SIZE (pages) - -Where the struct_size(pmd) is the size of the struct page structs of a -HugeTLB page of the pmd level mapping. - -E.g.: A 2MB HugeTLB page on x86_64 consists in 8 page frames while 1GB -HugeTLB page consists in 4096. - -Next, we take the pmd level mapping of the HugeTLB page as an example to -show the internal implementation of this optimization. There are 8 pages -struct page structs associated with a HugeTLB page which is pmd mapped. - -Here is how things look before optimization:: - - HugeTLB struct pages(8 pages) page frame(8 pages) - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | -------------> | 2 | - | | +-----------+ +-----------+ - | | | 3 | -------------> | 3 | - | | +-----------+ +-----------+ - | | | 4 | -------------> | 4 | - | PMD | +-----------+ +-----------+ - | level | | 5 | -------------> | 5 | - | mapping | +-----------+ +-----------+ - | | | 6 | -------------> | 6 | - | | +-----------+ +-----------+ - | | | 7 | -------------> | 7 | - | | +-----------+ +-----------+ - | | - | | - | | - +-----------+ - -The value of page->compound_head is the same for all tail pages. The first -page of page structs (page 0) associated with the HugeTLB page contains the 4 -page structs necessary to describe the HugeTLB. The only use of the remaining -pages of page structs (page 1 to page 7) is to point to page->compound_head. -Therefore, we can remap pages 1 to 7 to page 0. Only 1 page of page structs -will be used for each HugeTLB page. This will allow us to free the remaining -7 pages to the buddy allocator. - -Here is how things look after remapping:: - - HugeTLB struct pages(8 pages) page frame(8 pages) - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | ---------------^ ^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | | - | | | 2 | -----------------+ | | | | | - | | +-----------+ | | | | | - | | | 3 | -------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | ---------------------+ | | | - | PMD | +-----------+ | | | - | level | | 5 | -----------------------+ | | - | mapping | +-----------+ | | - | | | 6 | -------------------------+ | - | | +-----------+ | - | | | 7 | ---------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ - -When a HugeTLB is freed to the buddy system, we should allocate 7 pages for -vmemmap pages and restore the previous mapping relationship. - -For the HugeTLB page of the pud level mapping. It is similar to the former. -We also can use this approach to free (PAGE_SIZE - 1) vmemmap pages. - -Apart from the HugeTLB page of the pmd/pud level mapping, some architectures -(e.g. aarch64) provides a contiguous bit in the translation table entries -that hints to the MMU to indicate that it is one of a contiguous set of -entries that can be cached in a single TLB entry. - -The contiguous bit is used to increase the mapping size at the pmd and pte -(last) level. So this type of HugeTLB page can be optimized only when its -size of the struct page structs is greater than 1 page. - -Notice: The head vmemmap page is not freed to the buddy allocator and all -tail vmemmap pages are mapped to the head vmemmap page frame. So we can see -more than one struct page struct with PG_head (e.g. 8 per 2 MB HugeTLB page) -associated with each HugeTLB page. The compound_head() can handle this -correctly (more details refer to the comment above compound_head()). - -Device DAX -========== - -The device-dax interface uses the same tail deduplication technique explained -in the previous chapter, except when used with the vmemmap in -the device (altmap). - -The following page sizes are supported in DAX: PAGE_SIZE (4K on x86_64), -PMD_SIZE (2M on x86_64) and PUD_SIZE (1G on x86_64). - -The differences with HugeTLB are relatively minor. - -It only use 3 page structs for storing all information as opposed -to 4 on HugeTLB pages. - -There's no remapping of vmemmap given that device-dax memory is not part of -System RAM ranges initialized at boot. Thus the tail page deduplication -happens at a later stage when we populate the sections. HugeTLB reuses the -the head vmemmap page representing, whereas device-dax reuses the tail -vmemmap page. This results in only half of the savings compared to HugeTLB. - -Deduplicated tail pages are not mapped read-only. - -Here's how things look like on device-dax after the sections are populated:: - - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PMD | +-----------+ | | | - | level | | 5 | ----------------------+ | | - | mapping | +-----------+ | | - | | | 6 | ------------------------+ | - | | +-----------+ | - | | | 7 | --------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ diff --git a/Documentation/vm/z3fold.rst b/Documentation/vm/z3fold.rst deleted file mode 100644 index 224e3c61d686..000000000000 --- a/Documentation/vm/z3fold.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _z3fold: - -====== -z3fold -====== - -z3fold is a special purpose allocator for storing compressed pages. -It is designed to store up to three compressed pages per physical page. -It is a zbud derivative which allows for higher compression -ratio keeping the simplicity and determinism of its predecessor. - -The main differences between z3fold and zbud are: - -* unlike zbud, z3fold allows for up to PAGE_SIZE allocations -* z3fold can hold up to 3 compressed pages in its page -* z3fold doesn't export any API itself and is thus intended to be used - via the zpool API. - -To keep the determinism and simplicity, z3fold, just like zbud, always -stores an integral number of compressed pages per page, but it can store -up to 3 pages unlike zbud which can store at most 2. Therefore the -compression ratio goes to around 2.7x while zbud's one is around 1.7x. - -Unlike zbud (but like zsmalloc for that matter) z3fold_alloc() does not -return a dereferenceable pointer. Instead, it returns an unsigned long -handle which encodes actual location of the allocated object. - -Keeping effective compression ratio close to zsmalloc's, z3fold doesn't -depend on MMU enabled and provides more predictable reclaim behavior -which makes it a better fit for small and response-critical systems. diff --git a/Documentation/vm/zsmalloc.rst b/Documentation/vm/zsmalloc.rst deleted file mode 100644 index 6e79893d6132..000000000000 --- a/Documentation/vm/zsmalloc.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. _zsmalloc: - -======== -zsmalloc -======== - -This allocator is designed for use with zram. Thus, the allocator is -supposed to work well under low memory conditions. In particular, it -never attempts higher order page allocation which is very likely to -fail under memory pressure. On the other hand, if we just use single -(0-order) pages, it would suffer from very high fragmentation -- -any object of size PAGE_SIZE/2 or larger would occupy an entire page. -This was one of the major issues with its predecessor (xvmalloc). - -To overcome these issues, zsmalloc allocates a bunch of 0-order pages -and links them together using various 'struct page' fields. These linked -pages act as a single higher-order page i.e. an object can span 0-order -page boundaries. The code refers to these linked pages as a single entity -called zspage. - -For simplicity, zsmalloc can only allocate objects of size up to PAGE_SIZE -since this satisfies the requirements of all its current users (in the -worst case, page is incompressible and is thus stored "as-is" i.e. in -uncompressed form). For allocation requests larger than this size, failure -is returned (see zs_malloc). - -Additionally, zs_malloc() does not return a dereferenceable pointer. -Instead, it returns an opaque handle (unsigned long) which encodes actual -location of the allocated object. The reason for this indirection is that -zsmalloc does not keep zspages permanently mapped since that would cause -issues on 32-bit systems where the VA region for kernel space mappings -is very small. So, before using the allocating memory, the object has to -be mapped using zs_map_object() to get a usable pointer and subsequently -unmapped using zs_unmap_object(). - -stat -==== - -With CONFIG_ZSMALLOC_STAT, we could see zsmalloc internal information via -``/sys/kernel/debug/zsmalloc/``. Here is a sample of stat output:: - - # cat /sys/kernel/debug/zsmalloc/zram0/classes - - class size almost_full almost_empty obj_allocated obj_used pages_used pages_per_zspage - ... - ... - 9 176 0 1 186 129 8 4 - 10 192 1 0 2880 2872 135 3 - 11 208 0 1 819 795 42 2 - 12 224 0 1 219 159 12 4 - ... - ... - - -class - index -size - object size zspage stores -almost_empty - the number of ZS_ALMOST_EMPTY zspages(see below) -almost_full - the number of ZS_ALMOST_FULL zspages(see below) -obj_allocated - the number of objects allocated -obj_used - the number of objects allocated to the user -pages_used - the number of pages allocated for the class -pages_per_zspage - the number of 0-order pages to make a zspage - -We assign a zspage to ZS_ALMOST_EMPTY fullness group when n <= N / f, where - -* n = number of allocated objects -* N = total number of objects zspage can store -* f = fullness_threshold_frac(ie, 4 at the moment) - -Similarly, we assign zspage to: - -* ZS_ALMOST_FULL when n > N / f -* ZS_EMPTY when n == 0 -* ZS_FULL when n == N diff --git a/MAINTAINERS b/MAINTAINERS index fe5daf141501..55fb1daa9057 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5526,7 +5526,7 @@ L: linux-mm@kvack.org S: Maintained F: Documentation/ABI/testing/sysfs-kernel-mm-damon F: Documentation/admin-guide/mm/damon/ -F: Documentation/vm/damon/ +F: Documentation/mm/damon/ F: include/linux/damon.h F: include/trace/events/damon.h F: mm/damon/ @@ -9037,7 +9037,7 @@ HMM - Heterogeneous Memory Management M: Jérôme Glisse L: linux-mm@kvack.org S: Maintained -F: Documentation/vm/hmm.rst +F: Documentation/mm/hmm.rst F: include/linux/hmm* F: lib/test_hmm* F: mm/hmm* @@ -9135,8 +9135,8 @@ L: linux-mm@kvack.org S: Maintained F: Documentation/ABI/testing/sysfs-kernel-mm-hugepages F: Documentation/admin-guide/mm/hugetlbpage.rst -F: Documentation/vm/hugetlbfs_reserv.rst -F: Documentation/vm/vmemmap_dedup.rst +F: Documentation/mm/hugetlbfs_reserv.rst +F: Documentation/mm/vmemmap_dedup.rst F: fs/hugetlbfs/ F: include/linux/hugetlb.h F: mm/hugetlb.c @@ -15072,7 +15072,7 @@ M: Pasha Tatashin M: Andrew Morton L: linux-mm@kvack.org S: Maintained -F: Documentation/vm/page_table_check.rst +F: Documentation/mm/page_table_check.rst F: include/linux/page_table_check.h F: mm/page_table_check.c @@ -22158,7 +22158,7 @@ M: Nitin Gupta R: Sergey Senozhatsky L: linux-mm@kvack.org S: Maintained -F: Documentation/vm/zsmalloc.rst +F: Documentation/mm/zsmalloc.rst F: include/linux/zsmalloc.h F: mm/zsmalloc.c diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 1920d52653b4..db2838cf8c02 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -410,7 +410,7 @@ config ARCH_SPARSEMEM_ENABLE Say Y to support efficient handling of sparse physical memory, for architectures which are either NUMA (Non-Uniform Memory Access) or have huge holes in the physical address space for other reasons. - See for more. + See for more. config ARCH_ENABLE_THP_MIGRATION def_bool y diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index cb9d5fd39d7f..392ff48f77df 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -1273,7 +1273,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, * should return true. * We should not call this on a hugetlb entry. We should check for HugeTLB * entry using vma->vm_flags - * The page table walk rule is explained in Documentation/vm/transhuge.rst + * The page table walk rule is explained in Documentation/mm/transhuge.rst */ static inline int pmd_trans_huge(pmd_t pmd) { diff --git a/include/linux/hmm.h b/include/linux/hmm.h index d5a6f101f843..126a36571667 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -4,7 +4,7 @@ * * Authors: Jérôme Glisse * - * See Documentation/vm/hmm.rst for reasons and overview of what HMM is. + * See Documentation/mm/hmm.rst for reasons and overview of what HMM is. */ #ifndef LINUX_HMM_H #define LINUX_HMM_H @@ -100,7 +100,7 @@ struct hmm_range { }; /* - * Please see Documentation/vm/hmm.rst for how to use the range API. + * Please see Documentation/mm/hmm.rst for how to use the range API. */ int hmm_range_fault(struct hmm_range *range); diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 8af304f6b504..9f5ee49482de 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -39,7 +39,7 @@ struct vmem_altmap { * must be treated as an opaque object, rather than a "normal" struct page. * * A more complete discussion of unaddressable memory may be found in - * include/linux/hmm.h and Documentation/vm/hmm.rst. + * include/linux/hmm.h and Documentation/mm/hmm.rst. * * MEMORY_DEVICE_FS_DAX: * Host memory that has similar access semantics as System RAM i.e. DMA diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index 45fc2c81e370..d6c06e140277 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -198,7 +198,7 @@ struct mmu_notifier_ops { * invalidate_range_start()/end() notifiers, as * invalidate_range() already catches the points in time when an * external TLB range needs to be flushed. For more in depth - * discussion on this see Documentation/vm/mmu_notifier.rst + * discussion on this see Documentation/mm/mmu_notifier.rst * * Note that this function might be called with just a sub-range * of what was passed to invalidate_range_start()/end(), if diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 8cd975a8bfeb..2a243616f222 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -29,7 +29,7 @@ extern struct mm_struct *mm_alloc(void); * * Use mmdrop() to release the reference acquired by mmgrab(). * - * See also for an in-depth explanation + * See also for an in-depth explanation * of &mm_struct.mm_count vs &mm_struct.mm_users. */ static inline void mmgrab(struct mm_struct *mm) @@ -92,7 +92,7 @@ static inline void mmdrop_sched(struct mm_struct *mm) * * Use mmput() to release the reference acquired by mmget(). * - * See also for an in-depth explanation + * See also for an in-depth explanation * of &mm_struct.mm_count vs &mm_struct.mm_users. */ static inline void mmget(struct mm_struct *mm) diff --git a/include/linux/swap.h b/include/linux/swap.h index 0c0fed1b348f..95a5b7aa1ae9 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -74,7 +74,7 @@ static inline int current_is_kswapd(void) /* * Unaddressable device memory support. See include/linux/hmm.h and - * Documentation/vm/hmm.rst. Short description is we need struct pages for + * Documentation/mm/hmm.rst. Short description is we need struct pages for * device memory that is unaddressable (inaccessible) by CPU, so that we can * migrate part of a process memory to device memory. * diff --git a/mm/Kconfig b/mm/Kconfig index 169e64192e48..c1fa4993a56f 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -663,7 +663,7 @@ config KSM the many instances by a single page with that content, so saving memory until one or another app needs to modify the content. Recommended for use with KVM, or with other duplicative applications. - See Documentation/vm/ksm.rst for more information: KSM is inactive + See Documentation/mm/ksm.rst for more information: KSM is inactive until a program has madvised that an area is MADV_MERGEABLE, and root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set). diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 1ab091f49fc0..dc7df1254f0a 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -35,7 +35,7 @@ #include /* - * Please refer Documentation/vm/arch_pgtable_helpers.rst for the semantics + * Please refer Documentation/mm/arch_pgtable_helpers.rst for the semantics * expectations that are being validated here. All future changes in here * or the documentation need to be in sync. */ diff --git a/mm/frontswap.c b/mm/frontswap.c index 6f69b044a8cc..1a97610308cb 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -4,7 +4,7 @@ * * This code provides the generic "frontend" layer to call a matching * "backend" driver implementation of frontswap. See - * Documentation/vm/frontswap.rst for more information. + * Documentation/mm/frontswap.rst for more information. * * Copyright (C) 2009-2012 Oracle Corp. All rights reserved. * Author: Dan Magenheimer diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 834f288b3769..f9b90a8d7dfa 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1937,7 +1937,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, * replacing a zero pmd write protected page with a zero pte write * protected page. * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ pmdp_huge_clear_flush(vma, haddr, pmd); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a57e1be41401..b36a4ef87a2e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4875,7 +4875,7 @@ again: * table protection not changing it to point * to a new page. * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ huge_ptep_set_wrprotect(src, addr, src_pte); entry = huge_pte_wrprotect(entry); @@ -6403,7 +6403,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, * No need to call mmu_notifier_invalidate_range() we are downgrading * page table protection not changing it to point to a new page. * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ i_mmap_unlock_write(vma->vm_file->f_mapping); mmu_notifier_invalidate_range_end(&range); @@ -7102,7 +7102,7 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) i_mmap_unlock_write(vma->vm_file->f_mapping); /* * No need to call mmu_notifier_invalidate_range(), see - * Documentation/vm/mmu_notifier.rst. + * Documentation/mm/mmu_notifier.rst. */ mmu_notifier_invalidate_range_end(&range); } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 1089ea8a9c98..ba29c15c53d6 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -6,7 +6,7 @@ * * Author: Muchun Song * - * See Documentation/vm/vmemmap_dedup.rst + * See Documentation/mm/vmemmap_dedup.rst */ #define pr_fmt(fmt) "HugeTLB: " fmt diff --git a/mm/ksm.c b/mm/ksm.c index 54f78c9eecae..8d2dc501c92c 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1083,7 +1083,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, * No need to notify as we are downgrading page table to read * only not changing it to point to a new page. * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ entry = ptep_clear_flush(vma, pvmw.address, pvmw.pte); /* @@ -1186,7 +1186,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, * No need to notify as we are replacing a read only page with another * read only page with the same content. * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ ptep_clear_flush(vma, addr, ptep); set_pte_at_notify(mm, addr, ptep, newpte); diff --git a/mm/mmap.c b/mm/mmap.c index 61e6135c54ef..c14d7286a379 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2944,7 +2944,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, unsigned long ret = -EINVAL; struct file *file; - pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.rst.\n", + pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n", current->comm, current->pid); if (prot) diff --git a/mm/rmap.c b/mm/rmap.c index 5bcb334cd6f2..65e0a767b837 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -999,7 +999,7 @@ static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) * downgrading page table protection not changing it to point * to a new page. * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ if (ret) cleaned++; @@ -1765,7 +1765,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * to point at a new folio while a device is * still using this folio. * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ dec_mm_counter(mm, mm_counter_file(&folio->page)); } @@ -1775,7 +1775,7 @@ discard: * done above for all cases requiring it to happen under page * table lock before mmu_notifier_invalidate_range_end() * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) @@ -2093,7 +2093,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * done above for all cases requiring it to happen under page * table lock before mmu_notifier_invalidate_range_end() * - * See Documentation/vm/mmu_notifier.rst + * See Documentation/mm/mmu_notifier.rst */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 652f11a05749..3ff88a2eefb8 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -752,7 +752,7 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, /* * Reuse the previous page for the rest of tail pages - * See layout diagram in Documentation/vm/vmemmap_dedup.rst + * See layout diagram in Documentation/mm/vmemmap_dedup.rst */ next += PAGE_SIZE; rc = vmemmap_populate_range(next, last, node, NULL, diff --git a/mm/util.c b/mm/util.c index 0837570c9225..5df8f2db7ca9 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1005,7 +1005,7 @@ EXPORT_SYMBOL_GPL(vm_memory_committed); * succeed and -ENOMEM implies there is not. * * We currently support three overcommit policies, which are set via the - * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting.rst + * vm.overcommit_memory sysctl. See Documentation/mm/overcommit-accounting.rst * * Strict overcommit modes added 2002 Feb 26 by Alan Cox. * Additional code 2002 Jul 20 by Robert Love. diff --git a/tools/vm/page_owner_sort.c b/tools/vm/page_owner_sort.c index c149427eb1c9..74c3dcecf64d 100644 --- a/tools/vm/page_owner_sort.c +++ b/tools/vm/page_owner_sort.c @@ -8,7 +8,7 @@ * Or sort by total memory: * ./page_owner_sort -m page_owner_full.txt sorted_page_owner.txt * - * See Documentation/vm/page_owner.rst + * See Documentation/mm/page_owner.rst */ #include -- cgit v1.2.3 From 586b3b7600e44c1cad52c683ccfbb76fb2c10cc8 Mon Sep 17 00:00:00 2001 From: Sai Krishna Potthuri Date: Fri, 17 Jun 2022 16:16:56 +0530 Subject: firmware: xilinx: Add configuration values for tri-state Add configuration values(enable/disable) for tri-state parameter. Signed-off-by: Sai Krishna Potthuri Link: https://lore.kernel.org/r/1655462819-28801-2-git-send-email-lakshmi.sai.krishna.potthuri@xilinx.com Signed-off-by: Linus Walleij --- include/linux/firmware/xlnx-zynqmp.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 1ec73d5352c3..4901fb31de3e 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -368,6 +368,11 @@ enum pm_pinctrl_drive_strength { PM_PINCTRL_DRIVE_STRENGTH_12MA = 3, }; +enum pm_pinctrl_tri_state { + PM_PINCTRL_TRI_STATE_DISABLE = 0, + PM_PINCTRL_TRI_STATE_ENABLE = 1, +}; + enum zynqmp_pm_shutdown_type { ZYNQMP_PM_SHUTDOWN_TYPE_SHUTDOWN = 0, ZYNQMP_PM_SHUTDOWN_TYPE_RESET = 1, -- cgit v1.2.3 From 0e584d46218e3b9dc12a98e18e81a0cd3e0d5419 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 28 Jun 2022 10:46:22 +0100 Subject: regulator: fix a kernel-doc warning document n_ramp_values field at struct regulator_desc, in order to solve this warning: include/linux/regulator/driver.h:434: warning: Function parameter or member 'n_ramp_values' not described in 'regulator_desc' Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/15efc16e878aa327aa2769023bcdf959a795f41d.1656409369.git.mchehab@kernel.org Signed-off-by: Mark Brown --- include/linux/regulator/driver.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index 0228caaa6741..f9a7461e72b8 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -348,6 +348,7 @@ enum regulator_type { * @ramp_delay_table: Table for mapping the regulator ramp-rate values. Values * should be given in units of V/S (uV/uS). See the * regulator_set_ramp_delay_regmap(). + * @n_ramp_values: number of elements at @ramp_delay_table. * * @enable_time: Time taken for initial enable of regulator (in uS). * @off_on_delay: guard time (in uS), before re-enabling a regulator -- cgit v1.2.3 From 1f90307e5f0d7bc9a336ead528f616a5df8e5944 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 19 Jun 2022 08:05:49 +0200 Subject: block: remove QUEUE_FLAG_DEAD Disallow setting the blk-mq state on any queue that is already dying as setting the state even then is a bad idea, and remove the now unused QUEUE_FLAG_DEAD flag. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Link: https://lore.kernel.org/r/20220619060552.1850436-4-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-core.c | 3 --- block/blk-mq-debugfs.c | 8 +++----- include/linux/blkdev.h | 2 -- 3 files changed, 3 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index c2cec402d01c..f86df390afad 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -313,9 +313,6 @@ void blk_cleanup_queue(struct request_queue *q) * after draining finished. */ blk_freeze_queue(q); - - blk_queue_flag_set(QUEUE_FLAG_DEAD, q); - blk_sync_queue(q); if (queue_is_mq(q)) { blk_mq_cancel_work_sync(q); diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 4d1ce9ef4318..b80fae7ab1d9 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -116,7 +116,6 @@ static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(NOXMERGES), QUEUE_FLAG_NAME(ADD_RANDOM), QUEUE_FLAG_NAME(SAME_FORCE), - QUEUE_FLAG_NAME(DEAD), QUEUE_FLAG_NAME(INIT_DONE), QUEUE_FLAG_NAME(STABLE_WRITES), QUEUE_FLAG_NAME(POLL), @@ -151,11 +150,10 @@ static ssize_t queue_state_write(void *data, const char __user *buf, char opbuf[16] = { }, *op; /* - * The "state" attribute is removed after blk_cleanup_queue() has called - * blk_mq_free_queue(). Return if QUEUE_FLAG_DEAD has been set to avoid - * triggering a use-after-free. + * The "state" attribute is removed when the queue is removed. Don't + * allow setting the state on a dying queue to avoid a use-after-free. */ - if (blk_queue_dead(q)) + if (blk_queue_dying(q)) return -ENOENT; if (count >= sizeof(opbuf)) { diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b2d42201bd5d..f4632f4fe884 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -564,7 +564,6 @@ struct request_queue { #define QUEUE_FLAG_NOXMERGES 9 /* No extended merges */ #define QUEUE_FLAG_ADD_RANDOM 10 /* Contributes to random pool */ #define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */ -#define QUEUE_FLAG_DEAD 13 /* queue tear-down finished */ #define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */ #define QUEUE_FLAG_STABLE_WRITES 15 /* don't modify blks until WB is done */ #define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */ @@ -592,7 +591,6 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) #define blk_queue_has_srcu(q) test_bit(QUEUE_FLAG_HAS_SRCU, &(q)->queue_flags) -#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags) #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags) #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) #define blk_queue_noxmerges(q) \ -- cgit v1.2.3 From 6f8191fdf41d3a53cc1d63fe2234e812c55a0092 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 19 Jun 2022 08:05:51 +0200 Subject: block: simplify disk shutdown Set the queue dying flag and call blk_mq_exit_queue from del_gendisk for all disks that do not have separately allocated queues, and thus remove the need to call blk_cleanup_queue for them. Rename blk_cleanup_disk to blk_mq_destroy_queue to make it clear that this function is intended only for separately allocated blk-mq queues. This saves an extra queue freeze for devices without a separately allocated queue. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Link: https://lore.kernel.org/r/20220619060552.1850436-6-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-core.c | 37 ------------------------------- block/blk-mq.c | 43 +++++++++++++++++++++++++++++++++++-- block/blk-sysfs.c | 5 ----- block/blk.h | 3 +++ block/bsg-lib.c | 4 ++-- block/genhd.c | 23 +++++++++++--------- drivers/block/ataflop.c | 1 - drivers/block/loop.c | 1 - drivers/block/mtip32xx/mtip32xx.c | 2 -- drivers/block/rnbd/rnbd-clt.c | 2 +- drivers/block/sx8.c | 4 ++-- drivers/block/virtio_blk.c | 1 - drivers/block/z2ram.c | 1 - drivers/cdrom/gdrom.c | 1 - drivers/memstick/core/ms_block.c | 1 - drivers/memstick/core/mspro_block.c | 1 - drivers/mmc/core/block.c | 1 - drivers/mmc/core/queue.c | 1 - drivers/nvme/host/apple.c | 2 +- drivers/nvme/host/core.c | 1 - drivers/nvme/host/fc.c | 12 +++++------ drivers/nvme/host/pci.c | 2 +- drivers/nvme/host/rdma.c | 12 +++++------ drivers/nvme/host/tcp.c | 12 +++++------ drivers/nvme/target/loop.c | 12 +++++------ drivers/s390/block/dasd.c | 2 +- drivers/s390/block/dasd_genhd.c | 4 ++-- drivers/scsi/scsi_lib.c | 6 +++--- drivers/scsi/scsi_sysfs.c | 2 +- drivers/scsi/sd.c | 4 ++-- drivers/scsi/sr.c | 4 ++-- drivers/ufs/core/ufshcd.c | 4 ++-- include/linux/blk-mq.h | 3 +++ include/linux/blkdev.h | 4 +--- 34 files changed, 105 insertions(+), 113 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 04029ffea031..5ad7bd93077c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -284,43 +284,6 @@ void blk_queue_start_drain(struct request_queue *q) wake_up_all(&q->mq_freeze_wq); } -/** - * blk_cleanup_queue - shutdown a request queue - * @q: request queue to shutdown - * - * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and - * put it. All future requests will be failed immediately with -ENODEV. - * - * Context: can sleep - */ -void blk_cleanup_queue(struct request_queue *q) -{ - /* cannot be called from atomic context */ - might_sleep(); - - WARN_ON_ONCE(blk_queue_registered(q)); - - /* mark @q DYING, no new request or merges will be allowed afterwards */ - blk_queue_flag_set(QUEUE_FLAG_DYING, q); - blk_queue_start_drain(q); - - /* - * Drain all requests queued before DYING marking. Set DEAD flag to - * prevent that blk_mq_run_hw_queues() accesses the hardware queues - * after draining finished. - */ - blk_freeze_queue(q); - blk_sync_queue(q); - if (queue_is_mq(q)) { - blk_mq_cancel_work_sync(q); - blk_mq_exit_queue(q); - } - - /* @q is and will stay empty, shutdown and put */ - blk_put_queue(q); -} -EXPORT_SYMBOL(blk_cleanup_queue); - /** * blk_queue_enter() - try to increase q->q_usage_counter * @q: request queue pointer diff --git a/block/blk-mq.c b/block/blk-mq.c index 92aae03103b7..b1dbc4b2c2c9 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3902,7 +3902,7 @@ static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set, q->queuedata = queuedata; ret = blk_mq_init_allocated_queue(set, q); if (ret) { - blk_cleanup_queue(q); + blk_put_queue(q); return ERR_PTR(ret); } return q; @@ -3914,6 +3914,35 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) } EXPORT_SYMBOL(blk_mq_init_queue); +/** + * blk_mq_destroy_queue - shutdown a request queue + * @q: request queue to shutdown + * + * This shuts down a request queue allocated by blk_mq_init_queue() and drops + * the initial reference. All future requests will failed with -ENODEV. + * + * Context: can sleep + */ +void blk_mq_destroy_queue(struct request_queue *q) +{ + WARN_ON_ONCE(!queue_is_mq(q)); + WARN_ON_ONCE(blk_queue_registered(q)); + + might_sleep(); + + blk_queue_flag_set(QUEUE_FLAG_DYING, q); + blk_queue_start_drain(q); + blk_freeze_queue(q); + + blk_sync_queue(q); + blk_mq_cancel_work_sync(q); + blk_mq_exit_queue(q); + + /* @q is and will stay empty, shutdown and put */ + blk_put_queue(q); +} +EXPORT_SYMBOL(blk_mq_destroy_queue); + struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata, struct lock_class_key *lkclass) { @@ -3926,13 +3955,23 @@ struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata, disk = __alloc_disk_node(q, set->numa_node, lkclass); if (!disk) { - blk_cleanup_queue(q); + blk_put_queue(q); return ERR_PTR(-ENOMEM); } + set_bit(GD_OWNS_QUEUE, &disk->state); return disk; } EXPORT_SYMBOL(__blk_mq_alloc_disk); +struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q, + struct lock_class_key *lkclass) +{ + if (!blk_get_queue(q)) + return NULL; + return __alloc_disk_node(q, NUMA_NO_NODE, lkclass); +} +EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue); + static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( struct blk_mq_tag_set *set, struct request_queue *q, int hctx_idx, int node) diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 69e53d1a4f0e..9b211e519de8 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -755,11 +755,6 @@ static void blk_free_queue_rcu(struct rcu_head *rcu_head) * decremented with blk_put_queue(). Once the refcount reaches 0 this function * is called. * - * For drivers that have a request_queue on a gendisk and added with - * __device_add_disk() the refcount to request_queue will reach 0 with - * the last put_disk() called by the driver. For drivers which don't use - * __device_add_disk() this happens with blk_cleanup_queue(). - * * Drivers exist which depend on the release of the request_queue to be * synchronous, it should not be deferred. * diff --git a/block/blk.h b/block/blk.h index 8e79296ee97a..1a0d3e6a4a63 100644 --- a/block/blk.h +++ b/block/blk.h @@ -424,6 +424,9 @@ int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start, sector_t length); void blk_drop_partitions(struct gendisk *disk); +struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, + struct lock_class_key *lkclass); + int bio_add_hw_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned int max_sectors, bool *same_page); diff --git a/block/bsg-lib.c b/block/bsg-lib.c index acfe1357bf6c..fd4cd5e68282 100644 --- a/block/bsg-lib.c +++ b/block/bsg-lib.c @@ -324,7 +324,7 @@ void bsg_remove_queue(struct request_queue *q) container_of(q->tag_set, struct bsg_set, tag_set); bsg_unregister_queue(bset->bd); - blk_cleanup_queue(q); + blk_mq_destroy_queue(q); blk_mq_free_tag_set(&bset->tag_set); kfree(bset); } @@ -399,7 +399,7 @@ struct request_queue *bsg_setup_queue(struct device *dev, const char *name, return q; out_cleanup_queue: - blk_cleanup_queue(q); + blk_mq_destroy_queue(q); out_queue: blk_mq_free_tag_set(set); out_tag_set: diff --git a/block/genhd.c b/block/genhd.c index 278227ba1d53..4d15f828c449 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -617,6 +617,8 @@ void del_gendisk(struct gendisk *disk) * Fail any new I/O. */ set_bit(GD_DEAD, &disk->state); + if (test_bit(GD_OWNS_QUEUE, &disk->state)) + blk_queue_flag_set(QUEUE_FLAG_DYING, q); set_capacity(disk, 0); /* @@ -663,11 +665,16 @@ void del_gendisk(struct gendisk *disk) blk_mq_unquiesce_queue(q); /* - * Allow using passthrough request again after the queue is torn down. + * If the disk does not own the queue, allow using passthrough requests + * again. Else leave the queue frozen to fail all I/O. */ - blk_queue_flag_clear(QUEUE_FLAG_INIT_DONE, q); - __blk_mq_unfreeze_queue(q, true); - + if (!test_bit(GD_OWNS_QUEUE, &disk->state)) { + blk_queue_flag_clear(QUEUE_FLAG_INIT_DONE, q); + __blk_mq_unfreeze_queue(q, true); + } else { + if (queue_is_mq(q)) + blk_mq_exit_queue(q); + } } EXPORT_SYMBOL(del_gendisk); @@ -1338,9 +1345,6 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, { struct gendisk *disk; - if (!blk_get_queue(q)) - return NULL; - disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id); if (!disk) goto out_put_queue; @@ -1391,7 +1395,6 @@ out_put_queue: blk_put_queue(q); return NULL; } -EXPORT_SYMBOL(__alloc_disk_node); struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass) { @@ -1404,9 +1407,10 @@ struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass) disk = __alloc_disk_node(q, node, lkclass); if (!disk) { - blk_cleanup_queue(q); + blk_put_queue(q); return NULL; } + set_bit(GD_OWNS_QUEUE, &disk->state); return disk; } EXPORT_SYMBOL(__blk_alloc_disk); @@ -1439,7 +1443,6 @@ EXPORT_SYMBOL(put_disk); */ void blk_cleanup_disk(struct gendisk *disk) { - blk_cleanup_queue(disk->queue); put_disk(disk); } EXPORT_SYMBOL(blk_cleanup_disk); diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index e232cc4fd444..c6e41ee18aaa 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -2045,7 +2045,6 @@ static void atari_floppy_cleanup(void) if (!unit[i].disk[type]) continue; del_gendisk(unit[i].disk[type]); - blk_cleanup_queue(unit[i].disk[type]->queue); put_disk(unit[i].disk[type]); } blk_mq_free_tag_set(&unit[i].tag_set); diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 084f9b8a0ba3..cc608226c8c7 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -2057,7 +2057,6 @@ static void loop_remove(struct loop_device *lo) { /* Make this loop device unreachable from pathname. */ del_gendisk(lo->lo_disk); - blk_cleanup_queue(lo->lo_disk->queue); blk_mq_free_tag_set(&lo->tag_set); mutex_lock(&loop_ctl_mutex); diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index e7604b3bf8a7..1d0e0a9fdd7c 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3565,7 +3565,6 @@ static int mtip_block_shutdown(struct driver_data *dd) if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) del_gendisk(dd->disk); - blk_cleanup_queue(dd->queue); blk_mq_free_tag_set(&dd->tags); put_disk(dd->disk); return 0; @@ -3914,7 +3913,6 @@ static void mtip_pci_remove(struct pci_dev *pdev) dev_info(&dd->pdev->dev, "device %s surprise removal\n", dd->disk->disk_name); - blk_cleanup_queue(dd->queue); blk_mq_free_tag_set(&dd->tags); /* De-initialize the protocol layer. */ diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index 409c76b81aed..a4470374f54f 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -1755,7 +1755,7 @@ static void rnbd_destroy_sessions(void) list_for_each_entry_safe(dev, tn, &sess->devs_list, list) { /* * Here unmap happens in parallel for only one reason: - * blk_cleanup_queue() takes around half a second, so + * del_gendisk() takes around half a second, so * on huge amount of devices the whole module unload * procedure takes minutes. */ diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index 63b4f6431d2e..75057dbbcfbe 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -1536,7 +1536,7 @@ err_out_free_majors: clear_bit(0, &carm_major_alloc); else if (host->major == 161) clear_bit(1, &carm_major_alloc); - blk_cleanup_queue(host->oob_q); + blk_mq_destroy_queue(host->oob_q); blk_mq_free_tag_set(&host->tag_set); err_out_dma_free: dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma); @@ -1570,7 +1570,7 @@ static void carm_remove_one (struct pci_dev *pdev) clear_bit(0, &carm_major_alloc); else if (host->major == 161) clear_bit(1, &carm_major_alloc); - blk_cleanup_queue(host->oob_q); + blk_mq_destroy_queue(host->oob_q); blk_mq_free_tag_set(&host->tag_set); dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma); iounmap(host->mmio); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 6fc7850c2b0a..cff1b6f6b054 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -1111,7 +1111,6 @@ static void virtblk_remove(struct virtio_device *vdev) flush_work(&vblk->config_work); del_gendisk(vblk->disk); - blk_cleanup_queue(vblk->disk->queue); blk_mq_free_tag_set(&vblk->tag_set); mutex_lock(&vblk->vdev_mutex); diff --git a/drivers/block/z2ram.c b/drivers/block/z2ram.c index 7a6ed83481b8..18ad43d9933e 100644 --- a/drivers/block/z2ram.c +++ b/drivers/block/z2ram.c @@ -384,7 +384,6 @@ static void __exit z2_exit(void) for (i = 0; i < Z2MINOR_COUNT; i++) { del_gendisk(z2ram_gendisk[i]); - blk_cleanup_queue(z2ram_gendisk[i]->queue); put_disk(z2ram_gendisk[i]); } blk_mq_free_tag_set(&tag_set); diff --git a/drivers/cdrom/gdrom.c b/drivers/cdrom/gdrom.c index 8e78b37d0f6a..f4cc90ea6198 100644 --- a/drivers/cdrom/gdrom.c +++ b/drivers/cdrom/gdrom.c @@ -831,7 +831,6 @@ probe_fail_no_mem: static int remove_gdrom(struct platform_device *devptr) { - blk_cleanup_queue(gd.gdrom_rq); blk_mq_free_tag_set(&gd.tag_set); free_irq(HW_EVENT_GDROM_CMD, &gd); free_irq(HW_EVENT_GDROM_DMA, &gd); diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c index 3993bdd4b519..ba7e7249a3db 100644 --- a/drivers/memstick/core/ms_block.c +++ b/drivers/memstick/core/ms_block.c @@ -2187,7 +2187,6 @@ static void msb_remove(struct memstick_dev *card) /* Remove the disk */ del_gendisk(msb->disk); - blk_cleanup_queue(msb->queue); blk_mq_free_tag_set(&msb->tag_set); msb->queue = NULL; diff --git a/drivers/memstick/core/mspro_block.c b/drivers/memstick/core/mspro_block.c index 725ba74ded30..72e91c06c618 100644 --- a/drivers/memstick/core/mspro_block.c +++ b/drivers/memstick/core/mspro_block.c @@ -1294,7 +1294,6 @@ static void mspro_block_remove(struct memstick_dev *card) del_gendisk(msb->disk); dev_dbg(&card->dev, "mspro block remove\n"); - blk_cleanup_queue(msb->queue); blk_mq_free_tag_set(&msb->tag_set); msb->queue = NULL; diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index f4a1281658db..bda6c67ce93f 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -2509,7 +2509,6 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, return md; err_cleanup_queue: - blk_cleanup_queue(md->disk->queue); blk_mq_free_tag_set(&md->queue.tag_set); err_kfree: kfree(md); diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index fa5324ceeebe..f824cfdab75a 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -494,7 +494,6 @@ void mmc_cleanup_queue(struct mmc_queue *mq) if (blk_queue_quiesced(q)) blk_mq_unquiesce_queue(q); - blk_cleanup_queue(q); blk_mq_free_tag_set(&mq->tag_set); /* diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index d702d7d60235..2d23b7d41f7e 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1502,7 +1502,7 @@ static int apple_nvme_probe(struct platform_device *pdev) if (!blk_get_queue(anv->ctrl.admin_q)) { nvme_start_admin_queue(&anv->ctrl); - blk_cleanup_queue(anv->ctrl.admin_q); + blk_mq_destroy_queue(anv->ctrl.admin_q); anv->ctrl.admin_q = NULL; ret = -ENODEV; goto put_dev; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index b3d9c29aba1e..4e3a0f7bfc9c 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4103,7 +4103,6 @@ static void nvme_ns_remove(struct nvme_ns *ns) if (!nvme_ns_head_multipath(ns->head)) nvme_cdev_del(&ns->cdev, &ns->cdev_device); del_gendisk(ns->disk); - blk_cleanup_queue(ns->queue); down_write(&ns->ctrl->namespaces_rwsem); list_del_init(&ns->list); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 3c778bb0c294..a96aa831684c 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2392,7 +2392,7 @@ nvme_fc_ctrl_free(struct kref *ref) unsigned long flags; if (ctrl->ctrl.tagset) { - blk_cleanup_queue(ctrl->ctrl.connect_q); + blk_mq_destroy_queue(ctrl->ctrl.connect_q); blk_mq_free_tag_set(&ctrl->tag_set); } @@ -2402,8 +2402,8 @@ nvme_fc_ctrl_free(struct kref *ref) spin_unlock_irqrestore(&ctrl->rport->lock, flags); nvme_start_admin_queue(&ctrl->ctrl); - blk_cleanup_queue(ctrl->ctrl.admin_q); - blk_cleanup_queue(ctrl->ctrl.fabrics_q); + blk_mq_destroy_queue(ctrl->ctrl.admin_q); + blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); blk_mq_free_tag_set(&ctrl->admin_tag_set); kfree(ctrl->queues); @@ -2953,7 +2953,7 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) out_delete_hw_queues: nvme_fc_delete_hw_io_queues(ctrl); out_cleanup_blk_queue: - blk_cleanup_queue(ctrl->ctrl.connect_q); + blk_mq_destroy_queue(ctrl->ctrl.connect_q); out_free_tag_set: blk_mq_free_tag_set(&ctrl->tag_set); nvme_fc_free_io_queues(ctrl); @@ -3642,9 +3642,9 @@ fail_ctrl: return ERR_PTR(-EIO); out_cleanup_admin_q: - blk_cleanup_queue(ctrl->ctrl.admin_q); + blk_mq_destroy_queue(ctrl->ctrl.admin_q); out_cleanup_fabrics_q: - blk_cleanup_queue(ctrl->ctrl.fabrics_q); + blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); out_free_admin_tag_set: blk_mq_free_tag_set(&ctrl->admin_tag_set); out_free_queues: diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d7b24ee17285..247a74aba336 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1760,7 +1760,7 @@ static void nvme_dev_remove_admin(struct nvme_dev *dev) * queue to flush these to completion. */ nvme_start_admin_queue(&dev->ctrl); - blk_cleanup_queue(dev->ctrl.admin_q); + blk_mq_destroy_queue(dev->ctrl.admin_q); blk_mq_free_tag_set(&dev->admin_tagset); } } diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index f2a5e1ea508a..0fb7c8e7ab0b 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -840,8 +840,8 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl, bool remove) { if (remove) { - blk_cleanup_queue(ctrl->ctrl.admin_q); - blk_cleanup_queue(ctrl->ctrl.fabrics_q); + blk_mq_destroy_queue(ctrl->ctrl.admin_q); + blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); blk_mq_free_tag_set(ctrl->ctrl.admin_tagset); } if (ctrl->async_event_sqe.data) { @@ -935,10 +935,10 @@ out_stop_queue: nvme_cancel_admin_tagset(&ctrl->ctrl); out_cleanup_queue: if (new) - blk_cleanup_queue(ctrl->ctrl.admin_q); + blk_mq_destroy_queue(ctrl->ctrl.admin_q); out_cleanup_fabrics_q: if (new) - blk_cleanup_queue(ctrl->ctrl.fabrics_q); + blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); out_free_tagset: if (new) blk_mq_free_tag_set(ctrl->ctrl.admin_tagset); @@ -957,7 +957,7 @@ static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl, bool remove) { if (remove) { - blk_cleanup_queue(ctrl->ctrl.connect_q); + blk_mq_destroy_queue(ctrl->ctrl.connect_q); blk_mq_free_tag_set(ctrl->ctrl.tagset); } nvme_rdma_free_io_queues(ctrl); @@ -1012,7 +1012,7 @@ out_wait_freeze_timed_out: out_cleanup_connect_q: nvme_cancel_tagset(&ctrl->ctrl); if (new) - blk_cleanup_queue(ctrl->ctrl.connect_q); + blk_mq_destroy_queue(ctrl->ctrl.connect_q); out_free_tag_set: if (new) blk_mq_free_tag_set(ctrl->ctrl.tagset); diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index bb67538d241b..b81942fa5f95 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -1885,7 +1885,7 @@ static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) { nvme_tcp_stop_io_queues(ctrl); if (remove) { - blk_cleanup_queue(ctrl->connect_q); + blk_mq_destroy_queue(ctrl->connect_q); blk_mq_free_tag_set(ctrl->tagset); } nvme_tcp_free_io_queues(ctrl); @@ -1940,7 +1940,7 @@ out_wait_freeze_timed_out: out_cleanup_connect_q: nvme_cancel_tagset(ctrl); if (new) - blk_cleanup_queue(ctrl->connect_q); + blk_mq_destroy_queue(ctrl->connect_q); out_free_tag_set: if (new) blk_mq_free_tag_set(ctrl->tagset); @@ -1953,8 +1953,8 @@ static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) { nvme_tcp_stop_queue(ctrl, 0); if (remove) { - blk_cleanup_queue(ctrl->admin_q); - blk_cleanup_queue(ctrl->fabrics_q); + blk_mq_destroy_queue(ctrl->admin_q); + blk_mq_destroy_queue(ctrl->fabrics_q); blk_mq_free_tag_set(ctrl->admin_tagset); } nvme_tcp_free_admin_queue(ctrl); @@ -2012,10 +2012,10 @@ out_stop_queue: nvme_cancel_admin_tagset(ctrl); out_cleanup_queue: if (new) - blk_cleanup_queue(ctrl->admin_q); + blk_mq_destroy_queue(ctrl->admin_q); out_cleanup_fabrics_q: if (new) - blk_cleanup_queue(ctrl->fabrics_q); + blk_mq_destroy_queue(ctrl->fabrics_q); out_free_tagset: if (new) blk_mq_free_tag_set(ctrl->admin_tagset); diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index 59024af2da2e..0f5c77e22a0a 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -266,8 +266,8 @@ static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags)) return; nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); - blk_cleanup_queue(ctrl->ctrl.admin_q); - blk_cleanup_queue(ctrl->ctrl.fabrics_q); + blk_mq_destroy_queue(ctrl->ctrl.admin_q); + blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); blk_mq_free_tag_set(&ctrl->admin_tag_set); } @@ -283,7 +283,7 @@ static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) mutex_unlock(&nvme_loop_ctrl_mutex); if (nctrl->tagset) { - blk_cleanup_queue(ctrl->ctrl.connect_q); + blk_mq_destroy_queue(ctrl->ctrl.connect_q); blk_mq_free_tag_set(&ctrl->tag_set); } kfree(ctrl->queues); @@ -410,9 +410,9 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) out_cleanup_queue: clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); - blk_cleanup_queue(ctrl->ctrl.admin_q); + blk_mq_destroy_queue(ctrl->ctrl.admin_q); out_cleanup_fabrics_q: - blk_cleanup_queue(ctrl->ctrl.fabrics_q); + blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); out_free_tagset: blk_mq_free_tag_set(&ctrl->admin_tag_set); out_free_sq: @@ -554,7 +554,7 @@ static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) return 0; out_cleanup_connect_q: - blk_cleanup_queue(ctrl->ctrl.connect_q); + blk_mq_destroy_queue(ctrl->ctrl.connect_q); out_free_tagset: blk_mq_free_tag_set(&ctrl->tag_set); out_destroy_queues: diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index ba6d78789660..e8489331f12b 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -3280,7 +3280,7 @@ static int dasd_alloc_queue(struct dasd_block *block) static void dasd_free_queue(struct dasd_block *block) { if (block->request_queue) { - blk_cleanup_queue(block->request_queue); + blk_mq_destroy_queue(block->request_queue); blk_mq_free_tag_set(&block->tag_set); block->request_queue = NULL; } diff --git a/drivers/s390/block/dasd_genhd.c b/drivers/s390/block/dasd_genhd.c index a7a33ebf4bbe..5a83f0a39901 100644 --- a/drivers/s390/block/dasd_genhd.c +++ b/drivers/s390/block/dasd_genhd.c @@ -41,8 +41,8 @@ int dasd_gendisk_alloc(struct dasd_block *block) if (base->devindex >= DASD_PER_MAJOR) return -EBUSY; - gdp = __alloc_disk_node(block->request_queue, NUMA_NO_NODE, - &dasd_bio_compl_lkclass); + gdp = blk_mq_alloc_disk_for_queue(block->request_queue, + &dasd_bio_compl_lkclass); if (!gdp) return -ENOMEM; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 6ffc9e4258a8..cdf0056582d5 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -163,7 +163,7 @@ static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy) * Requeue this command. It will go before all other commands * that are already in the queue. Schedule requeue work under * lock such that the kblockd_schedule_work() call happens - * before blk_cleanup_queue() finishes. + * before blk_mq_destroy_queue() finishes. */ cmd->result = 0; @@ -424,9 +424,9 @@ static void scsi_starved_list_run(struct Scsi_Host *shost) * it and the queue. Mitigate by taking a reference to the * queue and never touching the sdev again after we drop the * host lock. Note: if __scsi_remove_device() invokes - * blk_cleanup_queue() before the queue is run from this + * blk_mq_destroy_queue() before the queue is run from this * function then blk_run_queue() will return immediately since - * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING. + * blk_mq_destroy_queue() marks the queue with QUEUE_FLAG_DYING. */ slq = sdev->request_queue; if (!blk_get_queue(slq)) diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index 43949798a2e4..aa70d9282161 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -1475,7 +1475,7 @@ void __scsi_remove_device(struct scsi_device *sdev) scsi_device_set_state(sdev, SDEV_DEL); mutex_unlock(&sdev->state_mutex); - blk_cleanup_queue(sdev->request_queue); + blk_mq_destroy_queue(sdev->request_queue); cancel_work_sync(&sdev->requeue_work); if (sdev->host->hostt->slave_destroy) diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index a1a2ac09066f..cb587e488601 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -3440,8 +3440,8 @@ static int sd_probe(struct device *dev) if (!sdkp) goto out; - gd = __alloc_disk_node(sdp->request_queue, NUMA_NO_NODE, - &sd_bio_compl_lkclass); + gd = blk_mq_alloc_disk_for_queue(sdp->request_queue, + &sd_bio_compl_lkclass); if (!gd) goto out_free; diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 32d3b8274f14..a278b739d0c5 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -624,8 +624,8 @@ static int sr_probe(struct device *dev) if (!cd) goto fail; - disk = __alloc_disk_node(sdev->request_queue, NUMA_NO_NODE, - &sr_bio_compl_lkclass); + disk = blk_mq_alloc_disk_for_queue(sdev->request_queue, + &sr_bio_compl_lkclass); if (!disk) goto fail_free; mutex_init(&cd->lock); diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index ce86d1b790c0..91d8852daaa9 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -9487,7 +9487,7 @@ void ufshcd_remove(struct ufs_hba *hba) ufs_bsg_remove(hba); ufshpb_remove(hba); ufs_sysfs_remove_nodes(hba->dev); - blk_cleanup_queue(hba->tmf_queue); + blk_mq_destroy_queue(hba->tmf_queue); blk_mq_free_tag_set(&hba->tmf_tag_set); scsi_remove_host(hba->host); /* disable interrupts */ @@ -9783,7 +9783,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) return 0; free_tmf_queue: - blk_cleanup_queue(hba->tmf_queue); + blk_mq_destroy_queue(hba->tmf_queue); free_tmf_tag_set: blk_mq_free_tag_set(&hba->tmf_tag_set); out_remove_scsi_host: diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index e2d9daf7e8dd..0fd96e92c6c6 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -686,10 +686,13 @@ struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata, \ __blk_mq_alloc_disk(set, queuedata, &__key); \ }) +struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q, + struct lock_class_key *lkclass); struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, struct request_queue *q); void blk_mq_unregister_dev(struct device *, struct request_queue *); +void blk_mq_destroy_queue(struct request_queue *); int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set, diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index f4632f4fe884..530eeccffda3 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -148,6 +148,7 @@ struct gendisk { #define GD_NATIVE_CAPACITY 3 #define GD_ADDED 4 #define GD_SUPPRESS_PART_SCAN 5 +#define GD_OWNS_QUEUE 6 struct mutex open_mutex; /* open/close mutex */ unsigned open_partitions; /* number of open partitions */ @@ -815,8 +816,6 @@ static inline u64 sb_bdev_nr_blocks(struct super_block *sb) int bdev_disk_changed(struct gendisk *disk, bool invalidate); -struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, - struct lock_class_key *lkclass); void put_disk(struct gendisk *disk); struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass); @@ -933,7 +932,6 @@ static inline unsigned int blk_chunk_sectors_left(sector_t offset, /* * Access functions for manipulating queue properties */ -extern void blk_cleanup_queue(struct request_queue *); void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce limit); extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int); -- cgit v1.2.3 From 8b9ab62662048a3274361c7e5f64037c2c133e2c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 19 Jun 2022 08:05:52 +0200 Subject: block: remove blk_cleanup_disk blk_cleanup_disk is nothing but a trivial wrapper for put_disk now, so remove it. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Link: https://lore.kernel.org/r/20220619060552.1850436-7-hch@lst.de Signed-off-by: Jens Axboe --- arch/m68k/emu/nfblock.c | 4 ++-- arch/um/drivers/ubd_kern.c | 4 ++-- arch/xtensa/platforms/iss/simdisk.c | 4 ++-- block/genhd.c | 15 --------------- drivers/block/amiflop.c | 2 +- drivers/block/aoe/aoeblk.c | 2 +- drivers/block/aoe/aoedev.c | 2 +- drivers/block/ataflop.c | 4 ++-- drivers/block/brd.c | 4 ++-- drivers/block/drbd/drbd_main.c | 4 ++-- drivers/block/floppy.c | 6 +++--- drivers/block/loop.c | 2 +- drivers/block/mtip32xx/mtip32xx.c | 2 +- drivers/block/n64cart.c | 2 +- drivers/block/nbd.c | 4 ++-- drivers/block/null_blk/main.c | 4 ++-- drivers/block/paride/pcd.c | 4 ++-- drivers/block/paride/pd.c | 4 ++-- drivers/block/paride/pf.c | 4 ++-- drivers/block/pktcdvd.c | 4 ++-- drivers/block/ps3disk.c | 4 ++-- drivers/block/ps3vram.c | 4 ++-- drivers/block/rbd.c | 2 +- drivers/block/rnbd/rnbd-clt.c | 4 ++-- drivers/block/sunvdc.c | 4 ++-- drivers/block/swim.c | 2 +- drivers/block/swim3.c | 2 +- drivers/block/sx8.c | 2 +- drivers/block/virtio_blk.c | 2 +- drivers/block/xen-blkfront.c | 4 ++-- drivers/block/z2ram.c | 2 +- drivers/block/zram/zram_drv.c | 4 ++-- drivers/cdrom/gdrom.c | 2 +- drivers/md/bcache/super.c | 2 +- drivers/md/dm.c | 2 +- drivers/md/md.c | 4 ++-- drivers/memstick/core/ms_block.c | 2 +- drivers/memstick/core/mspro_block.c | 2 +- drivers/mtd/mtd_blkdevs.c | 4 ++-- drivers/mtd/ubi/block.c | 4 ++-- drivers/nvdimm/btt.c | 4 ++-- drivers/nvdimm/pmem.c | 4 ++-- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/multipath.c | 2 +- drivers/s390/block/dcssblk.c | 8 ++++---- drivers/s390/block/scm_blk.c | 4 ++-- include/linux/blkdev.h | 1 - 47 files changed, 74 insertions(+), 90 deletions(-) (limited to 'include/linux') diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 267b02cc5655..a708fbd5a844 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -138,7 +138,7 @@ static int __init nfhd_init_one(int id, u32 blocks, u32 bsize) return 0; out_cleanup_disk: - blk_cleanup_disk(dev->disk); + put_disk(dev->disk); free_dev: kfree(dev); out: @@ -180,7 +180,7 @@ static void __exit nfhd_exit(void) list_for_each_entry_safe(dev, next, &nfhd_list, list) { list_del(&dev->list); del_gendisk(dev->disk); - blk_cleanup_disk(dev->disk); + put_disk(dev->disk); kfree(dev); } unregister_blkdev(major_num, "nfhd"); diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index c4344b67628d..479b79e11442 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -925,7 +925,7 @@ static int ubd_add(int n, char **error_out) return 0; out_cleanup_disk: - blk_cleanup_disk(disk); + put_disk(disk); out_cleanup_tags: blk_mq_free_tag_set(&ubd_dev->tag_set); out: @@ -1032,7 +1032,7 @@ static int ubd_remove(int n, char **error_out) ubd_gendisk[n] = NULL; if(disk != NULL){ del_gendisk(disk); - blk_cleanup_disk(disk); + put_disk(disk); } err = 0; diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 4255b92fa3eb..f50caaa1c249 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -290,7 +290,7 @@ static int __init simdisk_setup(struct simdisk *dev, int which, return 0; out_cleanup_disk: - blk_cleanup_disk(dev->gd); + put_disk(dev->gd); out: return err; } @@ -344,7 +344,7 @@ static void simdisk_teardown(struct simdisk *dev, int which, simdisk_detach(dev); if (dev->gd) { del_gendisk(dev->gd); - blk_cleanup_disk(dev->gd); + put_disk(dev->gd); } remove_proc_entry(tmp, procdir); } diff --git a/block/genhd.c b/block/genhd.c index 4d15f828c449..bf9be06af2c8 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1432,21 +1432,6 @@ void put_disk(struct gendisk *disk) } EXPORT_SYMBOL(put_disk); -/** - * blk_cleanup_disk - shutdown a gendisk allocated by blk_alloc_disk - * @disk: gendisk to shutdown - * - * Mark the queue hanging off @disk DYING, drain all pending requests, then mark - * the queue DEAD, destroy and put it and the gendisk structure. - * - * Context: can sleep - */ -void blk_cleanup_disk(struct gendisk *disk) -{ - put_disk(disk); -} -EXPORT_SYMBOL(blk_cleanup_disk); - static void set_disk_ro_uevent(struct gendisk *gd, int ro) { char event[] = "DISK_RO=1"; diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c index 5a566f2fd533..4c8b2ba579ee 100644 --- a/drivers/block/amiflop.c +++ b/drivers/block/amiflop.c @@ -1802,7 +1802,7 @@ static int fd_alloc_disk(int drive, int system) unit[drive].gendisk[system] = disk; err = add_disk(disk); if (err) - blk_cleanup_disk(disk); + put_disk(disk); return err; } diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c index 348adf335217..12b3ca8f6f4a 100644 --- a/drivers/block/aoe/aoeblk.c +++ b/drivers/block/aoe/aoeblk.c @@ -427,7 +427,7 @@ aoeblk_gdalloc(void *vp) return; out_disk_cleanup: - blk_cleanup_disk(gd); + put_disk(gd); err_tagset: blk_mq_free_tag_set(set); err_mempool: diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c index b381d1c3ef32..3523dd82d7a0 100644 --- a/drivers/block/aoe/aoedev.c +++ b/drivers/block/aoe/aoedev.c @@ -277,7 +277,7 @@ freedev(struct aoedev *d) if (d->gd) { aoedisk_rm_debugfs(d); del_gendisk(d->gd); - blk_cleanup_disk(d->gd); + put_disk(d->gd); blk_mq_free_tag_set(&d->tag_set); } t = d->targets; diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index c6e41ee18aaa..9deb4df6bdb8 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -2031,7 +2031,7 @@ static void ataflop_probe(dev_t dev) return; cleanup_disk: - blk_cleanup_disk(unit[drive].disk[type]); + put_disk(unit[drive].disk[type]); unit[drive].disk[type] = NULL; } @@ -2063,7 +2063,7 @@ static void atari_cleanup_floppy_disk(struct atari_floppy_struct *fs) continue; if (fs->registered[type]) del_gendisk(fs->disk[type]); - blk_cleanup_disk(fs->disk[type]); + put_disk(fs->disk[type]); } blk_mq_free_tag_set(&fs->tag_set); } diff --git a/drivers/block/brd.c b/drivers/block/brd.c index 6e3f2f0d2352..9e26d5e769f3 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -419,7 +419,7 @@ static int brd_alloc(int i) return 0; out_cleanup_disk: - blk_cleanup_disk(disk); + put_disk(disk); out_free_dev: list_del(&brd->brd_list); kfree(brd); @@ -439,7 +439,7 @@ static void brd_cleanup(void) list_for_each_entry_safe(brd, next, &brd_devices, brd_list) { del_gendisk(brd->brd_disk); - blk_cleanup_disk(brd->brd_disk); + put_disk(brd->brd_disk); brd_free_pages(brd); list_del(&brd->brd_list); kfree(brd); diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 2887350ae010..f3e4db16fd07 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2207,7 +2207,7 @@ void drbd_destroy_device(struct kref *kref) if (device->bitmap) /* should no longer be there. */ drbd_bm_cleanup(device); __free_page(device->md_io.page); - blk_cleanup_disk(device->vdisk); + put_disk(device->vdisk); kfree(device->rs_plan_s); /* not for_each_connection(connection, resource): @@ -2807,7 +2807,7 @@ out_no_minor_idr: out_no_bitmap: __free_page(device->md_io.page); out_no_io_page: - blk_cleanup_disk(disk); + put_disk(disk); out_no_disk: kref_put(&resource->kref, drbd_destroy_resource); kfree(device); diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 015841f50f4e..491e7205a0db 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4557,7 +4557,7 @@ out: return; cleanup_disk: - blk_cleanup_disk(disks[drive][type]); + put_disk(disks[drive][type]); disks[drive][type] = NULL; mutex_unlock(&floppy_probe_lock); } @@ -4753,7 +4753,7 @@ out_put_disk: if (!disks[drive][0]) break; del_timer_sync(&motor_off_timer[drive]); - blk_cleanup_disk(disks[drive][0]); + put_disk(disks[drive][0]); blk_mq_free_tag_set(&tag_sets[drive]); } return err; @@ -4985,7 +4985,7 @@ static void __exit floppy_module_exit(void) } for (i = 0; i < ARRAY_SIZE(floppy_type); i++) { if (disks[drive][i]) - blk_cleanup_disk(disks[drive][i]); + put_disk(disks[drive][i]); } blk_mq_free_tag_set(&tag_sets[drive]); } diff --git a/drivers/block/loop.c b/drivers/block/loop.c index cc608226c8c7..e3c0ba93c1a3 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -2040,7 +2040,7 @@ static int loop_add(int i) return i; out_cleanup_disk: - blk_cleanup_disk(disk); + put_disk(disk); out_cleanup_tags: blk_mq_free_tag_set(&lo->tag_set); out_free_idr: diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 1d0e0a9fdd7c..e116c6cf56f5 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3534,7 +3534,7 @@ init_hw_cmds_error: disk_index_error: ida_free(&rssd_index_ida, index); ida_get_error: - blk_cleanup_disk(dd->disk); + put_disk(dd->disk); block_queue_alloc_init_error: blk_mq_free_tag_set(&dd->tags); block_queue_alloc_tag_error: diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c index e094d2b8b5a9..d914156db2d8 100644 --- a/drivers/block/n64cart.c +++ b/drivers/block/n64cart.c @@ -157,7 +157,7 @@ static int __init n64cart_probe(struct platform_device *pdev) return 0; out_cleanup_disk: - blk_cleanup_disk(disk); + put_disk(disk); out: return err; } diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 07f3c139a3d7..5c4c9c45c6ac 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -250,7 +250,7 @@ static void nbd_dev_remove(struct nbd_device *nbd) struct gendisk *disk = nbd->disk; del_gendisk(disk); - blk_cleanup_disk(disk); + put_disk(disk); blk_mq_free_tag_set(&nbd->tag_set); /* @@ -1833,7 +1833,7 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs) out_free_work: destroy_workqueue(nbd->recv_workq); out_err_disk: - blk_cleanup_disk(disk); + put_disk(disk); out_free_idr: mutex_lock(&nbd_index_mutex); idr_remove(&nbd_index_idr, index); diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 6b67088f4ea7..d695ea29efa6 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1737,7 +1737,7 @@ static void null_del_dev(struct nullb *nullb) null_restart_queue_async(nullb); } - blk_cleanup_disk(nullb->disk); + put_disk(nullb->disk); if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set) blk_mq_free_tag_set(nullb->tag_set); @@ -2082,7 +2082,7 @@ static int null_add_dev(struct nullb_device *dev) out_cleanup_zone: null_free_zoned_dev(dev); out_cleanup_disk: - blk_cleanup_disk(nullb->disk); + put_disk(nullb->disk); out_cleanup_tags: if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set) blk_mq_free_tag_set(nullb->tag_set); diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index f462ad67931a..a5ab40784119 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -956,7 +956,7 @@ out_unreg_cdrom: out_pi_release: pi_release(cd->pi); out_free_disk: - blk_cleanup_disk(cd->disk); + put_disk(cd->disk); out_free_tag_set: blk_mq_free_tag_set(&cd->tag_set); return ret; @@ -1029,7 +1029,7 @@ static void __exit pcd_exit(void) unregister_cdrom(&cd->info); del_gendisk(cd->disk); pi_release(cd->pi); - blk_cleanup_disk(cd->disk); + put_disk(cd->disk); blk_mq_free_tag_set(&cd->tag_set); } diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index 3637c38c72f9..c8c14c6f5c3a 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -943,7 +943,7 @@ static int pd_probe_drive(struct pd_unit *disk, int autoprobe, int port, goto cleanup_disk; return 0; cleanup_disk: - blk_cleanup_disk(disk->gd); + put_disk(disk->gd); put_disk: put_disk(p); disk->gd = NULL; @@ -1018,7 +1018,7 @@ static void __exit pd_exit(void) if (p) { disk->gd = NULL; del_gendisk(p); - blk_cleanup_disk(p); + put_disk(p); blk_mq_free_tag_set(&disk->tag_set); pi_release(disk->pi); } diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index 292e9a4ce1b9..eec1b9fde245 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c @@ -975,7 +975,7 @@ static int __init pf_init_unit(struct pf_unit *pf, bool autoprobe, int port, out_pi_release: pi_release(pf->pi); out_free_disk: - blk_cleanup_disk(pf->disk); + put_disk(pf->disk); out_free_tag_set: blk_mq_free_tag_set(&pf->tag_set); return ret; @@ -1044,7 +1044,7 @@ static void __exit pf_exit(void) if (!pf->present) continue; del_gendisk(pf->disk); - blk_cleanup_disk(pf->disk); + put_disk(pf->disk); blk_mq_free_tag_set(&pf->tag_set); pi_release(pf->pi); } diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 789093375344..653d24231483 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -2733,7 +2733,7 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev) return 0; out_mem2: - blk_cleanup_disk(disk); + put_disk(disk); out_mem: mempool_exit(&pd->rb_pool); kfree(pd); @@ -2783,7 +2783,7 @@ static int pkt_remove_dev(dev_t pkt_dev) pkt_dbg(1, pd, "writer unmapped\n"); del_gendisk(pd->disk); - blk_cleanup_disk(pd->disk); + put_disk(pd->disk); mempool_exit(&pd->rb_pool); kfree(pd); diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c index 3054adf77460..36d7b36c60c7 100644 --- a/drivers/block/ps3disk.c +++ b/drivers/block/ps3disk.c @@ -473,7 +473,7 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) return 0; fail_cleanup_disk: - blk_cleanup_disk(gendisk); + put_disk(gendisk); fail_free_tag_set: blk_mq_free_tag_set(&priv->tag_set); fail_teardown: @@ -500,7 +500,7 @@ static void ps3disk_remove(struct ps3_system_bus_device *_dev) &ps3disk_mask); mutex_unlock(&ps3disk_mask_mutex); del_gendisk(priv->gendisk); - blk_cleanup_disk(priv->gendisk); + put_disk(priv->gendisk); blk_mq_free_tag_set(&priv->tag_set); dev_notice(&dev->sbd.core, "Synchronizing disk cache\n"); ps3disk_sync_cache(dev); diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index 4f90819e245e..d1e0fefec90b 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -761,7 +761,7 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev) return 0; out_cleanup_disk: - blk_cleanup_disk(gendisk); + put_disk(gendisk); out_cache_cleanup: remove_proc_entry(DEVICE_NAME, NULL); ps3vram_cache_cleanup(dev); @@ -792,7 +792,7 @@ static void ps3vram_remove(struct ps3_system_bus_device *dev) struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev); del_gendisk(priv->gendisk); - blk_cleanup_disk(priv->gendisk); + put_disk(priv->gendisk); remove_proc_entry(DEVICE_NAME, NULL); ps3vram_cache_cleanup(dev); iounmap(priv->reports); diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index ef9bc62e9afd..0d8ec2fe5740 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -4729,7 +4729,7 @@ static blk_status_t rbd_queue_rq(struct blk_mq_hw_ctx *hctx, static void rbd_free_disk(struct rbd_device *rbd_dev) { - blk_cleanup_disk(rbd_dev->disk); + put_disk(rbd_dev->disk); blk_mq_free_tag_set(&rbd_dev->tag_set); rbd_dev->disk = NULL; } diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index a4470374f54f..b8d9e2824d9c 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -1408,7 +1408,7 @@ static int rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx) blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue); err = add_disk(dev->gd); if (err) - blk_cleanup_disk(dev->gd); + put_disk(dev->gd); return err; } @@ -1630,7 +1630,7 @@ put_sess: static void destroy_gen_disk(struct rnbd_clt_dev *dev) { del_gendisk(dev->gd); - blk_cleanup_disk(dev->gd); + put_disk(dev->gd); } static void destroy_sysfs(struct rnbd_clt_dev *dev, diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index dd0a1a6fed29..fb855da971ee 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -886,7 +886,7 @@ static int probe_disk(struct vdc_port *port) return 0; out_cleanup_disk: - blk_cleanup_disk(g); + put_disk(g); out_free_tag: blk_mq_free_tag_set(&port->tag_set); return err; @@ -1070,7 +1070,7 @@ static void vdc_port_remove(struct vio_dev *vdev) del_timer_sync(&port->vio.timer); del_gendisk(port->disk); - blk_cleanup_disk(port->disk); + put_disk(port->disk); blk_mq_free_tag_set(&port->tag_set); vdc_free_tx_ring(port); diff --git a/drivers/block/swim.c b/drivers/block/swim.c index fef65a18d56f..42b4b6828690 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -783,7 +783,7 @@ static void swim_cleanup_floppy_disk(struct floppy_state *fs) if (fs->registered) del_gendisk(fs->disk); - blk_cleanup_disk(disk); + put_disk(disk); blk_mq_free_tag_set(&fs->tag_set); } diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index 6c39f2c9f806..da811a7da03f 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c @@ -1238,7 +1238,7 @@ static int swim3_attach(struct macio_dev *mdev, return 0; out_cleanup_disk: - blk_cleanup_disk(disk); + put_disk(disk); out_free_tag_set: blk_mq_free_tag_set(&fs->tag_set); out_unregister: diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index 75057dbbcfbe..0e1a484cab0b 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -1377,7 +1377,7 @@ static void carm_free_disk(struct carm_host *host, unsigned int port_no) if (host->state > HST_DEV_ACTIVATE) del_gendisk(disk); - blk_cleanup_disk(disk); + put_disk(disk); } static int carm_init_shm(struct carm_host *host) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index cff1b6f6b054..d7d72e8f6e55 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -1089,7 +1089,7 @@ static int virtblk_probe(struct virtio_device *vdev) return 0; out_cleanup_disk: - blk_cleanup_disk(vblk->disk); + put_disk(vblk->disk); out_free_tags: blk_mq_free_tag_set(&vblk->tag_set); out_free_vq: diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 33f04ef78984..3bc80f35418b 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -2384,7 +2384,7 @@ static void blkfront_connect(struct blkfront_info *info) err = device_add_disk(&info->xbdev->dev, info->gd, NULL); if (err) { - blk_cleanup_disk(info->gd); + put_disk(info->gd); blk_mq_free_tag_set(&info->tag_set); info->rq = NULL; goto fail; @@ -2469,7 +2469,7 @@ static int blkfront_remove(struct xenbus_device *xbdev) blkif_free(info, 0); if (info->gd) { xlbd_release_minors(info->gd->first_minor, info->gd->minors); - blk_cleanup_disk(info->gd); + put_disk(info->gd); blk_mq_free_tag_set(&info->tag_set); } diff --git a/drivers/block/z2ram.c b/drivers/block/z2ram.c index 18ad43d9933e..c1e85f356e4d 100644 --- a/drivers/block/z2ram.c +++ b/drivers/block/z2ram.c @@ -337,7 +337,7 @@ static int z2ram_register_disk(int minor) z2ram_gendisk[minor] = disk; err = add_disk(disk); if (err) - blk_cleanup_disk(disk); + put_disk(disk); return err; } diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index b8549c61ff2c..e5233c911e43 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1957,7 +1957,7 @@ static int zram_add(void) return device_id; out_cleanup_disk: - blk_cleanup_disk(zram->disk); + put_disk(zram->disk); out_free_idr: idr_remove(&zram_index_idr, device_id); out_free_dev: @@ -2008,7 +2008,7 @@ static int zram_remove(struct zram *zram) */ zram_reset_device(zram); - blk_cleanup_disk(zram->disk); + put_disk(zram->disk); kfree(zram); return 0; } diff --git a/drivers/cdrom/gdrom.c b/drivers/cdrom/gdrom.c index f4cc90ea6198..ceded5772aac 100644 --- a/drivers/cdrom/gdrom.c +++ b/drivers/cdrom/gdrom.c @@ -817,7 +817,7 @@ probe_fail_free_irqs: free_irq(HW_EVENT_GDROM_DMA, &gd); free_irq(HW_EVENT_GDROM_CMD, &gd); probe_fail_cleanup_disk: - blk_cleanup_disk(gd.disk); + put_disk(gd.disk); probe_fail_free_tag_set: blk_mq_free_tag_set(&gd.tag_set); probe_fail_free_cd_info: diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 3563d15dbaf2..9dd752d272f6 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -884,7 +884,7 @@ static void bcache_device_free(struct bcache_device *d) if (disk) { ida_simple_remove(&bcache_device_idx, first_minor_to_idx(disk->first_minor)); - blk_cleanup_disk(disk); + put_disk(disk); } bioset_exit(&d->bio_split); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 4c04a980fcd9..8872f9c63688 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1894,7 +1894,7 @@ static void cleanup_mapped_device(struct mapped_device *md) del_gendisk(md->disk); } dm_queue_destroy_crypto_profile(md->queue); - blk_cleanup_disk(md->disk); + put_disk(md->disk); } if (md->pending_io) { diff --git a/drivers/md/md.c b/drivers/md/md.c index c7ecb0bffda0..076255ec9ba1 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5579,7 +5579,7 @@ static void md_free(struct kobject *ko) if (mddev->gendisk) { del_gendisk(mddev->gendisk); - blk_cleanup_disk(mddev->gendisk); + put_disk(mddev->gendisk); } percpu_ref_exit(&mddev->writes_pending); @@ -5718,7 +5718,7 @@ static int md_alloc(dev_t dev, char *name) out_del_gendisk: del_gendisk(disk); out_cleanup_disk: - blk_cleanup_disk(disk); + put_disk(disk); out_unlock_disks_mutex: mutex_unlock(&disks_mutex); mddev_put(mddev); diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c index ba7e7249a3db..ed9a683b3ca8 100644 --- a/drivers/memstick/core/ms_block.c +++ b/drivers/memstick/core/ms_block.c @@ -2129,7 +2129,7 @@ static int msb_init_disk(struct memstick_dev *card) return 0; out_cleanup_disk: - blk_cleanup_disk(msb->disk); + put_disk(msb->disk); out_free_tag_set: blk_mq_free_tag_set(&msb->tag_set); out_release_id: diff --git a/drivers/memstick/core/mspro_block.c b/drivers/memstick/core/mspro_block.c index 72e91c06c618..61cf75d4a01e 100644 --- a/drivers/memstick/core/mspro_block.c +++ b/drivers/memstick/core/mspro_block.c @@ -1209,7 +1209,7 @@ static int mspro_block_init_disk(struct memstick_dev *card) return 0; out_cleanup_disk: - blk_cleanup_disk(msb->disk); + put_disk(msb->disk); out_free_tag_set: blk_mq_free_tag_set(&msb->tag_set); out_release_id: diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c index f73172111465..60b222799871 100644 --- a/drivers/mtd/mtd_blkdevs.c +++ b/drivers/mtd/mtd_blkdevs.c @@ -29,7 +29,7 @@ static void blktrans_dev_release(struct kref *kref) struct mtd_blktrans_dev *dev = container_of(kref, struct mtd_blktrans_dev, ref); - blk_cleanup_disk(dev->disk); + put_disk(dev->disk); blk_mq_free_tag_set(dev->tag_set); kfree(dev->tag_set); list_del(&dev->list); @@ -398,7 +398,7 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) return 0; out_cleanup_disk: - blk_cleanup_disk(new->disk); + put_disk(new->disk); out_free_tag_set: blk_mq_free_tag_set(new->tag_set); out_kfree_tag_set: diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c index a78fdf3b30f7..4cf67a2a0d04 100644 --- a/drivers/mtd/ubi/block.c +++ b/drivers/mtd/ubi/block.c @@ -467,7 +467,7 @@ out_destroy_wq: out_remove_minor: idr_remove(&ubiblock_minor_idr, gd->first_minor); out_cleanup_disk: - blk_cleanup_disk(dev->gd); + put_disk(dev->gd); out_free_tags: blk_mq_free_tag_set(&dev->tag_set); out_free_dev: @@ -486,7 +486,7 @@ static void ubiblock_cleanup(struct ubiblock *dev) destroy_workqueue(dev->wq); /* Finally destroy the blk queue */ dev_info(disk_to_dev(dev->gd), "released"); - blk_cleanup_disk(dev->gd); + put_disk(dev->gd); blk_mq_free_tag_set(&dev->tag_set); idr_remove(&ubiblock_minor_idr, dev->gd->first_minor); } diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 9613e54c7a67..5e622c0d4b66 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1548,14 +1548,14 @@ static int btt_blk_init(struct btt *btt) return 0; out_cleanup_disk: - blk_cleanup_disk(btt->btt_disk); + put_disk(btt->btt_disk); return rc; } static void btt_blk_cleanup(struct btt *btt) { del_gendisk(btt->btt_disk); - blk_cleanup_disk(btt->btt_disk); + put_disk(btt->btt_disk); } /** diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 629d10fcf53b..a72b81fa3242 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -450,7 +450,7 @@ static void pmem_release_disk(void *__pmem) put_dax(pmem->dax_dev); del_gendisk(pmem->disk); - blk_cleanup_disk(pmem->disk); + put_disk(pmem->disk); } static int pmem_attach_disk(struct device *dev, @@ -596,7 +596,7 @@ out_cleanup_dax: kill_dax(pmem->dax_dev); put_dax(pmem->dax_dev); out: - blk_cleanup_disk(pmem->disk); + put_disk(pmem->disk); return rc; } diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 4e3a0f7bfc9c..b5b24998a5ab 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4061,7 +4061,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid, mutex_unlock(&ctrl->subsys->lock); nvme_put_ns_head(ns->head); out_cleanup_disk: - blk_cleanup_disk(disk); + put_disk(disk); out_free_ns: kfree(ns); out_free_id: diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index d3e2440d8abb..ccf9a6da8f6e 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -853,7 +853,7 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head) /* make sure all pending bios are cleaned up */ kblockd_schedule_work(&head->requeue_work); flush_work(&head->requeue_work); - blk_cleanup_disk(head->disk); + put_disk(head->disk); } void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 8d0d0eaa3059..4d8d1759775a 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -414,7 +414,7 @@ removeseg: kill_dax(dev_info->dax_dev); put_dax(dev_info->dax_dev); del_gendisk(dev_info->gd); - blk_cleanup_disk(dev_info->gd); + put_disk(dev_info->gd); up_write(&dcssblk_devices_sem); if (device_remove_file_self(dev, attr)) { @@ -712,7 +712,7 @@ out_dax: put_dax(dev_info->dax_dev); put_dev: list_del(&dev_info->lh); - blk_cleanup_disk(dev_info->gd); + put_disk(dev_info->gd); list_for_each_entry(seg_info, &dev_info->seg_list, lh) { segment_unload(seg_info->segment_name); } @@ -722,7 +722,7 @@ put_dev: dev_list_del: list_del(&dev_info->lh); release_gd: - blk_cleanup_disk(dev_info->gd); + put_disk(dev_info->gd); up_write(&dcssblk_devices_sem); seg_list_del: if (dev_info == NULL) @@ -790,7 +790,7 @@ dcssblk_remove_store(struct device *dev, struct device_attribute *attr, const ch kill_dax(dev_info->dax_dev); put_dax(dev_info->dax_dev); del_gendisk(dev_info->gd); - blk_cleanup_disk(dev_info->gd); + put_disk(dev_info->gd); /* unload all related segments */ list_for_each_entry(entry, &dev_info->seg_list, lh) diff --git a/drivers/s390/block/scm_blk.c b/drivers/s390/block/scm_blk.c index 2a9c0ddcade5..0c1df1d5f1ac 100644 --- a/drivers/s390/block/scm_blk.c +++ b/drivers/s390/block/scm_blk.c @@ -501,7 +501,7 @@ int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev) return 0; out_cleanup_disk: - blk_cleanup_disk(bdev->gendisk); + put_disk(bdev->gendisk); out_tag: blk_mq_free_tag_set(&bdev->tag_set); out: @@ -512,7 +512,7 @@ out: void scm_blk_dev_cleanup(struct scm_blk_dev *bdev) { del_gendisk(bdev->gendisk); - blk_cleanup_disk(bdev->gendisk); + put_disk(bdev->gendisk); blk_mq_free_tag_set(&bdev->tag_set); } diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 530eeccffda3..22b12531aeb7 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -834,7 +834,6 @@ struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass); \ __blk_alloc_disk(node_id, &__key); \ }) -void blk_cleanup_disk(struct gendisk *disk); int __register_blkdev(unsigned int major, const char *name, void (*probe)(dev_t devt)); -- cgit v1.2.3 From cc5c516df028b221d94c65c47c5ae8d20f61b6f9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 28 Jun 2022 19:18:45 +0200 Subject: block: simplify blktrace sysfs attribute creation Add the trace attributes to the default gendisk attributes, just like we already do for partitions. Signed-off-by: Christoph Hellwig Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/20220628171850.1313069-2-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-sysfs.c | 11 +---------- block/blk.h | 2 ++ block/genhd.c | 3 +++ block/partitions/core.c | 1 - include/linux/blktrace_api.h | 10 ---------- kernel/trace/blktrace.c | 11 ----------- 6 files changed, 6 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 9b211e519de8..5f3f73115988 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -810,21 +810,14 @@ int blk_register_queue(struct gendisk *disk) struct device *dev = disk_to_dev(disk); struct request_queue *q = disk->queue; - ret = blk_trace_init_sysfs(dev); - if (ret) - return ret; - mutex_lock(&q->sysfs_dir_lock); ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue"); - if (ret < 0) { - blk_trace_remove_sysfs(dev); + if (ret < 0) goto unlock; - } ret = sysfs_create_group(&q->kobj, &queue_attr_group); if (ret) { - blk_trace_remove_sysfs(dev); kobject_del(&q->kobj); kobject_put(&dev->kobj); goto unlock; @@ -890,7 +883,6 @@ put_dev: mutex_unlock(&q->sysfs_lock); mutex_unlock(&q->sysfs_dir_lock); kobject_del(&q->kobj); - blk_trace_remove_sysfs(dev); kobject_put(&dev->kobj); return ret; @@ -931,7 +923,6 @@ void blk_unregister_queue(struct gendisk *disk) if (queue_is_mq(q)) blk_mq_unregister_dev(disk_to_dev(disk), q); blk_crypto_sysfs_unregister(q); - blk_trace_remove_sysfs(disk_to_dev(disk)); mutex_lock(&q->sysfs_lock); elv_unregister_queue(q); diff --git a/block/blk.h b/block/blk.h index 1a0d3e6a4a63..74d59435870c 100644 --- a/block/blk.h +++ b/block/blk.h @@ -452,6 +452,8 @@ extern struct device_attribute dev_attr_events; extern struct device_attribute dev_attr_events_async; extern struct device_attribute dev_attr_events_poll_msecs; +extern struct attribute_group blk_trace_attr_group; + long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg); long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg); diff --git a/block/genhd.c b/block/genhd.c index bf9be06af2c8..b1fb7e058b9c 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1134,6 +1134,9 @@ static struct attribute_group disk_attr_group = { static const struct attribute_group *disk_attr_groups[] = { &disk_attr_group, +#ifdef CONFIG_BLK_DEV_IO_TRACE + &blk_trace_attr_group, +#endif NULL }; diff --git a/block/partitions/core.c b/block/partitions/core.c index 8a0ec929023b..7dc487f5b03c 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include "check.h" diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h index 623e22492afa..f6f9b544365a 100644 --- a/include/linux/blktrace_api.h +++ b/include/linux/blktrace_api.h @@ -77,10 +77,6 @@ extern int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, char __user *arg); extern int blk_trace_startstop(struct request_queue *q, int start); extern int blk_trace_remove(struct request_queue *q); -extern void blk_trace_remove_sysfs(struct device *dev); -extern int blk_trace_init_sysfs(struct device *dev); - -extern struct attribute_group blk_trace_attr_group; #else /* !CONFIG_BLK_DEV_IO_TRACE */ # define blk_trace_ioctl(bdev, cmd, arg) (-ENOTTY) @@ -91,13 +87,7 @@ extern struct attribute_group blk_trace_attr_group; # define blk_trace_remove(q) (-ENOTTY) # define blk_add_trace_msg(q, fmt, ...) do { } while (0) # define blk_add_cgroup_trace_msg(q, cg, fmt, ...) do { } while (0) -# define blk_trace_remove_sysfs(dev) do { } while (0) # define blk_trace_note_message_enabled(q) (false) -static inline int blk_trace_init_sysfs(struct device *dev) -{ - return 0; -} - #endif /* CONFIG_BLK_DEV_IO_TRACE */ #ifdef CONFIG_COMPAT diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index fe04c6f96ca5..c584effe5fe9 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -1867,17 +1867,6 @@ out_unlock_bdev: out: return ret ? ret : count; } - -int blk_trace_init_sysfs(struct device *dev) -{ - return sysfs_create_group(&dev->kobj, &blk_trace_attr_group); -} - -void blk_trace_remove_sysfs(struct device *dev) -{ - sysfs_remove_group(&dev->kobj, &blk_trace_attr_group); -} - #endif /* CONFIG_BLK_DEV_IO_TRACE */ #ifdef CONFIG_EVENT_TRACING -- cgit v1.2.3 From 8682b92e5ab852b93739a0f2b261fff4c733be57 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 28 Jun 2022 19:18:50 +0200 Subject: blk-mq: cleanup disk sysfs registration Pass a gendisk to the sysfs register/unregister functions and give them descriptive names. Also move the unregistration helper next to the one doing the registration. Signed-off-by: Christoph Hellwig Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/20220628171850.1313069-7-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-mq-sysfs.c | 39 ++++++++++++++++++++------------------- block/blk-mq.h | 3 ++- block/blk-sysfs.c | 9 ++++----- include/linux/blk-mq.h | 1 - 4 files changed, 26 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c index ee6efe2b250d..93997d297d42 100644 --- a/block/blk-mq-sysfs.c +++ b/block/blk-mq-sysfs.c @@ -203,22 +203,6 @@ static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) return ret; } -void blk_mq_unregister_dev(struct device *dev, struct request_queue *q) -{ - struct blk_mq_hw_ctx *hctx; - unsigned long i; - - lockdep_assert_held(&q->sysfs_dir_lock); - - queue_for_each_hw_ctx(q, hctx, i) - blk_mq_unregister_hctx(hctx); - - kobject_uevent(q->mq_kobj, KOBJ_REMOVE); - kobject_del(q->mq_kobj); - - q->mq_sysfs_init_done = false; -} - void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx) { kobject_init(&hctx->kobj, &blk_mq_hw_ktype); @@ -251,16 +235,16 @@ void blk_mq_sysfs_init(struct request_queue *q) } } -int __blk_mq_register_dev(struct device *dev, struct request_queue *q) +int blk_mq_sysfs_register(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct blk_mq_hw_ctx *hctx; unsigned long i, j; int ret; - WARN_ON_ONCE(!q->kobj.parent); lockdep_assert_held(&q->sysfs_dir_lock); - ret = kobject_add(q->mq_kobj, &dev->kobj, "%s", "mq"); + ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq"); if (ret < 0) goto out; @@ -288,6 +272,23 @@ unreg: return ret; } +void blk_mq_sysfs_unregister(struct gendisk *disk) +{ + struct request_queue *q = disk->queue; + struct blk_mq_hw_ctx *hctx; + unsigned long i; + + lockdep_assert_held(&q->sysfs_dir_lock); + + queue_for_each_hw_ctx(q, hctx, i) + blk_mq_unregister_hctx(hctx); + + kobject_uevent(q->mq_kobj, KOBJ_REMOVE); + kobject_del(q->mq_kobj); + + q->mq_sysfs_init_done = false; +} + void blk_mq_sysfs_unregister_hctxs(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; diff --git a/block/blk-mq.h b/block/blk-mq.h index a92639f2bfd2..54e20edf0da3 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -118,7 +118,8 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, */ extern void blk_mq_sysfs_init(struct request_queue *q); extern void blk_mq_sysfs_deinit(struct request_queue *q); -extern int __blk_mq_register_dev(struct device *dev, struct request_queue *q); +int blk_mq_sysfs_register(struct gendisk *disk); +void blk_mq_sysfs_unregister(struct gendisk *disk); int blk_mq_sysfs_register_hctxs(struct request_queue *q); void blk_mq_sysfs_unregister_hctxs(struct request_queue *q); extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index b72506770b97..85ea43eff094 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -812,18 +812,17 @@ struct kobj_type blk_queue_ktype = { */ int blk_register_queue(struct gendisk *disk) { - int ret; - struct device *dev = disk_to_dev(disk); struct request_queue *q = disk->queue; + int ret; mutex_lock(&q->sysfs_dir_lock); - ret = kobject_add(&q->kobj, &dev->kobj, "%s", "queue"); + ret = kobject_add(&q->kobj, &disk_to_dev(disk)->kobj, "queue"); if (ret < 0) goto unlock; if (queue_is_mq(q)) - __blk_mq_register_dev(dev, q); + blk_mq_sysfs_register(disk); mutex_lock(&q->sysfs_lock); mutex_lock(&q->debugfs_mutex); @@ -919,7 +918,7 @@ void blk_unregister_queue(struct gendisk *disk) * structures that can be modified through sysfs. */ if (queue_is_mq(q)) - blk_mq_unregister_dev(disk_to_dev(disk), q); + blk_mq_sysfs_unregister(disk); blk_crypto_sysfs_unregister(q); mutex_lock(&q->sysfs_lock); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 0fd96e92c6c6..43aad0da3305 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -691,7 +691,6 @@ struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q, struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, struct request_queue *q); -void blk_mq_unregister_dev(struct device *, struct request_queue *); void blk_mq_destroy_queue(struct request_queue *); int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); -- cgit v1.2.3 From 8add9a3a2243166f8f60fc20e876caaf30a333f7 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Tue, 28 Jun 2022 15:18:21 +0100 Subject: efi: Simplify arch_efi_call_virt() macro Currently, the arch_efi_call_virt() assumes all users of it will have defined a type 'efi_##f##_t' to make use of it. Simplify the arch_efi_call_virt() macro by eliminating the explicit need for efi_##f##_t type for every user of this macro. Signed-off-by: Sudeep Holla Acked-by: Russell King (Oracle) [ardb: apply Sudeep's ARM fix to i686, Loongarch and RISC-V too] Signed-off-by: Ard Biesheuvel --- arch/arm/include/asm/efi.h | 7 ------- arch/arm64/include/asm/efi.h | 7 ++----- arch/loongarch/include/asm/efi.h | 16 ++-------------- arch/riscv/include/asm/efi.h | 2 -- arch/x86/include/asm/efi.h | 3 +-- include/linux/efi.h | 2 ++ 6 files changed, 7 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h index 27218eabbf9a..3088ef72704e 100644 --- a/arch/arm/include/asm/efi.h +++ b/arch/arm/include/asm/efi.h @@ -24,13 +24,6 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); #define arch_efi_call_virt_setup() efi_virtmap_load() #define arch_efi_call_virt_teardown() efi_virtmap_unload() -#define arch_efi_call_virt(p, f, args...) \ -({ \ - efi_##f##_t *__f; \ - __f = p->f; \ - __f(args); \ -}) - #define ARCH_EFI_IRQ_FLAGS_MASK \ (PSR_J_BIT | PSR_E_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | \ PSR_T_BIT | MODE_MASK) diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index ad55079abe47..439e2bc5d5d8 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -27,12 +27,9 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); __efi_fpsimd_begin(); \ }) +#undef arch_efi_call_virt #define arch_efi_call_virt(p, f, args...) \ -({ \ - efi_##f##_t *__f; \ - __f = p->f; \ - __efi_rt_asm_wrapper(__f, #f, args); \ -}) + __efi_rt_asm_wrapper((p)->f, #f, args) #define arch_efi_call_virt_teardown() \ ({ \ diff --git a/arch/loongarch/include/asm/efi.h b/arch/loongarch/include/asm/efi.h index 0127d84d5e1d..9d44c6948be1 100644 --- a/arch/loongarch/include/asm/efi.h +++ b/arch/loongarch/include/asm/efi.h @@ -13,20 +13,8 @@ void efifb_setup_from_dmi(struct screen_info *si, const char *opt); #define ARCH_EFI_IRQ_FLAGS_MASK 0x00000004 /* Bit 2: CSR.CRMD.IE */ -#define arch_efi_call_virt_setup() \ -({ \ -}) - -#define arch_efi_call_virt(p, f, args...) \ -({ \ - efi_##f##_t * __f; \ - __f = p->f; \ - __f(args); \ -}) - -#define arch_efi_call_virt_teardown() \ -({ \ -}) +#define arch_efi_call_virt_setup() +#define arch_efi_call_virt_teardown() #define EFI_ALLOC_ALIGN SZ_64K diff --git a/arch/riscv/include/asm/efi.h b/arch/riscv/include/asm/efi.h index cc4f6787f937..f74879a8f1ea 100644 --- a/arch/riscv/include/asm/efi.h +++ b/arch/riscv/include/asm/efi.h @@ -23,8 +23,6 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); #define arch_efi_call_virt_setup() efi_virtmap_load() #define arch_efi_call_virt_teardown() efi_virtmap_unload() -#define arch_efi_call_virt(p, f, args...) p->f(args) - #define ARCH_EFI_IRQ_FLAGS_MASK (SR_IE | SR_SPIE) /* Load initrd anywhere in system RAM */ diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index eb90206eae80..9a63dd1b794c 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -100,8 +100,6 @@ static inline void efi_fpu_end(void) efi_fpu_end(); \ }) -#define arch_efi_call_virt(p, f, args...) p->f(args) - #else /* !CONFIG_X86_32 */ #define EFI_LOADER_SIGNATURE "EL64" @@ -121,6 +119,7 @@ extern asmlinkage u64 __efi_call(void *fp, ...); efi_enter_mm(); \ }) +#undef arch_efi_call_virt #define arch_efi_call_virt(p, f, args...) ({ \ u64 ret, ibt = ibt_save(); \ ret = efi_call((void *)p->f, args); \ diff --git a/include/linux/efi.h b/include/linux/efi.h index 93ce85a14a46..9ff63acef1ec 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1200,6 +1200,8 @@ static inline void efi_check_for_embedded_firmwares(void) { } efi_status_t efi_random_get_seed(void); +#define arch_efi_call_virt(p, f, args...) ((p)->f(args)) + /* * Arch code can implement the following three template macros, avoiding * reptition for the void/non-void return cases of {__,}efi_call_virt(): -- cgit v1.2.3 From c3497fd009ef2c59eea60d21c3ac22de3585ed7d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 12 Jun 2022 19:50:29 -0400 Subject: fix short copy handling in copy_mc_pipe_to_iter() Unlike other copying operations on ITER_PIPE, copy_mc_to_iter() can result in a short copy. In that case we need to trim the unused buffers, as well as the length of partially filled one - it's not enough to set ->head, ->iov_offset and ->count to reflect how much had we copied. Not hard to fix, fortunately... I'd put a helper (pipe_discard_from(pipe, head)) into pipe_fs_i.h, rather than iov_iter.c - it has nothing to do with iov_iter and having it will allow us to avoid an ugly kludge in fs/splice.c. We could put it into lib/iov_iter.c for now and move it later, but I don't see the point going that way... Cc: stable@kernel.org # 4.19+ Fixes: ca146f6f091e "lib/iov_iter: Fix pipe handling in _copy_to_iter_mcsafe()" Reviewed-by: Jeff Layton Reviewed-by: Christian Brauner (Microsoft) Signed-off-by: Al Viro --- include/linux/pipe_fs_i.h | 9 +++++++++ lib/iov_iter.c | 15 +++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index cb0fd633a610..4ea496924106 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -229,6 +229,15 @@ static inline bool pipe_buf_try_steal(struct pipe_inode_info *pipe, return buf->ops->try_steal(pipe, buf); } +static inline void pipe_discard_from(struct pipe_inode_info *pipe, + unsigned int old_head) +{ + unsigned int mask = pipe->ring_size - 1; + + while (pipe->head > old_head) + pipe_buf_release(pipe, &pipe->bufs[--pipe->head & mask]); +} + /* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual memory allocation, whereas PIPE_BUF makes atomicity guarantees. */ #define PIPE_SIZE PAGE_SIZE diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 0b64695ab632..2bf20b48a04a 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -689,6 +689,7 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, struct pipe_inode_info *pipe = i->pipe; unsigned int p_mask = pipe->ring_size - 1; unsigned int i_head; + unsigned int valid = pipe->head; size_t n, off, xfer = 0; if (!sanity(i)) @@ -702,11 +703,17 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, rem = copy_mc_to_kernel(p + off, addr + xfer, chunk); chunk -= rem; kunmap_local(p); - i->head = i_head; - i->iov_offset = off + chunk; - xfer += chunk; - if (rem) + if (chunk) { + i->head = i_head; + i->iov_offset = off + chunk; + xfer += chunk; + valid = i_head + 1; + } + if (rem) { + pipe->bufs[i_head & p_mask].len -= rem; + pipe_discard_from(pipe, valid); break; + } n -= chunk; off = 0; i_head++; -- cgit v1.2.3 From acd6510dd7ab3664b69eb99e37c4fd6325a7d442 Mon Sep 17 00:00:00 2001 From: Tanmay Shah Date: Tue, 7 Jun 2022 15:42:54 -0700 Subject: firmware: xilinx: Add TF_A_PM_REGISTER_SGI SMC call SGI interrupt register and reset is performed by EEMI ioctl IOCTL_REGISTER_SGI. However, this is not correct use of EEMI call. SGI registration functionality does not qualify as energy management activity and so shouldn't be mapped to EEMI call. This new call will replace IOCTL_REGISTER_SGI and will be handled by TF-A specific handler in TF-A. To maintain backward compatibility for a while firmware driver will still use IOCTL_REGISTER_SGI as fallback strategy if new call fails or is not supported by TF-A. This new design also helps to make TF-A as pass through layer for EEMI calls. So we don't have to maintain PM_IOCTL as EEMI API ID in TF-A. Signed-off-by: Tanmay Shah Acked-by: Michal Simek Link: https://lore.kernel.org/r/20220607224253.54919-1-tanmay.shah@xilinx.com Signed-off-by: Michal Simek --- drivers/firmware/xilinx/zynqmp.c | 16 +++++++++++++++- drivers/soc/xilinx/xlnx_event_manager.c | 5 ++--- include/linux/firmware/xlnx-zynqmp.h | 7 +++++++ 3 files changed, 24 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index 7977a494a651..d1f652802181 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -2,7 +2,7 @@ /* * Xilinx Zynq MPSoC Firmware layer * - * Copyright (C) 2014-2021 Xilinx, Inc. + * Copyright (C) 2014-2022 Xilinx, Inc. * * Michal Simek * Davorin Mista @@ -340,6 +340,20 @@ int zynqmp_pm_invoke_fn(u32 pm_api_id, u32 arg0, u32 arg1, static u32 pm_api_version; static u32 pm_tz_version; +int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset) +{ + int ret; + + ret = zynqmp_pm_invoke_fn(TF_A_PM_REGISTER_SGI, sgi_num, reset, 0, 0, + NULL); + if (!ret) + return ret; + + /* try old implementation as fallback strategy if above fails */ + return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_REGISTER_SGI, sgi_num, + reset, NULL); +} + /** * zynqmp_pm_get_api_version() - Get version number of PMU PM firmware * @version: Returned version value diff --git a/drivers/soc/xilinx/xlnx_event_manager.c b/drivers/soc/xilinx/xlnx_event_manager.c index 5dcb7665fe22..2de082765bef 100644 --- a/drivers/soc/xilinx/xlnx_event_manager.c +++ b/drivers/soc/xilinx/xlnx_event_manager.c @@ -647,8 +647,7 @@ static int xlnx_event_manager_probe(struct platform_device *pdev) cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "soc/event:starting", xlnx_event_cpuhp_start, xlnx_event_cpuhp_down); - ret = zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_REGISTER_SGI, sgi_num, - 0, NULL); + ret = zynqmp_pm_register_sgi(sgi_num, 0); if (ret) { dev_err(&pdev->dev, "SGI %d Registration over TF-A failed with %d\n", sgi_num, ret); xlnx_event_cleanup_sgi(pdev); @@ -681,7 +680,7 @@ static int xlnx_event_manager_remove(struct platform_device *pdev) kfree(eve_data); } - ret = zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_REGISTER_SGI, 0, 1, NULL); + ret = zynqmp_pm_register_sgi(0, 1); if (ret) dev_err(&pdev->dev, "SGI unregistration over TF-A failed with %d\n", ret); diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 1ec73d5352c3..cbde3b1fa414 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -34,6 +34,7 @@ #define PM_API_VERSION_2 2 /* ATF only commands */ +#define TF_A_PM_REGISTER_SGI 0xa04 #define PM_GET_TRUSTZONE_VERSION 0xa03 #define PM_SET_SUSPEND_MODE 0xa02 #define GET_CALLBACK_DATA 0xa01 @@ -468,6 +469,7 @@ int zynqmp_pm_feature(const u32 api_id); int zynqmp_pm_is_function_supported(const u32 api_id, const u32 id); int zynqmp_pm_set_feature_config(enum pm_feature_config_id id, u32 value); int zynqmp_pm_get_feature_config(enum pm_feature_config_id id, u32 *payload); +int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset); #else static inline int zynqmp_pm_get_api_version(u32 *version) { @@ -733,6 +735,11 @@ static inline int zynqmp_pm_get_feature_config(enum pm_feature_config_id id, { return -ENODEV; } + +static inline int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset) +{ + return -ENODEV; +} #endif #endif /* __FIRMWARE_ZYNQMP_H__ */ -- cgit v1.2.3 From 6ffcd825e7d0416d78fd41cd5b7856a78122cc8c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 28 Jun 2022 20:41:40 -0400 Subject: mm: Remove __delete_from_page_cache() This wrapper is no longer used. Remove it and all references to it. Signed-off-by: Matthew Wilcox (Oracle) --- fs/inode.c | 2 +- include/linux/pagemap.h | 4 ---- mm/memory-failure.c | 2 +- mm/shmem.c | 2 +- mm/truncate.c | 2 +- 5 files changed, 4 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/fs/inode.c b/fs/inode.c index bd4da9c5207e..d2bdc407c94b 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -604,7 +604,7 @@ void clear_inode(struct inode *inode) { /* * We have to cycle the i_pages lock here because reclaim can be in the - * process of removing the last page (in __delete_from_page_cache()) + * process of removing the last page (in __filemap_remove_folio()) * and we must not free the mapping under it. */ xa_lock_irq(&inode->i_data.i_pages); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ce96866fbec4..44b9c265a234 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1107,10 +1107,6 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio, void filemap_remove_folio(struct folio *folio); void delete_from_page_cache(struct page *page); void __filemap_remove_folio(struct folio *folio, void *shadow); -static inline void __delete_from_page_cache(struct page *page, void *shadow) -{ - __filemap_remove_folio(page_folio(page), shadow); -} void replace_page_cache_page(struct page *old, struct page *new); void delete_from_page_cache_batch(struct address_space *mapping, struct folio_batch *fbatch); diff --git a/mm/memory-failure.c b/mm/memory-failure.c index b85661cbdc4a..a859486ddda9 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1935,7 +1935,7 @@ try_again: /* * Now take care of user space mappings. - * Abort on fail: __delete_from_page_cache() assumes unmapped page. + * Abort on fail: __filemap_remove_folio() assumes unmapped page. */ if (!hwpoison_user_mappings(p, pfn, flags, p)) { action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED); diff --git a/mm/shmem.c b/mm/shmem.c index a6f565308133..b42dfb01c634 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -392,7 +392,7 @@ void shmem_uncharge(struct inode *inode, long pages) struct shmem_inode_info *info = SHMEM_I(inode); unsigned long flags; - /* nrpages adjustment done by __delete_from_page_cache() or caller */ + /* nrpages adjustment done by __filemap_remove_folio() or caller */ spin_lock_irqsave(&info->lock, flags); info->alloced -= pages; diff --git a/mm/truncate.c b/mm/truncate.c index ab50d0d59a2a..0b0708bf935f 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -443,7 +443,7 @@ EXPORT_SYMBOL(truncate_inode_pages_range); * mapping->invalidate_lock. * * Note: When this function returns, there can be a page in the process of - * deletion (inside __delete_from_page_cache()) in the specified range. Thus + * deletion (inside __filemap_remove_folio()) in the specified range. Thus * mapping->nrpages can be non-zero when this function returns even after * truncation of the whole mapping. */ -- cgit v1.2.3 From 2bb876b58d593d7f2522ec0f41f20a74fde76822 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 1 Jun 2022 15:13:59 -0400 Subject: filemap: Remove add_to_page_cache() and add_to_page_cache_locked() These functions have no more users, so delete them. Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Mike Kravetz Reviewed-by: Muchun Song --- Documentation/admin-guide/cgroup-v1/memcg_test.rst | 2 +- include/linux/pagemap.h | 18 ------------------ mm/filemap.c | 20 -------------------- mm/shmem.c | 2 +- mm/swap_state.c | 2 +- 5 files changed, 3 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cgroup-v1/memcg_test.rst b/Documentation/admin-guide/cgroup-v1/memcg_test.rst index 45b94f7b3beb..a402359abb99 100644 --- a/Documentation/admin-guide/cgroup-v1/memcg_test.rst +++ b/Documentation/admin-guide/cgroup-v1/memcg_test.rst @@ -97,7 +97,7 @@ Under below explanation, we assume CONFIG_MEM_RES_CTRL_SWAP=y. ============= Page Cache is charged at - - add_to_page_cache_locked(). + - filemap_add_folio(). The logic is very clear. (About migration, see below) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 44b9c265a234..dee519ef7436 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1098,8 +1098,6 @@ size_t fault_in_subpage_writeable(char __user *uaddr, size_t size); size_t fault_in_safe_writeable(const char __user *uaddr, size_t size); size_t fault_in_readable(const char __user *uaddr, size_t size); -int add_to_page_cache_locked(struct page *page, struct address_space *mapping, - pgoff_t index, gfp_t gfp); int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp); int filemap_add_folio(struct address_space *mapping, struct folio *folio, @@ -1115,22 +1113,6 @@ bool filemap_release_folio(struct folio *folio, gfp_t gfp); loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end, int whence); -/* - * Like add_to_page_cache_locked, but used to add newly allocated pages: - * the page is new, so we can just run __SetPageLocked() against it. - */ -static inline int add_to_page_cache(struct page *page, - struct address_space *mapping, pgoff_t offset, gfp_t gfp_mask) -{ - int error; - - __SetPageLocked(page); - error = add_to_page_cache_locked(page, mapping, offset, gfp_mask); - if (unlikely(error)) - __ClearPageLocked(page); - return error; -} - /* Must be non-static for BPF error injection */ int __filemap_add_folio(struct address_space *mapping, struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp); diff --git a/mm/filemap.c b/mm/filemap.c index ffdfbc8b0e3c..22a17ab256f7 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -929,26 +929,6 @@ error: } ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO); -/** - * add_to_page_cache_locked - add a locked page to the pagecache - * @page: page to add - * @mapping: the page's address_space - * @offset: page index - * @gfp_mask: page allocation mode - * - * This function is used to add a page to the pagecache. It must be locked. - * This function does not add the page to the LRU. The caller must do that. - * - * Return: %0 on success, negative error code otherwise. - */ -int add_to_page_cache_locked(struct page *page, struct address_space *mapping, - pgoff_t offset, gfp_t gfp_mask) -{ - return __filemap_add_folio(mapping, page_folio(page), offset, - gfp_mask, NULL); -} -EXPORT_SYMBOL(add_to_page_cache_locked); - int filemap_add_folio(struct address_space *mapping, struct folio *folio, pgoff_t index, gfp_t gfp) { diff --git a/mm/shmem.c b/mm/shmem.c index b42dfb01c634..6a5e46f1a326 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -693,7 +693,7 @@ static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ /* - * Like add_to_page_cache_locked, but error if expected item has gone. + * Like filemap_add_folio, but error if expected item has gone. */ static int shmem_add_to_page_cache(struct folio *folio, struct address_space *mapping, diff --git a/mm/swap_state.c b/mm/swap_state.c index 778d57d2d92d..f5b6f5638908 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -95,7 +95,7 @@ void *get_shadow_from_swap_cache(swp_entry_t entry) } /* - * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, + * add_to_swap_cache resembles filemap_add_folio on swapper_space, * but sets SwapCache flag and private instead of mapping and index. */ int add_to_swap_cache(struct page *page, swp_entry_t entry, -- cgit v1.2.3 From be0ced5e9cb81a4c2edefd081a7794a1814fb60d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 3 Jun 2022 15:30:25 -0400 Subject: filemap: Add filemap_get_folios() This is the equivalent of find_get_pages() but fills a folio_batch instead of an array of pages. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Acked-by: Christian Brauner (Microsoft) --- include/linux/pagemap.h | 2 ++ mm/filemap.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index dee519ef7436..cfd0e8001b3b 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -718,6 +718,8 @@ static inline struct page *find_subpage(struct page *head, pgoff_t index) return head + (index & (thp_nr_pages(head) - 1)); } +unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, + pgoff_t end, struct folio_batch *fbatch); unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, pgoff_t end, unsigned int nr_pages, struct page **pages); diff --git a/mm/filemap.c b/mm/filemap.c index 22a17ab256f7..ce3209a8ad49 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2127,6 +2127,65 @@ put: return folio_batch_count(fbatch); } +/** + * filemap_get_folios - Get a batch of folios + * @mapping: The address_space to search + * @start: The starting page index + * @end: The final page index (inclusive) + * @fbatch: The batch to fill. + * + * Search for and return a batch of folios in the mapping starting at + * index @start and up to index @end (inclusive). The folios are returned + * in @fbatch with an elevated reference count. + * + * The first folio may start before @start; if it does, it will contain + * @start. The final folio may extend beyond @end; if it does, it will + * contain @end. The folios have ascending indices. There may be gaps + * between the folios if there are indices which have no folio in the + * page cache. If folios are added to or removed from the page cache + * while this is running, they may or may not be found by this call. + * + * Return: The number of folios which were found. + * We also update @start to index the next folio for the traversal. + */ +unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, + pgoff_t end, struct folio_batch *fbatch) +{ + XA_STATE(xas, &mapping->i_pages, *start); + struct folio *folio; + + rcu_read_lock(); + while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { + /* Skip over shadow, swap and DAX entries */ + if (xa_is_value(folio)) + continue; + if (!folio_batch_add(fbatch, folio)) { + unsigned long nr = folio_nr_pages(folio); + + if (folio_test_hugetlb(folio)) + nr = 1; + *start = folio->index + nr; + goto out; + } + } + + /* + * We come here when there is no page beyond @end. We take care to not + * overflow the index @start as it confuses some of the callers. This + * breaks the iteration when there is a page at index -1 but that is + * already broken anyway. + */ + if (end == (pgoff_t)-1) + *start = (pgoff_t)-1; + else + *start = end + 1; +out: + rcu_read_unlock(); + + return folio_batch_count(fbatch); +} +EXPORT_SYMBOL(filemap_get_folios); + static inline bool folio_more_pages(struct folio *folio, pgoff_t index, pgoff_t max) { -- cgit v1.2.3 From 77414d195f905dd43f58bce82118775ffa59575c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 4 Jun 2022 17:39:09 -0400 Subject: vmscan: Add check_move_unevictable_folios() Change the guts of check_move_unevictable_pages() over to use folios and add check_move_unevictable_pages() as a wrapper. Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Christian Brauner (Microsoft) --- include/linux/swap.h | 3 ++- mm/vmscan.c | 56 +++++++++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 0c0fed1b348f..8672a7123ccd 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -438,7 +438,8 @@ static inline bool node_reclaim_enabled(void) return node_reclaim_mode & (RECLAIM_ZONE|RECLAIM_WRITE|RECLAIM_UNMAP); } -extern void check_move_unevictable_pages(struct pagevec *pvec); +void check_move_unevictable_folios(struct folio_batch *fbatch); +void check_move_unevictable_pages(struct pagevec *pvec); extern void kswapd_run(int nid); extern void kswapd_stop(int nid); diff --git a/mm/vmscan.c b/mm/vmscan.c index f7d9a683e3a7..04f8671caad9 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4790,45 +4790,57 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) } #endif +void check_move_unevictable_pages(struct pagevec *pvec) +{ + struct folio_batch fbatch; + unsigned i; + + folio_batch_init(&fbatch); + for (i = 0; i < pvec->nr; i++) { + struct page *page = pvec->pages[i]; + + if (PageTransTail(page)) + continue; + folio_batch_add(&fbatch, page_folio(page)); + } + check_move_unevictable_folios(&fbatch); +} +EXPORT_SYMBOL_GPL(check_move_unevictable_pages); + /** - * check_move_unevictable_pages - check pages for evictability and move to - * appropriate zone lru list - * @pvec: pagevec with lru pages to check + * check_move_unevictable_folios - Move evictable folios to appropriate zone + * lru list + * @fbatch: Batch of lru folios to check. * - * Checks pages for evictability, if an evictable page is in the unevictable + * Checks folios for evictability, if an evictable folio is in the unevictable * lru list, moves it to the appropriate evictable lru list. This function - * should be only used for lru pages. + * should be only used for lru folios. */ -void check_move_unevictable_pages(struct pagevec *pvec) +void check_move_unevictable_folios(struct folio_batch *fbatch) { struct lruvec *lruvec = NULL; int pgscanned = 0; int pgrescued = 0; int i; - for (i = 0; i < pvec->nr; i++) { - struct page *page = pvec->pages[i]; - struct folio *folio = page_folio(page); - int nr_pages; - - if (PageTransTail(page)) - continue; + for (i = 0; i < fbatch->nr; i++) { + struct folio *folio = fbatch->folios[i]; + int nr_pages = folio_nr_pages(folio); - nr_pages = thp_nr_pages(page); pgscanned += nr_pages; - /* block memcg migration during page moving between lru */ - if (!TestClearPageLRU(page)) + /* block memcg migration while the folio moves between lrus */ + if (!folio_test_clear_lru(folio)) continue; lruvec = folio_lruvec_relock_irq(folio, lruvec); - if (page_evictable(page) && PageUnevictable(page)) { - del_page_from_lru_list(page, lruvec); - ClearPageUnevictable(page); - add_page_to_lru_list(page, lruvec); + if (folio_evictable(folio) && folio_test_unevictable(folio)) { + lruvec_del_folio(lruvec, folio); + folio_clear_unevictable(folio); + lruvec_add_folio(lruvec, folio); pgrescued += nr_pages; } - SetPageLRU(page); + folio_set_lru(folio); } if (lruvec) { @@ -4839,4 +4851,4 @@ void check_move_unevictable_pages(struct pagevec *pvec) count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); } } -EXPORT_SYMBOL_GPL(check_move_unevictable_pages); +EXPORT_SYMBOL_GPL(check_move_unevictable_folios); -- cgit v1.2.3 From bb4b42ba926224e26ed699e51def164b4b163935 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 4 Jun 2022 17:46:02 -0400 Subject: filemap: Remove find_get_pages_range() and associated functions All callers of find_get_pages_range(), pagevec_lookup_range() and pagevec_lookup() have now been removed. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Acked-by: Christian Brauner (Microsoft) --- include/linux/pagemap.h | 3 --- include/linux/pagevec.h | 10 -------- mm/filemap.c | 67 ------------------------------------------------- mm/swap.c | 29 --------------------- 4 files changed, 109 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index cfd0e8001b3b..87d4ea571240 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -720,9 +720,6 @@ static inline struct page *find_subpage(struct page *head, pgoff_t index) unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, pgoff_t end, struct folio_batch *fbatch); -unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, - pgoff_t end, unsigned int nr_pages, - struct page **pages); unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start, unsigned int nr_pages, struct page **pages); unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index 67b1246f136b..6649154a2115 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -27,16 +27,6 @@ struct pagevec { void __pagevec_release(struct pagevec *pvec); void __pagevec_lru_add(struct pagevec *pvec); -unsigned pagevec_lookup_range(struct pagevec *pvec, - struct address_space *mapping, - pgoff_t *start, pgoff_t end); -static inline unsigned pagevec_lookup(struct pagevec *pvec, - struct address_space *mapping, - pgoff_t *start) -{ - return pagevec_lookup_range(pvec, mapping, start, (pgoff_t)-1); -} - unsigned pagevec_lookup_range_tag(struct pagevec *pvec, struct address_space *mapping, pgoff_t *index, pgoff_t end, xa_mark_t tag); diff --git a/mm/filemap.c b/mm/filemap.c index ce3209a8ad49..15399e8cd281 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2196,73 +2196,6 @@ bool folio_more_pages(struct folio *folio, pgoff_t index, pgoff_t max) return index < folio->index + folio_nr_pages(folio) - 1; } -/** - * find_get_pages_range - gang pagecache lookup - * @mapping: The address_space to search - * @start: The starting page index - * @end: The final page index (inclusive) - * @nr_pages: The maximum number of pages - * @pages: Where the resulting pages are placed - * - * find_get_pages_range() will search for and return a group of up to @nr_pages - * pages in the mapping starting at index @start and up to index @end - * (inclusive). The pages are placed at @pages. find_get_pages_range() takes - * a reference against the returned pages. - * - * The search returns a group of mapping-contiguous pages with ascending - * indexes. There may be holes in the indices due to not-present pages. - * We also update @start to index the next page for the traversal. - * - * Return: the number of pages which were found. If this number is - * smaller than @nr_pages, the end of specified range has been - * reached. - */ -unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, - pgoff_t end, unsigned int nr_pages, - struct page **pages) -{ - XA_STATE(xas, &mapping->i_pages, *start); - struct folio *folio; - unsigned ret = 0; - - if (unlikely(!nr_pages)) - return 0; - - rcu_read_lock(); - while ((folio = find_get_entry(&xas, end, XA_PRESENT))) { - /* Skip over shadow, swap and DAX entries */ - if (xa_is_value(folio)) - continue; - -again: - pages[ret] = folio_file_page(folio, xas.xa_index); - if (++ret == nr_pages) { - *start = xas.xa_index + 1; - goto out; - } - if (folio_more_pages(folio, xas.xa_index, end)) { - xas.xa_index++; - folio_ref_inc(folio); - goto again; - } - } - - /* - * We come here when there is no page beyond @end. We take care to not - * overflow the index @start as it confuses some of the callers. This - * breaks the iteration when there is a page at index -1 but that is - * already broken anyway. - */ - if (end == (pgoff_t)-1) - *start = (pgoff_t)-1; - else - *start = end + 1; -out: - rcu_read_unlock(); - - return ret; -} - /** * find_get_pages_contig - gang contiguous pagecache lookup * @mapping: The address_space to search diff --git a/mm/swap.c b/mm/swap.c index f3922a96b2e9..f65e284247b2 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -1086,35 +1086,6 @@ void folio_batch_remove_exceptionals(struct folio_batch *fbatch) fbatch->nr = j; } -/** - * pagevec_lookup_range - gang pagecache lookup - * @pvec: Where the resulting pages are placed - * @mapping: The address_space to search - * @start: The starting page index - * @end: The final page index - * - * pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE - * pages in the mapping starting from index @start and upto index @end - * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a - * reference against the pages in @pvec. - * - * The search returns a group of mapping-contiguous pages with ascending - * indexes. There may be holes in the indices due to not-present pages. We - * also update @start to index the next page for the traversal. - * - * pagevec_lookup_range() returns the number of pages which were found. If this - * number is smaller than PAGEVEC_SIZE, the end of specified range has been - * reached. - */ -unsigned pagevec_lookup_range(struct pagevec *pvec, - struct address_space *mapping, pgoff_t *start, pgoff_t end) -{ - pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE, - pvec->pages); - return pagevec_count(pvec); -} -EXPORT_SYMBOL(pagevec_lookup_range); - unsigned pagevec_lookup_range_tag(struct pagevec *pvec, struct address_space *mapping, pgoff_t *index, pgoff_t end, xa_mark_t tag) -- cgit v1.2.3 From 0e8e08cca5e3256a6209f02b482bee96fb91ba1b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 29 Apr 2022 08:49:28 -0400 Subject: netfs: Remove extern from function prototypes The 'extern' keyword is not necessary and removing it lets us shorten some lines. Signed-off-by: Matthew Wilcox (Oracle) --- include/linux/netfs.h | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 1773e5df8e65..11df38d03359 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -276,19 +276,18 @@ struct netfs_cache_ops { }; struct readahead_control; -extern void netfs_readahead(struct readahead_control *); +void netfs_readahead(struct readahead_control *); int netfs_read_folio(struct file *, struct folio *); -extern int netfs_write_begin(struct netfs_inode *, - struct file *, struct address_space *, - loff_t, unsigned int, struct folio **, - void **); - -extern void netfs_subreq_terminated(struct netfs_io_subrequest *, ssize_t, bool); -extern void netfs_get_subrequest(struct netfs_io_subrequest *subreq, - enum netfs_sreq_ref_trace what); -extern void netfs_put_subrequest(struct netfs_io_subrequest *subreq, - bool was_async, enum netfs_sreq_ref_trace what); -extern void netfs_stats_show(struct seq_file *); +int netfs_write_begin(struct netfs_inode *, struct file *, + struct address_space *, loff_t pos, unsigned int len, + struct folio **, void **fsdata); + +void netfs_subreq_terminated(struct netfs_io_subrequest *, ssize_t, bool); +void netfs_get_subrequest(struct netfs_io_subrequest *subreq, + enum netfs_sreq_ref_trace what); +void netfs_put_subrequest(struct netfs_io_subrequest *subreq, + bool was_async, enum netfs_sreq_ref_trace what); +void netfs_stats_show(struct seq_file *); /** * netfs_inode - Get the netfs inode context from the inode -- cgit v1.2.3 From 6a27d28c81bc5843de2490688a04ee5baa6615e7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 29 Jun 2022 08:20:12 +0200 Subject: block: move ->ia_ranges from the request_queue to the gendisk Independent access ranges only matter for file system I/O and are only valid with a registered gendisk, so move them there. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Tested-by: Damien Le Moal Link: https://lore.kernel.org/r/20220629062013.1331068-2-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-ia-ranges.c | 18 +++++++++--------- include/linux/blkdev.h | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/block/blk-ia-ranges.c b/block/blk-ia-ranges.c index 47c89e65b57f..c1bf14bcd15f 100644 --- a/block/blk-ia-ranges.c +++ b/block/blk-ia-ranges.c @@ -106,7 +106,7 @@ static struct kobj_type blk_ia_ranges_ktype = { * * Register with sysfs a set of independent access ranges for @disk. * If @new_iars is not NULL, this set of ranges is registered and the old set - * specified by q->ia_ranges is unregistered. Otherwise, q->ia_ranges is + * specified by disk->ia_ranges is unregistered. Otherwise, disk->ia_ranges is * registered if it is not already. */ int disk_register_independent_access_ranges(struct gendisk *disk, @@ -121,12 +121,12 @@ int disk_register_independent_access_ranges(struct gendisk *disk, /* If a new range set is specified, unregister the old one */ if (new_iars) { - if (q->ia_ranges) + if (disk->ia_ranges) disk_unregister_independent_access_ranges(disk); - q->ia_ranges = new_iars; + disk->ia_ranges = new_iars; } - iars = q->ia_ranges; + iars = disk->ia_ranges; if (!iars) return 0; @@ -138,7 +138,7 @@ int disk_register_independent_access_ranges(struct gendisk *disk, ret = kobject_init_and_add(&iars->kobj, &blk_ia_ranges_ktype, &q->kobj, "%s", "independent_access_ranges"); if (ret) { - q->ia_ranges = NULL; + disk->ia_ranges = NULL; kobject_put(&iars->kobj); return ret; } @@ -164,7 +164,7 @@ int disk_register_independent_access_ranges(struct gendisk *disk, void disk_unregister_independent_access_ranges(struct gendisk *disk) { struct request_queue *q = disk->queue; - struct blk_independent_access_ranges *iars = q->ia_ranges; + struct blk_independent_access_ranges *iars = disk->ia_ranges; int i; lockdep_assert_held(&q->sysfs_dir_lock); @@ -182,7 +182,7 @@ void disk_unregister_independent_access_ranges(struct gendisk *disk) kfree(iars); } - q->ia_ranges = NULL; + disk->ia_ranges = NULL; } static struct blk_independent_access_range * @@ -242,7 +242,7 @@ static bool disk_check_ia_ranges(struct gendisk *disk, static bool disk_ia_ranges_changed(struct gendisk *disk, struct blk_independent_access_ranges *new) { - struct blk_independent_access_ranges *old = disk->queue->ia_ranges; + struct blk_independent_access_ranges *old = disk->ia_ranges; int i; if (!old) @@ -331,7 +331,7 @@ reg: if (blk_queue_registered(q)) { disk_register_independent_access_ranges(disk, iars); } else { - swap(q->ia_ranges, iars); + swap(disk->ia_ranges, iars); kfree(iars); } diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 22b12531aeb7..b9a94c53c6cd 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -171,6 +171,12 @@ struct gendisk { struct badblocks *bb; struct lockdep_map lockdep_map; u64 diskseq; + + /* + * Independent sector access ranges. This is always NULL for + * devices that do not have multiple independent access ranges. + */ + struct blk_independent_access_ranges *ia_ranges; }; static inline bool disk_live(struct gendisk *disk) @@ -539,12 +545,6 @@ struct request_queue { bool mq_sysfs_init_done; - /* - * Independent sector access ranges. This is always NULL for - * devices that do not have multiple independent access ranges. - */ - struct blk_independent_access_ranges *ia_ranges; - /** * @srcu: Sleepable RCU. Use as lock when type of the request queue * is blocking (BLK_MQ_F_BLOCKING). Must be the last member -- cgit v1.2.3 From 445cbd219ac3b8f451153c210aaf97adcbf4bd02 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:09 +0100 Subject: regmap-irq: Convert bool bitfields to unsigned int Use 'unsigned int' for bitfields for consistency with most other kernel code. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-2-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 2 +- include/linux/regmap.h | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index a6db605707b0..a58b29e9c7c7 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -43,7 +43,7 @@ struct regmap_irq_chip_data { unsigned int irq_reg_stride; unsigned int type_reg_stride; - bool clear_status:1; + unsigned int clear_status:1; }; static int sub_irq_reg(struct regmap_irq_chip_data *data, diff --git a/include/linux/regmap.h b/include/linux/regmap.h index d5b08f4f0dc0..bcbc5d4ffd30 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1534,19 +1534,19 @@ struct regmap_irq_chip { unsigned int type_base; unsigned int *virt_reg_base; unsigned int irq_reg_stride; - bool mask_writeonly:1; - bool init_ack_masked:1; - bool mask_invert:1; - bool use_ack:1; - bool ack_invert:1; - bool clear_ack:1; - bool wake_invert:1; - bool runtime_pm:1; - bool type_invert:1; - bool type_in_mask:1; - bool clear_on_unmask:1; - bool not_fixed_stride:1; - bool status_invert:1; + unsigned int mask_writeonly:1; + unsigned int init_ack_masked:1; + unsigned int mask_invert:1; + unsigned int use_ack:1; + unsigned int ack_invert:1; + unsigned int clear_ack:1; + unsigned int wake_invert:1; + unsigned int runtime_pm:1; + unsigned int type_invert:1; + unsigned int type_in_mask:1; + unsigned int clear_on_unmask:1; + unsigned int not_fixed_stride:1; + unsigned int status_invert:1; int num_regs; -- cgit v1.2.3 From 53a1a16dcc972163bd5816192d5d63ae433c9e56 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:10 +0100 Subject: regmap-irq: Remove unused type_reg_stride field It appears that no chip ever required a nonzero type_reg_stride and commit 1066cfbdfa3f ("regmap-irq: Extend sub-irq to support non-fixed reg strides") broke support. Just remove the field. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-3-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 6 ------ include/linux/regmap.h | 3 --- 2 files changed, 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index a58b29e9c7c7..475a959e2b8b 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -41,7 +41,6 @@ struct regmap_irq_chip_data { unsigned int **virt_buf; unsigned int irq_reg_stride; - unsigned int type_reg_stride; unsigned int clear_status:1; }; @@ -743,11 +742,6 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, else d->irq_reg_stride = 1; - if (chip->type_reg_stride) - d->type_reg_stride = chip->type_reg_stride; - else - d->type_reg_stride = 1; - if (!map->use_single_read && map->reg_stride == 1 && d->irq_reg_stride == 1) { d->status_reg_buf = kmalloc_array(chip->num_regs, diff --git a/include/linux/regmap.h b/include/linux/regmap.h index bcbc5d4ffd30..58db2206f193 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1503,8 +1503,6 @@ struct regmap_irq_sub_irq_map { * @num_type_reg: Number of type registers. * @num_virt_regs: Number of non-standard irq configuration registers. * If zero unsupported. - * @type_reg_stride: Stride to use for chips where type registers are not - * contiguous. * @handle_pre_irq: Driver specific callback to handle interrupt from device * before regmap_irq_handler process the interrupts. * @handle_post_irq: Driver specific callback to handle interrupt from device @@ -1555,7 +1553,6 @@ struct regmap_irq_chip { int num_type_reg; int num_virt_regs; - unsigned int type_reg_stride; int (*handle_pre_irq)(void *irq_drv_data); int (*handle_post_irq)(void *irq_drv_data); -- cgit v1.2.3 From 610fdd668e6af48fcae7908161d14eee3a95ec92 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:12 +0100 Subject: regmap-irq: Remove an unnecessary restriction on type_in_mask Check types_supported instead of checking type_rising/falling_val when using type_in_mask interrupts. This makes the intent clearer and allows a type_in_mask irq to support level or edge triggers, rather than only edge triggers. Update the documentation and comments to reflect the new behavior. This shouldn't affect existing drivers, because if they didn't set types_supported properly the type buffer wouldn't be updated. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-5-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 19 ++++++++----------- include/linux/regmap.h | 8 +++++--- 2 files changed, 13 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index dca27b4e29d3..fd7c4315d16b 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -252,22 +252,19 @@ static void regmap_irq_enable(struct irq_data *data) struct regmap *map = d->map; const struct regmap_irq *irq_data = irq_to_regmap_irq(d, data->hwirq); unsigned int reg = irq_data->reg_offset / map->reg_stride; - unsigned int mask, type; - - type = irq_data->type.type_falling_val | irq_data->type.type_rising_val; + unsigned int mask; /* * The type_in_mask flag means that the underlying hardware uses - * separate mask bits for rising and falling edge interrupts, but - * we want to make them into a single virtual interrupt with - * configurable edge. + * separate mask bits for each interrupt trigger type, but we want + * to have a single logical interrupt with a configurable type. * - * If the interrupt we're enabling defines the falling or rising - * masks then instead of using the regular mask bits for this - * interrupt, use the value previously written to the type buffer - * at the corresponding offset in regmap_irq_set_type(). + * If the interrupt we're enabling defines any supported types + * then instead of using the regular mask bits for this interrupt, + * use the value previously written to the type buffer at the + * corresponding offset in regmap_irq_set_type(). */ - if (d->chip->type_in_mask && type) + if (d->chip->type_in_mask && irq_data->type.types_supported) mask = d->type_buf[reg] & irq_data->mask; else mask = irq_data->mask; diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 58db2206f193..b404c59b3b20 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1484,9 +1484,11 @@ struct regmap_irq_sub_irq_map { * @clear_ack: Use this to set 1 and 0 or vice-versa to clear interrupts. * @wake_invert: Inverted wake register: cleared bits are wake enabled. * @type_invert: Invert the type flags. - * @type_in_mask: Use the mask registers for controlling irq type. For - * interrupts defining type_rising/falling_mask use mask_base - * for edge configuration and never update bits in type_base. + * @type_in_mask: Use the mask registers for controlling irq type. Use this if + * the hardware provides separate bits for rising/falling edge + * or low/high level interrupts and they should be combined into + * a single logical interrupt. Use &struct regmap_irq_type data + * to define the mask bit for each irq type. * @clear_on_unmask: For chips with interrupts cleared on read: read the status * registers before unmasking interrupts to clear any bits * set when they were masked. -- cgit v1.2.3 From ad22b3e98f9430896bd4bd8f4fbff4667f02a0c8 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:14 +0100 Subject: regmap-irq: Remove mask_writeonly and regmap_irq_update_bits() Commit a71411dbf6c8 ("regmap: irq: add chip option mask_writeonly") introduced the mask_writeonly option, but it isn't used now and it appears it's never been used by any in-tree drivers. The motivation for the option is mentioned in the commit message, Some irq controllers have writeonly/multipurpose register layouts. In those cases we read invalid data back. [...] The option causes mask register updates to use regmap_write_bits() instead of regmap_update_bits(). However, regmap_write_bits() doesn't solve the reading invalid data problem. It's still a read-modify-write op like regmap_update_bits(). The difference is that 'update bits' will only write the new value if it is different from the current value, while 'write bits' will write the new value unconditionally, even if it's the same as the current value. This seems like a bit of a specialized use case and probably isn't that useful for regmap-irq, so let's just remove the option and go back to using an 'update bits' op for the mask registers. We can always add the option back if some driver ends up needing it in the future. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-7-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 24 +++++++----------------- include/linux/regmap.h | 2 -- 2 files changed, 7 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index cb20ce6f91e7..7e93dd8af56b 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -80,16 +80,6 @@ static void regmap_irq_lock(struct irq_data *data) mutex_lock(&d->lock); } -static int regmap_irq_update_bits(struct regmap_irq_chip_data *d, - unsigned int reg, unsigned int mask, - unsigned int val) -{ - if (d->chip->mask_writeonly) - return regmap_write_bits(d->map, reg, mask, val); - else - return regmap_update_bits(d->map, reg, mask, val); -} - static void regmap_irq_sync_unlock(struct irq_data *data) { struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data); @@ -130,11 +120,11 @@ static void regmap_irq_sync_unlock(struct irq_data *data) reg = sub_irq_reg(d, d->chip->mask_base, i); if (d->chip->mask_invert) { - ret = regmap_irq_update_bits(d, reg, + ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], ~d->mask_buf[i]); } else if (d->chip->unmask_base) { /* set mask with mask_base register */ - ret = regmap_irq_update_bits(d, reg, + ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], ~d->mask_buf[i]); if (ret < 0) dev_err(d->map->dev, @@ -143,12 +133,12 @@ static void regmap_irq_sync_unlock(struct irq_data *data) unmask_offset = d->chip->unmask_base - d->chip->mask_base; /* clear mask with unmask_base register */ - ret = regmap_irq_update_bits(d, + ret = regmap_update_bits(d->map, reg + unmask_offset, d->mask_buf_def[i], d->mask_buf[i]); } else { - ret = regmap_irq_update_bits(d, reg, + ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], d->mask_buf[i]); } if (ret != 0) @@ -763,17 +753,17 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, reg = sub_irq_reg(d, d->chip->mask_base, i); if (chip->mask_invert) - ret = regmap_irq_update_bits(d, reg, + ret = regmap_update_bits(d->map, reg, d->mask_buf[i], ~d->mask_buf[i]); else if (d->chip->unmask_base) { unmask_offset = d->chip->unmask_base - d->chip->mask_base; - ret = regmap_irq_update_bits(d, + ret = regmap_update_bits(d->map, reg + unmask_offset, d->mask_buf[i], d->mask_buf[i]); } else - ret = regmap_irq_update_bits(d, reg, + ret = regmap_update_bits(d->map, reg, d->mask_buf[i], d->mask_buf[i]); if (ret != 0) { dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", diff --git a/include/linux/regmap.h b/include/linux/regmap.h index b404c59b3b20..6489b3610362 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1468,7 +1468,6 @@ struct regmap_irq_sub_irq_map { * * @status_base: Base status register address. * @mask_base: Base mask register address. - * @mask_writeonly: Base mask register is write only. * @unmask_base: Base unmask register address. for chips who have * separate mask and unmask registers * @ack_base: Base ack address. If zero then the chip is clear on read. @@ -1534,7 +1533,6 @@ struct regmap_irq_chip { unsigned int type_base; unsigned int *virt_reg_base; unsigned int irq_reg_stride; - unsigned int mask_writeonly:1; unsigned int init_ack_masked:1; unsigned int mask_invert:1; unsigned int use_ack:1; -- cgit v1.2.3 From faa87ce9196dbb074d75bd4aecb8bacf18f19b4e Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:16 +0100 Subject: regmap-irq: Introduce config registers for irq types Config registers provide a more uniform approach to handling irq type registers. They are essentially an extension of the virtual registers used by the qcom-pm8008 driver. Config registers can be represented as a 2D array: config_base[0] reg0,0 reg0,1 reg0,2 reg0,3 config_base[1] reg1,0 reg1,1 reg1,2 reg1,3 config_base[2] reg2,0 reg2,1 reg2,2 reg2,3 There are 'num_config_bases' base registers, each of which is used to address 'num_config_regs' registers. The addresses are calculated in the same way as for other bases. It is assumed that an irq's type is controlled by one column of registers; that column is identified by the irq's 'type_reg_offset'. The set_type_config() callback is responsible for updating the config register contents. It receives an array of buffers (each represents a row of registers) and the index of the column to update, along with the 'struct regmap_irq' description and requested irq type. Buffered values are written to registers in regmap_irq_sync_unlock(). Note that the entire register contents are overwritten, which is a minor change in behavior from type registers via 'type_base'. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-9-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 115 +++++++++++++++++++++++++++++++++++++-- include/linux/regmap.h | 12 ++++ 2 files changed, 122 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 5f9a5856c45e..e3dbf55a561f 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -39,6 +39,7 @@ struct regmap_irq_chip_data { unsigned int *type_buf; unsigned int *type_buf_def; unsigned int **virt_buf; + unsigned int **config_buf; unsigned int irq_reg_stride; @@ -228,6 +229,17 @@ static void regmap_irq_sync_unlock(struct irq_data *data) } } + for (i = 0; i < d->chip->num_config_bases; i++) { + for (j = 0; j < d->chip->num_config_regs; j++) { + reg = sub_irq_reg(d, d->chip->config_base[i], j); + ret = regmap_write(map, reg, d->config_buf[i][j]); + if (ret) + dev_err(d->map->dev, + "Failed to write config %x: %d\n", + reg, ret); + } + } + if (d->chip->runtime_pm) pm_runtime_put(map->dev); @@ -287,7 +299,7 @@ static int regmap_irq_set_type(struct irq_data *data, unsigned int type) struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data); struct regmap *map = d->map; const struct regmap_irq *irq_data = irq_to_regmap_irq(d, data->hwirq); - int reg; + int reg, ret; const struct regmap_irq_type *t = &irq_data->type; if ((t->types_supported & type) != type) @@ -327,9 +339,19 @@ static int regmap_irq_set_type(struct irq_data *data, unsigned int type) return -EINVAL; } - if (d->chip->set_type_virt) - return d->chip->set_type_virt(d->virt_buf, type, data->hwirq, - reg); + if (d->chip->set_type_virt) { + ret = d->chip->set_type_virt(d->virt_buf, type, data->hwirq, + reg); + if (ret) + return ret; + } + + if (d->chip->set_type_config) { + ret = d->chip->set_type_config(d->config_buf, type, + irq_data, reg); + if (ret) + return ret; + } return 0; } @@ -599,6 +621,61 @@ static const struct irq_domain_ops regmap_domain_ops = { .xlate = irq_domain_xlate_onetwocell, }; +/** + * regmap_irq_set_type_config_simple() - Simple IRQ type configuration callback. + * @buf: Buffer containing configuration register values, this is a 2D array of + * `num_config_bases` rows, each of `num_config_regs` elements. + * @type: The requested IRQ type. + * @irq_data: The IRQ being configured. + * @idx: Index of the irq's config registers within each array `buf[i]` + * + * This is a &struct regmap_irq_chip->set_type_config callback suitable for + * chips with one config register. Register values are updated according to + * the &struct regmap_irq_type data associated with an IRQ. + */ +int regmap_irq_set_type_config_simple(unsigned int **buf, unsigned int type, + const struct regmap_irq *irq_data, int idx) +{ + const struct regmap_irq_type *t = &irq_data->type; + + if (t->type_reg_mask) + buf[0][idx] &= ~t->type_reg_mask; + else + buf[0][idx] &= ~(t->type_falling_val | + t->type_rising_val | + t->type_level_low_val | + t->type_level_high_val); + + switch (type) { + case IRQ_TYPE_EDGE_FALLING: + buf[0][idx] |= t->type_falling_val; + break; + + case IRQ_TYPE_EDGE_RISING: + buf[0][idx] |= t->type_rising_val; + break; + + case IRQ_TYPE_EDGE_BOTH: + buf[0][idx] |= (t->type_falling_val | + t->type_rising_val); + break; + + case IRQ_TYPE_LEVEL_HIGH: + buf[0][idx] |= t->type_level_high_val; + break; + + case IRQ_TYPE_LEVEL_LOW: + buf[0][idx] |= t->type_level_low_val; + break; + + default: + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL_GPL(regmap_irq_set_type_config_simple); + /** * regmap_add_irq_chip_fwnode() - Use standard regmap IRQ controller handling * @@ -724,6 +801,24 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, } } + if (chip->num_config_bases && chip->num_config_regs) { + /* + * Create config_buf[num_config_bases][num_config_regs] + */ + d->config_buf = kcalloc(chip->num_config_bases, + sizeof(*d->config_buf), GFP_KERNEL); + if (!d->config_buf) + goto err_alloc; + + for (i = 0; i < chip->num_config_regs; i++) { + d->config_buf[i] = kcalloc(chip->num_config_regs, + sizeof(**d->config_buf), + GFP_KERNEL); + if (!d->config_buf[i]) + goto err_alloc; + } + } + d->irq_chip = regmap_irq_chip; d->irq_chip.name = chip->name; d->irq = irq; @@ -894,6 +989,11 @@ err_alloc: kfree(d->virt_buf[i]); kfree(d->virt_buf); } + if (d->config_buf) { + for (i = 0; i < chip->num_config_bases; i++) + kfree(d->config_buf[i]); + kfree(d->config_buf); + } kfree(d); return ret; } @@ -934,7 +1034,7 @@ EXPORT_SYMBOL_GPL(regmap_add_irq_chip); void regmap_del_irq_chip(int irq, struct regmap_irq_chip_data *d) { unsigned int virq; - int hwirq; + int i, hwirq; if (!d) return; @@ -964,6 +1064,11 @@ void regmap_del_irq_chip(int irq, struct regmap_irq_chip_data *d) kfree(d->mask_buf); kfree(d->status_reg_buf); kfree(d->status_buf); + if (d->config_buf) { + for (i = 0; i < d->chip->num_config_bases; i++) + kfree(d->config_buf[i]); + kfree(d->config_buf); + } kfree(d); } EXPORT_SYMBOL_GPL(regmap_del_irq_chip); diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 6489b3610362..791c15ad1f63 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1475,6 +1475,7 @@ struct regmap_irq_sub_irq_map { * @wake_base: Base address for wake enables. If zero unsupported. * @type_base: Base address for irq type. If zero unsupported. * @virt_reg_base: Base addresses for extra config regs. + * @config_base: Base address for IRQ type config regs. If null unsupported. * @irq_reg_stride: Stride to use for chips where registers are not contiguous. * @init_ack_masked: Ack all masked interrupts once during initalization. * @mask_invert: Inverted mask register: cleared bits are masked out. @@ -1504,12 +1505,15 @@ struct regmap_irq_sub_irq_map { * @num_type_reg: Number of type registers. * @num_virt_regs: Number of non-standard irq configuration registers. * If zero unsupported. + * @num_config_bases: Number of config base registers. + * @num_config_regs: Number of config registers for each config base register. * @handle_pre_irq: Driver specific callback to handle interrupt from device * before regmap_irq_handler process the interrupts. * @handle_post_irq: Driver specific callback to handle interrupt from device * after handling the interrupts in regmap_irq_handler(). * @set_type_virt: Driver specific callback to extend regmap_irq_set_type() * and configure virt regs. + * @set_type_config: Callback used for configuring irq types. * @irq_drv_data: Driver specific IRQ data which is passed as parameter when * driver specific pre/post interrupt handler is called. * @@ -1532,6 +1536,7 @@ struct regmap_irq_chip { unsigned int wake_base; unsigned int type_base; unsigned int *virt_reg_base; + const unsigned int *config_base; unsigned int irq_reg_stride; unsigned int init_ack_masked:1; unsigned int mask_invert:1; @@ -1553,16 +1558,23 @@ struct regmap_irq_chip { int num_type_reg; int num_virt_regs; + int num_config_bases; + int num_config_regs; int (*handle_pre_irq)(void *irq_drv_data); int (*handle_post_irq)(void *irq_drv_data); int (*set_type_virt)(unsigned int **buf, unsigned int type, unsigned long hwirq, int reg); + int (*set_type_config)(unsigned int **buf, unsigned int type, + const struct regmap_irq *irq_data, int idx); void *irq_drv_data; }; struct regmap_irq_chip_data; +int regmap_irq_set_type_config_simple(unsigned int **buf, unsigned int type, + const struct regmap_irq *irq_data, int idx); + int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, int irq_base, const struct regmap_irq_chip *chip, struct regmap_irq_chip_data **data); -- cgit v1.2.3 From 9edd4f5aee8470dcfd0db04005908f61fbfae8e0 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:17 +0100 Subject: regmap-irq: Deprecate type registers and virtual registers Config registers can be used to replace both type and virtual registers, so mark both features are deprecated and issue a warning if they're used. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-10-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 6 ++++++ include/linux/regmap.h | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index e3dbf55a561f..8cbc62c3d638 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -726,6 +726,12 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, return -EINVAL; } + if (chip->num_type_reg) + dev_warn(map->dev, "type registers are deprecated; use config registers instead"); + + if (chip->num_virt_regs || chip->virt_reg_base || chip->set_type_virt) + dev_warn(map->dev, "virtual registers are deprecated; use config registers instead"); + if (irq_base) { irq_base = irq_alloc_descs(irq_base, 0, chip->num_irqs, 0); if (irq_base < 0) { diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 791c15ad1f63..c9a9bbbb6261 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1473,8 +1473,10 @@ struct regmap_irq_sub_irq_map { * @ack_base: Base ack address. If zero then the chip is clear on read. * Using zero value is possible with @use_ack bit. * @wake_base: Base address for wake enables. If zero unsupported. - * @type_base: Base address for irq type. If zero unsupported. - * @virt_reg_base: Base addresses for extra config regs. + * @type_base: Base address for irq type. If zero unsupported. Deprecated, + * use @config_base instead. + * @virt_reg_base: Base addresses for extra config regs. Deprecated, use + * @config_base instead. * @config_base: Base address for IRQ type config regs. If null unsupported. * @irq_reg_stride: Stride to use for chips where registers are not contiguous. * @init_ack_masked: Ack all masked interrupts once during initalization. @@ -1483,7 +1485,8 @@ struct regmap_irq_sub_irq_map { * @ack_invert: Inverted ack register: cleared bits for ack. * @clear_ack: Use this to set 1 and 0 or vice-versa to clear interrupts. * @wake_invert: Inverted wake register: cleared bits are wake enabled. - * @type_invert: Invert the type flags. + * @type_invert: Invert the type flags. Deprecated, use config registers + * instead. * @type_in_mask: Use the mask registers for controlling irq type. Use this if * the hardware provides separate bits for rising/falling edge * or low/high level interrupts and they should be combined into @@ -1502,9 +1505,11 @@ struct regmap_irq_sub_irq_map { * @irqs: Descriptors for individual IRQs. Interrupt numbers are * assigned based on the index in the array of the interrupt. * @num_irqs: Number of descriptors. - * @num_type_reg: Number of type registers. + * @num_type_reg: Number of type registers. Deprecated, use config registers + * instead. * @num_virt_regs: Number of non-standard irq configuration registers. - * If zero unsupported. + * If zero unsupported. Deprecated, use config registers + * instead. * @num_config_bases: Number of config base registers. * @num_config_regs: Number of config registers for each config base register. * @handle_pre_irq: Driver specific callback to handle interrupt from device @@ -1512,7 +1517,8 @@ struct regmap_irq_sub_irq_map { * @handle_post_irq: Driver specific callback to handle interrupt from device * after handling the interrupts in regmap_irq_handler(). * @set_type_virt: Driver specific callback to extend regmap_irq_set_type() - * and configure virt regs. + * and configure virt regs. Deprecated, use @set_type_config + * callback and config registers instead. * @set_type_config: Callback used for configuring irq types. * @irq_drv_data: Driver specific IRQ data which is passed as parameter when * driver specific pre/post interrupt handler is called. -- cgit v1.2.3 From e8ffb12e7f065db616a3eba79ff138bececf0825 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:18 +0100 Subject: regmap-irq: Fix inverted handling of unmask registers To me "unmask" suggests that we write 1s to the register when an interrupt is enabled. This also makes sense because it's the opposite of what the "mask" register does (write 1s to disable an interrupt). But regmap-irq does the opposite: for a disabled interrupt, it writes 1s to "unmask" and 0s to "mask". This is surprising and deviates from the usual way mask registers are handled. Additionally, mask_invert didn't interact with unmask registers properly -- it caused them to be ignored entirely. Fix this by making mask and unmask registers orthogonal, using the following behavior: * Mask registers are written with 1s for disabled interrupts. * Unmask registers are written with 1s for enabled interrupts. This behavior supports both normal or inverted mask registers and separate set/clear registers via different combinations of mask_base/unmask_base. The old unmask register behavior is deprecated. Drivers need to opt-in to the new behavior by setting mask_unmask_non_inverted. Warnings are issued if the driver relies on deprecated behavior. Chips that only set one of mask_base/unmask_base don't have to use the mask_unmask_non_inverted flag because that use case was previously not supported. The mask_invert flag is also deprecated in favor of describing inverted mask registers as unmask registers. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-11-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 114 +++++++++++++++++++++++---------------- include/linux/regmap.h | 18 +++++-- 2 files changed, 84 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 8cbc62c3d638..2c724ae185c4 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -30,6 +30,9 @@ struct regmap_irq_chip_data { int irq; int wake_count; + unsigned int mask_base; + unsigned int unmask_base; + void *status_reg_buf; unsigned int *main_status_buf; unsigned int *status_buf; @@ -95,7 +98,6 @@ static void regmap_irq_sync_unlock(struct irq_data *data) struct regmap *map = d->map; int i, j, ret; u32 reg; - u32 unmask_offset; u32 val; if (d->chip->runtime_pm) { @@ -124,35 +126,23 @@ static void regmap_irq_sync_unlock(struct irq_data *data) * suppress pointless writes. */ for (i = 0; i < d->chip->num_regs; i++) { - if (!d->chip->mask_base) - continue; - - reg = sub_irq_reg(d, d->chip->mask_base, i); - if (d->chip->mask_invert) { + if (d->mask_base) { + reg = sub_irq_reg(d, d->mask_base, i); ret = regmap_update_bits(d->map, reg, - d->mask_buf_def[i], ~d->mask_buf[i]); - } else if (d->chip->unmask_base) { - /* set mask with mask_base register */ + d->mask_buf_def[i], d->mask_buf[i]); + if (ret) + dev_err(d->map->dev, "Failed to sync masks in %x\n", + reg); + } + + if (d->unmask_base) { + reg = sub_irq_reg(d, d->unmask_base, i); ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], ~d->mask_buf[i]); - if (ret < 0) - dev_err(d->map->dev, - "Failed to sync unmasks in %x\n", + if (ret) + dev_err(d->map->dev, "Failed to sync masks in %x\n", reg); - unmask_offset = d->chip->unmask_base - - d->chip->mask_base; - /* clear mask with unmask_base register */ - ret = regmap_update_bits(d->map, - reg + unmask_offset, - d->mask_buf_def[i], - d->mask_buf[i]); - } else { - ret = regmap_update_bits(d->map, reg, - d->mask_buf_def[i], d->mask_buf[i]); } - if (ret != 0) - dev_err(d->map->dev, "Failed to sync masks in %x\n", - reg); reg = sub_irq_reg(d, d->chip->wake_base, i); if (d->wake_buf) { @@ -704,7 +694,6 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, int ret = -ENOMEM; int num_type_reg; u32 reg; - u32 unmask_offset; if (chip->num_regs <= 0) return -EINVAL; @@ -832,6 +821,42 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, d->chip = chip; d->irq_base = irq_base; + if (chip->mask_base && chip->unmask_base && + !chip->mask_unmask_non_inverted) { + /* + * Chips that specify both mask_base and unmask_base used to + * get inverted mask behavior by default, with no way to ask + * for the normal, non-inverted behavior. This "inverted by + * default" behavior is deprecated, but we have to support it + * until existing drivers have been fixed. + * + * Existing drivers should be updated by swapping mask_base + * and unmask_base and setting mask_unmask_non_inverted=true. + * New drivers should always set the flag. + */ + dev_warn(map->dev, "mask_base and unmask_base are inverted, please fix it"); + + /* Might as well warn about mask_invert while we're at it... */ + if (chip->mask_invert) + dev_warn(map->dev, "mask_invert=true ignored"); + + d->mask_base = chip->unmask_base; + d->unmask_base = chip->mask_base; + } else if (chip->mask_invert) { + /* + * Swap the roles of mask_base and unmask_base if the bits are + * inverted. This is deprecated, drivers should use unmask_base + * directly. + */ + dev_warn(map->dev, "mask_invert=true is deprecated; please switch to unmask_base"); + + d->mask_base = chip->unmask_base; + d->unmask_base = chip->mask_base; + } else { + d->mask_base = chip->mask_base; + d->unmask_base = chip->unmask_base; + } + if (chip->irq_reg_stride) d->irq_reg_stride = chip->irq_reg_stride; else @@ -854,28 +879,27 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, /* Mask all the interrupts by default */ for (i = 0; i < chip->num_regs; i++) { d->mask_buf[i] = d->mask_buf_def[i]; - if (!chip->mask_base) - continue; - - reg = sub_irq_reg(d, d->chip->mask_base, i); - if (chip->mask_invert) + if (d->mask_base) { + reg = sub_irq_reg(d, d->mask_base, i); ret = regmap_update_bits(d->map, reg, - d->mask_buf[i], ~d->mask_buf[i]); - else if (d->chip->unmask_base) { - unmask_offset = d->chip->unmask_base - - d->chip->mask_base; - ret = regmap_update_bits(d->map, - reg + unmask_offset, - d->mask_buf[i], - d->mask_buf[i]); - } else + d->mask_buf_def[i], d->mask_buf[i]); + if (ret) { + dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", + reg, ret); + goto err_alloc; + } + } + + if (d->unmask_base) { + reg = sub_irq_reg(d, d->unmask_base, i); ret = regmap_update_bits(d->map, reg, - d->mask_buf[i], d->mask_buf[i]); - if (ret != 0) { - dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", - reg, ret); - goto err_alloc; + d->mask_buf_def[i], ~d->mask_buf[i]); + if (ret) { + dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", + reg, ret); + goto err_alloc; + } } if (!chip->init_ack_masked) diff --git a/include/linux/regmap.h b/include/linux/regmap.h index c9a9bbbb6261..c9c9e39750ed 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1467,9 +1467,10 @@ struct regmap_irq_sub_irq_map { * main_status set. * * @status_base: Base status register address. - * @mask_base: Base mask register address. - * @unmask_base: Base unmask register address. for chips who have - * separate mask and unmask registers + * @mask_base: Base mask register address. Mask bits are set to 1 when an + * interrupt is masked, 0 when unmasked. + * @unmask_base: Base unmask register address. Unmask bits are set to 1 when + * an interrupt is unmasked and 0 when masked. * @ack_base: Base ack address. If zero then the chip is clear on read. * Using zero value is possible with @use_ack bit. * @wake_base: Base address for wake enables. If zero unsupported. @@ -1481,6 +1482,16 @@ struct regmap_irq_sub_irq_map { * @irq_reg_stride: Stride to use for chips where registers are not contiguous. * @init_ack_masked: Ack all masked interrupts once during initalization. * @mask_invert: Inverted mask register: cleared bits are masked out. + * Deprecated; prefer describing an inverted mask register as + * an unmask register. + * @mask_unmask_non_inverted: Controls mask bit inversion for chips that set + * both @mask_base and @unmask_base. If false, mask and unmask bits are + * inverted (which is deprecated behavior); if true, bits will not be + * inverted and the registers keep their normal behavior. Note that if + * you use only one of @mask_base or @unmask_base, this flag has no + * effect and is unnecessary. Any new drivers that set both @mask_base + * and @unmask_base should set this to true to avoid relying on the + * deprecated behavior. * @use_ack: Use @ack register even if it is zero. * @ack_invert: Inverted ack register: cleared bits for ack. * @clear_ack: Use this to set 1 and 0 or vice-versa to clear interrupts. @@ -1546,6 +1557,7 @@ struct regmap_irq_chip { unsigned int irq_reg_stride; unsigned int init_ack_masked:1; unsigned int mask_invert:1; + unsigned int mask_unmask_non_inverted:1; unsigned int use_ack:1; unsigned int ack_invert:1; unsigned int clear_ack:1; -- cgit v1.2.3 From bdf9b86cd3adbbcf590ab82b74ab8554534c9b6e Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:19 +0100 Subject: regmap-irq: Add get_irq_reg() callback Replace the internal sub_irq_reg() function with a public callback that drivers can use when they have more complex register layouts. The default implementation is regmap_irq_get_irq_reg_linear(), used if the chip doesn't provide its own callback. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-12-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 126 +++++++++++++++++++++++++-------------- include/linux/regmap.h | 15 ++++- 2 files changed, 93 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 2c724ae185c4..71de097847a7 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -46,30 +46,12 @@ struct regmap_irq_chip_data { unsigned int irq_reg_stride; + unsigned int (*get_irq_reg)(struct regmap_irq_chip_data *data, + unsigned int base, int index); + unsigned int clear_status:1; }; -static int sub_irq_reg(struct regmap_irq_chip_data *data, - unsigned int base_reg, int i) -{ - const struct regmap_irq_chip *chip = data->chip; - struct regmap *map = data->map; - struct regmap_irq_sub_irq_map *subreg; - unsigned int offset; - int reg = 0; - - if (!chip->sub_reg_offsets || !chip->not_fixed_stride) { - /* Assume linear mapping */ - reg = base_reg + (i * map->reg_stride * data->irq_reg_stride); - } else { - subreg = &chip->sub_reg_offsets[i]; - offset = subreg->offset[0]; - reg = base_reg + offset; - } - - return reg; -} - static inline const struct regmap_irq *irq_to_regmap_irq(struct regmap_irq_chip_data *data, int irq) @@ -81,7 +63,13 @@ static bool regmap_irq_can_bulk_read_status(struct regmap_irq_chip_data *data) { struct regmap *map = data->map; + /* + * While possible that a user-defined ->get_irq_reg() callback might + * be linear enough to support bulk reads, most of the time it won't. + * Therefore only allow them if the default callback is being used. + */ return data->irq_reg_stride == 1 && map->reg_stride == 1 && + data->get_irq_reg == regmap_irq_get_irq_reg_linear && !map->use_single_read; } @@ -109,7 +97,7 @@ static void regmap_irq_sync_unlock(struct irq_data *data) if (d->clear_status) { for (i = 0; i < d->chip->num_regs; i++) { - reg = sub_irq_reg(d, d->chip->status_base, i); + reg = d->get_irq_reg(d, d->chip->status_base, i); ret = regmap_read(map, reg, &val); if (ret) @@ -127,7 +115,7 @@ static void regmap_irq_sync_unlock(struct irq_data *data) */ for (i = 0; i < d->chip->num_regs; i++) { if (d->mask_base) { - reg = sub_irq_reg(d, d->mask_base, i); + reg = d->get_irq_reg(d, d->mask_base, i); ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], d->mask_buf[i]); if (ret) @@ -136,7 +124,7 @@ static void regmap_irq_sync_unlock(struct irq_data *data) } if (d->unmask_base) { - reg = sub_irq_reg(d, d->unmask_base, i); + reg = d->get_irq_reg(d, d->unmask_base, i); ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], ~d->mask_buf[i]); if (ret) @@ -144,7 +132,7 @@ static void regmap_irq_sync_unlock(struct irq_data *data) reg); } - reg = sub_irq_reg(d, d->chip->wake_base, i); + reg = d->get_irq_reg(d, d->chip->wake_base, i); if (d->wake_buf) { if (d->chip->wake_invert) ret = regmap_update_bits(d->map, reg, @@ -168,7 +156,7 @@ static void regmap_irq_sync_unlock(struct irq_data *data) * it'll be ignored in irq handler, then may introduce irq storm */ if (d->mask_buf[i] && (d->chip->ack_base || d->chip->use_ack)) { - reg = sub_irq_reg(d, d->chip->ack_base, i); + reg = d->get_irq_reg(d, d->chip->ack_base, i); /* some chips ack by write 0 */ if (d->chip->ack_invert) @@ -192,7 +180,7 @@ static void regmap_irq_sync_unlock(struct irq_data *data) for (i = 0; i < d->chip->num_type_reg; i++) { if (!d->type_buf_def[i]) continue; - reg = sub_irq_reg(d, d->chip->type_base, i); + reg = d->get_irq_reg(d, d->chip->type_base, i); if (d->chip->type_invert) ret = regmap_update_bits(d->map, reg, d->type_buf_def[i], ~d->type_buf[i]); @@ -208,8 +196,8 @@ static void regmap_irq_sync_unlock(struct irq_data *data) if (d->chip->num_virt_regs) { for (i = 0; i < d->chip->num_virt_regs; i++) { for (j = 0; j < d->chip->num_regs; j++) { - reg = sub_irq_reg(d, d->chip->virt_reg_base[i], - j); + reg = d->get_irq_reg(d, d->chip->virt_reg_base[i], + j); ret = regmap_write(map, reg, d->virt_buf[i][j]); if (ret != 0) dev_err(d->map->dev, @@ -221,7 +209,7 @@ static void regmap_irq_sync_unlock(struct irq_data *data) for (i = 0; i < d->chip->num_config_bases; i++) { for (j = 0; j < d->chip->num_config_regs; j++) { - reg = sub_irq_reg(d, d->chip->config_base[i], j); + reg = d->get_irq_reg(d, d->chip->config_base[i], j); ret = regmap_write(map, reg, d->config_buf[i][j]); if (ret) dev_err(d->map->dev, @@ -382,14 +370,17 @@ static inline int read_sub_irq_data(struct regmap_irq_chip_data *data, const struct regmap_irq_chip *chip = data->chip; struct regmap *map = data->map; struct regmap_irq_sub_irq_map *subreg; + unsigned int reg; int i, ret = 0; if (!chip->sub_reg_offsets) { - /* Assume linear mapping */ - ret = regmap_read(map, chip->status_base + - (b * map->reg_stride * data->irq_reg_stride), - &data->status_buf[b]); + reg = data->get_irq_reg(data, chip->status_base, b); + ret = regmap_read(map, reg, &data->status_buf[b]); } else { + /* + * Note we can't use ->get_irq_reg() here because the offsets + * in 'subreg' are *not* interchangeable with indices. + */ subreg = &chip->sub_reg_offsets[b]; for (i = 0; i < subreg->num_regs; i++) { unsigned int offset = subreg->offset[i]; @@ -455,10 +446,18 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) * sake of simplicity. and add bulk reads only if needed */ for (i = 0; i < chip->num_main_regs; i++) { - ret = regmap_read(map, chip->main_status + - (i * map->reg_stride - * data->irq_reg_stride), - &data->main_status_buf[i]); + /* + * For not_fixed_stride, don't use ->get_irq_reg(). + * It would produce an incorrect result. + */ + if (data->chip->not_fixed_stride) + reg = chip->main_status + + i * map->reg_stride * data->irq_reg_stride; + else + reg = data->get_irq_reg(data, + chip->main_status, i); + + ret = regmap_read(map, reg, &data->main_status_buf[i]); if (ret) { dev_err(map->dev, "Failed to read IRQ status %d\n", @@ -523,7 +522,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) } else { for (i = 0; i < data->chip->num_regs; i++) { - unsigned int reg = sub_irq_reg(data, + unsigned int reg = data->get_irq_reg(data, data->chip->status_base, i); ret = regmap_read(map, reg, &data->status_buf[i]); @@ -551,7 +550,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) data->status_buf[i] &= ~data->mask_buf[i]; if (data->status_buf[i] && (chip->ack_base || chip->use_ack)) { - reg = sub_irq_reg(data, data->chip->ack_base, i); + reg = data->get_irq_reg(data, data->chip->ack_base, i); if (chip->ack_invert) ret = regmap_write(map, reg, @@ -611,6 +610,36 @@ static const struct irq_domain_ops regmap_domain_ops = { .xlate = irq_domain_xlate_onetwocell, }; +/** + * regmap_irq_get_irq_reg_linear() - Linear IRQ register mapping callback. + * @data: Data for the &struct regmap_irq_chip + * @base: Base register + * @index: Register index + * + * Returns the register address corresponding to the given @base and @index + * by the formula ``base + index * regmap_stride * irq_reg_stride``. + */ +unsigned int regmap_irq_get_irq_reg_linear(struct regmap_irq_chip_data *data, + unsigned int base, int index) +{ + const struct regmap_irq_chip *chip = data->chip; + struct regmap *map = data->map; + + /* + * FIXME: This is for backward compatibility and should be removed + * when not_fixed_stride is dropped (it's only used by qcom-pm8008). + */ + if (chip->not_fixed_stride && chip->sub_reg_offsets) { + struct regmap_irq_sub_irq_map *subreg; + + subreg = &chip->sub_reg_offsets[0]; + return base + subreg->offset[0]; + } + + return base + index * map->reg_stride * chip->irq_reg_stride; +} +EXPORT_SYMBOL_GPL(regmap_irq_get_irq_reg_linear); + /** * regmap_irq_set_type_config_simple() - Simple IRQ type configuration callback. * @buf: Buffer containing configuration register values, this is a 2D array of @@ -862,6 +891,11 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, else d->irq_reg_stride = 1; + if (chip->get_irq_reg) + d->get_irq_reg = chip->get_irq_reg; + else + d->get_irq_reg = regmap_irq_get_irq_reg_linear; + if (regmap_irq_can_bulk_read_status(d)) { d->status_reg_buf = kmalloc_array(chip->num_regs, map->format.val_bytes, @@ -881,7 +915,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, d->mask_buf[i] = d->mask_buf_def[i]; if (d->mask_base) { - reg = sub_irq_reg(d, d->mask_base, i); + reg = d->get_irq_reg(d, d->mask_base, i); ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], d->mask_buf[i]); if (ret) { @@ -892,7 +926,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, } if (d->unmask_base) { - reg = sub_irq_reg(d, d->unmask_base, i); + reg = d->get_irq_reg(d, d->unmask_base, i); ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], ~d->mask_buf[i]); if (ret) { @@ -906,7 +940,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, continue; /* Ack masked but set interrupts */ - reg = sub_irq_reg(d, d->chip->status_base, i); + reg = d->get_irq_reg(d, d->chip->status_base, i); ret = regmap_read(map, reg, &d->status_buf[i]); if (ret != 0) { dev_err(map->dev, "Failed to read IRQ status: %d\n", @@ -918,7 +952,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, d->status_buf[i] = ~d->status_buf[i]; if (d->status_buf[i] && (chip->ack_base || chip->use_ack)) { - reg = sub_irq_reg(d, d->chip->ack_base, i); + reg = d->get_irq_reg(d, d->chip->ack_base, i); if (chip->ack_invert) ret = regmap_write(map, reg, ~(d->status_buf[i] & d->mask_buf[i])); @@ -943,7 +977,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (d->wake_buf) { for (i = 0; i < chip->num_regs; i++) { d->wake_buf[i] = d->mask_buf_def[i]; - reg = sub_irq_reg(d, d->chip->wake_base, i); + reg = d->get_irq_reg(d, d->chip->wake_base, i); if (chip->wake_invert) ret = regmap_update_bits(d->map, reg, @@ -963,7 +997,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (chip->num_type_reg && !chip->type_in_mask) { for (i = 0; i < chip->num_type_reg; ++i) { - reg = sub_irq_reg(d, d->chip->type_base, i); + reg = d->get_irq_reg(d, d->chip->type_base, i); ret = regmap_read(map, reg, &d->type_buf_def[i]); diff --git a/include/linux/regmap.h b/include/linux/regmap.h index c9c9e39750ed..0d240922d4da 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1440,6 +1440,8 @@ struct regmap_irq_sub_irq_map { unsigned int *offset; }; +struct regmap_irq_chip_data; + /** * struct regmap_irq_chip - Description of a generic regmap irq_chip. * @@ -1531,6 +1533,13 @@ struct regmap_irq_sub_irq_map { * and configure virt regs. Deprecated, use @set_type_config * callback and config registers instead. * @set_type_config: Callback used for configuring irq types. + * @get_irq_reg: Callback for mapping (base register, index) pairs to register + * addresses. The base register will be one of @status_base, + * @mask_base, etc., @main_status, or any of @config_base. + * The index will be in the range [0, num_main_regs[ for the + * main status base, [0, num_type_settings[ for any config + * register base, and [0, num_regs[ for any other base. + * If unspecified then regmap_irq_get_irq_reg_linear() is used. * @irq_drv_data: Driver specific IRQ data which is passed as parameter when * driver specific pre/post interrupt handler is called. * @@ -1585,11 +1594,13 @@ struct regmap_irq_chip { unsigned long hwirq, int reg); int (*set_type_config)(unsigned int **buf, unsigned int type, const struct regmap_irq *irq_data, int idx); + unsigned int (*get_irq_reg)(struct regmap_irq_chip_data *data, + unsigned int base, int index); void *irq_drv_data; }; -struct regmap_irq_chip_data; - +unsigned int regmap_irq_get_irq_reg_linear(struct regmap_irq_chip_data *data, + unsigned int base, int index); int regmap_irq_set_type_config_simple(unsigned int **buf, unsigned int type, const struct regmap_irq *irq_data, int idx); -- cgit v1.2.3 From 48e014ee9a61e8f4700987b82f7cb1dc3c89fa76 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 23 Jun 2022 22:14:20 +0100 Subject: regmap-irq: Deprecate the not_fixed_stride flag This flag is a bit of a hack and the same thing can be accomplished using a custom ->get_irq_reg() callback. Add a warning to catch any use of the flag. Signed-off-by: Aidan MacDonald Link: https://lore.kernel.org/r/20220623211420.918875-13-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-irq.c | 2 ++ include/linux/regmap.h | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 71de097847a7..a691553a0d6f 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -739,6 +739,8 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, } if (chip->not_fixed_stride) { + dev_warn(map->dev, "not_fixed_stride is deprecated; use ->get_irq_reg() instead"); + for (i = 0; i < chip->num_regs; i++) if (chip->sub_reg_offsets[i].num_regs != 1) return -EINVAL; diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 0d240922d4da..7cf2157134ac 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1509,8 +1509,10 @@ struct regmap_irq_chip_data; * registers before unmasking interrupts to clear any bits * set when they were masked. * @not_fixed_stride: Used when chip peripherals are not laid out with fixed - * stride. Must be used with sub_reg_offsets containing the - * offsets to each peripheral. + * stride. Must be used with sub_reg_offsets containing the + * offsets to each peripheral. Deprecated; the same thing + * can be accomplished with a @get_irq_reg callback, without + * the need for a @sub_reg_offsets table. * @status_invert: Inverted status register: cleared bits are active interrupts. * @runtime_pm: Hold a runtime PM lock on the device when accessing it. * -- cgit v1.2.3 From ae92b1c84306a68c361d90b46ad3acef99c10e29 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 28 Jun 2022 10:46:24 +0100 Subject: usb: typec_altmode: add a missing "@" at a kernel-doc parameter Without that, the parameter is not properly parsed: include/linux/usb/typec_altmode.h:132: warning: Function parameter or member 'altmode' not described in 'typec_altmode_get_orientation' Signed-off-by: Mauro Carvalho Chehab Acked-by: Heikki Krogerus Link: https://lore.kernel.org/r/70dc4c5d744cf1fe9a0efe6b85deaa0489628282.1656409369.git.mchehab@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/typec_altmode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index 65933cbe9129..350d49012659 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -124,7 +124,7 @@ struct typec_altmode *typec_match_altmode(struct typec_altmode **altmodes, /** * typec_altmode_get_orientation - Get cable plug orientation - * altmode: Handle to the alternate mode + * @altmode: Handle to the alternate mode */ static inline enum typec_orientation typec_altmode_get_orientation(struct typec_altmode *altmode) -- cgit v1.2.3 From af3f4134006bf3bf894179c08aaf98ed5938cf4e Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Tue, 28 Jun 2022 10:43:04 -0700 Subject: bpf: add bpf_func_t and trampoline helpers I'll be adding lsm cgroup specific helpers that grab trampoline mutex. No functional changes. Reviewed-by: Martin KaFai Lau Signed-off-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220628174314.1216643-2-sdf@google.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 11 ++++----- kernel/bpf/trampoline.c | 63 ++++++++++++++++++++++++++----------------------- 2 files changed, 38 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d05e1495a06e..d547be9db75f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -56,6 +56,8 @@ typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); +typedef unsigned int (*bpf_func_t)(const void *, + const struct bpf_insn *); struct bpf_iter_seq_info { const struct seq_operations *seq_ops; bpf_iter_init_seq_priv_t init_seq_private; @@ -879,8 +881,7 @@ struct bpf_dispatcher { static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( const void *ctx, const struct bpf_insn *insnsi, - unsigned int (*bpf_func)(const void *, - const struct bpf_insn *)) + bpf_func_t bpf_func) { return bpf_func(ctx, insnsi); } @@ -909,8 +910,7 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs); noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ - unsigned int (*bpf_func)(const void *, \ - const struct bpf_insn *)) \ + bpf_func_t bpf_func) \ { \ return bpf_func(ctx, insnsi); \ } \ @@ -921,8 +921,7 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs); unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ - unsigned int (*bpf_func)(const void *, \ - const struct bpf_insn *)); \ + bpf_func_t bpf_func); \ extern struct bpf_dispatcher bpf_dispatcher_##name; #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 93c7675f0c9e..5466e15be61f 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -410,7 +410,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) +static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) { enum bpf_tramp_prog_type kind; struct bpf_tramp_link *link_exiting; @@ -418,44 +418,33 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline int cnt = 0, i; kind = bpf_attach_type_to_tramp(link->link.prog); - mutex_lock(&tr->mutex); - if (tr->extension_prog) { + if (tr->extension_prog) /* cannot attach fentry/fexit if extension prog is attached. * cannot overwrite extension prog either. */ - err = -EBUSY; - goto out; - } + return -EBUSY; for (i = 0; i < BPF_TRAMP_MAX; i++) cnt += tr->progs_cnt[i]; if (kind == BPF_TRAMP_REPLACE) { /* Cannot attach extension if fentry/fexit are in use. */ - if (cnt) { - err = -EBUSY; - goto out; - } + if (cnt) + return -EBUSY; tr->extension_prog = link->link.prog; - err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL, - link->link.prog->bpf_func); - goto out; - } - if (cnt >= BPF_MAX_TRAMP_LINKS) { - err = -E2BIG; - goto out; + return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL, + link->link.prog->bpf_func); } - if (!hlist_unhashed(&link->tramp_hlist)) { + if (cnt >= BPF_MAX_TRAMP_LINKS) + return -E2BIG; + if (!hlist_unhashed(&link->tramp_hlist)) /* prog already linked */ - err = -EBUSY; - goto out; - } + return -EBUSY; hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) { if (link_exiting->link.prog != link->link.prog) continue; /* prog already linked */ - err = -EBUSY; - goto out; + return -EBUSY; } hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]); @@ -465,30 +454,44 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline hlist_del_init(&link->tramp_hlist); tr->progs_cnt[kind]--; } -out: + return err; +} + +int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) +{ + int err; + + mutex_lock(&tr->mutex); + err = __bpf_trampoline_link_prog(link, tr); mutex_unlock(&tr->mutex); return err; } -/* bpf_trampoline_unlink_prog() should never fail. */ -int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) +static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) { enum bpf_tramp_prog_type kind; int err; kind = bpf_attach_type_to_tramp(link->link.prog); - mutex_lock(&tr->mutex); if (kind == BPF_TRAMP_REPLACE) { WARN_ON_ONCE(!tr->extension_prog); err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, tr->extension_prog->bpf_func, NULL); tr->extension_prog = NULL; - goto out; + return err; } hlist_del_init(&link->tramp_hlist); tr->progs_cnt[kind]--; - err = bpf_trampoline_update(tr); -out: + return bpf_trampoline_update(tr); +} + +/* bpf_trampoline_unlink_prog() should never fail. */ +int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) +{ + int err; + + mutex_lock(&tr->mutex); + err = __bpf_trampoline_unlink_prog(link, tr); mutex_unlock(&tr->mutex); return err; } -- cgit v1.2.3 From 00442143a2ab7f1da46fbf4d2a99c85df767d49a Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Tue, 28 Jun 2022 10:43:05 -0700 Subject: bpf: convert cgroup_bpf.progs to hlist This lets us reclaim some space to be used by new cgroup lsm slots. Before: struct cgroup_bpf { struct bpf_prog_array * effective[23]; /* 0 184 */ /* --- cacheline 2 boundary (128 bytes) was 56 bytes ago --- */ struct list_head progs[23]; /* 184 368 */ /* --- cacheline 8 boundary (512 bytes) was 40 bytes ago --- */ u32 flags[23]; /* 552 92 */ /* XXX 4 bytes hole, try to pack */ /* --- cacheline 10 boundary (640 bytes) was 8 bytes ago --- */ struct list_head storages; /* 648 16 */ struct bpf_prog_array * inactive; /* 664 8 */ struct percpu_ref refcnt; /* 672 16 */ struct work_struct release_work; /* 688 32 */ /* size: 720, cachelines: 12, members: 7 */ /* sum members: 716, holes: 1, sum holes: 4 */ /* last cacheline: 16 bytes */ }; After: struct cgroup_bpf { struct bpf_prog_array * effective[23]; /* 0 184 */ /* --- cacheline 2 boundary (128 bytes) was 56 bytes ago --- */ struct hlist_head progs[23]; /* 184 184 */ /* --- cacheline 5 boundary (320 bytes) was 48 bytes ago --- */ u8 flags[23]; /* 368 23 */ /* XXX 1 byte hole, try to pack */ /* --- cacheline 6 boundary (384 bytes) was 8 bytes ago --- */ struct list_head storages; /* 392 16 */ struct bpf_prog_array * inactive; /* 408 8 */ struct percpu_ref refcnt; /* 416 16 */ struct work_struct release_work; /* 432 72 */ /* size: 504, cachelines: 8, members: 7 */ /* sum members: 503, holes: 1, sum holes: 1 */ /* last cacheline: 56 bytes */ }; Suggested-by: Jakub Sitnicki Reviewed-by: Jakub Sitnicki Reviewed-by: Martin KaFai Lau Signed-off-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220628174314.1216643-3-sdf@google.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf-cgroup-defs.h | 4 +-- include/linux/bpf-cgroup.h | 2 +- kernel/bpf/cgroup.c | 76 ++++++++++++++++++++++++----------------- 3 files changed, 47 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index 695d1224a71b..5d268e76d8e6 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h @@ -47,8 +47,8 @@ struct cgroup_bpf { * have either zero or one element * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS */ - struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE]; - u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE]; + struct hlist_head progs[MAX_CGROUP_BPF_ATTACH_TYPE]; + u8 flags[MAX_CGROUP_BPF_ATTACH_TYPE]; /* list of cgroup shared storages */ struct list_head storages; diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 669d96d074ad..6673acfbf2ef 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -95,7 +95,7 @@ struct bpf_cgroup_link { }; struct bpf_prog_list { - struct list_head node; + struct hlist_node node; struct bpf_prog *prog; struct bpf_cgroup_link *link; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 7a394f7c205c..4adb4f3ecb7f 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -157,11 +157,12 @@ static void cgroup_bpf_release(struct work_struct *work) mutex_lock(&cgroup_mutex); for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) { - struct list_head *progs = &cgrp->bpf.progs[atype]; - struct bpf_prog_list *pl, *pltmp; + struct hlist_head *progs = &cgrp->bpf.progs[atype]; + struct bpf_prog_list *pl; + struct hlist_node *pltmp; - list_for_each_entry_safe(pl, pltmp, progs, node) { - list_del(&pl->node); + hlist_for_each_entry_safe(pl, pltmp, progs, node) { + hlist_del(&pl->node); if (pl->prog) bpf_prog_put(pl->prog); if (pl->link) @@ -217,12 +218,12 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl) /* count number of elements in the list. * it's slow but the list cannot be long */ -static u32 prog_list_length(struct list_head *head) +static u32 prog_list_length(struct hlist_head *head) { struct bpf_prog_list *pl; u32 cnt = 0; - list_for_each_entry(pl, head, node) { + hlist_for_each_entry(pl, head, node) { if (!prog_list_prog(pl)) continue; cnt++; @@ -291,7 +292,7 @@ static int compute_effective_progs(struct cgroup *cgrp, if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) continue; - list_for_each_entry(pl, &p->bpf.progs[atype], node) { + hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { if (!prog_list_prog(pl)) continue; @@ -342,7 +343,7 @@ int cgroup_bpf_inherit(struct cgroup *cgrp) cgroup_bpf_get(p); for (i = 0; i < NR; i++) - INIT_LIST_HEAD(&cgrp->bpf.progs[i]); + INIT_HLIST_HEAD(&cgrp->bpf.progs[i]); INIT_LIST_HEAD(&cgrp->bpf.storages); @@ -418,7 +419,7 @@ cleanup: #define BPF_CGROUP_MAX_PROGS 64 -static struct bpf_prog_list *find_attach_entry(struct list_head *progs, +static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs, struct bpf_prog *prog, struct bpf_cgroup_link *link, struct bpf_prog *replace_prog, @@ -428,12 +429,12 @@ static struct bpf_prog_list *find_attach_entry(struct list_head *progs, /* single-attach case */ if (!allow_multi) { - if (list_empty(progs)) + if (hlist_empty(progs)) return NULL; - return list_first_entry(progs, typeof(*pl), node); + return hlist_entry(progs->first, typeof(*pl), node); } - list_for_each_entry(pl, progs, node) { + hlist_for_each_entry(pl, progs, node) { if (prog && pl->prog == prog && prog != replace_prog) /* disallow attaching the same prog twice */ return ERR_PTR(-EINVAL); @@ -444,7 +445,7 @@ static struct bpf_prog_list *find_attach_entry(struct list_head *progs, /* direct prog multi-attach w/ replacement case */ if (replace_prog) { - list_for_each_entry(pl, progs, node) { + hlist_for_each_entry(pl, progs, node) { if (pl->prog == replace_prog) /* a match found */ return pl; @@ -480,7 +481,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; enum cgroup_bpf_attach_type atype; struct bpf_prog_list *pl; - struct list_head *progs; + struct hlist_head *progs; int err; if (((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI)) || @@ -503,7 +504,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (!hierarchy_allows_attach(cgrp, atype)) return -EPERM; - if (!list_empty(progs) && cgrp->bpf.flags[atype] != saved_flags) + if (!hlist_empty(progs) && cgrp->bpf.flags[atype] != saved_flags) /* Disallow attaching non-overridable on top * of existing overridable in this cgroup. * Disallow attaching multi-prog if overridable or none @@ -525,12 +526,22 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (pl) { old_prog = pl->prog; } else { + struct hlist_node *last = NULL; + pl = kmalloc(sizeof(*pl), GFP_KERNEL); if (!pl) { bpf_cgroup_storages_free(new_storage); return -ENOMEM; } - list_add_tail(&pl->node, progs); + if (hlist_empty(progs)) + hlist_add_head(&pl->node, progs); + else + hlist_for_each(last, progs) { + if (last->next) + continue; + hlist_add_behind(&pl->node, last); + break; + } } pl->prog = prog; @@ -556,7 +567,7 @@ cleanup: } bpf_cgroup_storages_free(new_storage); if (!old_prog) { - list_del(&pl->node); + hlist_del(&pl->node); kfree(pl); } return err; @@ -587,7 +598,7 @@ static void replace_effective_prog(struct cgroup *cgrp, struct cgroup_subsys_state *css; struct bpf_prog_array *progs; struct bpf_prog_list *pl; - struct list_head *head; + struct hlist_head *head; struct cgroup *cg; int pos; @@ -603,7 +614,7 @@ static void replace_effective_prog(struct cgroup *cgrp, continue; head = &cg->bpf.progs[atype]; - list_for_each_entry(pl, head, node) { + hlist_for_each_entry(pl, head, node) { if (!prog_list_prog(pl)) continue; if (pl->link == link) @@ -637,7 +648,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, enum cgroup_bpf_attach_type atype; struct bpf_prog *old_prog; struct bpf_prog_list *pl; - struct list_head *progs; + struct hlist_head *progs; bool found = false; atype = to_cgroup_bpf_attach_type(link->type); @@ -649,7 +660,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, if (link->link.prog->type != new_prog->type) return -EINVAL; - list_for_each_entry(pl, progs, node) { + hlist_for_each_entry(pl, progs, node) { if (pl->link == link) { found = true; break; @@ -688,7 +699,7 @@ out_unlock: return ret; } -static struct bpf_prog_list *find_detach_entry(struct list_head *progs, +static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs, struct bpf_prog *prog, struct bpf_cgroup_link *link, bool allow_multi) @@ -696,14 +707,14 @@ static struct bpf_prog_list *find_detach_entry(struct list_head *progs, struct bpf_prog_list *pl; if (!allow_multi) { - if (list_empty(progs)) + if (hlist_empty(progs)) /* report error when trying to detach and nothing is attached */ return ERR_PTR(-ENOENT); /* to maintain backward compatibility NONE and OVERRIDE cgroups * allow detaching with invalid FD (prog==NULL) in legacy mode */ - return list_first_entry(progs, typeof(*pl), node); + return hlist_entry(progs->first, typeof(*pl), node); } if (!prog && !link) @@ -713,7 +724,7 @@ static struct bpf_prog_list *find_detach_entry(struct list_head *progs, return ERR_PTR(-EINVAL); /* find the prog or link and detach it */ - list_for_each_entry(pl, progs, node) { + hlist_for_each_entry(pl, progs, node) { if (pl->prog == prog && pl->link == link) return pl; } @@ -737,7 +748,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, struct cgroup_subsys_state *css; struct bpf_prog_array *progs; struct bpf_prog_list *pl; - struct list_head *head; + struct hlist_head *head; struct cgroup *cg; int pos; @@ -754,7 +765,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, continue; head = &cg->bpf.progs[atype]; - list_for_each_entry(pl, head, node) { + hlist_for_each_entry(pl, head, node) { if (!prog_list_prog(pl)) continue; if (pl->prog == prog && pl->link == link) @@ -791,7 +802,7 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, enum cgroup_bpf_attach_type atype; struct bpf_prog *old_prog; struct bpf_prog_list *pl; - struct list_head *progs; + struct hlist_head *progs; u32 flags; atype = to_cgroup_bpf_attach_type(type); @@ -822,9 +833,10 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, } /* now can actually delete it from this cgroup list */ - list_del(&pl->node); + hlist_del(&pl->node); + kfree(pl); - if (list_empty(progs)) + if (hlist_empty(progs)) /* last program was detached, reset flags to zero */ cgrp->bpf.flags[atype] = 0; if (old_prog) @@ -852,7 +864,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, enum bpf_attach_type type = attr->query.attach_type; enum cgroup_bpf_attach_type atype; struct bpf_prog_array *effective; - struct list_head *progs; + struct hlist_head *progs; struct bpf_prog *prog; int cnt, ret = 0, i; u32 flags; @@ -891,7 +903,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, u32 id; i = 0; - list_for_each_entry(pl, progs, node) { + hlist_for_each_entry(pl, progs, node) { prog = prog_list_prog(pl); id = prog->aux->id; if (copy_to_user(prog_ids + i, &id, sizeof(id))) -- cgit v1.2.3 From 69fd337a975c7e690dfe49d9cb4fe5ba1e6db44e Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Tue, 28 Jun 2022 10:43:06 -0700 Subject: bpf: per-cgroup lsm flavor Allow attaching to lsm hooks in the cgroup context. Attaching to per-cgroup LSM works exactly like attaching to other per-cgroup hooks. New BPF_LSM_CGROUP is added to trigger new mode; the actual lsm hook we attach to is signaled via existing attach_btf_id. For the hooks that have 'struct socket' or 'struct sock' as its first argument, we use the cgroup associated with that socket. For the rest, we use 'current' cgroup (this is all on default hierarchy == v2 only). Note that for some hooks that work on 'struct sock' we still take the cgroup from 'current' because some of them work on the socket that hasn't been properly initialized yet. Behind the scenes, we allocate a shim program that is attached to the trampoline and runs cgroup effective BPF programs array. This shim has some rudimentary ref counting and can be shared between several programs attaching to the same lsm hook from different cgroups. Note that this patch bloats cgroup size because we add 211 cgroup_bpf_attach_type(s) for simplicity sake. This will be addressed in the subsequent patch. Also note that we only add non-sleepable flavor for now. To enable sleepable use-cases, bpf_prog_run_array_cg has to grab trace rcu, shim programs have to be freed via trace rcu, cgroup_bpf.effective should be also trace-rcu-managed + maybe some other changes that I'm not aware of. Reviewed-by: Martin KaFai Lau Signed-off-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220628174314.1216643-4-sdf@google.com Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 24 +++-- include/linux/bpf-cgroup-defs.h | 8 ++ include/linux/bpf-cgroup.h | 7 ++ include/linux/bpf.h | 24 +++++ include/linux/bpf_lsm.h | 13 +++ include/linux/btf_ids.h | 3 +- include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_lsm.c | 48 ++++++++++ kernel/bpf/btf.c | 11 +++ kernel/bpf/cgroup.c | 136 ++++++++++++++++++++++++--- kernel/bpf/core.c | 2 + kernel/bpf/syscall.c | 10 ++ kernel/bpf/trampoline.c | 198 ++++++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 32 +++++++ tools/include/uapi/linux/bpf.h | 1 + 15 files changed, 498 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2c51ca9f7cec..2f460c67f9c7 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1770,6 +1770,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, int run_ctx_off, bool save_ret) { + void (*exit)(struct bpf_prog *prog, u64 start, + struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_exit; + u64 (*enter)(struct bpf_prog *prog, + struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_enter; u8 *prog = *pprog; u8 *jmp_insn; int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); @@ -1788,15 +1792,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off); + if (p->aux->sleepable) { + enter = __bpf_prog_enter_sleepable; + exit = __bpf_prog_exit_sleepable; + } else if (p->expected_attach_type == BPF_LSM_CGROUP) { + enter = __bpf_prog_enter_lsm_cgroup; + exit = __bpf_prog_exit_lsm_cgroup; + } + /* arg1: mov rdi, progs[i] */ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); /* arg2: lea rsi, [rbp - ctx_cookie_off] */ EMIT4(0x48, 0x8D, 0x75, -run_ctx_off); - if (emit_call(&prog, - p->aux->sleepable ? __bpf_prog_enter_sleepable : - __bpf_prog_enter, prog)) - return -EINVAL; + if (emit_call(&prog, enter, prog)) + return -EINVAL; /* remember prog start time returned by __bpf_prog_enter */ emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); @@ -1840,10 +1850,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); /* arg3: lea rdx, [rbp - run_ctx_off] */ EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); - if (emit_call(&prog, - p->aux->sleepable ? __bpf_prog_exit_sleepable : - __bpf_prog_exit, prog)) - return -EINVAL; + if (emit_call(&prog, exit, prog)) + return -EINVAL; *pprog = prog; return 0; diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index 5d268e76d8e6..b99f8c3e37ea 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h @@ -10,6 +10,12 @@ struct bpf_prog_array; +#ifdef CONFIG_BPF_LSM +#define CGROUP_LSM_NUM 211 /* will be addressed in the next patch */ +#else +#define CGROUP_LSM_NUM 0 +#endif + enum cgroup_bpf_attach_type { CGROUP_BPF_ATTACH_TYPE_INVALID = -1, CGROUP_INET_INGRESS = 0, @@ -35,6 +41,8 @@ enum cgroup_bpf_attach_type { CGROUP_INET4_GETSOCKNAME, CGROUP_INET6_GETSOCKNAME, CGROUP_INET_SOCK_RELEASE, + CGROUP_LSM_START, + CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1, MAX_CGROUP_BPF_ATTACH_TYPE }; diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 6673acfbf2ef..2bd1b5f8de9b 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -23,6 +23,13 @@ struct ctl_table; struct ctl_table_header; struct task_struct; +unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx, + const struct bpf_insn *insn); +unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx, + const struct bpf_insn *insn); +unsigned int __cgroup_bpf_run_lsm_current(const void *ctx, + const struct bpf_insn *insn); + #ifdef CONFIG_CGROUP_BPF #define CGROUP_ATYPE(type) \ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d547be9db75f..77cd613a00bd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -794,6 +794,10 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_ u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx); +u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog, + struct bpf_tramp_run_ctx *run_ctx); +void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start, + struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); @@ -1060,6 +1064,7 @@ struct bpf_prog_aux { struct user_struct *user; u64 load_time; /* ns since boottime */ u32 verified_insns; + int cgroup_atype; /* enum cgroup_bpf_attach_type */ struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; char name[BPF_OBJ_NAME_LEN]; #ifdef CONFIG_SECURITY @@ -1167,6 +1172,11 @@ struct bpf_tramp_link { u64 cookie; }; +struct bpf_shim_tramp_link { + struct bpf_tramp_link link; + struct bpf_trampoline *trampoline; +}; + struct bpf_tracing_link { struct bpf_tramp_link link; enum bpf_attach_type attach_type; @@ -1245,6 +1255,9 @@ struct bpf_dummy_ops { int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); #endif +int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, + int cgroup_atype); +void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); #else static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) { @@ -1268,6 +1281,14 @@ static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, { return -EINVAL; } +static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, + int cgroup_atype) +{ + return -EOPNOTSUPP; +} +static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) +{ +} #endif struct bpf_array { @@ -2368,6 +2389,8 @@ extern const struct bpf_func_proto bpf_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_find_vma_proto; extern const struct bpf_func_proto bpf_loop_proto; extern const struct bpf_func_proto bpf_copy_from_user_task_proto; +extern const struct bpf_func_proto bpf_set_retval_proto; +extern const struct bpf_func_proto bpf_get_retval_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); @@ -2485,6 +2508,7 @@ int bpf_arch_text_invalidate(void *dst, size_t len); struct btf_id_set; bool btf_id_set_contains(const struct btf_id_set *set, u32 id); +int btf_id_set_index(const struct btf_id_set *set, u32 id); #define MAX_BPRINTF_VARARGS 12 diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index 479c101546ad..61787a5f6af9 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -42,6 +42,9 @@ extern const struct bpf_func_proto bpf_inode_storage_get_proto; extern const struct bpf_func_proto bpf_inode_storage_delete_proto; void bpf_inode_storage_free(struct inode *inode); +int bpf_lsm_hook_idx(u32 btf_id); +void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func); + #else /* !CONFIG_BPF_LSM */ static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id) @@ -65,6 +68,16 @@ static inline void bpf_inode_storage_free(struct inode *inode) { } +static inline void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, + bpf_func_t *bpf_func) +{ +} + +static inline int bpf_lsm_hook_idx(u32 btf_id) +{ + return -EINVAL; +} + #endif /* CONFIG_BPF_LSM */ #endif /* _LINUX_BPF_LSM_H */ diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 335a19092368..252a4befeab1 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -179,7 +179,8 @@ extern struct btf_id_set name; BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP, udp_sock) \ BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP6, udp6_sock) \ BTF_SOCK_TYPE(BTF_SOCK_TYPE_UNIX, unix_sock) \ - BTF_SOCK_TYPE(BTF_SOCK_TYPE_MPTCP, mptcp_sock) + BTF_SOCK_TYPE(BTF_SOCK_TYPE_MPTCP, mptcp_sock) \ + BTF_SOCK_TYPE(BTF_SOCK_TYPE_SOCKET, socket) enum { #define BTF_SOCK_TYPE(name, str) name, diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index e81362891596..b7479898c879 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -998,6 +998,7 @@ enum bpf_attach_type { BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, BPF_PERF_EVENT, BPF_TRACE_KPROBE_MULTI, + BPF_LSM_CGROUP, __MAX_BPF_ATTACH_TYPE }; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index c1351df9f7ee..0f72020bfdcf 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -16,6 +16,7 @@ #include #include #include +#include /* For every LSM hook that allows attachment of BPF programs, declare a nop * function where a BPF program can be attached. @@ -35,6 +36,44 @@ BTF_SET_START(bpf_lsm_hooks) #undef LSM_HOOK BTF_SET_END(bpf_lsm_hooks) +/* List of LSM hooks that should operate on 'current' cgroup regardless + * of function signature. + */ +BTF_SET_START(bpf_lsm_current_hooks) +/* operate on freshly allocated sk without any cgroup association */ +BTF_ID(func, bpf_lsm_sk_alloc_security) +BTF_ID(func, bpf_lsm_sk_free_security) +BTF_SET_END(bpf_lsm_current_hooks) + +void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, + bpf_func_t *bpf_func) +{ + const struct btf_param *args; + + if (btf_type_vlen(prog->aux->attach_func_proto) < 1 || + btf_id_set_contains(&bpf_lsm_current_hooks, + prog->aux->attach_btf_id)) { + *bpf_func = __cgroup_bpf_run_lsm_current; + return; + } + + args = btf_params(prog->aux->attach_func_proto); + +#ifdef CONFIG_NET + if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCKET]) + *bpf_func = __cgroup_bpf_run_lsm_socket; + else if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCK]) + *bpf_func = __cgroup_bpf_run_lsm_sock; + else +#endif + *bpf_func = __cgroup_bpf_run_lsm_current; +} + +int bpf_lsm_hook_idx(u32 btf_id) +{ + return btf_id_set_index(&bpf_lsm_hooks, btf_id); +} + int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, const struct bpf_prog *prog) { @@ -158,6 +197,15 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL; case BPF_FUNC_get_attach_cookie: return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL; + case BPF_FUNC_get_local_storage: + return prog->expected_attach_type == BPF_LSM_CGROUP ? + &bpf_get_local_storage_proto : NULL; + case BPF_FUNC_set_retval: + return prog->expected_attach_type == BPF_LSM_CGROUP ? + &bpf_set_retval_proto : NULL; + case BPF_FUNC_get_retval: + return prog->expected_attach_type == BPF_LSM_CGROUP ? + &bpf_get_retval_proto : NULL; default: return tracing_prog_func_proto(func_id, prog); } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 2e2066d6af94..7c1fe422ed3f 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5363,6 +5363,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, if (arg == nr_args) { switch (prog->expected_attach_type) { + case BPF_LSM_CGROUP: case BPF_LSM_MAC: case BPF_TRACE_FEXIT: /* When LSM programs are attached to void LSM hooks @@ -6842,6 +6843,16 @@ static int btf_id_cmp_func(const void *a, const void *b) return *pa - *pb; } +int btf_id_set_index(const struct btf_id_set *set, u32 id) +{ + const u32 *p; + + p = bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func); + if (!p) + return -1; + return p - set->ids; +} + bool btf_id_set_contains(const struct btf_id_set *set, u32 id) { return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4adb4f3ecb7f..9cf41dd4f96f 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -61,6 +63,87 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp, return run_ctx.retval; } +unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx, + const struct bpf_insn *insn) +{ + const struct bpf_prog *shim_prog; + struct sock *sk; + struct cgroup *cgrp; + int ret = 0; + u64 *args; + + args = (u64 *)ctx; + sk = (void *)(unsigned long)args[0]; + /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/ + shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi)); + + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); + if (likely(cgrp)) + ret = bpf_prog_run_array_cg(&cgrp->bpf, + shim_prog->aux->cgroup_atype, + ctx, bpf_prog_run, 0, NULL); + return ret; +} + +unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx, + const struct bpf_insn *insn) +{ + const struct bpf_prog *shim_prog; + struct socket *sock; + struct cgroup *cgrp; + int ret = 0; + u64 *args; + + args = (u64 *)ctx; + sock = (void *)(unsigned long)args[0]; + /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/ + shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi)); + + cgrp = sock_cgroup_ptr(&sock->sk->sk_cgrp_data); + if (likely(cgrp)) + ret = bpf_prog_run_array_cg(&cgrp->bpf, + shim_prog->aux->cgroup_atype, + ctx, bpf_prog_run, 0, NULL); + return ret; +} + +unsigned int __cgroup_bpf_run_lsm_current(const void *ctx, + const struct bpf_insn *insn) +{ + const struct bpf_prog *shim_prog; + struct cgroup *cgrp; + int ret = 0; + + /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/ + shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi)); + + /* We rely on trampoline's __bpf_prog_enter_lsm_cgroup to grab RCU read lock. */ + cgrp = task_dfl_cgroup(current); + if (likely(cgrp)) + ret = bpf_prog_run_array_cg(&cgrp->bpf, + shim_prog->aux->cgroup_atype, + ctx, bpf_prog_run, 0, NULL); + return ret; +} + +#ifdef CONFIG_BPF_LSM +static enum cgroup_bpf_attach_type +bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id) +{ + if (attach_type != BPF_LSM_CGROUP) + return to_cgroup_bpf_attach_type(attach_type); + return CGROUP_LSM_START + bpf_lsm_hook_idx(attach_btf_id); +} +#else +static enum cgroup_bpf_attach_type +bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id) +{ + if (attach_type != BPF_LSM_CGROUP) + return to_cgroup_bpf_attach_type(attach_type); + return -EOPNOTSUPP; +} +#endif /* CONFIG_BPF_LSM */ + void cgroup_bpf_offline(struct cgroup *cgrp) { cgroup_get(cgrp); @@ -163,10 +246,16 @@ static void cgroup_bpf_release(struct work_struct *work) hlist_for_each_entry_safe(pl, pltmp, progs, node) { hlist_del(&pl->node); - if (pl->prog) + if (pl->prog) { + if (pl->prog->expected_attach_type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(pl->prog); bpf_prog_put(pl->prog); - if (pl->link) + } + if (pl->link) { + if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog); bpf_cgroup_link_auto_detach(pl->link); + } kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key[atype]); } @@ -479,6 +568,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; + struct bpf_prog *new_prog = prog ? : link->link.prog; enum cgroup_bpf_attach_type atype; struct bpf_prog_list *pl; struct hlist_head *progs; @@ -495,7 +585,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, /* replace_prog implies BPF_F_REPLACE, and vice versa */ return -EINVAL; - atype = to_cgroup_bpf_attach_type(type); + atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id); if (atype < 0) return -EINVAL; @@ -549,17 +639,30 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, bpf_cgroup_storages_assign(pl->storage, storage); cgrp->bpf.flags[atype] = saved_flags; + if (type == BPF_LSM_CGROUP) { + err = bpf_trampoline_link_cgroup_shim(new_prog, atype); + if (err) + goto cleanup; + } + err = update_effective_progs(cgrp, atype); if (err) - goto cleanup; + goto cleanup_trampoline; - if (old_prog) + if (old_prog) { + if (type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(old_prog); bpf_prog_put(old_prog); - else + } else { static_branch_inc(&cgroup_bpf_enabled_key[atype]); + } bpf_cgroup_storages_link(new_storage, cgrp, type); return 0; +cleanup_trampoline: + if (type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(new_prog); + cleanup: if (old_prog) { pl->prog = old_prog; @@ -651,7 +754,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, struct hlist_head *progs; bool found = false; - atype = to_cgroup_bpf_attach_type(link->type); + atype = bpf_cgroup_atype_find(link->type, new_prog->aux->attach_btf_id); if (atype < 0) return -EINVAL; @@ -803,9 +906,15 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, struct bpf_prog *old_prog; struct bpf_prog_list *pl; struct hlist_head *progs; + u32 attach_btf_id = 0; u32 flags; - atype = to_cgroup_bpf_attach_type(type); + if (prog) + attach_btf_id = prog->aux->attach_btf_id; + if (link) + attach_btf_id = link->link.prog->aux->attach_btf_id; + + atype = bpf_cgroup_atype_find(type, attach_btf_id); if (atype < 0) return -EINVAL; @@ -839,8 +948,11 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, if (hlist_empty(progs)) /* last program was detached, reset flags to zero */ cgrp->bpf.flags[atype] = 0; - if (old_prog) + if (old_prog) { + if (type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(old_prog); bpf_prog_put(old_prog); + } static_branch_dec(&cgroup_bpf_enabled_key[atype]); return 0; } @@ -999,6 +1111,8 @@ static void bpf_cgroup_link_release(struct bpf_link *link) WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link, cg_link->type)); + if (cg_link->type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog); cg = cg_link->cgroup; cg_link->cgroup = NULL; @@ -1343,7 +1457,7 @@ BPF_CALL_0(bpf_get_retval) return ctx->retval; } -static const struct bpf_func_proto bpf_get_retval_proto = { +const struct bpf_func_proto bpf_get_retval_proto = { .func = bpf_get_retval, .gpl_only = false, .ret_type = RET_INTEGER, @@ -1358,7 +1472,7 @@ BPF_CALL_1(bpf_set_retval, int, retval) return 0; } -static const struct bpf_func_proto bpf_set_retval_proto = { +const struct bpf_func_proto bpf_set_retval_proto = { .func = bpf_set_retval, .gpl_only = false, .ret_type = RET_INTEGER, diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index f023cb399e3f..4cc10b942a3c 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2666,6 +2666,8 @@ const struct bpf_func_proto bpf_get_local_storage_proto __weak; const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak; const struct bpf_func_proto bpf_snprintf_btf_proto __weak; const struct bpf_func_proto bpf_seq_printf_btf_proto __weak; +const struct bpf_func_proto bpf_set_retval_proto __weak; +const struct bpf_func_proto bpf_get_retval_proto __weak; const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 7d5af5b99f0d..626b8f7d237b 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3416,6 +3416,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) return BPF_PROG_TYPE_SK_LOOKUP; case BPF_XDP: return BPF_PROG_TYPE_XDP; + case BPF_LSM_CGROUP: + return BPF_PROG_TYPE_LSM; default: return BPF_PROG_TYPE_UNSPEC; } @@ -3469,6 +3471,11 @@ static int bpf_prog_attach(const union bpf_attr *attr) case BPF_PROG_TYPE_CGROUP_SOCKOPT: case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_SOCK_OPS: + case BPF_PROG_TYPE_LSM: + if (ptype == BPF_PROG_TYPE_LSM && + prog->expected_attach_type != BPF_LSM_CGROUP) + return -EINVAL; + ret = cgroup_bpf_prog_attach(attr, ptype, prog); break; default: @@ -3506,6 +3513,7 @@ static int bpf_prog_detach(const union bpf_attr *attr) case BPF_PROG_TYPE_CGROUP_SOCKOPT: case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_SOCK_OPS: + case BPF_PROG_TYPE_LSM: return cgroup_bpf_prog_detach(attr, ptype); default: return -EINVAL; @@ -4540,6 +4548,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) ret = bpf_raw_tp_link_attach(prog, NULL); else if (prog->expected_attach_type == BPF_TRACE_ITER) ret = bpf_iter_link_attach(attr, uattr, prog); + else if (prog->expected_attach_type == BPF_LSM_CGROUP) + ret = cgroup_bpf_link_attach(attr, prog); else ret = bpf_tracing_prog_attach(prog, attr->link_create.target_fd, diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 5466e15be61f..d7c251d7fbcd 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include /* dummy _ops. The verifier will operate on target program's ops. */ const struct bpf_verifier_ops bpf_extension_verifier_ops = { @@ -496,6 +498,177 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin return err; } +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) +static void bpf_shim_tramp_link_release(struct bpf_link *link) +{ + struct bpf_shim_tramp_link *shim_link = + container_of(link, struct bpf_shim_tramp_link, link.link); + + /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */ + if (!shim_link->trampoline) + return; + + WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline)); + bpf_trampoline_put(shim_link->trampoline); +} + +static void bpf_shim_tramp_link_dealloc(struct bpf_link *link) +{ + struct bpf_shim_tramp_link *shim_link = + container_of(link, struct bpf_shim_tramp_link, link.link); + + kfree(shim_link); +} + +static const struct bpf_link_ops bpf_shim_tramp_link_lops = { + .release = bpf_shim_tramp_link_release, + .dealloc = bpf_shim_tramp_link_dealloc, +}; + +static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog, + bpf_func_t bpf_func, + int cgroup_atype) +{ + struct bpf_shim_tramp_link *shim_link = NULL; + struct bpf_prog *p; + + shim_link = kzalloc(sizeof(*shim_link), GFP_USER); + if (!shim_link) + return NULL; + + p = bpf_prog_alloc(1, 0); + if (!p) { + kfree(shim_link); + return NULL; + } + + p->jited = false; + p->bpf_func = bpf_func; + + p->aux->cgroup_atype = cgroup_atype; + p->aux->attach_func_proto = prog->aux->attach_func_proto; + p->aux->attach_btf_id = prog->aux->attach_btf_id; + p->aux->attach_btf = prog->aux->attach_btf; + btf_get(p->aux->attach_btf); + p->type = BPF_PROG_TYPE_LSM; + p->expected_attach_type = BPF_LSM_MAC; + bpf_prog_inc(p); + bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC, + &bpf_shim_tramp_link_lops, p); + + return shim_link; +} + +static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr, + bpf_func_t bpf_func) +{ + struct bpf_tramp_link *link; + int kind; + + for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { + hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { + struct bpf_prog *p = link->link.prog; + + if (p->bpf_func == bpf_func) + return container_of(link, struct bpf_shim_tramp_link, link); + } + } + + return NULL; +} + +int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, + int cgroup_atype) +{ + struct bpf_shim_tramp_link *shim_link = NULL; + struct bpf_attach_target_info tgt_info = {}; + struct bpf_trampoline *tr; + bpf_func_t bpf_func; + u64 key; + int err; + + err = bpf_check_attach_target(NULL, prog, NULL, + prog->aux->attach_btf_id, + &tgt_info); + if (err) + return err; + + key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, + prog->aux->attach_btf_id); + + bpf_lsm_find_cgroup_shim(prog, &bpf_func); + tr = bpf_trampoline_get(key, &tgt_info); + if (!tr) + return -ENOMEM; + + mutex_lock(&tr->mutex); + + shim_link = cgroup_shim_find(tr, bpf_func); + if (shim_link) { + /* Reusing existing shim attached by the other program. */ + bpf_link_inc(&shim_link->link.link); + + mutex_unlock(&tr->mutex); + bpf_trampoline_put(tr); /* bpf_trampoline_get above */ + return 0; + } + + /* Allocate and install new shim. */ + + shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype); + if (!shim_link) { + err = -ENOMEM; + goto err; + } + + err = __bpf_trampoline_link_prog(&shim_link->link, tr); + if (err) + goto err; + + shim_link->trampoline = tr; + /* note, we're still holding tr refcnt from above */ + + mutex_unlock(&tr->mutex); + + return 0; +err: + mutex_unlock(&tr->mutex); + + if (shim_link) + bpf_link_put(&shim_link->link.link); + + /* have to release tr while _not_ holding its mutex */ + bpf_trampoline_put(tr); /* bpf_trampoline_get above */ + + return err; +} + +void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) +{ + struct bpf_shim_tramp_link *shim_link = NULL; + struct bpf_trampoline *tr; + bpf_func_t bpf_func; + u64 key; + + key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, + prog->aux->attach_btf_id); + + bpf_lsm_find_cgroup_shim(prog, &bpf_func); + tr = bpf_trampoline_lookup(key); + if (WARN_ON_ONCE(!tr)) + return; + + mutex_lock(&tr->mutex); + shim_link = cgroup_shim_find(tr, bpf_func); + mutex_unlock(&tr->mutex); + + if (shim_link) + bpf_link_put(&shim_link->link.link); + + bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */ +} +#endif + struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info) { @@ -628,6 +801,31 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_ rcu_read_unlock(); } +u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog, + struct bpf_tramp_run_ctx *run_ctx) + __acquires(RCU) +{ + /* Runtime stats are exported via actual BPF_LSM_CGROUP + * programs, not the shims. + */ + rcu_read_lock(); + migrate_disable(); + + run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); + + return NO_START_TIME; +} + +void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start, + struct bpf_tramp_run_ctx *run_ctx) + __releases(RCU) +{ + bpf_reset_run_ctx(run_ctx->saved_run_ctx); + + migrate_enable(); + rcu_read_unlock(); +} + u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx) { rcu_read_lock_trace(); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4938477912cd..df3ec6b05f05 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7322,6 +7322,18 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn reg_type_str(env, regs[BPF_REG_1].type)); return -EACCES; } + break; + case BPF_FUNC_set_retval: + if (env->prog->expected_attach_type == BPF_LSM_CGROUP) { + if (!env->prog->aux->attach_func_proto->type) { + /* Make sure programs that attach to void + * hooks don't try to modify return value. + */ + verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); + return -EINVAL; + } + } + break; } if (err) @@ -10527,6 +10539,22 @@ static int check_return_code(struct bpf_verifier_env *env) case BPF_PROG_TYPE_SK_LOOKUP: range = tnum_range(SK_DROP, SK_PASS); break; + + case BPF_PROG_TYPE_LSM: + if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { + /* Regular BPF_PROG_TYPE_LSM programs can return + * any value. + */ + return 0; + } + if (!env->prog->aux->attach_func_proto->type) { + /* Make sure programs that attach to void + * hooks don't try to modify return value. + */ + range = tnum_range(1, 1); + } + break; + case BPF_PROG_TYPE_EXT: /* freplace program can return anything as its return value * depends on the to-be-replaced kernel func or bpf program. @@ -10543,6 +10571,9 @@ static int check_return_code(struct bpf_verifier_env *env) if (!tnum_in(range, reg->var_off)) { verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); + if (prog->expected_attach_type == BPF_LSM_CGROUP && + !prog->aux->attach_func_proto->type) + verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); return -EINVAL; } @@ -14902,6 +14933,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, fallthrough; case BPF_MODIFY_RETURN: case BPF_LSM_MAC: + case BPF_LSM_CGROUP: case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: if (!btf_type_is_func(t)) { diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index e81362891596..b7479898c879 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -998,6 +998,7 @@ enum bpf_attach_type { BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, BPF_PERF_EVENT, BPF_TRACE_KPROBE_MULTI, + BPF_LSM_CGROUP, __MAX_BPF_ATTACH_TYPE }; -- cgit v1.2.3 From c0e19f2c9a3edd38e4b1bdae98eb44555d02bc31 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Tue, 28 Jun 2022 10:43:07 -0700 Subject: bpf: minimize number of allocated lsm slots per program Previous patch adds 1:1 mapping between all 211 LSM hooks and bpf_cgroup program array. Instead of reserving a slot per possible hook, reserve 10 slots per cgroup for lsm programs. Those slots are dynamically allocated on demand and reclaimed. struct cgroup_bpf { struct bpf_prog_array * effective[33]; /* 0 264 */ /* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */ struct hlist_head progs[33]; /* 264 264 */ /* --- cacheline 8 boundary (512 bytes) was 16 bytes ago --- */ u8 flags[33]; /* 528 33 */ /* XXX 7 bytes hole, try to pack */ struct list_head storages; /* 568 16 */ /* --- cacheline 9 boundary (576 bytes) was 8 bytes ago --- */ struct bpf_prog_array * inactive; /* 584 8 */ struct percpu_ref refcnt; /* 592 16 */ struct work_struct release_work; /* 608 72 */ /* size: 680, cachelines: 11, members: 7 */ /* sum members: 673, holes: 1, sum holes: 7 */ /* last cacheline: 40 bytes */ }; Reviewed-by: Martin KaFai Lau Signed-off-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220628174314.1216643-5-sdf@google.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf-cgroup-defs.h | 3 ++- include/linux/bpf.h | 9 +++++++- include/linux/bpf_lsm.h | 6 ------ kernel/bpf/bpf_lsm.c | 5 ----- kernel/bpf/btf.c | 10 --------- kernel/bpf/cgroup.c | 47 ++++++++++++++++++++++++++++++++++++++++- kernel/bpf/core.c | 7 ++++++ kernel/bpf/trampoline.c | 1 + 8 files changed, 64 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index b99f8c3e37ea..7b121bd780eb 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h @@ -11,7 +11,8 @@ struct bpf_prog_array; #ifdef CONFIG_BPF_LSM -#define CGROUP_LSM_NUM 211 /* will be addressed in the next patch */ +/* Maximum number of concurrently attachable per-cgroup LSM hooks. */ +#define CGROUP_LSM_NUM 10 #else #define CGROUP_LSM_NUM 0 #endif diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 77cd613a00bd..5d2afa55c7c3 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2508,7 +2508,6 @@ int bpf_arch_text_invalidate(void *dst, size_t len); struct btf_id_set; bool btf_id_set_contains(const struct btf_id_set *set, u32 id); -int btf_id_set_index(const struct btf_id_set *set, u32 id); #define MAX_BPRINTF_VARARGS 12 @@ -2545,4 +2544,12 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr); int bpf_dynptr_check_size(u32 size); +#ifdef CONFIG_BPF_LSM +void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype); +void bpf_cgroup_atype_put(int cgroup_atype); +#else +static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {} +static inline void bpf_cgroup_atype_put(int cgroup_atype) {} +#endif /* CONFIG_BPF_LSM */ + #endif /* _LINUX_BPF_H */ diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index 61787a5f6af9..4bcf76a9bb06 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -42,7 +42,6 @@ extern const struct bpf_func_proto bpf_inode_storage_get_proto; extern const struct bpf_func_proto bpf_inode_storage_delete_proto; void bpf_inode_storage_free(struct inode *inode); -int bpf_lsm_hook_idx(u32 btf_id); void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func); #else /* !CONFIG_BPF_LSM */ @@ -73,11 +72,6 @@ static inline void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, { } -static inline int bpf_lsm_hook_idx(u32 btf_id) -{ - return -EINVAL; -} - #endif /* CONFIG_BPF_LSM */ #endif /* _LINUX_BPF_LSM_H */ diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 0f72020bfdcf..83aa431dd52e 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -69,11 +69,6 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, *bpf_func = __cgroup_bpf_run_lsm_current; } -int bpf_lsm_hook_idx(u32 btf_id) -{ - return btf_id_set_index(&bpf_lsm_hooks, btf_id); -} - int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, const struct bpf_prog *prog) { diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7c1fe422ed3f..8d3c7ab8af46 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6843,16 +6843,6 @@ static int btf_id_cmp_func(const void *a, const void *b) return *pa - *pb; } -int btf_id_set_index(const struct btf_id_set *set, u32 id) -{ - const u32 *p; - - p = bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func); - if (!p) - return -1; - return p - set->ids; -} - bool btf_id_set_contains(const struct btf_id_set *set, u32 id) { return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 9cf41dd4f96f..169cbd0de797 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -127,12 +127,57 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx, } #ifdef CONFIG_BPF_LSM +struct cgroup_lsm_atype { + u32 attach_btf_id; + int refcnt; +}; + +static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM]; + static enum cgroup_bpf_attach_type bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id) { + int i; + + lockdep_assert_held(&cgroup_mutex); + if (attach_type != BPF_LSM_CGROUP) return to_cgroup_bpf_attach_type(attach_type); - return CGROUP_LSM_START + bpf_lsm_hook_idx(attach_btf_id); + + for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++) + if (cgroup_lsm_atype[i].attach_btf_id == attach_btf_id) + return CGROUP_LSM_START + i; + + for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++) + if (cgroup_lsm_atype[i].attach_btf_id == 0) + return CGROUP_LSM_START + i; + + return -E2BIG; + +} + +void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) +{ + int i = cgroup_atype - CGROUP_LSM_START; + + lockdep_assert_held(&cgroup_mutex); + + WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id && + cgroup_lsm_atype[i].attach_btf_id != attach_btf_id); + + cgroup_lsm_atype[i].attach_btf_id = attach_btf_id; + cgroup_lsm_atype[i].refcnt++; +} + +void bpf_cgroup_atype_put(int cgroup_atype) +{ + int i = cgroup_atype - CGROUP_LSM_START; + + mutex_lock(&cgroup_mutex); + if (--cgroup_lsm_atype[i].refcnt <= 0) + cgroup_lsm_atype[i].attach_btf_id = 0; + WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0); + mutex_unlock(&cgroup_mutex); } #else static enum cgroup_bpf_attach_type diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 4cc10b942a3c..805c2ad5c793 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -107,6 +107,9 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->aux->prog = fp; fp->jit_requested = ebpf_jit_enabled(); fp->blinding_requested = bpf_jit_blinding_enabled(fp); +#ifdef CONFIG_CGROUP_BPF + aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; +#endif INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); mutex_init(&fp->aux->used_maps_mutex); @@ -2569,6 +2572,10 @@ static void bpf_prog_free_deferred(struct work_struct *work) aux = container_of(work, struct bpf_prog_aux, work); #ifdef CONFIG_BPF_SYSCALL bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); +#endif +#ifdef CONFIG_CGROUP_BPF + if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) + bpf_cgroup_atype_put(aux->cgroup_atype); #endif bpf_free_used_maps(aux); bpf_free_used_btfs(aux); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index d7c251d7fbcd..6cd226584c33 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -555,6 +555,7 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog bpf_prog_inc(p); bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC, &bpf_shim_tramp_link_lops, p); + bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype); return shim_link; } -- cgit v1.2.3 From 9113d7e48e9128522b9f5a54dfd30dff10509a92 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Tue, 28 Jun 2022 10:43:09 -0700 Subject: bpf: expose bpf_{g,s}etsockopt to lsm cgroup I don't see how to make it nice without introducing btf id lists for the hooks where these helpers are allowed. Some LSM hooks work on the locked sockets, some are triggering early and don't grab any locks, so have two lists for now: 1. LSM hooks which trigger under socket lock - minority of the hooks, but ideal case for us, we can expose existing BTF-based helpers 2. LSM hooks which trigger without socket lock, but they trigger early in the socket creation path where it should be safe to do setsockopt without any locks 3. The rest are prohibited. I'm thinking that this use-case might be a good gateway to sleeping lsm cgroup hooks in the future. We can either expose lock/unlock operations (and add tracking to the verifier) or have another set of bpf_setsockopt wrapper that grab the locks and might sleep. Reviewed-by: Martin KaFai Lau Signed-off-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220628174314.1216643-7-sdf@google.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 ++ kernel/bpf/bpf_lsm.c | 38 +++++++++++++++++++++++++++++++++ net/core/filter.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 93 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 5d2afa55c7c3..2b21f2a3452f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2386,6 +2386,8 @@ extern const struct bpf_func_proto bpf_for_each_map_elem_proto; extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; extern const struct bpf_func_proto bpf_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_sk_getsockopt_proto; +extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto; +extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_find_vma_proto; extern const struct bpf_func_proto bpf_loop_proto; extern const struct bpf_func_proto bpf_copy_from_user_task_proto; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 83aa431dd52e..d469b7f3deef 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -45,6 +45,24 @@ BTF_ID(func, bpf_lsm_sk_alloc_security) BTF_ID(func, bpf_lsm_sk_free_security) BTF_SET_END(bpf_lsm_current_hooks) +/* List of LSM hooks that trigger while the socket is properly locked. + */ +BTF_SET_START(bpf_lsm_locked_sockopt_hooks) +BTF_ID(func, bpf_lsm_socket_sock_rcv_skb) +BTF_ID(func, bpf_lsm_sock_graft) +BTF_ID(func, bpf_lsm_inet_csk_clone) +BTF_ID(func, bpf_lsm_inet_conn_established) +BTF_SET_END(bpf_lsm_locked_sockopt_hooks) + +/* List of LSM hooks that trigger while the socket is _not_ locked, + * but it's ok to call bpf_{g,s}etsockopt because the socket is still + * in the early init phase. + */ +BTF_SET_START(bpf_lsm_unlocked_sockopt_hooks) +BTF_ID(func, bpf_lsm_socket_post_create) +BTF_ID(func, bpf_lsm_socket_socketpair) +BTF_SET_END(bpf_lsm_unlocked_sockopt_hooks) + void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func) { @@ -201,6 +219,26 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_get_retval: return prog->expected_attach_type == BPF_LSM_CGROUP ? &bpf_get_retval_proto : NULL; + case BPF_FUNC_setsockopt: + if (prog->expected_attach_type != BPF_LSM_CGROUP) + return NULL; + if (btf_id_set_contains(&bpf_lsm_locked_sockopt_hooks, + prog->aux->attach_btf_id)) + return &bpf_sk_setsockopt_proto; + if (btf_id_set_contains(&bpf_lsm_unlocked_sockopt_hooks, + prog->aux->attach_btf_id)) + return &bpf_unlocked_sk_setsockopt_proto; + return NULL; + case BPF_FUNC_getsockopt: + if (prog->expected_attach_type != BPF_LSM_CGROUP) + return NULL; + if (btf_id_set_contains(&bpf_lsm_locked_sockopt_hooks, + prog->aux->attach_btf_id)) + return &bpf_sk_getsockopt_proto; + if (btf_id_set_contains(&bpf_lsm_unlocked_sockopt_hooks, + prog->aux->attach_btf_id)) + return &bpf_unlocked_sk_getsockopt_proto; + return NULL; default: return tracing_prog_func_proto(func_id, prog); } diff --git a/net/core/filter.c b/net/core/filter.c index 151aa4756bd6..c6941ab0eb52 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -5012,8 +5012,8 @@ static const struct bpf_func_proto bpf_get_socket_uid_proto = { .arg1_type = ARG_PTR_TO_CTX, }; -static int _bpf_setsockopt(struct sock *sk, int level, int optname, - char *optval, int optlen) +static int __bpf_setsockopt(struct sock *sk, int level, int optname, + char *optval, int optlen) { char devname[IFNAMSIZ]; int val, valbool; @@ -5024,8 +5024,6 @@ static int _bpf_setsockopt(struct sock *sk, int level, int optname, if (!sk_fullsock(sk)) return -EINVAL; - sock_owned_by_me(sk); - if (level == SOL_SOCKET) { if (optlen != sizeof(int) && optname != SO_BINDTODEVICE) return -EINVAL; @@ -5258,14 +5256,20 @@ static int _bpf_setsockopt(struct sock *sk, int level, int optname, return ret; } -static int _bpf_getsockopt(struct sock *sk, int level, int optname, +static int _bpf_setsockopt(struct sock *sk, int level, int optname, char *optval, int optlen) +{ + if (sk_fullsock(sk)) + sock_owned_by_me(sk); + return __bpf_setsockopt(sk, level, optname, optval, optlen); +} + +static int __bpf_getsockopt(struct sock *sk, int level, int optname, + char *optval, int optlen) { if (!sk_fullsock(sk)) goto err_clear; - sock_owned_by_me(sk); - if (level == SOL_SOCKET) { if (optlen != sizeof(int)) goto err_clear; @@ -5360,6 +5364,14 @@ err_clear: return -EINVAL; } +static int _bpf_getsockopt(struct sock *sk, int level, int optname, + char *optval, int optlen) +{ + if (sk_fullsock(sk)) + sock_owned_by_me(sk); + return __bpf_getsockopt(sk, level, optname, optval, optlen); +} + BPF_CALL_5(bpf_sk_setsockopt, struct sock *, sk, int, level, int, optname, char *, optval, int, optlen) { @@ -5400,6 +5412,40 @@ const struct bpf_func_proto bpf_sk_getsockopt_proto = { .arg5_type = ARG_CONST_SIZE, }; +BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level, + int, optname, char *, optval, int, optlen) +{ + return __bpf_setsockopt(sk, level, optname, optval, optlen); +} + +const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto = { + .func = bpf_unlocked_sk_setsockopt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg5_type = ARG_CONST_SIZE, +}; + +BPF_CALL_5(bpf_unlocked_sk_getsockopt, struct sock *, sk, int, level, + int, optname, char *, optval, int, optlen) +{ + return __bpf_getsockopt(sk, level, optname, optval, optlen); +} + +const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto = { + .func = bpf_unlocked_sk_getsockopt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, + .arg4_type = ARG_PTR_TO_UNINIT_MEM, + .arg5_type = ARG_CONST_SIZE, +}; + BPF_CALL_5(bpf_sock_addr_setsockopt, struct bpf_sock_addr_kern *, ctx, int, level, int, optname, char *, optval, int, optlen) { -- cgit v1.2.3 From f163f0302ab69722c052519f4014814bf10026a9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:21 +0200 Subject: context_tracking: Rename context_tracking_user_enter/exit() to user_enter/exit_callable() context_tracking_user_enter() and context_tracking_user_exit() are ASM callable versions of user_enter() and user_exit() for architectures that didn't manage to check the context tracking static key from ASM. Change those function names to better reflect their purpose. [ frederic: Apply Max Filippov feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- arch/arm/kernel/entry-header.S | 8 ++++---- arch/csky/kernel/entry.S | 4 ++-- arch/riscv/kernel/entry.S | 6 +++--- arch/xtensa/kernel/entry.S | 4 ++-- include/linux/context_tracking.h | 4 ++-- kernel/context_tracking.c | 28 +++++++++++++++++----------- 6 files changed, 30 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 5865621bf691..95def2b38d1c 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -369,10 +369,10 @@ ALT_UP_B(.L1_\@) #ifdef CONFIG_CONTEXT_TRACKING .if \save stmdb sp!, {r0-r3, ip, lr} - bl context_tracking_user_exit + bl user_exit_callable ldmia sp!, {r0-r3, ip, lr} .else - bl context_tracking_user_exit + bl user_exit_callable .endif #endif .endm @@ -381,10 +381,10 @@ ALT_UP_B(.L1_\@) #ifdef CONFIG_CONTEXT_TRACKING .if \save stmdb sp!, {r0-r3, ip, lr} - bl context_tracking_user_enter + bl user_enter_callable ldmia sp!, {r0-r3, ip, lr} .else - bl context_tracking_user_enter + bl user_enter_callable .endif #endif .endm diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index a4ababf25e24..bc734d17c16f 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -23,7 +23,7 @@ mfcr a0, epsr btsti a0, 31 bt 1f - jbsr context_tracking_user_exit + jbsr user_exit_callable ldw a0, (sp, LSAVE_A0) ldw a1, (sp, LSAVE_A1) ldw a2, (sp, LSAVE_A2) @@ -160,7 +160,7 @@ ret_from_exception: cmpnei r10, 0 bt exit_work #ifdef CONFIG_CONTEXT_TRACKING - jbsr context_tracking_user_enter + jbsr user_enter_callable #endif 1: #ifdef CONFIG_PREEMPTION diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 2e5b88ca11ce..12f6bba57e33 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -112,11 +112,11 @@ _save_context: #endif #ifdef CONFIG_CONTEXT_TRACKING - /* If previous state is in user mode, call context_tracking_user_exit. */ + /* If previous state is in user mode, call user_exit_callable(). */ li a0, SR_PP and a0, s1, a0 bnez a0, skip_context_tracking - call context_tracking_user_exit + call user_exit_callable skip_context_tracking: #endif @@ -270,7 +270,7 @@ resume_userspace: bnez s1, work_pending #ifdef CONFIG_CONTEXT_TRACKING - call context_tracking_user_enter + call user_enter_callable #endif /* Save unwound kernel stack pointer in thread_info */ diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index e3eae648ba2e..d72bcafae90c 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -458,7 +458,7 @@ KABI_W or a3, a3, a2 #ifdef CONFIG_CONTEXT_TRACKING l32i abi_tmp0, a1, PT_PS bbci.l abi_tmp0, PS_UM_BIT, 1f - abi_call context_tracking_user_exit + abi_call user_exit_callable 1: #endif @@ -545,7 +545,7 @@ common_exception_return: .Lexit_tif_loop_user: #ifdef CONFIG_CONTEXT_TRACKING - abi_call context_tracking_user_enter + abi_call user_enter_callable #endif #ifdef CONFIG_HAVE_HW_BREAKPOINT _bbci.l abi_saved0, TIF_DB_DISABLED, 1f diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 773035124bad..69532cd18f72 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -19,8 +19,8 @@ extern void __ct_user_exit(enum ctx_state state); extern void context_tracking_enter(enum ctx_state state); extern void context_tracking_exit(enum ctx_state state); -extern void context_tracking_user_enter(void); -extern void context_tracking_user_exit(void); +extern void user_enter_callable(void); +extern void user_exit_callable(void); static inline void user_enter(void) { diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 88c60ab39fbb..8f7dd5799bda 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -134,19 +134,22 @@ void context_tracking_enter(enum ctx_state state) NOKPROBE_SYMBOL(context_tracking_enter); EXPORT_SYMBOL_GPL(context_tracking_enter); -/* - * OBSOLETE: - * This function should be noinstr but it unsafely calls local_irq_restore(), - * involving illegal RCU uses through tracing and lockdep. +/** + * user_enter_callable() - Unfortunate ASM callable version of user_enter() for + * archs that didn't manage to check the context tracking + * static key from low level code. + * + * This OBSOLETE function should be noinstr but it unsafely calls + * local_irq_restore(), involving illegal RCU uses through tracing and lockdep. * This is unlikely to be fixed as this function is obsolete. The preferred * way is to call user_enter_irqoff(). It should be the arch entry code * responsibility to call into context tracking with IRQs disabled. */ -void context_tracking_user_enter(void) +void user_enter_callable(void) { user_enter(); } -NOKPROBE_SYMBOL(context_tracking_user_enter); +NOKPROBE_SYMBOL(user_enter_callable); /** * __ct_user_exit - Inform the context tracking that the CPU is @@ -208,19 +211,22 @@ void context_tracking_exit(enum ctx_state state) NOKPROBE_SYMBOL(context_tracking_exit); EXPORT_SYMBOL_GPL(context_tracking_exit); -/* - * OBSOLETE: - * This function should be noinstr but it unsafely calls local_irq_save(), +/** + * user_exit_callable() - Unfortunate ASM callable version of user_exit() for + * archs that didn't manage to check the context tracking + * static key from low level code. + * + * This OBSOLETE function should be noinstr but it unsafely calls local_irq_save(), * involving illegal RCU uses through tracing and lockdep. This is unlikely * to be fixed as this function is obsolete. The preferred way is to call * user_exit_irqoff(). It should be the arch entry code responsibility to * call into context tracking with IRQs disabled. */ -void context_tracking_user_exit(void) +void user_exit_callable(void) { user_exit(); } -NOKPROBE_SYMBOL(context_tracking_user_exit); +NOKPROBE_SYMBOL(user_exit_callable); void __init context_tracking_cpu_set(int cpu) { -- cgit v1.2.3 From fe98db1c6d1ad7349e61a5a2766ad64975bc9ae4 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:22 +0200 Subject: context_tracking: Rename context_tracking_enter/exit() to ct_user_enter/exit() context_tracking_enter() and context_tracking_exit() have confusing names that don't explain the fact they are referring to user/guest state. Use more self-explanatory names and shrink to the new context tracking prefix instead. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 13 +++++++------ kernel/context_tracking.c | 12 ++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 69532cd18f72..7a5f04ae1758 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -17,21 +17,22 @@ extern void context_tracking_cpu_set(int cpu); extern void __ct_user_enter(enum ctx_state state); extern void __ct_user_exit(enum ctx_state state); -extern void context_tracking_enter(enum ctx_state state); -extern void context_tracking_exit(enum ctx_state state); +extern void ct_user_enter(enum ctx_state state); +extern void ct_user_exit(enum ctx_state state); + extern void user_enter_callable(void); extern void user_exit_callable(void); static inline void user_enter(void) { if (context_tracking_enabled()) - context_tracking_enter(CONTEXT_USER); + ct_user_enter(CONTEXT_USER); } static inline void user_exit(void) { if (context_tracking_enabled()) - context_tracking_exit(CONTEXT_USER); + ct_user_exit(CONTEXT_USER); } /* Called with interrupts disabled. */ @@ -57,7 +58,7 @@ static inline enum ctx_state exception_enter(void) prev_ctx = this_cpu_read(context_tracking.state); if (prev_ctx != CONTEXT_KERNEL) - context_tracking_exit(prev_ctx); + ct_user_exit(prev_ctx); return prev_ctx; } @@ -67,7 +68,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) && context_tracking_enabled()) { if (prev_ctx != CONTEXT_KERNEL) - context_tracking_enter(prev_ctx); + ct_user_enter(prev_ctx); } } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 8f7dd5799bda..590c920ad57f 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -112,7 +112,7 @@ EXPORT_SYMBOL_GPL(__ct_user_enter); * or context_tracking_guest_enter(). It should be the arch entry code * responsibility to call into context tracking with IRQs disabled. */ -void context_tracking_enter(enum ctx_state state) +void ct_user_enter(enum ctx_state state) { unsigned long flags; @@ -131,8 +131,8 @@ void context_tracking_enter(enum ctx_state state) __ct_user_enter(state); local_irq_restore(flags); } -NOKPROBE_SYMBOL(context_tracking_enter); -EXPORT_SYMBOL_GPL(context_tracking_enter); +NOKPROBE_SYMBOL(ct_user_enter); +EXPORT_SYMBOL_GPL(ct_user_enter); /** * user_enter_callable() - Unfortunate ASM callable version of user_enter() for @@ -197,7 +197,7 @@ EXPORT_SYMBOL_GPL(__ct_user_exit); * or context_tracking_guest_exit(). It should be the arch entry code * responsibility to call into context tracking with IRQs disabled. */ -void context_tracking_exit(enum ctx_state state) +void ct_user_exit(enum ctx_state state) { unsigned long flags; @@ -208,8 +208,8 @@ void context_tracking_exit(enum ctx_state state) __ct_user_exit(state); local_irq_restore(flags); } -NOKPROBE_SYMBOL(context_tracking_exit); -EXPORT_SYMBOL_GPL(context_tracking_exit); +NOKPROBE_SYMBOL(ct_user_exit); +EXPORT_SYMBOL_GPL(ct_user_exit); /** * user_exit_callable() - Unfortunate ASM callable version of user_exit() for -- cgit v1.2.3 From 2a0aafce963dab996b843f6cdb49237b0ae4a118 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:23 +0200 Subject: context_tracking: Rename context_tracking_cpu_set() to ct_cpu_track_user() context_tracking_cpu_set() is called in order to tell a CPU to track user/kernel transitions. Since context tracking is going to expand in to also track transitions from/to idle/IRQ/NMIs, the scope of this function name becomes too broad and needs to be made more specific. Also shorten the prefix to align with the new namespace. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 2 +- kernel/context_tracking.c | 4 ++-- kernel/time/tick-sched.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 7a5f04ae1758..63259fece7c7 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -11,7 +11,7 @@ #ifdef CONFIG_CONTEXT_TRACKING -extern void context_tracking_cpu_set(int cpu); +extern void ct_cpu_track_user(int cpu); /* Called with interrupts disabled. */ extern void __ct_user_enter(enum ctx_state state); diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 590c920ad57f..d361bd52e4e1 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -228,7 +228,7 @@ void user_exit_callable(void) } NOKPROBE_SYMBOL(user_exit_callable); -void __init context_tracking_cpu_set(int cpu) +void __init ct_cpu_track_user(int cpu) { static __initdata bool initialized = false; @@ -258,6 +258,6 @@ void __init context_tracking_init(void) int cpu; for_each_possible_cpu(cpu) - context_tracking_cpu_set(cpu); + ct_cpu_track_user(cpu); } #endif diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 58a11f859ac7..de192dcff828 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -571,7 +571,7 @@ void __init tick_nohz_init(void) } for_each_cpu(cpu, tick_nohz_full_mask) - context_tracking_cpu_set(cpu); + ct_cpu_track_user(cpu); ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "kernel/nohz:predown", NULL, -- cgit v1.2.3 From 24a9c54182b3758801b8ca6c8c237cc2ff654732 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:24 +0200 Subject: context_tracking: Split user tracking Kconfig Context tracking is going to be used not only to track user transitions but also idle/IRQs/NMIs. The user tracking part will then become a separate feature. Prepare Kconfig for that. [ frederic: Apply Max Filippov feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- .../time/context-tracking/arch-support.txt | 6 ++--- arch/Kconfig | 4 +-- arch/arm/Kconfig | 2 +- arch/arm/kernel/entry-common.S | 4 +-- arch/arm/kernel/entry-header.S | 4 +-- arch/arm64/Kconfig | 2 +- arch/csky/Kconfig | 2 +- arch/csky/kernel/entry.S | 4 +-- arch/loongarch/Kconfig | 2 +- arch/mips/Kconfig | 2 +- arch/powerpc/Kconfig | 2 +- arch/powerpc/include/asm/context_tracking.h | 2 +- arch/riscv/Kconfig | 2 +- arch/riscv/kernel/entry.S | 6 ++--- arch/sparc/Kconfig | 2 +- arch/sparc/kernel/rtrap_64.S | 2 +- arch/x86/Kconfig | 4 +-- arch/xtensa/Kconfig | 2 +- arch/xtensa/kernel/entry.S | 4 +-- include/linux/context_tracking.h | 12 ++++----- include/linux/context_tracking_state.h | 4 +-- init/Kconfig | 4 +-- kernel/context_tracking.c | 6 ++++- kernel/sched/core.c | 2 +- kernel/time/Kconfig | 31 ++++++++++++++-------- 25 files changed, 65 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/Documentation/features/time/context-tracking/arch-support.txt b/Documentation/features/time/context-tracking/arch-support.txt index c9e0a16290e6..e59071a49090 100644 --- a/Documentation/features/time/context-tracking/arch-support.txt +++ b/Documentation/features/time/context-tracking/arch-support.txt @@ -1,7 +1,7 @@ # -# Feature name: context-tracking -# Kconfig: HAVE_CONTEXT_TRACKING -# description: arch supports context tracking for NO_HZ_FULL +# Feature name: user-context-tracking +# Kconfig: HAVE_CONTEXT_TRACKING_USER +# description: arch supports user context tracking for NO_HZ_FULL # ----------------------- | arch |status| diff --git a/arch/Kconfig b/arch/Kconfig index fcf9a41a4ef5..154b7b78da09 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -774,7 +774,7 @@ config HAVE_ARCH_WITHIN_STACK_FRAMES and similar) by implementing an inline arch_within_stack_frames(), which is used by CONFIG_HARDENED_USERCOPY. -config HAVE_CONTEXT_TRACKING +config HAVE_CONTEXT_TRACKING_USER bool help Provide kernel/user boundaries probes necessary for subsystems @@ -785,7 +785,7 @@ config HAVE_CONTEXT_TRACKING protected inside rcu_irq_enter/rcu_irq_exit() but preemption or signal handling on irq exit still need to be protected. -config HAVE_CONTEXT_TRACKING_OFFSTACK +config HAVE_CONTEXT_TRACKING_USER_OFFSTACK bool help Architecture neither relies on exception_enter()/exception_exit() diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 7630ba9cb6cc..9acc6aac5912 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -84,7 +84,7 @@ config ARM select HAVE_ARCH_TRANSPARENT_HUGEPAGE if ARM_LPAE select HAVE_ARM_SMCCC if CPU_V7 select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT select HAVE_BUILDTIME_MCOUNT_SORT select HAVE_DEBUG_KMEMLEAK if !XIP_KERNEL diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 7aa3ded4af92..37a0125fc926 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -28,7 +28,7 @@ #include "entry-header.S" saved_psr .req r8 -#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) +#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING_USER) saved_pc .req r9 #define TRACE(x...) x #else @@ -38,7 +38,7 @@ saved_pc .req lr .section .entry.text,"ax",%progbits .align 5 -#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING) || \ +#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING_USER) || \ IS_ENABLED(CONFIG_DEBUG_RSEQ)) /* * This is the fast syscall return path. We do as little as possible here, diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 95def2b38d1c..99411fa91350 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -366,7 +366,7 @@ ALT_UP_B(.L1_\@) * between user and kernel mode. */ .macro ct_user_exit, save = 1 -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER .if \save stmdb sp!, {r0-r3, ip, lr} bl user_exit_callable @@ -378,7 +378,7 @@ ALT_UP_B(.L1_\@) .endm .macro ct_user_enter, save = 1 -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER .if \save stmdb sp!, {r0-r3, ip, lr} bl user_enter_callable diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1652a9800ebe..7c5dd2af9ca9 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -174,7 +174,7 @@ config ARM64 select HAVE_C_RECORDMCOUNT select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 21d72b078eef..f55ba1745f7b 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -42,7 +42,7 @@ config CSKY select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_MMAP_RND_BITS select HAVE_ARCH_SECCOMP_FILTER - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_VIRT_CPU_ACCOUNTING_GEN select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index bc734d17c16f..547b4cd1b24b 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -19,7 +19,7 @@ .endm .macro context_tracking -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER mfcr a0, epsr btsti a0, 31 bt 1f @@ -159,7 +159,7 @@ ret_from_exception: and r10, r9 cmpnei r10, 0 bt exit_work -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER jbsr user_enter_callable #endif 1: diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 1920d52653b4..130dc65f3c85 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -76,7 +76,7 @@ config LOONGARCH select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_COPY_THREAD_TLS select HAVE_DEBUG_STACKOVERFLOW select HAVE_DMA_CONTIGUOUS diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index db09d45d59ec..9457894db237 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -56,7 +56,7 @@ config MIPS select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_TIF_NOHZ select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_KMEMLEAK diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index c2ce2e60c8f0..874c8d81284a 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -202,7 +202,7 @@ config PPC select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING if PPC64 + select HAVE_CONTEXT_TRACKING_USER if PPC64 select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW diff --git a/arch/powerpc/include/asm/context_tracking.h b/arch/powerpc/include/asm/context_tracking.h index f2682b28b050..4b63931c49e0 100644 --- a/arch/powerpc/include/asm/context_tracking.h +++ b/arch/powerpc/include/asm/context_tracking.h @@ -2,7 +2,7 @@ #ifndef _ASM_POWERPC_CONTEXT_TRACKING_H #define _ASM_POWERPC_CONTEXT_TRACKING_H -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER #define SCHEDULE_USER bl schedule_user #else #define SCHEDULE_USER bl schedule diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 32ffef9f6e5b..29b46f217345 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -86,7 +86,7 @@ config RISCV select HAVE_ARCH_THREAD_STRUCT_WHITELIST select HAVE_ARCH_VMAP_STACK if MMU && 64BIT select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 12f6bba57e33..b9eda3fcbd6d 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -111,7 +111,7 @@ _save_context: call __trace_hardirqs_off #endif -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER /* If previous state is in user mode, call user_exit_callable(). */ li a0, SR_PP and a0, s1, a0 @@ -176,7 +176,7 @@ handle_syscall: */ csrs CSR_STATUS, SR_IE #endif -#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) +#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING_USER) /* Recover a0 - a7 for system calls */ REG_L a0, PT_A0(sp) REG_L a1, PT_A1(sp) @@ -269,7 +269,7 @@ resume_userspace: andi s1, s0, _TIF_WORK_MASK bnez s1, work_pending -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER call user_enter_callable #endif diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ba449c47effd..9232411a8821 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -71,7 +71,7 @@ config SPARC64 select HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD select HAVE_SYSCALL_TRACEPOINTS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_TIF_NOHZ select HAVE_DEBUG_KMEMLEAK select IOMMU_HELPER diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index c5fd4b450d9b..eef102765a7e 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -15,7 +15,7 @@ #include #include -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER # define SCHEDULE_USER schedule_user #else # define SCHEDULE_USER schedule diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index be0b95e51df6..b0a6dbbb760b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -186,8 +186,8 @@ config X86 select HAVE_ASM_MODVERSIONS select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL - select HAVE_CONTEXT_TRACKING if X86_64 - select HAVE_CONTEXT_TRACKING_OFFSTACK if HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER if X86_64 + select HAVE_CONTEXT_TRACKING_USER_OFFSTACK if HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT select HAVE_OBJTOOL_MCOUNT if HAVE_OBJTOOL select HAVE_BUILDTIME_MCOUNT_SORT diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 0b0f0172cced..7927fed7bc83 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -33,7 +33,7 @@ config XTENSA select HAVE_ARCH_KCSAN select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_EXIT_THREAD diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index d72bcafae90c..fb67d85116e4 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -455,7 +455,7 @@ KABI_W or a3, a3, a2 abi_call trace_hardirqs_off 1: #endif -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER l32i abi_tmp0, a1, PT_PS bbci.l abi_tmp0, PS_UM_BIT, 1f abi_call user_exit_callable @@ -544,7 +544,7 @@ common_exception_return: j .Lrestore_state .Lexit_tif_loop_user: -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER abi_call user_enter_callable #endif #ifdef CONFIG_HAVE_HW_BREAKPOINT diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 63259fece7c7..e35ae66b4794 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -10,7 +10,7 @@ #include -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER extern void ct_cpu_track_user(int cpu); /* Called with interrupts disabled. */ @@ -52,7 +52,7 @@ static inline enum ctx_state exception_enter(void) { enum ctx_state prev_ctx; - if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) || + if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) || !context_tracking_enabled()) return 0; @@ -65,7 +65,7 @@ static inline enum ctx_state exception_enter(void) static inline void exception_exit(enum ctx_state prev_ctx) { - if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) && + if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) && context_tracking_enabled()) { if (prev_ctx != CONTEXT_KERNEL) ct_user_enter(prev_ctx); @@ -109,14 +109,14 @@ static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; } static __always_inline bool context_tracking_guest_enter(void) { return false; } static inline void context_tracking_guest_exit(void) { } -#endif /* !CONFIG_CONTEXT_TRACKING */ +#endif /* !CONFIG_CONTEXT_TRACKING_USER */ #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) -#ifdef CONFIG_CONTEXT_TRACKING_FORCE +#ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE extern void context_tracking_init(void); #else static inline void context_tracking_init(void) { } -#endif /* CONFIG_CONTEXT_TRACKING_FORCE */ +#endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */ #endif diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index edc7b46376a6..2b46afe105a9 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -22,7 +22,7 @@ struct context_tracking { } state; }; -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER extern struct static_key_false context_tracking_key; DECLARE_PER_CPU(struct context_tracking, context_tracking); @@ -45,6 +45,6 @@ static inline bool context_tracking_enabled_this_cpu(void) static __always_inline bool context_tracking_enabled(void) { return false; } static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; } static __always_inline bool context_tracking_enabled_this_cpu(void) { return false; } -#endif /* CONFIG_CONTEXT_TRACKING */ +#endif /* CONFIG_CONTEXT_TRACKING_USER */ #endif diff --git a/init/Kconfig b/init/Kconfig index c7900e8975f1..06454d19e2f0 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -494,11 +494,11 @@ config VIRT_CPU_ACCOUNTING_NATIVE config VIRT_CPU_ACCOUNTING_GEN bool "Full dynticks CPU time accounting" - depends on HAVE_CONTEXT_TRACKING + depends on HAVE_CONTEXT_TRACKING_USER depends on HAVE_VIRT_CPU_ACCOUNTING_GEN depends on GENERIC_CLOCKEVENTS select VIRT_CPU_ACCOUNTING - select CONTEXT_TRACKING + select CONTEXT_TRACKING_USER help Select this option to enable task and CPU time accounting on full dynticks systems. This accounting is implemented by watching every diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index d361bd52e4e1..f3dec1be2bf6 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -22,6 +22,8 @@ #include #include +#ifdef CONFIG_CONTEXT_TRACKING_USER + #define CREATE_TRACE_POINTS #include @@ -252,7 +254,7 @@ void __init ct_cpu_track_user(int cpu) initialized = true; } -#ifdef CONFIG_CONTEXT_TRACKING_FORCE +#ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE void __init context_tracking_init(void) { int cpu; @@ -261,3 +263,5 @@ void __init context_tracking_init(void) ct_cpu_track_user(cpu); } #endif + +#endif /* #ifdef CONFIG_CONTEXT_TRACKING_USER */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index da0bf6fe9ecd..883167a57bf9 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6559,7 +6559,7 @@ void __sched schedule_idle(void) } while (need_resched()); } -#if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) +#if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) asmlinkage __visible void __sched schedule_user(void) { /* diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index 27b7868b5c30..41f99bcfe9e6 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -73,6 +73,9 @@ config TIME_KUNIT_TEST If unsure, say N. +config CONTEXT_TRACKING + bool + if GENERIC_CLOCKEVENTS menu "Timers subsystem" @@ -111,7 +114,7 @@ config NO_HZ_FULL # NO_HZ_COMMON dependency # We need at least one periodic CPU for timekeeping depends on SMP - depends on HAVE_CONTEXT_TRACKING + depends on HAVE_CONTEXT_TRACKING_USER # VIRT_CPU_ACCOUNTING_GEN dependency depends on HAVE_VIRT_CPU_ACCOUNTING_GEN select NO_HZ_COMMON @@ -137,31 +140,37 @@ config NO_HZ_FULL endchoice -config CONTEXT_TRACKING - bool +config CONTEXT_TRACKING_USER + bool + depends on HAVE_CONTEXT_TRACKING_USER + select CONTEXT_TRACKING + help + Track transitions between kernel and user on behalf of RCU and + tickless cputime accounting. The former case relies on context + tracking to enter/exit RCU extended quiescent states. -config CONTEXT_TRACKING_FORCE - bool "Force context tracking" - depends on CONTEXT_TRACKING +config CONTEXT_TRACKING_USER_FORCE + bool "Force user context tracking" + depends on CONTEXT_TRACKING_USER default y if !NO_HZ_FULL help The major pre-requirement for full dynticks to work is to - support the context tracking subsystem. But there are also + support the user context tracking subsystem. But there are also other dependencies to provide in order to make the full dynticks working. This option stands for testing when an arch implements the - context tracking backend but doesn't yet fulfill all the + user context tracking backend but doesn't yet fulfill all the requirements to make the full dynticks feature working. Without the full dynticks, there is no way to test the support - for context tracking and the subsystems that rely on it: RCU + for user context tracking and the subsystems that rely on it: RCU userspace extended quiescent state and tickless cputime accounting. This option copes with the absence of the full - dynticks subsystem by forcing the context tracking on all + dynticks subsystem by forcing the user context tracking on all CPUs in the system. Say Y only if you're working on the development of an - architecture backend for the context tracking. + architecture backend for the user context tracking. Say N otherwise, this option brings an overhead that you don't want in production. -- cgit v1.2.3 From 5f278dbd540b7548bc5193552e6d478255c14c2d Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Tue, 28 Jun 2022 12:10:15 -0700 Subject: iosys-map: Add per-word read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of always falling back to memcpy_fromio() for any size, prefer using read{b,w,l}(). When reading struct members it's common to read individual integer variables individually. Going through memcpy_fromio() for each of them poses a high penalty. Employ a similar trick as __seqprop() by using _Generic() to generate only the specific call based on a type-compatible variable. For a pariticular i915 workload producing GPU context switches, __get_engine_usage_record() is particularly hot since the engine usage is read from device local memory with dgfx, possibly multiple times since it's racy. Test execution time for this test shows a ~12.5% improvement with DG2: Before: nrepeats = 1000; min = 7.63243e+06; max = 1.01817e+07; median = 9.52548e+06; var = 526149; After: nrepeats = 1000; min = 7.03402e+06; max = 8.8832e+06; median = 8.33955e+06; var = 333113; Other things attempted that didn't prove very useful: 1) Change the _Generic() on x86 to just dereference the memory address 2) Change __get_engine_usage_record() to do just 1 read per loop, comparing with the previous value read 3) Change __get_engine_usage_record() to access the fields directly as it was before the conversion to iosys-map (3) did gave a small improvement (~3%), but doesn't seem to scale well to other similar cases in the driver. Additional test by Chris Wilson using gem_create from igt with some changes to track object creation time. This happens to accidentally stress this code path: Pre iosys_map conversion of engine busyness: lmem0: Creating 262144 4KiB objects took 59274.2ms Unpatched: lmem0: Creating 262144 4KiB objects took 108830.2ms With readl (this patch): lmem0: Creating 262144 4KiB objects took 61348.6ms s/readl/READ_ONCE/ lmem0: Creating 262144 4KiB objects took 61333.2ms So we do take a little bit more time than before the conversion, but that is due to other factors: bringing the READ_ONCE back would be as good as just doing this conversion. v2: - Remove default from _Generic() - callers wanting to read more than u64 should use iosys_map_memcpy_from() - Add READ_ONCE() cases dereferencing the pointer when using system memory v3: - Fix precedence issue when casting inside READ_ONCE(). By not using () around vaddr__ the offset was not part of the cast, but rather added to it, producing a wrong address - Remove compiletime_assert() as READ_ONCE() already contains it Signed-off-by: Lucas De Marchi Reviewed-by: Christian König # v1 Reviewed-by: Thomas Zimmermann Link: https://patchwork.freedesktop.org/patch/msgid/20220628191016.3899428-1-lucas.demarchi@intel.com --- include/linux/iosys-map.h | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/iosys-map.h b/include/linux/iosys-map.h index 4b8406ee8bc4..d19977e011ae 100644 --- a/include/linux/iosys-map.h +++ b/include/linux/iosys-map.h @@ -6,6 +6,7 @@ #ifndef __IOSYS_MAP_H__ #define __IOSYS_MAP_H__ +#include #include #include @@ -333,6 +334,23 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset, memset(dst->vaddr + offset, value, len); } +#ifdef CONFIG_64BIT +#define __iosys_map_rd_io_u64_case(val_, vaddr_iomem_) \ + u64: val_ = readq(vaddr_iomem_) +#else +#define __iosys_map_rd_io_u64_case(val_, vaddr_iomem_) \ + u64: memcpy_fromio(&(val_), vaddr_iomem_, sizeof(u64)) +#endif + +#define __iosys_map_rd_io(val__, vaddr_iomem__, type__) _Generic(val__, \ + u8: val__ = readb(vaddr_iomem__), \ + u16: val__ = readw(vaddr_iomem__), \ + u32: val__ = readl(vaddr_iomem__), \ + __iosys_map_rd_io_u64_case(val__, vaddr_iomem__)) + +#define __iosys_map_rd_sys(val__, vaddr__, type__) \ + val__ = READ_ONCE(*(type__ *)(vaddr__)) + /** * iosys_map_rd - Read a C-type value from the iosys_map * @@ -340,16 +358,21 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset, * @offset__: The offset from which to read * @type__: Type of the value being read * - * Read a C type value from iosys_map, handling possible un-aligned accesses to - * the mapping. + * Read a C type value (u8, u16, u32 and u64) from iosys_map. For other types or + * if pointer may be unaligned (and problematic for the architecture supported), + * use iosys_map_memcpy_from(). * * Returns: * The value read from the mapping. */ -#define iosys_map_rd(map__, offset__, type__) ({ \ - type__ val; \ - iosys_map_memcpy_from(&val, map__, offset__, sizeof(val)); \ - val; \ +#define iosys_map_rd(map__, offset__, type__) ({ \ + type__ val; \ + if ((map__)->is_iomem) { \ + __iosys_map_rd_io(val, (map__)->vaddr_iomem + (offset__), type__);\ + } else { \ + __iosys_map_rd_sys(val, (map__)->vaddr + (offset__), type__); \ + } \ + val; \ }) /** @@ -379,9 +402,10 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset, * * Read a value from iosys_map considering its layout is described by a C struct * starting at @struct_offset__. The field offset and size is calculated and its - * value read handling possible un-aligned memory accesses. For example: suppose - * there is a @struct foo defined as below and the value ``foo.field2.inner2`` - * needs to be read from the iosys_map: + * value read. If the field access would incur in un-aligned access, then either + * iosys_map_memcpy_from() needs to be used or the architecture must support it. + * For example: suppose there is a @struct foo defined as below and the value + * ``foo.field2.inner2`` needs to be read from the iosys_map: * * .. code-block:: c * -- cgit v1.2.3 From 6fb5ee7cec06266a29f25ecc01a23b9d107f64e1 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Tue, 28 Jun 2022 12:10:16 -0700 Subject: iosys-map: Add per-word write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Like was done for read, provide the equivalent for write. Even if current users are not in the hot path, this should future-proof it. v2: - Remove default from _Generic() - callers wanting to write more than u64 should use iosys_map_memcpy_to() - Add WRITE_ONCE() cases dereferencing the pointer when using system memory v3: - Fix precedence issue when casting inside WRITE_ONCE(). By not using () around vaddr__ the offset was not part of the cast, but rather added to it, producing a wrong address - Remove compiletime_assert() as WRITE_ONCE() already contains it Signed-off-by: Lucas De Marchi Reviewed-by: Reviewed-by: Christian König # v1 Reviewed-by: Thomas Zimmermann Link: https://patchwork.freedesktop.org/patch/msgid/20220628191016.3899428-2-lucas.demarchi@intel.com --- include/linux/iosys-map.h | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/iosys-map.h b/include/linux/iosys-map.h index d19977e011ae..a533cae189d7 100644 --- a/include/linux/iosys-map.h +++ b/include/linux/iosys-map.h @@ -337,9 +337,13 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset, #ifdef CONFIG_64BIT #define __iosys_map_rd_io_u64_case(val_, vaddr_iomem_) \ u64: val_ = readq(vaddr_iomem_) +#define __iosys_map_wr_io_u64_case(val_, vaddr_iomem_) \ + u64: writeq(val_, vaddr_iomem_) #else #define __iosys_map_rd_io_u64_case(val_, vaddr_iomem_) \ u64: memcpy_fromio(&(val_), vaddr_iomem_, sizeof(u64)) +#define __iosys_map_wr_io_u64_case(val_, vaddr_iomem_) \ + u64: memcpy_toio(vaddr_iomem_, &(val_), sizeof(u64)) #endif #define __iosys_map_rd_io(val__, vaddr_iomem__, type__) _Generic(val__, \ @@ -351,6 +355,15 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset, #define __iosys_map_rd_sys(val__, vaddr__, type__) \ val__ = READ_ONCE(*(type__ *)(vaddr__)) +#define __iosys_map_wr_io(val__, vaddr_iomem__, type__) _Generic(val__, \ + u8: writeb(val__, vaddr_iomem__), \ + u16: writew(val__, vaddr_iomem__), \ + u32: writel(val__, vaddr_iomem__), \ + __iosys_map_wr_io_u64_case(val__, vaddr_iomem__)) + +#define __iosys_map_wr_sys(val__, vaddr__, type__) \ + WRITE_ONCE(*(type__ *)(vaddr__), val__) + /** * iosys_map_rd - Read a C-type value from the iosys_map * @@ -383,12 +396,17 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset, * @type__: Type of the value being written * @val__: Value to write * - * Write a C-type value to the iosys_map, handling possible un-aligned accesses - * to the mapping. + * Write a C type value (u8, u16, u32 and u64) to the iosys_map. For other types + * or if pointer may be unaligned (and problematic for the architecture + * supported), use iosys_map_memcpy_to() */ -#define iosys_map_wr(map__, offset__, type__, val__) ({ \ - type__ val = (val__); \ - iosys_map_memcpy_to(map__, offset__, &val, sizeof(val)); \ +#define iosys_map_wr(map__, offset__, type__, val__) ({ \ + type__ val = (val__); \ + if ((map__)->is_iomem) { \ + __iosys_map_wr_io(val, (map__)->vaddr_iomem + (offset__), type__);\ + } else { \ + __iosys_map_wr_sys(val, (map__)->vaddr + (offset__), type__); \ + } \ }) /** @@ -469,10 +487,12 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset, * @field__: Member of the struct to read * @val__: Value to write * - * Write a value to the iosys_map considering its layout is described by a C struct - * starting at @struct_offset__. The field offset and size is calculated and the - * @val__ is written handling possible un-aligned memory accesses. Refer to - * iosys_map_rd_field() for expected usage and memory layout. + * Write a value to the iosys_map considering its layout is described by a C + * struct starting at @struct_offset__. The field offset and size is calculated + * and the @val__ is written. If the field access would incur in un-aligned + * access, then either iosys_map_memcpy_to() needs to be used or the + * architecture must support it. Refer to iosys_map_rd_field() for expected + * usage and memory layout. */ #define iosys_map_wr_field(map__, struct_offset__, struct_type__, field__, val__) ({ \ struct_type__ *s; \ -- cgit v1.2.3 From c381d02b2fd5f82d2207db1b9b25ff60d0d9c27c Mon Sep 17 00:00:00 2001 From: Yuwei Wang Date: Wed, 29 Jun 2022 08:48:31 +0000 Subject: sysctl: add proc_dointvec_ms_jiffies_minmax add proc_dointvec_ms_jiffies_minmax to fit read msecs value to jiffies with a limited range of values Signed-off-by: Yuwei Wang Signed-off-by: Paolo Abeni --- include/linux/sysctl.h | 2 ++ kernel/sysctl.c | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 80263f7cdb77..17b42ce89d3e 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -75,6 +75,8 @@ int proc_douintvec_minmax(struct ctl_table *table, int write, void *buffer, int proc_dou8vec_minmax(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int proc_dointvec_jiffies(struct ctl_table *, int, void *, size_t *, loff_t *); +int proc_dointvec_ms_jiffies_minmax(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos); int proc_dointvec_userhz_jiffies(struct ctl_table *, int, void *, size_t *, loff_t *); int proc_dointvec_ms_jiffies(struct ctl_table *, int, void *, size_t *, diff --git a/kernel/sysctl.c b/kernel/sysctl.c index e52b6e372c60..85c92e2c2570 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1237,6 +1237,30 @@ static int do_proc_dointvec_ms_jiffies_conv(bool *negp, unsigned long *lvalp, return 0; } +static int do_proc_dointvec_ms_jiffies_minmax_conv(bool *negp, unsigned long *lvalp, + int *valp, int write, void *data) +{ + int tmp, ret; + struct do_proc_dointvec_minmax_conv_param *param = data; + /* + * If writing, first do so via a temporary local int so we can + * bounds-check it before touching *valp. + */ + int *ip = write ? &tmp : valp; + + ret = do_proc_dointvec_ms_jiffies_conv(negp, lvalp, ip, write, data); + if (ret) + return ret; + + if (write) { + if ((param->min && *param->min > tmp) || + (param->max && *param->max < tmp)) + return -EINVAL; + *valp = tmp; + } + return 0; +} + /** * proc_dointvec_jiffies - read a vector of integers as seconds * @table: the sysctl table @@ -1259,6 +1283,17 @@ int proc_dointvec_jiffies(struct ctl_table *table, int write, do_proc_dointvec_jiffies_conv,NULL); } +int proc_dointvec_ms_jiffies_minmax(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct do_proc_dointvec_minmax_conv_param param = { + .min = (int *) table->extra1, + .max = (int *) table->extra2, + }; + return do_proc_dointvec(table, write, buffer, lenp, ppos, + do_proc_dointvec_ms_jiffies_minmax_conv, ¶m); +} + /** * proc_dointvec_userhz_jiffies - read a vector of integers as 1/USER_HZ seconds * @table: the sysctl table @@ -1523,6 +1558,12 @@ int proc_dointvec_jiffies(struct ctl_table *table, int write, return -ENOSYS; } +int proc_dointvec_ms_jiffies_minmax(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + int proc_dointvec_userhz_jiffies(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { -- cgit v1.2.3 From 95c8222f0e52b09b7607616274e7cae84d519a9b Mon Sep 17 00:00:00 2001 From: David Jander Date: Wed, 29 Jun 2022 16:25:18 +0200 Subject: spi: spi.c: Fix comment style Capitalize first word in comment where appropriate and add parentheses to function names. Reported-by: Andy Shevchenko Signed-off-by: David Jander Link: https://lore.kernel.org/r/20220629142519.3985486-3-david@protonic.nl Signed-off-by: Mark Brown --- drivers/spi/spi.c | 94 ++++++++++++++++++++++++------------------------- include/linux/spi/spi.h | 82 +++++++++++++++++++++--------------------- 2 files changed, 88 insertions(+), 88 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 031606251b58..6d69f5dc62bc 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1354,7 +1354,7 @@ int spi_delay_to_ns(struct spi_delay *_delay, struct spi_transfer *xfer) /* Nothing to do here */ break; case SPI_DELAY_UNIT_SCK: - /* clock cycles need to be obtained from spi_transfer */ + /* Clock cycles need to be obtained from spi_transfer */ if (!xfer) return -EINVAL; /* @@ -1403,7 +1403,7 @@ static void _spi_transfer_cs_change_delay(struct spi_message *msg, u32 unit = xfer->cs_change_delay.unit; int ret; - /* return early on "fast" mode - for everything but USECS */ + /* Return early on "fast" mode - for everything but USECS */ if (!delay) { if (unit == SPI_DELAY_UNIT_USECS) _spi_transfer_delay_ns(default_delay_ns); @@ -1629,7 +1629,7 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, WRITE_ONCE(ctlr->cur_msg_incomplete, true); WRITE_ONCE(ctlr->cur_msg_need_completion, false); reinit_completion(&ctlr->cur_msg_completion); - smp_wmb(); /* make these available to spi_finalize_current_message */ + smp_wmb(); /* Make these available to spi_finalize_current_message() */ ret = ctlr->transfer_one_message(ctlr, msg); if (ret) { @@ -1905,7 +1905,7 @@ struct spi_message *spi_get_next_queued_message(struct spi_controller *ctlr) struct spi_message *next; unsigned long flags; - /* get a pointer to the next message, if any */ + /* Get a pointer to the next message, if any */ spin_lock_irqsave(&ctlr->queue_lock, flags); next = list_first_entry_or_null(&ctlr->queue, struct spi_message, queue); @@ -2558,7 +2558,7 @@ struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr, acpi_dev_free_resource_list(&resource_list); if (ret < 0) - /* found SPI in _CRS but it points to another controller */ + /* Found SPI in _CRS but it points to another controller */ return ERR_PTR(-ENODEV); if (!lookup.max_speed_hz && @@ -3014,7 +3014,7 @@ int spi_register_controller(struct spi_controller *ctlr) return status; if (ctlr->bus_num >= 0) { - /* devices with a fixed bus num must check-in with the num */ + /* Devices with a fixed bus num must check-in with the num */ mutex_lock(&board_lock); id = idr_alloc(&spi_master_idr, ctlr, ctlr->bus_num, ctlr->bus_num + 1, GFP_KERNEL); @@ -3023,7 +3023,7 @@ int spi_register_controller(struct spi_controller *ctlr) return id == -ENOSPC ? -EBUSY : id; ctlr->bus_num = id; } else if (ctlr->dev.of_node) { - /* allocate dynamic bus number using Linux idr */ + /* Allocate dynamic bus number using Linux idr */ id = of_alias_get_id(ctlr->dev.of_node, "spi"); if (id >= 0) { ctlr->bus_num = id; @@ -3082,7 +3082,7 @@ int spi_register_controller(struct spi_controller *ctlr) goto free_bus_id; } - /* setting last_cs to -1 means no chip selected */ + /* Setting last_cs to -1 means no chip selected */ ctlr->last_cs = -1; status = device_add(&ctlr->dev); @@ -3106,7 +3106,7 @@ int spi_register_controller(struct spi_controller *ctlr) goto free_bus_id; } } - /* add statistics */ + /* Add statistics */ ctlr->pcpu_statistics = spi_alloc_pcpu_stats(dev); if (!ctlr->pcpu_statistics) { dev_err(dev, "Error allocating per-cpu statistics\n"); @@ -3209,7 +3209,7 @@ void spi_unregister_controller(struct spi_controller *ctlr) device_del(&ctlr->dev); - /* free bus id */ + /* Free bus id */ mutex_lock(&board_lock); if (found == ctlr) idr_remove(&spi_master_idr, id); @@ -3268,14 +3268,14 @@ static void __spi_replace_transfers_release(struct spi_controller *ctlr, struct spi_replaced_transfers *rxfer = res; size_t i; - /* call extra callback if requested */ + /* Call extra callback if requested */ if (rxfer->release) rxfer->release(ctlr, msg, res); - /* insert replaced transfers back into the message */ + /* Insert replaced transfers back into the message */ list_splice(&rxfer->replaced_transfers, rxfer->replaced_after); - /* remove the formerly inserted entries */ + /* Remove the formerly inserted entries */ for (i = 0; i < rxfer->inserted; i++) list_del(&rxfer->inserted_transfers[i].transfer_list); } @@ -3308,7 +3308,7 @@ static struct spi_replaced_transfers *spi_replace_transfers( struct spi_transfer *xfer; size_t i; - /* allocate the structure using spi_res */ + /* Allocate the structure using spi_res */ rxfer = spi_res_alloc(msg->spi, __spi_replace_transfers_release, struct_size(rxfer, inserted_transfers, insert) + extradatasize, @@ -3316,15 +3316,15 @@ static struct spi_replaced_transfers *spi_replace_transfers( if (!rxfer) return ERR_PTR(-ENOMEM); - /* the release code to invoke before running the generic release */ + /* The release code to invoke before running the generic release */ rxfer->release = release; - /* assign extradata */ + /* Assign extradata */ if (extradatasize) rxfer->extradata = &rxfer->inserted_transfers[insert]; - /* init the replaced_transfers list */ + /* Init the replaced_transfers list */ INIT_LIST_HEAD(&rxfer->replaced_transfers); /* @@ -3333,7 +3333,7 @@ static struct spi_replaced_transfers *spi_replace_transfers( */ rxfer->replaced_after = xfer_first->transfer_list.prev; - /* remove the requested number of transfers */ + /* Remove the requested number of transfers */ for (i = 0; i < remove; i++) { /* * If the entry after replaced_after it is msg->transfers @@ -3343,14 +3343,14 @@ static struct spi_replaced_transfers *spi_replace_transfers( if (rxfer->replaced_after->next == &msg->transfers) { dev_err(&msg->spi->dev, "requested to remove more spi_transfers than are available\n"); - /* insert replaced transfers back into the message */ + /* Insert replaced transfers back into the message */ list_splice(&rxfer->replaced_transfers, rxfer->replaced_after); - /* free the spi_replace_transfer structure */ + /* Free the spi_replace_transfer structure... */ spi_res_free(rxfer); - /* and return with an error */ + /* ...and return with an error */ return ERR_PTR(-EINVAL); } @@ -3367,26 +3367,26 @@ static struct spi_replaced_transfers *spi_replace_transfers( * based on the first transfer to get removed. */ for (i = 0; i < insert; i++) { - /* we need to run in reverse order */ + /* We need to run in reverse order */ xfer = &rxfer->inserted_transfers[insert - 1 - i]; - /* copy all spi_transfer data */ + /* Copy all spi_transfer data */ memcpy(xfer, xfer_first, sizeof(*xfer)); - /* add to list */ + /* Add to list */ list_add(&xfer->transfer_list, rxfer->replaced_after); - /* clear cs_change and delay for all but the last */ + /* Clear cs_change and delay for all but the last */ if (i) { xfer->cs_change = false; xfer->delay.value = 0; } } - /* set up inserted */ + /* Set up inserted... */ rxfer->inserted = insert; - /* and register it with spi_res/spi_message */ + /* ...and register it with spi_res/spi_message */ spi_res_add(msg, rxfer); return rxfer; @@ -3403,10 +3403,10 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, size_t offset; size_t count, i; - /* calculate how many we have to replace */ + /* Calculate how many we have to replace */ count = DIV_ROUND_UP(xfer->len, maxsize); - /* create replacement */ + /* Create replacement */ srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, gfp); if (IS_ERR(srt)) return PTR_ERR(srt); @@ -3429,9 +3429,9 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, */ xfers[0].len = min_t(size_t, maxsize, xfer[0].len); - /* all the others need rx_buf/tx_buf also set */ + /* All the others need rx_buf/tx_buf also set */ for (i = 1, offset = maxsize; i < count; offset += maxsize, i++) { - /* update rx_buf, tx_buf and dma */ + /* Update rx_buf, tx_buf and dma */ if (xfers[i].rx_buf) xfers[i].rx_buf += offset; if (xfers[i].rx_dma) @@ -3441,7 +3441,7 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, if (xfers[i].tx_dma) xfers[i].tx_dma += offset; - /* update length */ + /* Update length */ xfers[i].len = min(maxsize, xfers[i].len - offset); } @@ -3451,7 +3451,7 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, */ *xferp = &xfers[count - 1]; - /* increment statistics counters */ + /* Increment statistics counters */ SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, transfers_split_maxsize); SPI_STATISTICS_INCREMENT_FIELD(msg->spi->pcpu_statistics, @@ -3713,7 +3713,7 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) return ret; list_for_each_entry(xfer, &message->transfers, transfer_list) { - /* don't change cs_change on the last entry in the list */ + /* Don't change cs_change on the last entry in the list */ if (list_is_last(&xfer->transfer_list, &message->transfers)) break; xfer->cs_change = 1; @@ -3806,7 +3806,7 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) !(spi->mode & SPI_TX_QUAD)) return -EINVAL; } - /* check transfer rx_nbits */ + /* Check transfer rx_nbits */ if (xfer->rx_buf) { if (spi->mode & SPI_NO_RX) return -EINVAL; @@ -4144,7 +4144,7 @@ int spi_bus_lock(struct spi_controller *ctlr) ctlr->bus_lock_flag = 1; spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); - /* mutex remains locked until spi_bus_unlock is called */ + /* Mutex remains locked until spi_bus_unlock() is called */ return 0; } @@ -4173,7 +4173,7 @@ int spi_bus_unlock(struct spi_controller *ctlr) } EXPORT_SYMBOL_GPL(spi_bus_unlock); -/* portable code must never pass more than 32 bytes */ +/* Portable code must never pass more than 32 bytes */ #define SPI_BUFSIZ max(32, SMP_CACHE_BYTES) static u8 *buf; @@ -4239,7 +4239,7 @@ int spi_write_then_read(struct spi_device *spi, x[0].tx_buf = local_buf; x[1].rx_buf = local_buf + n_tx; - /* do the i/o */ + /* Do the i/o */ status = spi_sync(spi, &message); if (status == 0) memcpy(rxbuf, x[1].rx_buf, n_rx); @@ -4256,7 +4256,7 @@ EXPORT_SYMBOL_GPL(spi_write_then_read); /*-------------------------------------------------------------------------*/ #if IS_ENABLED(CONFIG_OF_DYNAMIC) -/* must call put_device() when done with returned spi_device device */ +/* Must call put_device() when done with returned spi_device device */ static struct spi_device *of_find_spi_device_by_node(struct device_node *node) { struct device *dev = bus_find_device_by_of_node(&spi_bus_type, node); @@ -4264,7 +4264,7 @@ static struct spi_device *of_find_spi_device_by_node(struct device_node *node) return dev ? to_spi_device(dev) : NULL; } -/* the spi controllers are not using spi_bus, so we find it with another way */ +/* The spi controllers are not using spi_bus, so we find it with another way */ static struct spi_controller *of_find_spi_controller_by_node(struct device_node *node) { struct device *dev; @@ -4275,7 +4275,7 @@ static struct spi_controller *of_find_spi_controller_by_node(struct device_node if (!dev) return NULL; - /* reference got in class_find_device */ + /* Reference got in class_find_device */ return container_of(dev, struct spi_controller, dev); } @@ -4290,7 +4290,7 @@ static int of_spi_notify(struct notifier_block *nb, unsigned long action, case OF_RECONFIG_CHANGE_ADD: ctlr = of_find_spi_controller_by_node(rd->dn->parent); if (ctlr == NULL) - return NOTIFY_OK; /* not for us */ + return NOTIFY_OK; /* Not for us */ if (of_node_test_and_set_flag(rd->dn, OF_POPULATED)) { put_device(&ctlr->dev); @@ -4309,19 +4309,19 @@ static int of_spi_notify(struct notifier_block *nb, unsigned long action, break; case OF_RECONFIG_CHANGE_REMOVE: - /* already depopulated? */ + /* Already depopulated? */ if (!of_node_check_flag(rd->dn, OF_POPULATED)) return NOTIFY_OK; - /* find our device by node */ + /* Find our device by node */ spi = of_find_spi_device_by_node(rd->dn); if (spi == NULL) - return NOTIFY_OK; /* no? not meant for us */ + return NOTIFY_OK; /* No? not meant for us */ - /* unregister takes one ref away */ + /* Unregister takes one ref away */ spi_unregister_device(spi); - /* and put the reference of the find */ + /* And put the reference of the find */ put_device(&spi->dev); break; } diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index eb0d316e3c36..e6c73d5ff1a8 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -176,13 +176,13 @@ extern int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer); struct spi_device { struct device dev; struct spi_controller *controller; - struct spi_controller *master; /* compatibility layer */ + struct spi_controller *master; /* Compatibility layer */ u32 max_speed_hz; u8 chip_select; u8 bits_per_word; bool rt; -#define SPI_NO_TX BIT(31) /* no transmit wire */ -#define SPI_NO_RX BIT(30) /* no receive wire */ +#define SPI_NO_TX BIT(31) /* No transmit wire */ +#define SPI_NO_RX BIT(30) /* No receive wire */ /* * All bits defined above should be covered by SPI_MODE_KERNEL_MASK. * The SPI_MODE_KERNEL_MASK has the SPI_MODE_USER_MASK counterpart, @@ -199,14 +199,14 @@ struct spi_device { void *controller_data; char modalias[SPI_NAME_SIZE]; const char *driver_override; - struct gpio_desc *cs_gpiod; /* chip select gpio desc */ - struct spi_delay word_delay; /* inter-word delay */ + struct gpio_desc *cs_gpiod; /* Chip select gpio desc */ + struct spi_delay word_delay; /* Inter-word delay */ /* CS delays */ struct spi_delay cs_setup; struct spi_delay cs_hold; struct spi_delay cs_inactive; - /* the statistics */ + /* The statistics */ struct spi_statistics __percpu *pcpu_statistics; /* @@ -228,7 +228,7 @@ static inline struct spi_device *to_spi_device(struct device *dev) return dev ? container_of(dev, struct spi_device, dev) : NULL; } -/* most drivers won't need to care about device refcounting */ +/* Most drivers won't need to care about device refcounting */ static inline struct spi_device *spi_dev_get(struct spi_device *spi) { return (spi && get_device(&spi->dev)) ? spi : NULL; @@ -251,7 +251,7 @@ static inline void spi_set_ctldata(struct spi_device *spi, void *state) spi->controller_state = state; } -/* device driver data */ +/* Device driver data */ static inline void spi_set_drvdata(struct spi_device *spi, void *data) { @@ -318,7 +318,7 @@ static inline void spi_unregister_driver(struct spi_driver *sdrv) extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 chip_select); -/* use a define to avoid include chaining to get THIS_MODULE */ +/* Use a define to avoid include chaining to get THIS_MODULE */ #define spi_register_driver(driver) \ __spi_register_driver(THIS_MODULE, driver) @@ -486,7 +486,7 @@ struct spi_controller { struct list_head list; - /* other than negative (== assign one dynamically), bus_num is fully + /* Other than negative (== assign one dynamically), bus_num is fully * board-specific. usually that simplifies to being SOC-specific. * example: one SOC has three SPI controllers, numbered 0..2, * and one board's schematics might show it using SPI-2. software @@ -499,7 +499,7 @@ struct spi_controller { */ u16 num_chipselect; - /* some SPI controllers pose alignment requirements on DMAable + /* Some SPI controllers pose alignment requirements on DMAable * buffers; let protocol drivers know about these requirements. */ u16 dma_alignment; @@ -510,29 +510,29 @@ struct spi_controller { /* spi_device.mode flags override flags for this controller */ u32 buswidth_override_bits; - /* bitmask of supported bits_per_word for transfers */ + /* Bitmask of supported bits_per_word for transfers */ u32 bits_per_word_mask; #define SPI_BPW_MASK(bits) BIT((bits) - 1) #define SPI_BPW_RANGE_MASK(min, max) GENMASK((max) - 1, (min) - 1) - /* limits on transfer speed */ + /* Limits on transfer speed */ u32 min_speed_hz; u32 max_speed_hz; - /* other constraints relevant to this driver */ + /* Other constraints relevant to this driver */ u16 flags; -#define SPI_CONTROLLER_HALF_DUPLEX BIT(0) /* can't do full duplex */ -#define SPI_CONTROLLER_NO_RX BIT(1) /* can't do buffer read */ -#define SPI_CONTROLLER_NO_TX BIT(2) /* can't do buffer write */ -#define SPI_CONTROLLER_MUST_RX BIT(3) /* requires rx */ -#define SPI_CONTROLLER_MUST_TX BIT(4) /* requires tx */ +#define SPI_CONTROLLER_HALF_DUPLEX BIT(0) /* Can't do full duplex */ +#define SPI_CONTROLLER_NO_RX BIT(1) /* Can't do buffer read */ +#define SPI_CONTROLLER_NO_TX BIT(2) /* Can't do buffer write */ +#define SPI_CONTROLLER_MUST_RX BIT(3) /* Requires rx */ +#define SPI_CONTROLLER_MUST_TX BIT(4) /* Requires tx */ #define SPI_MASTER_GPIO_SS BIT(5) /* GPIO CS must select slave */ - /* flag indicating if the allocation of this struct is devres-managed */ + /* Flag indicating if the allocation of this struct is devres-managed */ bool devm_allocated; - /* flag indicating this is an SPI slave controller */ + /* Flag indicating this is an SPI slave controller */ bool slave; /* @@ -548,11 +548,11 @@ struct spi_controller { /* Used to avoid adding the same CS twice */ struct mutex add_lock; - /* lock and mutex for SPI bus locking */ + /* Lock and mutex for SPI bus locking */ spinlock_t bus_lock_spinlock; struct mutex bus_lock_mutex; - /* flag indicating that the SPI bus is locked for exclusive use */ + /* Flag indicating that the SPI bus is locked for exclusive use */ bool bus_lock_flag; /* Setup mode and clock, etc (spi driver may call many times). @@ -573,7 +573,7 @@ struct spi_controller { */ int (*set_cs_timing)(struct spi_device *spi); - /* bidirectional bulk transfers + /* Bidirectional bulk transfers * * + The transfer() method may not sleep; its main role is * just to add the message to the queue. @@ -595,7 +595,7 @@ struct spi_controller { int (*transfer)(struct spi_device *spi, struct spi_message *mesg); - /* called on release() to free memory provided by spi_controller */ + /* Called on release() to free memory provided by spi_controller */ void (*cleanup)(struct spi_device *spi); /* @@ -666,14 +666,14 @@ struct spi_controller { s8 unused_native_cs; s8 max_native_cs; - /* statistics */ + /* Statistics */ struct spi_statistics __percpu *pcpu_statistics; /* DMA channels for use with core dmaengine helpers */ struct dma_chan *dma_tx; struct dma_chan *dma_rx; - /* dummy data for full duplex devices */ + /* Dummy data for full duplex devices */ void *dummy_rx; void *dummy_tx; @@ -738,7 +738,7 @@ void spi_take_timestamp_post(struct spi_controller *ctlr, struct spi_transfer *xfer, size_t progress, bool irqs_off); -/* the spi driver core manages memory for the spi_controller classdev */ +/* The spi driver core manages memory for the spi_controller classdev */ extern struct spi_controller *__spi_alloc_controller(struct device *host, unsigned int size, bool slave); @@ -808,7 +808,7 @@ typedef void (*spi_res_release_t)(struct spi_controller *ctlr, struct spi_res { struct list_head entry; spi_res_release_t release; - unsigned long long data[]; /* guarantee ull alignment */ + unsigned long long data[]; /* Guarantee ull alignment */ }; /*---------------------------------------------------------------------------*/ @@ -941,7 +941,7 @@ struct spi_res { * and its transfers, ignore them until its completion callback. */ struct spi_transfer { - /* it's ok if tx_buf == rx_buf (right?) + /* It's ok if tx_buf == rx_buf (right?) * for MicroWire, one buffer must be null * buffers must work with dma_*map_single() calls, unless * spi_message.is_dma_mapped reports a pre-existing mapping @@ -1032,24 +1032,24 @@ struct spi_message { * tell them about such special cases. */ - /* completion is reported through a callback */ + /* Completion is reported through a callback */ void (*complete)(void *context); void *context; unsigned frame_length; unsigned actual_length; int status; - /* for optional use by whatever driver currently owns the + /* For optional use by whatever driver currently owns the * spi_message ... between calls to spi_async and then later * complete(), that's the spi_controller controller driver. */ struct list_head queue; void *state; - /* list of spi_res reources when the spi message is processed */ + /* List of spi_res reources when the spi message is processed */ struct list_head resources; - /* spi_prepare_message was called for this message */ + /* spi_prepare_message() was called for this message */ bool prepared; }; @@ -1154,7 +1154,7 @@ spi_max_transfer_size(struct spi_device *spi) if (ctlr->max_transfer_size) tr_max = ctlr->max_transfer_size(spi); - /* transfer size limit must not be greater than message size limit */ + /* Transfer size limit must not be greater than message size limit */ return min(tr_max, msg_max); } @@ -1305,7 +1305,7 @@ spi_read(struct spi_device *spi, void *buf, size_t len) return spi_sync_transfer(spi, &t, 1); } -/* this copies txbuf and rxbuf data; for small transfers only! */ +/* This copies txbuf and rxbuf data; for small transfers only! */ extern int spi_write_then_read(struct spi_device *spi, const void *txbuf, unsigned n_tx, void *rxbuf, unsigned n_rx); @@ -1328,7 +1328,7 @@ static inline ssize_t spi_w8r8(struct spi_device *spi, u8 cmd) status = spi_write_then_read(spi, &cmd, 1, &result, 1); - /* return negative errno or unsigned value */ + /* Return negative errno or unsigned value */ return (status < 0) ? status : result; } @@ -1353,7 +1353,7 @@ static inline ssize_t spi_w8r16(struct spi_device *spi, u8 cmd) status = spi_write_then_read(spi, &cmd, 1, &result, 2); - /* return negative errno or unsigned value */ + /* Return negative errno or unsigned value */ return (status < 0) ? status : result; } @@ -1433,7 +1433,7 @@ static inline ssize_t spi_w8r16be(struct spi_device *spi, u8 cmd) * are active in some dynamic board configuration models. */ struct spi_board_info { - /* the device name and module name are coupled, like platform_bus; + /* The device name and module name are coupled, like platform_bus; * "modalias" is normally the driver name. * * platform_data goes to spi_device.dev.platform_data, @@ -1446,7 +1446,7 @@ struct spi_board_info { void *controller_data; int irq; - /* slower signaling on noisy or low voltage boards */ + /* Slower signaling on noisy or low voltage boards */ u32 max_speed_hz; @@ -1475,7 +1475,7 @@ struct spi_board_info { extern int spi_register_board_info(struct spi_board_info const *info, unsigned n); #else -/* board init code may ignore whether SPI is configured or not */ +/* Board init code may ignore whether SPI is configured or not */ static inline int spi_register_board_info(struct spi_board_info const *info, unsigned n) { return 0; } -- cgit v1.2.3 From 72a43046b61a3fe7164a622224bcdfc3cf6b795d Mon Sep 17 00:00:00 2001 From: Chanho Park Date: Wed, 29 Jun 2022 09:41:41 +0900 Subject: tty: serial: samsung_tty: loopback mode support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal loopback mode can be supported by setting UCON register's Loopback Mode bit. The mode & bit can be supported since s3c2410 and later SoCs. The prefix of LOOPBACK / BIT(5) naming should be also changed to S3C2410_ in order to avoid confusion. Reviewed-by: Krzysztof Kozlowski Reviewed-by: Ilpo Järvinen Signed-off-by: Chanho Park Link: https://lore.kernel.org/r/20220629004141.51484-1-chanho61.park@samsung.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/samsung_tty.c | 8 ++++++++ include/linux/serial_s3c.h | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index 3e0aa2923605..8971fbb49fa3 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -1018,6 +1018,7 @@ static unsigned int s3c24xx_serial_get_mctrl(struct uart_port *port) static void s3c24xx_serial_set_mctrl(struct uart_port *port, unsigned int mctrl) { unsigned int umcon = rd_regl(port, S3C2410_UMCON); + unsigned int ucon = rd_reg(port, S3C2410_UCON); if (mctrl & TIOCM_RTS) umcon |= S3C2410_UMCOM_RTS_LOW; @@ -1025,6 +1026,13 @@ static void s3c24xx_serial_set_mctrl(struct uart_port *port, unsigned int mctrl) umcon &= ~S3C2410_UMCOM_RTS_LOW; wr_regl(port, S3C2410_UMCON, umcon); + + if (mctrl & TIOCM_LOOP) + ucon |= S3C2410_UCON_LOOPBACK; + else + ucon &= ~S3C2410_UCON_LOOPBACK; + + wr_regl(port, S3C2410_UCON, ucon); } static void s3c24xx_serial_break_ctl(struct uart_port *port, int break_state) diff --git a/include/linux/serial_s3c.h b/include/linux/serial_s3c.h index dec15f5b3dec..1672cf0810ef 100644 --- a/include/linux/serial_s3c.h +++ b/include/linux/serial_s3c.h @@ -83,7 +83,7 @@ #define S3C2410_UCON_RXIRQMODE (1<<0) #define S3C2410_UCON_RXFIFO_TOI (1<<7) #define S3C2443_UCON_RXERR_IRQEN (1<<6) -#define S3C2443_UCON_LOOPBACK (1<<5) +#define S3C2410_UCON_LOOPBACK (1<<5) #define S3C2410_UCON_DEFAULT (S3C2410_UCON_TXILEVEL | \ S3C2410_UCON_RXILEVEL | \ -- cgit v1.2.3 From 6e97eba8ad8748fabb795cffc5d9e1a7dcfd7367 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 28 Jun 2022 18:59:10 +0300 Subject: vfio: Split migration ops from main device ops vfio core checks whether the driver sets some migration op (e.g. set_state/get_state) and accordingly calls its op. However, currently mlx5 driver sets the above ops without regards to its migration caps. This might lead to unexpected usage/Oops if user space may call to the above ops even if the driver doesn't support migration. As for example, the migration state_mutex is not initialized in that case. The cleanest way to manage that seems to split the migration ops from the main device ops, this will let the driver setting them separately from the main ops when it's applicable. As part of that, validate ops construction on registration and include a check for VFIO_MIGRATION_STOP_COPY since the uAPI claims it must be set in migration_flags. HISI driver was changed as well to match this scheme. This scheme may enable down the road to come with some extra group of ops (e.g. DMA log) that can be set without regards to the other options based on driver caps. Fixes: 6fadb021266d ("vfio/mlx5: Implement vfio_pci driver for mlx5 devices") Reviewed-by: Kevin Tian Signed-off-by: Yishai Hadas Link: https://lore.kernel.org/r/20220628155910.171454-3-yishaih@nvidia.com Signed-off-by: Alex Williamson --- drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 11 +++++++--- drivers/vfio/pci/mlx5/cmd.c | 4 +++- drivers/vfio/pci/mlx5/cmd.h | 3 ++- drivers/vfio/pci/mlx5/main.c | 9 +++++--- drivers/vfio/pci/vfio_pci_core.c | 7 ++++++ drivers/vfio/vfio.c | 11 +++++----- include/linux/vfio.h | 30 +++++++++++++++++--------- 7 files changed, 51 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index 4def43f5f7b6..ea762e28c1cc 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -1185,7 +1185,7 @@ static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) if (ret) return ret; - if (core_vdev->ops->migration_set_state) { + if (core_vdev->mig_ops) { ret = hisi_acc_vf_qm_init(hisi_acc_vdev); if (ret) { vfio_pci_core_disable(vdev); @@ -1208,6 +1208,11 @@ static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev) vfio_pci_core_close_device(core_vdev); } +static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = { + .migration_set_state = hisi_acc_vfio_pci_set_device_state, + .migration_get_state = hisi_acc_vfio_pci_get_device_state, +}; + static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { .name = "hisi-acc-vfio-pci-migration", .open_device = hisi_acc_vfio_pci_open_device, @@ -1219,8 +1224,6 @@ static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { .mmap = hisi_acc_vfio_pci_mmap, .request = vfio_pci_core_request, .match = vfio_pci_core_match, - .migration_set_state = hisi_acc_vfio_pci_set_device_state, - .migration_get_state = hisi_acc_vfio_pci_get_device_state, }; static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { @@ -1272,6 +1275,8 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device if (!ret) { vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, &hisi_acc_vfio_pci_migrn_ops); + hisi_acc_vdev->core_device.vdev.mig_ops = + &hisi_acc_vfio_pci_migrn_state_ops; } else { pci_warn(pdev, "migration support failed, continue with generic interface\n"); vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c index cdd0c667dc77..dd5d7bfe0a49 100644 --- a/drivers/vfio/pci/mlx5/cmd.c +++ b/drivers/vfio/pci/mlx5/cmd.c @@ -108,7 +108,8 @@ void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev) destroy_workqueue(mvdev->cb_wq); } -void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev) +void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, + const struct vfio_migration_ops *mig_ops) { struct pci_dev *pdev = mvdev->core_device.pdev; int ret; @@ -149,6 +150,7 @@ void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev) mvdev->core_device.vdev.migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P; + mvdev->core_device.vdev.mig_ops = mig_ops; end: mlx5_vf_put_core_dev(mvdev->mdev); diff --git a/drivers/vfio/pci/mlx5/cmd.h b/drivers/vfio/pci/mlx5/cmd.h index aa692d9ce656..8208f4701a90 100644 --- a/drivers/vfio/pci/mlx5/cmd.h +++ b/drivers/vfio/pci/mlx5/cmd.h @@ -62,7 +62,8 @@ int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, size_t *state_size); -void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev); +void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, + const struct vfio_migration_ops *mig_ops); void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c index d754990f0662..a9b63d15c5d3 100644 --- a/drivers/vfio/pci/mlx5/main.c +++ b/drivers/vfio/pci/mlx5/main.c @@ -574,6 +574,11 @@ static void mlx5vf_pci_close_device(struct vfio_device *core_vdev) vfio_pci_core_close_device(core_vdev); } +static const struct vfio_migration_ops mlx5vf_pci_mig_ops = { + .migration_set_state = mlx5vf_pci_set_device_state, + .migration_get_state = mlx5vf_pci_get_device_state, +}; + static const struct vfio_device_ops mlx5vf_pci_ops = { .name = "mlx5-vfio-pci", .open_device = mlx5vf_pci_open_device, @@ -585,8 +590,6 @@ static const struct vfio_device_ops mlx5vf_pci_ops = { .mmap = vfio_pci_core_mmap, .request = vfio_pci_core_request, .match = vfio_pci_core_match, - .migration_set_state = mlx5vf_pci_set_device_state, - .migration_get_state = mlx5vf_pci_get_device_state, }; static int mlx5vf_pci_probe(struct pci_dev *pdev, @@ -599,7 +602,7 @@ static int mlx5vf_pci_probe(struct pci_dev *pdev, if (!mvdev) return -ENOMEM; vfio_pci_core_init_device(&mvdev->core_device, pdev, &mlx5vf_pci_ops); - mlx5vf_cmd_set_migratable(mvdev); + mlx5vf_cmd_set_migratable(mvdev, &mlx5vf_pci_mig_ops); dev_set_drvdata(&pdev->dev, &mvdev->core_device); ret = vfio_pci_core_register_device(&mvdev->core_device); if (ret) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index a0d69ddaf90d..2efa06b1fafa 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -1855,6 +1855,13 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL) return -EINVAL; + if (vdev->vdev.mig_ops) { + if (!(vdev->vdev.mig_ops->migration_get_state && + vdev->vdev.mig_ops->migration_set_state) || + !(vdev->vdev.migration_flags & VFIO_MIGRATION_STOP_COPY)) + return -EINVAL; + } + /* * Prevent binding to PFs with VFs enabled, the VFs might be in use * by the host or other users. We cannot capture the VFs if they diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 61e71c1154be..aac9213a783d 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1541,8 +1541,7 @@ vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, struct file *filp = NULL; int ret; - if (!device->ops->migration_set_state || - !device->ops->migration_get_state) + if (!device->mig_ops) return -ENOTTY; ret = vfio_check_feature(flags, argsz, @@ -1558,7 +1557,8 @@ vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, if (flags & VFIO_DEVICE_FEATURE_GET) { enum vfio_device_mig_state curr_state; - ret = device->ops->migration_get_state(device, &curr_state); + ret = device->mig_ops->migration_get_state(device, + &curr_state); if (ret) return ret; mig.device_state = curr_state; @@ -1566,7 +1566,7 @@ vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, } /* Handle the VFIO_DEVICE_FEATURE_SET */ - filp = device->ops->migration_set_state(device, mig.device_state); + filp = device->mig_ops->migration_set_state(device, mig.device_state); if (IS_ERR(filp) || !filp) goto out_copy; @@ -1589,8 +1589,7 @@ static int vfio_ioctl_device_feature_migration(struct vfio_device *device, }; int ret; - if (!device->ops->migration_set_state || - !device->ops->migration_get_state) + if (!device->mig_ops) return -ENOTTY; ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 49580fa2073a..4d26e149db81 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -32,6 +32,11 @@ struct vfio_device_set { struct vfio_device { struct device *dev; const struct vfio_device_ops *ops; + /* + * mig_ops is a static property of the vfio_device which must be set + * prior to registering the vfio_device. + */ + const struct vfio_migration_ops *mig_ops; struct vfio_group *group; struct vfio_device_set *dev_set; struct list_head dev_set_list; @@ -61,16 +66,6 @@ struct vfio_device { * match, -errno for abort (ex. match with insufficient or incorrect * additional args) * @device_feature: Optional, fill in the VFIO_DEVICE_FEATURE ioctl - * @migration_set_state: Optional callback to change the migration state for - * devices that support migration. It's mandatory for - * VFIO_DEVICE_FEATURE_MIGRATION migration support. - * The returned FD is used for data transfer according to the FSM - * definition. The driver is responsible to ensure that FD reaches end - * of stream or error whenever the migration FSM leaves a data transfer - * state or before close_device() returns. - * @migration_get_state: Optional callback to get the migration state for - * devices that support migration. It's mandatory for - * VFIO_DEVICE_FEATURE_MIGRATION migration support. */ struct vfio_device_ops { char *name; @@ -87,6 +82,21 @@ struct vfio_device_ops { int (*match)(struct vfio_device *vdev, char *buf); int (*device_feature)(struct vfio_device *device, u32 flags, void __user *arg, size_t argsz); +}; + +/** + * @migration_set_state: Optional callback to change the migration state for + * devices that support migration. It's mandatory for + * VFIO_DEVICE_FEATURE_MIGRATION migration support. + * The returned FD is used for data transfer according to the FSM + * definition. The driver is responsible to ensure that FD reaches end + * of stream or error whenever the migration FSM leaves a data transfer + * state or before close_device() returns. + * @migration_get_state: Optional callback to get the migration state for + * devices that support migration. It's mandatory for + * VFIO_DEVICE_FEATURE_MIGRATION migration support. + */ +struct vfio_migration_ops { struct file *(*migration_set_state)( struct vfio_device *device, enum vfio_device_mig_state new_state); -- cgit v1.2.3 From 0e862838f290147ea9c16db852d8d494b552d38d Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 24 Jun 2022 14:13:07 +0200 Subject: bitops: unify non-atomic bitops prototypes across architectures Currently, there is a mess with the prototypes of the non-atomic bitops across the different architectures: ret bool, int, unsigned long nr int, long, unsigned int, unsigned long addr volatile unsigned long *, volatile void * Thankfully, it doesn't provoke any bugs, but can sometimes make the compiler angry when it's not handy at all. Adjust all the prototypes to the following standard: ret bool retval can be only 0 or 1 nr unsigned long native; signed makes no sense addr volatile unsigned long * bitmaps are arrays of ulongs Next, some architectures don't define 'arch_' versions as they don't support instrumentation, others do. To make sure there is always the same set of callables present and to ease any potential future changes, make them all follow the rule: * architecture-specific files define only 'arch_' versions; * non-prefixed versions can be defined only in asm-generic files; and place the non-prefixed definitions into a new file in asm-generic to be included by non-instrumented architectures. Finally, add some static assertions in order to prevent people from making a mess in this room again. I also used the %__always_inline attribute consistently, so that they always get resolved to the actual operations. Suggested-by: Andy Shevchenko Signed-off-by: Alexander Lobakin Acked-by: Mark Rutland Reviewed-by: Yury Norov Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov --- arch/alpha/include/asm/bitops.h | 32 ++++++------ arch/hexagon/include/asm/bitops.h | 24 +++++---- arch/ia64/include/asm/bitops.h | 42 ++++++++------- arch/m68k/include/asm/bitops.h | 49 +++++++++++------ arch/s390/include/asm/bitops.h | 61 +++++++++++----------- arch/sh/include/asm/bitops-op32.h | 34 +++++++----- arch/x86/include/asm/bitops.h | 22 ++++---- include/asm-generic/bitops/generic-non-atomic.h | 24 ++++----- .../asm-generic/bitops/instrumented-non-atomic.h | 21 +++++--- include/asm-generic/bitops/non-atomic.h | 13 +---- .../bitops/non-instrumented-non-atomic.h | 16 ++++++ include/linux/bitops.h | 17 ++++++ tools/include/asm-generic/bitops/non-atomic.h | 24 +++++---- 13 files changed, 229 insertions(+), 150 deletions(-) create mode 100644 include/asm-generic/bitops/non-instrumented-non-atomic.h (limited to 'include/linux') diff --git a/arch/alpha/include/asm/bitops.h b/arch/alpha/include/asm/bitops.h index e1d8483a45f2..492c7713ddae 100644 --- a/arch/alpha/include/asm/bitops.h +++ b/arch/alpha/include/asm/bitops.h @@ -46,8 +46,8 @@ set_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static inline void -__set_bit(unsigned long nr, volatile void * addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { int *m = ((int *) addr) + (nr >> 5); @@ -82,8 +82,8 @@ clear_bit_unlock(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static __inline__ void -__clear_bit(unsigned long nr, volatile void * addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { int *m = ((int *) addr) + (nr >> 5); @@ -94,7 +94,7 @@ static inline void __clear_bit_unlock(unsigned long nr, volatile void * addr) { smp_mb(); - __clear_bit(nr, addr); + arch___clear_bit(nr, addr); } static inline void @@ -118,8 +118,8 @@ change_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static __inline__ void -__change_bit(unsigned long nr, volatile void * addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { int *m = ((int *) addr) + (nr >> 5); @@ -186,8 +186,8 @@ test_and_set_bit_lock(unsigned long nr, volatile void *addr) /* * WARNING: non atomic version. */ -static inline int -__test_and_set_bit(unsigned long nr, volatile void * addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = 1 << (nr & 0x1f); int *m = ((int *) addr) + (nr >> 5); @@ -230,8 +230,8 @@ test_and_clear_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static inline int -__test_and_clear_bit(unsigned long nr, volatile void * addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = 1 << (nr & 0x1f); int *m = ((int *) addr) + (nr >> 5); @@ -272,8 +272,8 @@ test_and_change_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static __inline__ int -__test_and_change_bit(unsigned long nr, volatile void * addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = 1 << (nr & 0x1f); int *m = ((int *) addr) + (nr >> 5); @@ -283,8 +283,8 @@ __test_and_change_bit(unsigned long nr, volatile void * addr) return (old & mask) != 0; } -static inline int -test_bit(int nr, const volatile void * addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL; } @@ -450,6 +450,8 @@ sched_find_first_bit(const unsigned long b[2]) return __ffs(tmp) + ofs; } +#include + #include #include diff --git a/arch/hexagon/include/asm/bitops.h b/arch/hexagon/include/asm/bitops.h index 75d6ba3643b8..da500471ac73 100644 --- a/arch/hexagon/include/asm/bitops.h +++ b/arch/hexagon/include/asm/bitops.h @@ -127,38 +127,45 @@ static inline void change_bit(int nr, volatile void *addr) * be atomic, particularly for things like slab_lock and slab_unlock. * */ -static inline void __clear_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { test_and_clear_bit(nr, addr); } -static inline void __set_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { test_and_set_bit(nr, addr); } -static inline void __change_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { test_and_change_bit(nr, addr); } /* Apparently, at least some of these are allowed to be non-atomic */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { return test_and_clear_bit(nr, addr); } -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { return test_and_set_bit(nr, addr); } -static inline int __test_and_change_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { return test_and_change_bit(nr, addr); } -static inline int __test_bit(int nr, const volatile unsigned long *addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { int retval; @@ -172,8 +179,6 @@ static inline int __test_bit(int nr, const volatile unsigned long *addr) return retval; } -#define test_bit(nr, addr) __test_bit(nr, addr) - /* * ffz - find first zero in word. * @word: The word to search @@ -271,6 +276,7 @@ static inline unsigned long __fls(unsigned long word) } #include +#include #include #include diff --git a/arch/ia64/include/asm/bitops.h b/arch/ia64/include/asm/bitops.h index 577be93c0818..9f62af7fd7c4 100644 --- a/arch/ia64/include/asm/bitops.h +++ b/arch/ia64/include/asm/bitops.h @@ -53,7 +53,7 @@ set_bit (int nr, volatile void *addr) } /** - * __set_bit - Set a bit in memory + * arch___set_bit - Set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -61,8 +61,8 @@ set_bit (int nr, volatile void *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static __inline__ void -__set_bit (int nr, volatile void *addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { *((__u32 *) addr + (nr >> 5)) |= (1 << (nr & 31)); } @@ -135,7 +135,7 @@ __clear_bit_unlock(int nr, void *addr) } /** - * __clear_bit - Clears a bit in memory (non-atomic version) + * arch___clear_bit - Clears a bit in memory (non-atomic version) * @nr: the bit to clear * @addr: the address to start counting from * @@ -143,8 +143,8 @@ __clear_bit_unlock(int nr, void *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static __inline__ void -__clear_bit (int nr, volatile void *addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { *((__u32 *) addr + (nr >> 5)) &= ~(1 << (nr & 31)); } @@ -175,7 +175,7 @@ change_bit (int nr, volatile void *addr) } /** - * __change_bit - Toggle a bit in memory + * arch___change_bit - Toggle a bit in memory * @nr: the bit to toggle * @addr: the address to start counting from * @@ -183,8 +183,8 @@ change_bit (int nr, volatile void *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static __inline__ void -__change_bit (int nr, volatile void *addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { *((__u32 *) addr + (nr >> 5)) ^= (1 << (nr & 31)); } @@ -224,7 +224,7 @@ test_and_set_bit (int nr, volatile void *addr) #define test_and_set_bit_lock test_and_set_bit /** - * __test_and_set_bit - Set a bit and return its old value + * arch___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -232,8 +232,8 @@ test_and_set_bit (int nr, volatile void *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static __inline__ int -__test_and_set_bit (int nr, volatile void *addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { __u32 *p = (__u32 *) addr + (nr >> 5); __u32 m = 1 << (nr & 31); @@ -269,7 +269,7 @@ test_and_clear_bit (int nr, volatile void *addr) } /** - * __test_and_clear_bit - Clear a bit and return its old value + * arch___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -277,8 +277,8 @@ test_and_clear_bit (int nr, volatile void *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static __inline__ int -__test_and_clear_bit(int nr, volatile void * addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { __u32 *p = (__u32 *) addr + (nr >> 5); __u32 m = 1 << (nr & 31); @@ -314,14 +314,14 @@ test_and_change_bit (int nr, volatile void *addr) } /** - * __test_and_change_bit - Change a bit and return its old value + * arch___test_and_change_bit - Change a bit and return its old value * @nr: Bit to change * @addr: Address to count from * * This operation is non-atomic and can be reordered. */ -static __inline__ int -__test_and_change_bit (int nr, void *addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { __u32 old, bit = (1 << (nr & 31)); __u32 *m = (__u32 *) addr + (nr >> 5); @@ -331,8 +331,8 @@ __test_and_change_bit (int nr, void *addr) return (old & bit) != 0; } -static __inline__ int -test_bit (int nr, const volatile void *addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31)); } @@ -443,6 +443,8 @@ static __inline__ unsigned long __arch_hweight64(unsigned long x) #ifdef __KERNEL__ +#include + #include #include diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index 51283db53667..71495faf2a90 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -65,8 +65,11 @@ static inline void bfset_mem_set_bit(int nr, volatile unsigned long *vaddr) bfset_mem_set_bit(nr, vaddr)) #endif -#define __set_bit(nr, vaddr) set_bit(nr, vaddr) - +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) +{ + set_bit(nr, addr); +} static inline void bclr_reg_clear_bit(int nr, volatile unsigned long *vaddr) { @@ -105,8 +108,11 @@ static inline void bfclr_mem_clear_bit(int nr, volatile unsigned long *vaddr) bfclr_mem_clear_bit(nr, vaddr)) #endif -#define __clear_bit(nr, vaddr) clear_bit(nr, vaddr) - +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + clear_bit(nr, addr); +} static inline void bchg_reg_change_bit(int nr, volatile unsigned long *vaddr) { @@ -145,14 +151,17 @@ static inline void bfchg_mem_change_bit(int nr, volatile unsigned long *vaddr) bfchg_mem_change_bit(nr, vaddr)) #endif -#define __change_bit(nr, vaddr) change_bit(nr, vaddr) - - -static inline int test_bit(int nr, const volatile unsigned long *vaddr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { - return (vaddr[nr >> 5] & (1UL << (nr & 31))) != 0; + change_bit(nr, addr); } +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + return (addr[nr >> 5] & (1UL << (nr & 31))) != 0; +} static inline int bset_reg_test_and_set_bit(int nr, volatile unsigned long *vaddr) @@ -201,8 +210,11 @@ static inline int bfset_mem_test_and_set_bit(int nr, bfset_mem_test_and_set_bit(nr, vaddr)) #endif -#define __test_and_set_bit(nr, vaddr) test_and_set_bit(nr, vaddr) - +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + return test_and_set_bit(nr, addr); +} static inline int bclr_reg_test_and_clear_bit(int nr, volatile unsigned long *vaddr) @@ -251,8 +263,11 @@ static inline int bfclr_mem_test_and_clear_bit(int nr, bfclr_mem_test_and_clear_bit(nr, vaddr)) #endif -#define __test_and_clear_bit(nr, vaddr) test_and_clear_bit(nr, vaddr) - +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + return test_and_clear_bit(nr, addr); +} static inline int bchg_reg_test_and_change_bit(int nr, volatile unsigned long *vaddr) @@ -301,8 +316,11 @@ static inline int bfchg_mem_test_and_change_bit(int nr, bfchg_mem_test_and_change_bit(nr, vaddr)) #endif -#define __test_and_change_bit(nr, vaddr) test_and_change_bit(nr, vaddr) - +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + return test_and_change_bit(nr, addr); +} /* * The true 68020 and more advanced processors support the "bfffo" @@ -522,6 +540,7 @@ static inline int __fls(int x) #define clear_bit_unlock clear_bit #define __clear_bit_unlock clear_bit_unlock +#include #include #include #include diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index 191dc7898b0f..9a7d15da966e 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -113,75 +113,76 @@ static inline bool arch_test_and_change_bit(unsigned long nr, return old & mask; } -static inline void arch___set_bit(unsigned long nr, volatile unsigned long *ptr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - *addr |= mask; + *p |= mask; } -static inline void arch___clear_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - *addr &= ~mask; + *p &= ~mask; } -static inline void arch___change_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - *addr ^= mask; + *p ^= mask; } -static inline bool arch___test_and_set_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); unsigned long old; - old = *addr; - *addr |= mask; + old = *p; + *p |= mask; return old & mask; } -static inline bool arch___test_and_clear_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); unsigned long old; - old = *addr; - *addr &= ~mask; + old = *p; + *p &= ~mask; return old & mask; } -static inline bool arch___test_and_change_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); unsigned long old; - old = *addr; - *addr ^= mask; + old = *p; + *p ^= mask; return old & mask; } -static inline bool arch_test_bit(unsigned long nr, - const volatile unsigned long *ptr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { - const volatile unsigned long *addr = __bitops_word(nr, ptr); + const volatile unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - return *addr & mask; + return *p & mask; } static inline bool arch_test_and_set_bit_lock(unsigned long nr, diff --git a/arch/sh/include/asm/bitops-op32.h b/arch/sh/include/asm/bitops-op32.h index cfe5465acce7..565a85d8b7fb 100644 --- a/arch/sh/include/asm/bitops-op32.h +++ b/arch/sh/include/asm/bitops-op32.h @@ -2,6 +2,8 @@ #ifndef __ASM_SH_BITOPS_OP32_H #define __ASM_SH_BITOPS_OP32_H +#include + /* * The bit modifying instructions on SH-2A are only capable of working * with a 3-bit immediate, which signifies the shift position for the bit @@ -16,7 +18,8 @@ #define BYTE_OFFSET(nr) ((nr) % BITS_PER_BYTE) #endif -static inline void __set_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { if (__builtin_constant_p(nr)) { __asm__ __volatile__ ( @@ -33,7 +36,8 @@ static inline void __set_bit(int nr, volatile unsigned long *addr) } } -static inline void __clear_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { if (__builtin_constant_p(nr)) { __asm__ __volatile__ ( @@ -52,7 +56,7 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) } /** - * __change_bit - Toggle a bit in memory + * arch___change_bit - Toggle a bit in memory * @nr: the bit to change * @addr: the address to start counting from * @@ -60,7 +64,8 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __change_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { if (__builtin_constant_p(nr)) { __asm__ __volatile__ ( @@ -79,7 +84,7 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_set_bit - Set a bit and return its old value + * arch___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -87,7 +92,8 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -98,7 +104,7 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_clear_bit - Clear a bit and return its old value + * arch___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -106,7 +112,8 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -117,8 +124,8 @@ static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) } /* WARNING: non atomic and it can be reordered! */ -static inline int __test_and_change_bit(int nr, - volatile unsigned long *addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -129,13 +136,16 @@ static inline int __test_and_change_bit(int nr, } /** - * test_bit - Determine whether a bit is set + * arch_test_bit - Determine whether a bit is set * @nr: bit number to test * @addr: Address to start counting from */ -static inline int test_bit(int nr, const volatile unsigned long *addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +#include + #endif /* __ASM_SH_BITOPS_OP32_H */ diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index a288ecd230ab..973c6bd17f98 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -63,7 +63,7 @@ arch_set_bit(long nr, volatile unsigned long *addr) } static __always_inline void -arch___set_bit(long nr, volatile unsigned long *addr) +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { asm volatile(__ASM_SIZE(bts) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); } @@ -89,7 +89,7 @@ arch_clear_bit_unlock(long nr, volatile unsigned long *addr) } static __always_inline void -arch___clear_bit(long nr, volatile unsigned long *addr) +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); } @@ -114,7 +114,7 @@ arch___clear_bit_unlock(long nr, volatile unsigned long *addr) } static __always_inline void -arch___change_bit(long nr, volatile unsigned long *addr) +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { asm volatile(__ASM_SIZE(btc) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); } @@ -145,7 +145,7 @@ arch_test_and_set_bit_lock(long nr, volatile unsigned long *addr) } static __always_inline bool -arch___test_and_set_bit(long nr, volatile unsigned long *addr) +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { bool oldbit; @@ -171,7 +171,7 @@ arch_test_and_clear_bit(long nr, volatile unsigned long *addr) * this without also updating arch/x86/kernel/kvm.c */ static __always_inline bool -arch___test_and_clear_bit(long nr, volatile unsigned long *addr) +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { bool oldbit; @@ -183,7 +183,7 @@ arch___test_and_clear_bit(long nr, volatile unsigned long *addr) } static __always_inline bool -arch___test_and_change_bit(long nr, volatile unsigned long *addr) +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { bool oldbit; @@ -219,10 +219,12 @@ static __always_inline bool variable_test_bit(long nr, volatile const unsigned l return oldbit; } -#define arch_test_bit(nr, addr) \ - (__builtin_constant_p((nr)) \ - ? constant_test_bit((nr), (addr)) \ - : variable_test_bit((nr), (addr))) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + return __builtin_constant_p(nr) ? constant_test_bit(nr, addr) : + variable_test_bit(nr, addr); +} /** * __ffs - find first set bit in word diff --git a/include/asm-generic/bitops/generic-non-atomic.h b/include/asm-generic/bitops/generic-non-atomic.h index 7226488810e5..b85b8a2ac239 100644 --- a/include/asm-generic/bitops/generic-non-atomic.h +++ b/include/asm-generic/bitops/generic-non-atomic.h @@ -24,7 +24,7 @@ * may be that only one operation succeeds. */ static __always_inline void -generic___set_bit(unsigned int nr, volatile unsigned long *addr) +generic___set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -33,7 +33,7 @@ generic___set_bit(unsigned int nr, volatile unsigned long *addr) } static __always_inline void -generic___clear_bit(unsigned int nr, volatile unsigned long *addr) +generic___clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -50,8 +50,8 @@ generic___clear_bit(unsigned int nr, volatile unsigned long *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static __always_inline -void generic___change_bit(unsigned int nr, volatile unsigned long *addr) +static __always_inline void +generic___change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -68,8 +68,8 @@ void generic___change_bit(unsigned int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static __always_inline int -generic___test_and_set_bit(unsigned int nr, volatile unsigned long *addr) +static __always_inline bool +generic___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -88,8 +88,8 @@ generic___test_and_set_bit(unsigned int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static __always_inline int -generic___test_and_clear_bit(unsigned int nr, volatile unsigned long *addr) +static __always_inline bool +generic___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -100,8 +100,8 @@ generic___test_and_clear_bit(unsigned int nr, volatile unsigned long *addr) } /* WARNING: non atomic and it can be reordered! */ -static __always_inline int -generic___test_and_change_bit(unsigned int nr, volatile unsigned long *addr) +static __always_inline bool +generic___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -116,8 +116,8 @@ generic___test_and_change_bit(unsigned int nr, volatile unsigned long *addr) * @nr: bit number to test * @addr: Address to start counting from */ -static __always_inline int -generic_test_bit(unsigned int nr, const volatile unsigned long *addr) +static __always_inline bool +generic_test_bit(unsigned long nr, const volatile unsigned long *addr) { /* * Unlike the bitops with the '__' prefix above, this one *is* atomic, diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h index 7ab1ecc37782..b019f77ef21c 100644 --- a/include/asm-generic/bitops/instrumented-non-atomic.h +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -22,7 +22,8 @@ * region of memory concurrently, the effect may be that only one operation * succeeds. */ -static __always_inline void __set_bit(long nr, volatile unsigned long *addr) +static __always_inline void +__set_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___set_bit(nr, addr); @@ -37,7 +38,8 @@ static __always_inline void __set_bit(long nr, volatile unsigned long *addr) * region of memory concurrently, the effect may be that only one operation * succeeds. */ -static __always_inline void __clear_bit(long nr, volatile unsigned long *addr) +static __always_inline void +__clear_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___clear_bit(nr, addr); @@ -52,7 +54,8 @@ static __always_inline void __clear_bit(long nr, volatile unsigned long *addr) * region of memory concurrently, the effect may be that only one operation * succeeds. */ -static __always_inline void __change_bit(long nr, volatile unsigned long *addr) +static __always_inline void +__change_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___change_bit(nr, addr); @@ -90,7 +93,8 @@ static __always_inline void __instrument_read_write_bitop(long nr, volatile unsi * This operation is non-atomic. If two instances of this operation race, one * can appear to succeed but actually fail. */ -static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) +static __always_inline bool +__test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_set_bit(nr, addr); @@ -104,7 +108,8 @@ static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long * * This operation is non-atomic. If two instances of this operation race, one * can appear to succeed but actually fail. */ -static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) +static __always_inline bool +__test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_clear_bit(nr, addr); @@ -118,7 +123,8 @@ static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long * This operation is non-atomic. If two instances of this operation race, one * can appear to succeed but actually fail. */ -static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) +static __always_inline bool +__test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_change_bit(nr, addr); @@ -129,7 +135,8 @@ static __always_inline bool __test_and_change_bit(long nr, volatile unsigned lon * @nr: bit number to test * @addr: Address to start counting from */ -static __always_inline bool test_bit(long nr, const volatile unsigned long *addr) +static __always_inline bool +test_bit(unsigned long nr, const volatile unsigned long *addr) { instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); return arch_test_bit(nr, addr); diff --git a/include/asm-generic/bitops/non-atomic.h b/include/asm-generic/bitops/non-atomic.h index 23d3abc1e10d..5c37ced343ae 100644 --- a/include/asm-generic/bitops/non-atomic.h +++ b/include/asm-generic/bitops/non-atomic.h @@ -5,24 +5,15 @@ #include #define arch___set_bit generic___set_bit -#define __set_bit arch___set_bit - #define arch___clear_bit generic___clear_bit -#define __clear_bit arch___clear_bit - #define arch___change_bit generic___change_bit -#define __change_bit arch___change_bit #define arch___test_and_set_bit generic___test_and_set_bit -#define __test_and_set_bit arch___test_and_set_bit - #define arch___test_and_clear_bit generic___test_and_clear_bit -#define __test_and_clear_bit arch___test_and_clear_bit - #define arch___test_and_change_bit generic___test_and_change_bit -#define __test_and_change_bit arch___test_and_change_bit #define arch_test_bit generic_test_bit -#define test_bit arch_test_bit + +#include #endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */ diff --git a/include/asm-generic/bitops/non-instrumented-non-atomic.h b/include/asm-generic/bitops/non-instrumented-non-atomic.h new file mode 100644 index 000000000000..e0fd7bf72a56 --- /dev/null +++ b/include/asm-generic/bitops/non-instrumented-non-atomic.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H + +#define __set_bit arch___set_bit +#define __clear_bit arch___clear_bit +#define __change_bit arch___change_bit + +#define __test_and_set_bit arch___test_and_set_bit +#define __test_and_clear_bit arch___test_and_clear_bit +#define __test_and_change_bit arch___test_and_change_bit + +#define test_bit arch_test_bit + +#endif /* __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 7aaed501f768..87087454a288 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -26,12 +26,29 @@ extern unsigned int __sw_hweight16(unsigned int w); extern unsigned int __sw_hweight32(unsigned int w); extern unsigned long __sw_hweight64(__u64 w); +#include + /* * Include this here because some architectures need generic_ffs/fls in * scope */ #include +/* Check that the bitops prototypes are sane */ +#define __check_bitop_pr(name) \ + static_assert(__same_type(arch_##name, generic_##name) && \ + __same_type(name, generic_##name)) + +__check_bitop_pr(__set_bit); +__check_bitop_pr(__clear_bit); +__check_bitop_pr(__change_bit); +__check_bitop_pr(__test_and_set_bit); +__check_bitop_pr(__test_and_clear_bit); +__check_bitop_pr(__test_and_change_bit); +__check_bitop_pr(test_bit); + +#undef __check_bitop_pr + static inline int get_bitmask_order(unsigned int count) { int order; diff --git a/tools/include/asm-generic/bitops/non-atomic.h b/tools/include/asm-generic/bitops/non-atomic.h index 7e10c4b50c5d..e5e78e42e57b 100644 --- a/tools/include/asm-generic/bitops/non-atomic.h +++ b/tools/include/asm-generic/bitops/non-atomic.h @@ -2,7 +2,7 @@ #ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ -#include +#include /** * __set_bit - Set a bit in memory @@ -13,7 +13,8 @@ * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __set_bit(int nr, volatile unsigned long *addr) +static __always_inline void +__set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -21,7 +22,8 @@ static inline void __set_bit(int nr, volatile unsigned long *addr) *p |= mask; } -static inline void __clear_bit(int nr, volatile unsigned long *addr) +static __always_inline void +__clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -38,7 +40,8 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __change_bit(int nr, volatile unsigned long *addr) +static __always_inline void +__change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -55,7 +58,8 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +__test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -74,7 +78,8 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +__test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -85,8 +90,8 @@ static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) } /* WARNING: non atomic and it can be reordered! */ -static inline int __test_and_change_bit(int nr, - volatile unsigned long *addr) +static __always_inline bool +__test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -101,7 +106,8 @@ static inline int __test_and_change_bit(int nr, * @nr: bit number to test * @addr: Address to start counting from */ -static inline int test_bit(int nr, const volatile unsigned long *addr) +static __always_inline bool +test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } -- cgit v1.2.3 From bb7379bfa680bd48b468e856475778db2ad866c1 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 24 Jun 2022 14:13:08 +0200 Subject: bitops: define const_*() versions of the non-atomics Define const_*() variants of the non-atomic bitops to be used when the input arguments are compile-time constants, so that the compiler will be always able to resolve those to compile-time constants as well. Those are mostly direct aliases for generic_*() with one exception for const_test_bit(): the original one is declared atomic-safe and thus doesn't discard the `volatile` qualifier, so in order to let optimize code, define it separately disregarding the qualifier. Add them to the compile-time type checks as well just in case. Suggested-by: Marco Elver Signed-off-by: Alexander Lobakin Reviewed-by: Marco Elver Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov --- include/asm-generic/bitops/generic-non-atomic.h | 31 +++++++++++++++++++++++++ include/linux/bitops.h | 1 + 2 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/include/asm-generic/bitops/generic-non-atomic.h b/include/asm-generic/bitops/generic-non-atomic.h index b85b8a2ac239..3d5ebd24652b 100644 --- a/include/asm-generic/bitops/generic-non-atomic.h +++ b/include/asm-generic/bitops/generic-non-atomic.h @@ -127,4 +127,35 @@ generic_test_bit(unsigned long nr, const volatile unsigned long *addr) return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +/* + * const_*() definitions provide good compile-time optimizations when + * the passed arguments can be resolved at compile time. + */ +#define const___set_bit generic___set_bit +#define const___clear_bit generic___clear_bit +#define const___change_bit generic___change_bit +#define const___test_and_set_bit generic___test_and_set_bit +#define const___test_and_clear_bit generic___test_and_clear_bit +#define const___test_and_change_bit generic___test_and_change_bit + +/** + * const_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + * + * A version of generic_test_bit() which discards the `volatile` qualifier to + * allow a compiler to optimize code harder. Non-atomic and to be called only + * for testing compile-time constants, e.g. by the corresponding macros, not + * directly from "regular" code. + */ +static __always_inline bool +const_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + const unsigned long *p = (const unsigned long *)addr + BIT_WORD(nr); + unsigned long mask = BIT_MASK(nr); + unsigned long val = *p; + + return !!(val & mask); +} + #endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */ diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 87087454a288..d393297287d5 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -37,6 +37,7 @@ extern unsigned long __sw_hweight64(__u64 w); /* Check that the bitops prototypes are sane */ #define __check_bitop_pr(name) \ static_assert(__same_type(arch_##name, generic_##name) && \ + __same_type(const_##name, generic_##name) && \ __same_type(name, generic_##name)) __check_bitop_pr(__set_bit); -- cgit v1.2.3 From e69eb9c460f128b71c6b995d75a05244e4b6cc3e Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 24 Jun 2022 14:13:09 +0200 Subject: bitops: wrap non-atomic bitops with a transparent macro In preparation for altering the non-atomic bitops with a macro, wrap them in a transparent definition. This requires prepending one more '_' to their names in order to be able to do that seamlessly. It is a simple change, given that all the non-prefixed definitions are now in asm-generic. sparc32 already has several triple-underscored functions, so I had to rename them ('___' -> 'sp32_'). Signed-off-by: Alexander Lobakin Reviewed-by: Marco Elver Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov --- arch/sparc/include/asm/bitops_32.h | 18 +++++++------- arch/sparc/lib/atomic32.c | 12 +++++----- .../asm-generic/bitops/instrumented-non-atomic.h | 28 +++++++++++----------- .../bitops/non-instrumented-non-atomic.h | 14 +++++------ include/linux/bitops.h | 18 +++++++++++++- tools/include/asm-generic/bitops/non-atomic.h | 24 +++++++++---------- tools/include/linux/bitops.h | 16 +++++++++++++ 7 files changed, 81 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h index 889afa9f990f..3448c191b484 100644 --- a/arch/sparc/include/asm/bitops_32.h +++ b/arch/sparc/include/asm/bitops_32.h @@ -19,9 +19,9 @@ #error only can be included directly #endif -unsigned long ___set_bit(unsigned long *addr, unsigned long mask); -unsigned long ___clear_bit(unsigned long *addr, unsigned long mask); -unsigned long ___change_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask); /* * Set bit 'nr' in 32-bit quantity at address 'addr' where bit '0' @@ -36,7 +36,7 @@ static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *add ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___set_bit(ADDR, mask) != 0; + return sp32___set_bit(ADDR, mask) != 0; } static inline void set_bit(unsigned long nr, volatile unsigned long *addr) @@ -46,7 +46,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___set_bit(ADDR, mask); + (void) sp32___set_bit(ADDR, mask); } static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) @@ -56,7 +56,7 @@ static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *a ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___clear_bit(ADDR, mask) != 0; + return sp32___clear_bit(ADDR, mask) != 0; } static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) @@ -66,7 +66,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___clear_bit(ADDR, mask); + (void) sp32___clear_bit(ADDR, mask); } static inline int test_and_change_bit(unsigned long nr, volatile unsigned long *addr) @@ -76,7 +76,7 @@ static inline int test_and_change_bit(unsigned long nr, volatile unsigned long * ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___change_bit(ADDR, mask) != 0; + return sp32___change_bit(ADDR, mask) != 0; } static inline void change_bit(unsigned long nr, volatile unsigned long *addr) @@ -86,7 +86,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___change_bit(ADDR, mask); + (void) sp32___change_bit(ADDR, mask); } #include diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 8b81d0f00c97..cf80d1ae352b 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -120,7 +120,7 @@ void arch_atomic_set(atomic_t *v, int i) } EXPORT_SYMBOL(arch_atomic_set); -unsigned long ___set_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -131,9 +131,9 @@ unsigned long ___set_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___set_bit); +EXPORT_SYMBOL(sp32___set_bit); -unsigned long ___clear_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -144,9 +144,9 @@ unsigned long ___clear_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___clear_bit); +EXPORT_SYMBOL(sp32___clear_bit); -unsigned long ___change_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -157,7 +157,7 @@ unsigned long ___change_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___change_bit); +EXPORT_SYMBOL(sp32___change_bit); unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new) { diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h index b019f77ef21c..988a3bbfba34 100644 --- a/include/asm-generic/bitops/instrumented-non-atomic.h +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -14,7 +14,7 @@ #include /** - * __set_bit - Set a bit in memory + * ___set_bit - Set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -23,14 +23,14 @@ * succeeds. */ static __always_inline void -__set_bit(unsigned long nr, volatile unsigned long *addr) +___set_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___set_bit(nr, addr); } /** - * __clear_bit - Clears a bit in memory + * ___clear_bit - Clears a bit in memory * @nr: the bit to clear * @addr: the address to start counting from * @@ -39,14 +39,14 @@ __set_bit(unsigned long nr, volatile unsigned long *addr) * succeeds. */ static __always_inline void -__clear_bit(unsigned long nr, volatile unsigned long *addr) +___clear_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___clear_bit(nr, addr); } /** - * __change_bit - Toggle a bit in memory + * ___change_bit - Toggle a bit in memory * @nr: the bit to change * @addr: the address to start counting from * @@ -55,7 +55,7 @@ __clear_bit(unsigned long nr, volatile unsigned long *addr) * succeeds. */ static __always_inline void -__change_bit(unsigned long nr, volatile unsigned long *addr) +___change_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___change_bit(nr, addr); @@ -86,7 +86,7 @@ static __always_inline void __instrument_read_write_bitop(long nr, volatile unsi } /** - * __test_and_set_bit - Set a bit and return its old value + * ___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -94,14 +94,14 @@ static __always_inline void __instrument_read_write_bitop(long nr, volatile unsi * can appear to succeed but actually fail. */ static __always_inline bool -__test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_set_bit(nr, addr); } /** - * __test_and_clear_bit - Clear a bit and return its old value + * ___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -109,14 +109,14 @@ __test_and_set_bit(unsigned long nr, volatile unsigned long *addr) * can appear to succeed but actually fail. */ static __always_inline bool -__test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_clear_bit(nr, addr); } /** - * __test_and_change_bit - Change a bit and return its old value + * ___test_and_change_bit - Change a bit and return its old value * @nr: Bit to change * @addr: Address to count from * @@ -124,19 +124,19 @@ __test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) * can appear to succeed but actually fail. */ static __always_inline bool -__test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_change_bit(nr, addr); } /** - * test_bit - Determine whether a bit is set + * _test_bit - Determine whether a bit is set * @nr: bit number to test * @addr: Address to start counting from */ static __always_inline bool -test_bit(unsigned long nr, const volatile unsigned long *addr) +_test_bit(unsigned long nr, const volatile unsigned long *addr) { instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); return arch_test_bit(nr, addr); diff --git a/include/asm-generic/bitops/non-instrumented-non-atomic.h b/include/asm-generic/bitops/non-instrumented-non-atomic.h index e0fd7bf72a56..bdb9b1ffaee9 100644 --- a/include/asm-generic/bitops/non-instrumented-non-atomic.h +++ b/include/asm-generic/bitops/non-instrumented-non-atomic.h @@ -3,14 +3,14 @@ #ifndef __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H #define __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H -#define __set_bit arch___set_bit -#define __clear_bit arch___clear_bit -#define __change_bit arch___change_bit +#define ___set_bit arch___set_bit +#define ___clear_bit arch___clear_bit +#define ___change_bit arch___change_bit -#define __test_and_set_bit arch___test_and_set_bit -#define __test_and_clear_bit arch___test_and_clear_bit -#define __test_and_change_bit arch___test_and_change_bit +#define ___test_and_set_bit arch___test_and_set_bit +#define ___test_and_clear_bit arch___test_and_clear_bit +#define ___test_and_change_bit arch___test_and_change_bit -#define test_bit arch_test_bit +#define _test_bit arch_test_bit #endif /* __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/linux/bitops.h b/include/linux/bitops.h index d393297287d5..3c3afbae1533 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -26,8 +26,24 @@ extern unsigned int __sw_hweight16(unsigned int w); extern unsigned int __sw_hweight32(unsigned int w); extern unsigned long __sw_hweight64(__u64 w); +/* + * Defined here because those may be needed by architecture-specific static + * inlines. + */ + #include +#define bitop(op, nr, addr) \ + op(nr, addr) + +#define __set_bit(nr, addr) bitop(___set_bit, nr, addr) +#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr) +#define __change_bit(nr, addr) bitop(___change_bit, nr, addr) +#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr) +#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr) +#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr) +#define test_bit(nr, addr) bitop(_test_bit, nr, addr) + /* * Include this here because some architectures need generic_ffs/fls in * scope @@ -38,7 +54,7 @@ extern unsigned long __sw_hweight64(__u64 w); #define __check_bitop_pr(name) \ static_assert(__same_type(arch_##name, generic_##name) && \ __same_type(const_##name, generic_##name) && \ - __same_type(name, generic_##name)) + __same_type(_##name, generic_##name)) __check_bitop_pr(__set_bit); __check_bitop_pr(__clear_bit); diff --git a/tools/include/asm-generic/bitops/non-atomic.h b/tools/include/asm-generic/bitops/non-atomic.h index e5e78e42e57b..0c472a833408 100644 --- a/tools/include/asm-generic/bitops/non-atomic.h +++ b/tools/include/asm-generic/bitops/non-atomic.h @@ -5,7 +5,7 @@ #include /** - * __set_bit - Set a bit in memory + * ___set_bit - Set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -14,7 +14,7 @@ * may be that only one operation succeeds. */ static __always_inline void -__set_bit(unsigned long nr, volatile unsigned long *addr) +___set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -23,7 +23,7 @@ __set_bit(unsigned long nr, volatile unsigned long *addr) } static __always_inline void -__clear_bit(unsigned long nr, volatile unsigned long *addr) +___clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -32,7 +32,7 @@ __clear_bit(unsigned long nr, volatile unsigned long *addr) } /** - * __change_bit - Toggle a bit in memory + * ___change_bit - Toggle a bit in memory * @nr: the bit to change * @addr: the address to start counting from * @@ -41,7 +41,7 @@ __clear_bit(unsigned long nr, volatile unsigned long *addr) * may be that only one operation succeeds. */ static __always_inline void -__change_bit(unsigned long nr, volatile unsigned long *addr) +___change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -50,7 +50,7 @@ __change_bit(unsigned long nr, volatile unsigned long *addr) } /** - * __test_and_set_bit - Set a bit and return its old value + * ___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -59,7 +59,7 @@ __change_bit(unsigned long nr, volatile unsigned long *addr) * but actually fail. You must protect multiple accesses with a lock. */ static __always_inline bool -__test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -70,7 +70,7 @@ __test_and_set_bit(unsigned long nr, volatile unsigned long *addr) } /** - * __test_and_clear_bit - Clear a bit and return its old value + * ___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -79,7 +79,7 @@ __test_and_set_bit(unsigned long nr, volatile unsigned long *addr) * but actually fail. You must protect multiple accesses with a lock. */ static __always_inline bool -__test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -91,7 +91,7 @@ __test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) /* WARNING: non atomic and it can be reordered! */ static __always_inline bool -__test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -102,12 +102,12 @@ __test_and_change_bit(unsigned long nr, volatile unsigned long *addr) } /** - * test_bit - Determine whether a bit is set + * _test_bit - Determine whether a bit is set * @nr: bit number to test * @addr: Address to start counting from */ static __always_inline bool -test_bit(unsigned long nr, const volatile unsigned long *addr) +_test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } diff --git a/tools/include/linux/bitops.h b/tools/include/linux/bitops.h index 5fca38fe1ba8..f18683b95ea6 100644 --- a/tools/include/linux/bitops.h +++ b/tools/include/linux/bitops.h @@ -25,6 +25,22 @@ extern unsigned int __sw_hweight16(unsigned int w); extern unsigned int __sw_hweight32(unsigned int w); extern unsigned long __sw_hweight64(__u64 w); +/* + * Defined here because those may be needed by architecture-specific static + * inlines. + */ + +#define bitop(op, nr, addr) \ + op(nr, addr) + +#define __set_bit(nr, addr) bitop(___set_bit, nr, addr) +#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr) +#define __change_bit(nr, addr) bitop(___change_bit, nr, addr) +#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr) +#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr) +#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr) +#define test_bit(nr, addr) bitop(_test_bit, nr, addr) + /* * Include this here because some architectures need generic_ffs/fls in * scope -- cgit v1.2.3 From b03fc1173c0c2bb8fad61902a862985cecdc4b1b Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 24 Jun 2022 14:13:10 +0200 Subject: bitops: let optimize out non-atomic bitops on compile-time constants Currently, many architecture-specific non-atomic bitop implementations use inline asm or other hacks which are faster or more robust when working with "real" variables (i.e. fields from the structures etc.), but the compilers have no clue how to optimize them out when called on compile-time constants. That said, the following code: DECLARE_BITMAP(foo, BITS_PER_LONG) = { }; // -> unsigned long foo[1]; unsigned long bar = BIT(BAR_BIT); unsigned long baz = 0; __set_bit(FOO_BIT, foo); baz |= BIT(BAZ_BIT); BUILD_BUG_ON(!__builtin_constant_p(test_bit(FOO_BIT, foo)); BUILD_BUG_ON(!__builtin_constant_p(bar & BAR_BIT)); BUILD_BUG_ON(!__builtin_constant_p(baz & BAZ_BIT)); triggers the first assertion on x86_64, which means that the compiler is unable to evaluate it to a compile-time initializer when the architecture-specific bitop is used even if it's obvious. In order to let the compiler optimize out such cases, expand the bitop() macro to use the "constant" C non-atomic bitop implementations when all of the arguments passed are compile-time constants, which means that the result will be a compile-time constant as well, so that it produces more efficient and simple code in 100% cases, comparing to the architecture-specific counterparts. The savings are architecture, compiler and compiler flags dependent, for example, on x86_64 -O2: GCC 12: add/remove: 78/29 grow/shrink: 332/525 up/down: 31325/-61560 (-30235) LLVM 13: add/remove: 79/76 grow/shrink: 184/537 up/down: 55076/-141892 (-86816) LLVM 14: add/remove: 10/3 grow/shrink: 93/138 up/down: 3705/-6992 (-3287) and ARM64 (courtesy of Mark): GCC 11: add/remove: 92/29 grow/shrink: 933/2766 up/down: 39340/-82580 (-43240) LLVM 14: add/remove: 21/11 grow/shrink: 620/651 up/down: 12060/-15824 (-3764) Cc: Mark Rutland Signed-off-by: Alexander Lobakin Reviewed-by: Marco Elver Signed-off-by: Yury Norov --- include/linux/bitops.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 3c3afbae1533..cf9bf65039f2 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -33,8 +33,24 @@ extern unsigned long __sw_hweight64(__u64 w); #include +/* + * Many architecture-specific non-atomic bitops contain inline asm code and due + * to that the compiler can't optimize them to compile-time expressions or + * constants. In contrary, generic_*() helpers are defined in pure C and + * compilers optimize them just well. + * Therefore, to make `unsigned long foo = 0; __set_bit(BAR, &foo)` effectively + * equal to `unsigned long foo = BIT(BAR)`, pick the generic C alternative when + * the arguments can be resolved at compile time. That expression itself is a + * constant and doesn't bring any functional changes to the rest of cases. + * The casts to `uintptr_t` are needed to mitigate `-Waddress` warnings when + * passing a bitmap from .bss or .data (-> `!!addr` is always true). + */ #define bitop(op, nr, addr) \ - op(nr, addr) + ((__builtin_constant_p(nr) && \ + __builtin_constant_p((uintptr_t)(addr) != (uintptr_t)NULL) && \ + (uintptr_t)(addr) != (uintptr_t)NULL && \ + __builtin_constant_p(*(const unsigned long *)(addr))) ? \ + const##op(nr, addr) : op(nr, addr)) #define __set_bit(nr, addr) bitop(___set_bit, nr, addr) #define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr) -- cgit v1.2.3 From 3e7e5baaaba78075a7f3a57432609e363bf2a486 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 24 Jun 2022 14:13:12 +0200 Subject: bitmap: don't assume compiler evaluates small mem*() builtins calls Intel kernel bot triggered the build bug on ARC architecture that in fact is as follows: DECLARE_BITMAP(bitmap, BITS_PER_LONG); bitmap_clear(bitmap, 0, BITS_PER_LONG); BUILD_BUG_ON(!__builtin_constant_p(*bitmap)); which can be expanded to: unsigned long bitmap[1]; memset(bitmap, 0, sizeof(*bitmap)); BUILD_BUG_ON(!__builtin_constant_p(*bitmap)); In most cases, a compiler is able to expand small/simple mem*() calls to simple assignments or bitops, in this case that would mean: unsigned long bitmap[1] = { 0 }; BUILD_BUG_ON(!__builtin_constant_p(*bitmap)); and on most architectures this works, but not on ARC, despite having -O3 for every build. So, to make this work, in case when the last bit to modify is still within the first long (small_const_nbits()), just use plain assignments for the rest of bitmap_*() functions which still use mem*(), but didn't receive such compile-time optimizations yet. This doesn't have the same coverage as compilers provide, but at least something to start: text: add/remove: 3/7 grow/shrink: 43/78 up/down: 1848/-3370 (-1546) data: add/remove: 1/11 grow/shrink: 0/8 up/down: 4/-356 (-352) notably cpumask_*() family when NR_CPUS <= BITS_PER_LONG: netif_get_num_default_rss_queues 38 4 -34 cpumask_copy 90 - -90 cpumask_clear 146 - -146 and the abovementioned assertion started passing. Signed-off-by: Alexander Lobakin Signed-off-by: Yury Norov --- include/linux/bitmap.h | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index f091a1664bf1..c91638e507f2 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -238,20 +238,32 @@ extern int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) { unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long); - memset(dst, 0, len); + + if (small_const_nbits(nbits)) + *dst = 0; + else + memset(dst, 0, len); } static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) { unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long); - memset(dst, 0xff, len); + + if (small_const_nbits(nbits)) + *dst = ~0UL; + else + memset(dst, 0xff, len); } static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits) { unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long); - memcpy(dst, src, len); + + if (small_const_nbits(nbits)) + *dst = *src; + else + memcpy(dst, src, len); } /* @@ -431,6 +443,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, { if (__builtin_constant_p(nbits) && nbits == 1) __set_bit(start, map); + else if (small_const_nbits(start + nbits)) + *map |= GENMASK(start + nbits - 1, start); else if (__builtin_constant_p(start & BITMAP_MEM_MASK) && IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) && __builtin_constant_p(nbits & BITMAP_MEM_MASK) && @@ -445,6 +459,8 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, { if (__builtin_constant_p(nbits) && nbits == 1) __clear_bit(start, map); + else if (small_const_nbits(start + nbits)) + *map &= ~GENMASK(start + nbits - 1, start); else if (__builtin_constant_p(start & BITMAP_MEM_MASK) && IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) && __builtin_constant_p(nbits & BITMAP_MEM_MASK) && -- cgit v1.2.3 From 837ced3a1a5d8bb1a637dd584711f31ae6b54d93 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Tue, 28 Jun 2022 17:52:38 +0300 Subject: time64.h: consolidate uses of PSEC_PER_NSEC Time-sensitive networking code needs to work with PTP times expressed in nanoseconds, and with packet transmission times expressed in picoseconds, since those would be fractional at higher than gigabit speed when expressed in nanoseconds. Convert the existing uses in tc-taprio and the ocelot/felix DSA driver to a PSEC_PER_NSEC macro. This macro is placed in include/linux/time64.h as opposed to its relatives (PSEC_PER_SEC etc) from include/vdso/time64.h because the vDSO library does not (yet) need/use it. Cc: Andy Lutomirski Cc: Thomas Gleixner Signed-off-by: Vladimir Oltean Reviewed-by: Vincenzo Frascino # for the vDSO parts Signed-off-by: Jakub Kicinski --- drivers/net/dsa/ocelot/felix_vsc9959.c | 5 +++-- include/linux/time64.h | 3 +++ net/sched/sch_taprio.c | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index f51d9d5f0d82..61ed317602e7 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "felix.h" #define VSC9959_NUM_PORTS 6 @@ -1235,7 +1236,7 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) u32 max_sdu; if (min_gate_len[tc] == U64_MAX /* Gate always open */ || - min_gate_len[tc] * 1000 > needed_bit_time_ps) { + min_gate_len[tc] * PSEC_PER_NSEC > needed_bit_time_ps) { /* Setting QMAXSDU_CFG to 0 disables oversized frame * dropping. */ @@ -1249,7 +1250,7 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) * frame, make sure to enable oversize frame dropping * for frames larger than the smallest that would fit. */ - max_sdu = div_u64(min_gate_len[tc] * 1000, + max_sdu = div_u64(min_gate_len[tc] * PSEC_PER_NSEC, picos_per_byte); /* A TC gate may be completely closed, which is a * special case where all packets are oversized. diff --git a/include/linux/time64.h b/include/linux/time64.h index 81b9686a2079..2fb8232cff1d 100644 --- a/include/linux/time64.h +++ b/include/linux/time64.h @@ -20,6 +20,9 @@ struct itimerspec64 { struct timespec64 it_value; }; +/* Parameters used to convert the timespec values: */ +#define PSEC_PER_NSEC 1000L + /* Located here for timespec[64]_valid_strict */ #define TIME64_MAX ((s64)~((u64)1 << 63)) #define TIME64_MIN (-TIME64_MAX - 1) diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index b9c71a304d39..0b941dd63d26 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -176,7 +177,7 @@ static ktime_t get_interval_end_time(struct sched_gate_list *sched, static int length_to_duration(struct taprio_sched *q, int len) { - return div_u64(len * atomic64_read(&q->picos_per_byte), 1000); + return div_u64(len * atomic64_read(&q->picos_per_byte), PSEC_PER_NSEC); } /* Returns the entry corresponding to next available interval. If @@ -551,7 +552,7 @@ static struct sk_buff *taprio_peek(struct Qdisc *sch) static void taprio_set_budget(struct taprio_sched *q, struct sched_entry *entry) { atomic_set(&entry->budget, - div64_u64((u64)entry->interval * 1000, + div64_u64((u64)entry->interval * PSEC_PER_NSEC, atomic64_read(&q->picos_per_byte))); } -- cgit v1.2.3 From 6708be40047789aa3587a3866b782d5cda7b2a31 Mon Sep 17 00:00:00 2001 From: Peter Chiu Date: Wed, 22 Jun 2022 09:08:20 +0800 Subject: wifi: ieee80211: s1g action frames are not robust S1g action frame with code 22 is not protected so update the robust action frame list. Signed-off-by: Peter Chiu Link: https://lore.kernel.org/r/20220622010820.17522-1-chui-hao.chiu@mediatek.com Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 870f659087d6..f386f9ed41f3 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4094,6 +4094,7 @@ static inline bool _ieee80211_is_robust_mgmt_frame(struct ieee80211_hdr *hdr) *category != WLAN_CATEGORY_SELF_PROTECTED && *category != WLAN_CATEGORY_UNPROT_DMG && *category != WLAN_CATEGORY_VHT && + *category != WLAN_CATEGORY_S1G && *category != WLAN_CATEGORY_VENDOR_SPECIFIC; } -- cgit v1.2.3 From 80fc671bcc0173836e9032b0c698ea74c13b9d7c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 1 Jul 2022 11:48:43 +0800 Subject: uacce: Handle parent device removal or parent driver module rmmod The uacce driver must deal with a possible removal of the parent device or parent driver module rmmod at any time. Although uacce_remove(), called on device removal and on driver unbind, prevents future use of the uacce fops by removing the cdev, fops that were called before that point may still be running. Serialize uacce_fops_open() and uacce_remove() with uacce->mutex. Serialize other fops against uacce_remove() with q->mutex. Since we need to protect uacce_fops_poll() which gets called on the fast path, replace uacce->queues_lock with q->mutex to improve scalability. The other fops are only used during setup. uacce_queue_is_valid(), checked under q->mutex or uacce->mutex, denotes whether uacce_remove() has disabled all queues. If that is the case, don't go any further since the parent device is being removed and uacce->ops should not be called anymore. Reported-by: Yang Shen Signed-off-by: Zhangfei Gao Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20220701034843.7502-1-zhangfei.gao@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/uacce/uacce.c | 133 +++++++++++++++++++++++++++++---------------- include/linux/uacce.h | 6 +- 2 files changed, 91 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c index 281c54003edc..b70a013139c7 100644 --- a/drivers/misc/uacce/uacce.c +++ b/drivers/misc/uacce/uacce.c @@ -9,43 +9,38 @@ static struct class *uacce_class; static dev_t uacce_devt; -static DEFINE_MUTEX(uacce_mutex); static DEFINE_XARRAY_ALLOC(uacce_xa); -static int uacce_start_queue(struct uacce_queue *q) +/* + * If the parent driver or the device disappears, the queue state is invalid and + * ops are not usable anymore. + */ +static bool uacce_queue_is_valid(struct uacce_queue *q) { - int ret = 0; + return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED; +} - mutex_lock(&uacce_mutex); +static int uacce_start_queue(struct uacce_queue *q) +{ + int ret; - if (q->state != UACCE_Q_INIT) { - ret = -EINVAL; - goto out_with_lock; - } + if (q->state != UACCE_Q_INIT) + return -EINVAL; if (q->uacce->ops->start_queue) { ret = q->uacce->ops->start_queue(q); if (ret < 0) - goto out_with_lock; + return ret; } q->state = UACCE_Q_STARTED; - -out_with_lock: - mutex_unlock(&uacce_mutex); - - return ret; + return 0; } static int uacce_put_queue(struct uacce_queue *q) { struct uacce_device *uacce = q->uacce; - mutex_lock(&uacce_mutex); - - if (q->state == UACCE_Q_ZOMBIE) - goto out; - if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue) uacce->ops->stop_queue(q); @@ -54,8 +49,6 @@ static int uacce_put_queue(struct uacce_queue *q) uacce->ops->put_queue(q); q->state = UACCE_Q_ZOMBIE; -out: - mutex_unlock(&uacce_mutex); return 0; } @@ -65,20 +58,36 @@ static long uacce_fops_unl_ioctl(struct file *filep, { struct uacce_queue *q = filep->private_data; struct uacce_device *uacce = q->uacce; + long ret = -ENXIO; + + /* + * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from + * user. Avoid a circular lock dependency with uacce_fops_mmap(), which + * gets called with mmap_lock held, by taking uacce->mutex instead of + * q->mutex. Doing this in uacce_fops_mmap() is not possible because + * uacce_fops_open() calls iommu_sva_bind_device(), which takes + * mmap_lock, while holding uacce->mutex. + */ + mutex_lock(&uacce->mutex); + if (!uacce_queue_is_valid(q)) + goto out_unlock; switch (cmd) { case UACCE_CMD_START_Q: - return uacce_start_queue(q); - + ret = uacce_start_queue(q); + break; case UACCE_CMD_PUT_Q: - return uacce_put_queue(q); - + ret = uacce_put_queue(q); + break; default: - if (!uacce->ops->ioctl) - return -EINVAL; - - return uacce->ops->ioctl(q, cmd, arg); + if (uacce->ops->ioctl) + ret = uacce->ops->ioctl(q, cmd, arg); + else + ret = -EINVAL; } +out_unlock: + mutex_unlock(&uacce->mutex); + return ret; } #ifdef CONFIG_COMPAT @@ -136,6 +145,13 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) if (!q) return -ENOMEM; + mutex_lock(&uacce->mutex); + + if (!uacce->parent) { + ret = -EINVAL; + goto out_with_mem; + } + ret = uacce_bind_queue(uacce, q); if (ret) goto out_with_mem; @@ -152,10 +168,9 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) filep->private_data = q; uacce->inode = inode; q->state = UACCE_Q_INIT; - - mutex_lock(&uacce->queues_lock); + mutex_init(&q->mutex); list_add(&q->list, &uacce->queues); - mutex_unlock(&uacce->queues_lock); + mutex_unlock(&uacce->mutex); return 0; @@ -163,18 +178,20 @@ out_with_bond: uacce_unbind_queue(q); out_with_mem: kfree(q); + mutex_unlock(&uacce->mutex); return ret; } static int uacce_fops_release(struct inode *inode, struct file *filep) { struct uacce_queue *q = filep->private_data; + struct uacce_device *uacce = q->uacce; - mutex_lock(&q->uacce->queues_lock); - list_del(&q->list); - mutex_unlock(&q->uacce->queues_lock); + mutex_lock(&uacce->mutex); uacce_put_queue(q); uacce_unbind_queue(q); + list_del(&q->list); + mutex_unlock(&uacce->mutex); kfree(q); return 0; @@ -217,10 +234,9 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) vma->vm_private_data = q; qfr->type = type; - mutex_lock(&uacce_mutex); - - if (q->state != UACCE_Q_INIT && q->state != UACCE_Q_STARTED) { - ret = -EINVAL; + mutex_lock(&q->mutex); + if (!uacce_queue_is_valid(q)) { + ret = -ENXIO; goto out_with_lock; } @@ -248,12 +264,12 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) } q->qfrs[type] = qfr; - mutex_unlock(&uacce_mutex); + mutex_unlock(&q->mutex); return ret; out_with_lock: - mutex_unlock(&uacce_mutex); + mutex_unlock(&q->mutex); kfree(qfr); return ret; } @@ -262,12 +278,20 @@ static __poll_t uacce_fops_poll(struct file *file, poll_table *wait) { struct uacce_queue *q = file->private_data; struct uacce_device *uacce = q->uacce; + __poll_t ret = 0; + + mutex_lock(&q->mutex); + if (!uacce_queue_is_valid(q)) + goto out_unlock; poll_wait(file, &q->wait, wait); + if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q)) - return EPOLLIN | EPOLLRDNORM; + ret = EPOLLIN | EPOLLRDNORM; - return 0; +out_unlock: + mutex_unlock(&q->mutex); + return ret; } static const struct file_operations uacce_fops = { @@ -450,7 +474,7 @@ struct uacce_device *uacce_alloc(struct device *parent, goto err_with_uacce; INIT_LIST_HEAD(&uacce->queues); - mutex_init(&uacce->queues_lock); + mutex_init(&uacce->mutex); device_initialize(&uacce->dev); uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id); uacce->dev.class = uacce_class; @@ -507,13 +531,23 @@ void uacce_remove(struct uacce_device *uacce) if (uacce->inode) unmap_mapping_range(uacce->inode->i_mapping, 0, 0, 1); + /* + * uacce_fops_open() may be running concurrently, even after we remove + * the cdev. Holding uacce->mutex ensures that open() does not obtain a + * removed uacce device. + */ + mutex_lock(&uacce->mutex); /* ensure no open queue remains */ - mutex_lock(&uacce->queues_lock); list_for_each_entry_safe(q, next_q, &uacce->queues, list) { + /* + * Taking q->mutex ensures that fops do not use the defunct + * uacce->ops after the queue is disabled. + */ + mutex_lock(&q->mutex); uacce_put_queue(q); + mutex_unlock(&q->mutex); uacce_unbind_queue(q); } - mutex_unlock(&uacce->queues_lock); /* disable sva now since no opened queues */ uacce_disable_sva(uacce); @@ -521,6 +555,13 @@ void uacce_remove(struct uacce_device *uacce) if (uacce->cdev) cdev_device_del(uacce->cdev, &uacce->dev); xa_erase(&uacce_xa, uacce->dev_id); + /* + * uacce exists as long as there are open fds, but ops will be freed + * now. Ensure that bugs cause NULL deref rather than use-after-free. + */ + uacce->ops = NULL; + uacce->parent = NULL; + mutex_unlock(&uacce->mutex); put_device(&uacce->dev); } EXPORT_SYMBOL_GPL(uacce_remove); diff --git a/include/linux/uacce.h b/include/linux/uacce.h index 48e319f40275..9ce88c28b0a8 100644 --- a/include/linux/uacce.h +++ b/include/linux/uacce.h @@ -70,6 +70,7 @@ enum uacce_q_state { * @wait: wait queue head * @list: index into uacce queues list * @qfrs: pointer of qfr regions + * @mutex: protects queue state * @state: queue state machine * @pasid: pasid associated to the mm * @handle: iommu_sva handle returned by iommu_sva_bind_device() @@ -80,6 +81,7 @@ struct uacce_queue { wait_queue_head_t wait; struct list_head list; struct uacce_qfile_region *qfrs[UACCE_MAX_REGION]; + struct mutex mutex; enum uacce_q_state state; u32 pasid; struct iommu_sva *handle; @@ -97,9 +99,9 @@ struct uacce_queue { * @dev_id: id of the uacce device * @cdev: cdev of the uacce * @dev: dev of the uacce + * @mutex: protects uacce operation * @priv: private pointer of the uacce * @queues: list of queues - * @queues_lock: lock for queues list * @inode: core vfs */ struct uacce_device { @@ -113,9 +115,9 @@ struct uacce_device { u32 dev_id; struct cdev *cdev; struct device dev; + struct mutex mutex; void *priv; struct list_head queues; - struct mutex queues_lock; struct inode *inode; }; -- cgit v1.2.3 From c882716b6d411495f221bdd73e9137357c16a3ea Mon Sep 17 00:00:00 2001 From: Liang He Date: Tue, 28 Jun 2022 10:16:40 +0800 Subject: firmware: Hold a reference for of_find_compatible_node() In of_register_trusted_foundations(), we need to hold the reference returned by of_find_compatible_node() and then use it to call of_node_put() for refcount balance. Signed-off-by: Liang He Link: https://lore.kernel.org/r/20220628021640.4015-1-windhl@126.com Signed-off-by: Greg Kroah-Hartman --- include/linux/firmware/trusted_foundations.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/firmware/trusted_foundations.h b/include/linux/firmware/trusted_foundations.h index be5984bda592..931b6c5c72df 100644 --- a/include/linux/firmware/trusted_foundations.h +++ b/include/linux/firmware/trusted_foundations.h @@ -71,12 +71,16 @@ static inline void register_trusted_foundations( static inline void of_register_trusted_foundations(void) { + struct device_node *np = of_find_compatible_node(NULL, NULL, "tlm,trusted-foundations"); + + if (!np) + return; + of_node_put(np); /* * If we find the target should enable TF but does not support it, * fail as the system won't be able to do much anyway */ - if (of_find_compatible_node(NULL, NULL, "tlm,trusted-foundations")) - register_trusted_foundations(NULL); + register_trusted_foundations(NULL); } static inline bool trusted_foundations_registered(void) -- cgit v1.2.3 From 2852ca7fba9f77b204f0fe953b31fadd0057c936 Mon Sep 17 00:00:00 2001 From: David Gow Date: Fri, 1 Jul 2022 16:47:41 +0800 Subject: panic: Taint kernel if tests are run Most in-kernel tests (such as KUnit tests) are not supposed to run on production systems: they may do deliberately illegal things to trigger errors, and have security implications (for example, KUnit assertions will often deliberately leak kernel addresses). Add a new taint type, TAINT_TEST to signal that a test has been run. This will be printed as 'N' (originally for kuNit, as every other sensible letter was taken.) This should discourage people from running these tests on production systems, and to make it easier to tell if tests have been run accidentally (by loading the wrong configuration, etc.) Acked-by: Luis Chamberlain Reviewed-by: Brendan Higgins Signed-off-by: David Gow Signed-off-by: Shuah Khan --- Documentation/admin-guide/tainted-kernels.rst | 1 + include/linux/panic.h | 3 ++- kernel/panic.c | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/tainted-kernels.rst b/Documentation/admin-guide/tainted-kernels.rst index ceeed7b0798d..7d80e8c307d1 100644 --- a/Documentation/admin-guide/tainted-kernels.rst +++ b/Documentation/admin-guide/tainted-kernels.rst @@ -100,6 +100,7 @@ Bit Log Number Reason that got the kernel tainted 15 _/K 32768 kernel has been live patched 16 _/X 65536 auxiliary taint, defined for and used by distros 17 _/T 131072 kernel was built with the struct randomization plugin + 18 _/N 262144 an in-kernel test has been run === === ====== ======================================================== Note: The character ``_`` is representing a blank in this table to make reading diff --git a/include/linux/panic.h b/include/linux/panic.h index e71161da69c4..c7759b3f2045 100644 --- a/include/linux/panic.h +++ b/include/linux/panic.h @@ -68,7 +68,8 @@ static inline void set_arch_panic_timeout(int timeout, int arch_default_timeout) #define TAINT_LIVEPATCH 15 #define TAINT_AUX 16 #define TAINT_RANDSTRUCT 17 -#define TAINT_FLAGS_COUNT 18 +#define TAINT_TEST 18 +#define TAINT_FLAGS_COUNT 19 #define TAINT_FLAGS_MAX ((1UL << TAINT_FLAGS_COUNT) - 1) struct taint_flag { diff --git a/kernel/panic.c b/kernel/panic.c index a3c758dba15a..6b3369e21026 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -428,6 +428,7 @@ const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = { [ TAINT_LIVEPATCH ] = { 'K', ' ', true }, [ TAINT_AUX ] = { 'X', ' ', true }, [ TAINT_RANDSTRUCT ] = { 'T', ' ', true }, + [ TAINT_TEST ] = { 'N', ' ', true }, }; /** -- cgit v1.2.3 From eb003bf3ba221bb3d21d1fdcddaa36c158fd2d8f Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 24 Jun 2022 20:36:39 +0200 Subject: platform/surface: aggregator: Add helper macros for requests with argument and return value Add helper macros for synchronous stack-allocated Surface Aggregator request with both argument and return value, similar to the current argument-only and return-value-only ones. Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20220624183642.910893-2-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- include/linux/surface_aggregator/controller.h | 125 ++++++++++++++++++++++++++ include/linux/surface_aggregator/device.h | 36 ++++++++ 2 files changed, 161 insertions(+) (limited to 'include/linux') diff --git a/include/linux/surface_aggregator/controller.h b/include/linux/surface_aggregator/controller.h index 50a2b4926c06..d11a1c6e3186 100644 --- a/include/linux/surface_aggregator/controller.h +++ b/include/linux/surface_aggregator/controller.h @@ -469,6 +469,67 @@ struct ssam_request_spec_md { return 0; \ } +/** + * SSAM_DEFINE_SYNC_REQUEST_WR() - Define synchronous SAM request function with + * both argument and return value. + * @name: Name of the generated function. + * @atype: Type of the request's argument. + * @rtype: Type of the request's return value. + * @spec: Specification (&struct ssam_request_spec) defining the request. + * + * Defines a function executing the synchronous SAM request specified by @spec, + * with the request taking an argument of type @atype and having a return value + * of type @rtype. The generated function takes care of setting up the request + * and response structs, buffer allocation, as well as execution of the request + * itself, returning once the request has been fully completed. The required + * transport buffer will be allocated on the stack. + * + * The generated function is defined as ``static int name(struct + * ssam_controller *ctrl, const atype *arg, rtype *ret)``, returning the status + * of the request, which is zero on success and negative on failure. The + * ``ctrl`` parameter is the controller via which the request is sent. The + * request argument is specified via the ``arg`` pointer. The request's return + * value is written to the memory pointed to by the ``ret`` parameter. + * + * Refer to ssam_request_sync_onstack() for more details on the behavior of + * the generated function. + */ +#define SSAM_DEFINE_SYNC_REQUEST_WR(name, atype, rtype, spec...) \ + static int name(struct ssam_controller *ctrl, const atype *arg, rtype *ret) \ + { \ + struct ssam_request_spec s = (struct ssam_request_spec)spec; \ + struct ssam_request rqst; \ + struct ssam_response rsp; \ + int status; \ + \ + rqst.target_category = s.target_category; \ + rqst.target_id = s.target_id; \ + rqst.command_id = s.command_id; \ + rqst.instance_id = s.instance_id; \ + rqst.flags = s.flags | SSAM_REQUEST_HAS_RESPONSE; \ + rqst.length = sizeof(atype); \ + rqst.payload = (u8 *)arg; \ + \ + rsp.capacity = sizeof(rtype); \ + rsp.length = 0; \ + rsp.pointer = (u8 *)ret; \ + \ + status = ssam_request_sync_onstack(ctrl, &rqst, &rsp, sizeof(atype)); \ + if (status) \ + return status; \ + \ + if (rsp.length != sizeof(rtype)) { \ + struct device *dev = ssam_controller_device(ctrl); \ + dev_err(dev, \ + "rqst: invalid response length, expected %zu, got %zu (tc: %#04x, cid: %#04x)", \ + sizeof(rtype), rsp.length, rqst.target_category,\ + rqst.command_id); \ + return -EIO; \ + } \ + \ + return 0; \ + } + /** * SSAM_DEFINE_SYNC_REQUEST_MD_N() - Define synchronous multi-device SAM * request function with neither argument nor return value. @@ -613,6 +674,70 @@ struct ssam_request_spec_md { return 0; \ } +/** + * SSAM_DEFINE_SYNC_REQUEST_MD_WR() - Define synchronous multi-device SAM + * request function with both argument and return value. + * @name: Name of the generated function. + * @atype: Type of the request's argument. + * @rtype: Type of the request's return value. + * @spec: Specification (&struct ssam_request_spec_md) defining the request. + * + * Defines a function executing the synchronous SAM request specified by @spec, + * with the request taking an argument of type @atype and having a return value + * of type @rtype. Device specifying parameters are not hard-coded, but instead + * must be provided to the function. The generated function takes care of + * setting up the request and response structs, buffer allocation, as well as + * execution of the request itself, returning once the request has been fully + * completed. The required transport buffer will be allocated on the stack. + * + * The generated function is defined as ``static int name(struct + * ssam_controller *ctrl, u8 tid, u8 iid, const atype *arg, rtype *ret)``, + * returning the status of the request, which is zero on success and negative + * on failure. The ``ctrl`` parameter is the controller via which the request + * is sent, ``tid`` the target ID for the request, and ``iid`` the instance ID. + * The request argument is specified via the ``arg`` pointer. The request's + * return value is written to the memory pointed to by the ``ret`` parameter. + * + * Refer to ssam_request_sync_onstack() for more details on the behavior of + * the generated function. + */ +#define SSAM_DEFINE_SYNC_REQUEST_MD_WR(name, atype, rtype, spec...) \ + static int name(struct ssam_controller *ctrl, u8 tid, u8 iid, \ + const atype *arg, rtype *ret) \ + { \ + struct ssam_request_spec_md s = (struct ssam_request_spec_md)spec; \ + struct ssam_request rqst; \ + struct ssam_response rsp; \ + int status; \ + \ + rqst.target_category = s.target_category; \ + rqst.target_id = tid; \ + rqst.command_id = s.command_id; \ + rqst.instance_id = iid; \ + rqst.flags = s.flags | SSAM_REQUEST_HAS_RESPONSE; \ + rqst.length = sizeof(atype); \ + rqst.payload = (u8 *)arg; \ + \ + rsp.capacity = sizeof(rtype); \ + rsp.length = 0; \ + rsp.pointer = (u8 *)ret; \ + \ + status = ssam_request_sync_onstack(ctrl, &rqst, &rsp, sizeof(atype)); \ + if (status) \ + return status; \ + \ + if (rsp.length != sizeof(rtype)) { \ + struct device *dev = ssam_controller_device(ctrl); \ + dev_err(dev, \ + "rqst: invalid response length, expected %zu, got %zu (tc: %#04x, cid: %#04x)", \ + sizeof(rtype), rsp.length, rqst.target_category,\ + rqst.command_id); \ + return -EIO; \ + } \ + \ + return 0; \ + } + /* -- Event notifier/callbacks. --------------------------------------------- */ diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index c418f7f2732d..6cf7e80312d5 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -483,6 +483,42 @@ static inline void ssam_remove_clients(struct device *dev) {} sdev->uid.instance, ret); \ } +/** + * SSAM_DEFINE_SYNC_REQUEST_CL_WR() - Define synchronous client-device SAM + * request function with argument and return value. + * @name: Name of the generated function. + * @atype: Type of the request's argument. + * @rtype: Type of the request's return value. + * @spec: Specification (&struct ssam_request_spec_md) defining the request. + * + * Defines a function executing the synchronous SAM request specified by @spec, + * with the request taking an argument of type @atype and having a return value + * of type @rtype. Device specifying parameters are not hard-coded, but instead + * are provided via the client device, specifically its UID, supplied when + * calling this function. The generated function takes care of setting up the + * request struct, buffer allocation, as well as execution of the request + * itself, returning once the request has been fully completed. The required + * transport buffer will be allocated on the stack. + * + * The generated function is defined as ``static int name(struct ssam_device + * *sdev, const atype *arg, rtype *ret)``, returning the status of the request, + * which is zero on success and negative on failure. The ``sdev`` parameter + * specifies both the target device of the request and by association the + * controller via which the request is sent. The request's argument is + * specified via the ``arg`` pointer. The request's return value is written to + * the memory pointed to by the ``ret`` parameter. + * + * Refer to ssam_request_sync_onstack() for more details on the behavior of + * the generated function. + */ +#define SSAM_DEFINE_SYNC_REQUEST_CL_WR(name, atype, rtype, spec...) \ + SSAM_DEFINE_SYNC_REQUEST_MD_WR(__raw_##name, atype, rtype, spec) \ + static int name(struct ssam_device *sdev, const atype *arg, rtype *ret) \ + { \ + return __raw_##name(sdev->ctrl, sdev->uid.target, \ + sdev->uid.instance, arg, ret); \ + } + /* -- Helpers for client-device notifiers. ---------------------------------- */ -- cgit v1.2.3 From 4a4ab610b8ae912c28a4fd28442ef24ed7a1a5bd Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 24 Jun 2022 22:57:58 +0200 Subject: platform/surface: aggregator: Move device registry helper functions to core module Move helper functions for client device registration to the core module. This simplifies addition of future DT/OF support and also allows us to split out the device hub drivers into their own module. At the same time, also improve device node validation a bit by not silently skipping devices with invalid device UID specifiers. Further, ensure proper lifetime management for the firmware/software nodes associated with the added devices. Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20220624205800.1355621-2-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/surface/aggregator/bus.c | 149 ++++++++++++++++++--- .../platform/surface/surface_aggregator_registry.c | 75 +---------- include/linux/surface_aggregator/device.h | 52 +++++++ 3 files changed, 187 insertions(+), 89 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/surface/aggregator/bus.c b/drivers/platform/surface/aggregator/bus.c index abbbb5b08b07..e0b0381a2834 100644 --- a/drivers/platform/surface/aggregator/bus.c +++ b/drivers/platform/surface/aggregator/bus.c @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -14,6 +15,9 @@ #include "bus.h" #include "controller.h" + +/* -- Device and bus functions. --------------------------------------------- */ + static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -46,6 +50,7 @@ static void ssam_device_release(struct device *dev) struct ssam_device *sdev = to_ssam_device(dev); ssam_controller_put(sdev->ctrl); + fwnode_handle_put(sdev->dev.fwnode); kfree(sdev); } @@ -363,6 +368,134 @@ void ssam_device_driver_unregister(struct ssam_device_driver *sdrv) } EXPORT_SYMBOL_GPL(ssam_device_driver_unregister); + +/* -- Bus registration. ----------------------------------------------------- */ + +/** + * ssam_bus_register() - Register and set-up the SSAM client device bus. + */ +int ssam_bus_register(void) +{ + return bus_register(&ssam_bus_type); +} + +/** + * ssam_bus_unregister() - Unregister the SSAM client device bus. + */ +void ssam_bus_unregister(void) +{ + return bus_unregister(&ssam_bus_type); +} + + +/* -- Helpers for controller and hub devices. ------------------------------- */ + +static int ssam_device_uid_from_string(const char *str, struct ssam_device_uid *uid) +{ + u8 d, tc, tid, iid, fn; + int n; + + n = sscanf(str, "%hhx:%hhx:%hhx:%hhx:%hhx", &d, &tc, &tid, &iid, &fn); + if (n != 5) + return -EINVAL; + + uid->domain = d; + uid->category = tc; + uid->target = tid; + uid->instance = iid; + uid->function = fn; + + return 0; +} + +static int ssam_get_uid_for_node(struct fwnode_handle *node, struct ssam_device_uid *uid) +{ + const char *str = fwnode_get_name(node); + + /* + * To simplify definitions of firmware nodes, we set the device name + * based on the UID of the device, prefixed with "ssam:". + */ + if (strncmp(str, "ssam:", strlen("ssam:")) != 0) + return -ENODEV; + + str += strlen("ssam:"); + return ssam_device_uid_from_string(str, uid); +} + +static int ssam_add_client_device(struct device *parent, struct ssam_controller *ctrl, + struct fwnode_handle *node) +{ + struct ssam_device_uid uid; + struct ssam_device *sdev; + int status; + + status = ssam_get_uid_for_node(node, &uid); + if (status) + return status; + + sdev = ssam_device_alloc(ctrl, uid); + if (!sdev) + return -ENOMEM; + + sdev->dev.parent = parent; + sdev->dev.fwnode = fwnode_handle_get(node); + + status = ssam_device_add(sdev); + if (status) + ssam_device_put(sdev); + + return status; +} + +/** + * __ssam_register_clients() - Register client devices defined under the + * given firmware node as children of the given device. + * @parent: The parent device under which clients should be registered. + * @ctrl: The controller with which client should be registered. + * @node: The firmware node holding definitions of the devices to be added. + * + * Register all clients that have been defined as children of the given root + * firmware node as children of the given parent device. The respective child + * firmware nodes will be associated with the correspondingly created child + * devices. + * + * The given controller will be used to instantiate the new devices. See + * ssam_device_add() for details. + * + * Note that, generally, the use of either ssam_device_register_clients() or + * ssam_register_clients() should be preferred as they directly use the + * firmware node and/or controller associated with the given device. This + * function is only intended for use when different device specifications (e.g. + * ACPI and firmware nodes) need to be combined (as is done in the platform hub + * of the device registry). + * + * Return: Returns zero on success, nonzero on failure. + */ +int __ssam_register_clients(struct device *parent, struct ssam_controller *ctrl, + struct fwnode_handle *node) +{ + struct fwnode_handle *child; + int status; + + fwnode_for_each_child_node(node, child) { + /* + * Try to add the device specified in the firmware node. If + * this fails with -ENODEV, the node does not specify any SSAM + * device, so ignore it and continue with the next one. + */ + status = ssam_add_client_device(parent, ctrl, child); + if (status && status != -ENODEV) + goto err; + } + + return 0; +err: + ssam_remove_clients(parent); + return status; +} +EXPORT_SYMBOL_GPL(__ssam_register_clients); + static int ssam_remove_device(struct device *dev, void *_data) { struct ssam_device *sdev = to_ssam_device(dev); @@ -387,19 +520,3 @@ void ssam_remove_clients(struct device *dev) device_for_each_child_reverse(dev, NULL, ssam_remove_device); } EXPORT_SYMBOL_GPL(ssam_remove_clients); - -/** - * ssam_bus_register() - Register and set-up the SSAM client device bus. - */ -int ssam_bus_register(void) -{ - return bus_register(&ssam_bus_type); -} - -/** - * ssam_bus_unregister() - Unregister the SSAM client device bus. - */ -void ssam_bus_unregister(void) -{ - return bus_unregister(&ssam_bus_type); -} diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c index f1c5905f1c16..c680792a037e 100644 --- a/drivers/platform/surface/surface_aggregator_registry.c +++ b/drivers/platform/surface/surface_aggregator_registry.c @@ -286,76 +286,6 @@ static const struct software_node *ssam_node_group_sp8[] = { }; -/* -- Device registry helper functions. ------------------------------------- */ - -static int ssam_uid_from_string(const char *str, struct ssam_device_uid *uid) -{ - u8 d, tc, tid, iid, fn; - int n; - - n = sscanf(str, "ssam:%hhx:%hhx:%hhx:%hhx:%hhx", &d, &tc, &tid, &iid, &fn); - if (n != 5) - return -EINVAL; - - uid->domain = d; - uid->category = tc; - uid->target = tid; - uid->instance = iid; - uid->function = fn; - - return 0; -} - -static int ssam_hub_add_device(struct device *parent, struct ssam_controller *ctrl, - struct fwnode_handle *node) -{ - struct ssam_device_uid uid; - struct ssam_device *sdev; - int status; - - status = ssam_uid_from_string(fwnode_get_name(node), &uid); - if (status) - return status; - - sdev = ssam_device_alloc(ctrl, uid); - if (!sdev) - return -ENOMEM; - - sdev->dev.parent = parent; - sdev->dev.fwnode = node; - - status = ssam_device_add(sdev); - if (status) - ssam_device_put(sdev); - - return status; -} - -static int ssam_hub_register_clients(struct device *parent, struct ssam_controller *ctrl, - struct fwnode_handle *node) -{ - struct fwnode_handle *child; - int status; - - fwnode_for_each_child_node(node, child) { - /* - * Try to add the device specified in the firmware node. If - * this fails with -EINVAL, the node does not specify any SSAM - * device, so ignore it and continue with the next one. - */ - - status = ssam_hub_add_device(parent, ctrl, child); - if (status && status != -EINVAL) - goto err; - } - - return 0; -err: - ssam_remove_clients(parent); - return status; -} - - /* -- SSAM generic subsystem hub driver framework. -------------------------- */ enum ssam_hub_state { @@ -385,7 +315,6 @@ struct ssam_hub { static void ssam_hub_update_workfn(struct work_struct *work) { struct ssam_hub *hub = container_of(work, struct ssam_hub, update_work.work); - struct fwnode_handle *node = dev_fwnode(&hub->sdev->dev); enum ssam_hub_state state; int status = 0; @@ -425,7 +354,7 @@ static void ssam_hub_update_workfn(struct work_struct *work) hub->state = state; if (hub->state == SSAM_HUB_CONNECTED) - status = ssam_hub_register_clients(&hub->sdev->dev, hub->sdev->ctrl, node); + status = ssam_device_register_clients(hub->sdev); else ssam_remove_clients(&hub->sdev->dev); @@ -769,7 +698,7 @@ static int ssam_platform_hub_probe(struct platform_device *pdev) set_secondary_fwnode(&pdev->dev, root); - status = ssam_hub_register_clients(&pdev->dev, ctrl, root); + status = __ssam_register_clients(&pdev->dev, ctrl, root); if (status) { set_secondary_fwnode(&pdev->dev, NULL); software_node_unregister_node_group(nodes); diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index 6cf7e80312d5..46c45d1b6368 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -375,11 +376,62 @@ void ssam_device_driver_unregister(struct ssam_device_driver *d); /* -- Helpers for controller and hub devices. ------------------------------- */ #ifdef CONFIG_SURFACE_AGGREGATOR_BUS + +int __ssam_register_clients(struct device *parent, struct ssam_controller *ctrl, + struct fwnode_handle *node); void ssam_remove_clients(struct device *dev); + #else /* CONFIG_SURFACE_AGGREGATOR_BUS */ + +static inline int __ssam_register_clients(struct device *parent, struct ssam_controller *ctrl, + struct fwnode_handle *node) +{ + return 0; +} + static inline void ssam_remove_clients(struct device *dev) {} + #endif /* CONFIG_SURFACE_AGGREGATOR_BUS */ +/** + * ssam_register_clients() - Register all client devices defined under the + * given parent device. + * @dev: The parent device under which clients should be registered. + * @ctrl: The controller with which client should be registered. + * + * Register all clients that have via firmware nodes been defined as children + * of the given (parent) device. The respective child firmware nodes will be + * associated with the correspondingly created child devices. + * + * The given controller will be used to instantiate the new devices. See + * ssam_device_add() for details. + * + * Return: Returns zero on success, nonzero on failure. + */ +static inline int ssam_register_clients(struct device *dev, struct ssam_controller *ctrl) +{ + return __ssam_register_clients(dev, ctrl, dev_fwnode(dev)); +} + +/** + * ssam_device_register_clients() - Register all client devices defined under + * the given SSAM parent device. + * @sdev: The parent device under which clients should be registered. + * + * Register all clients that have via firmware nodes been defined as children + * of the given (parent) device. The respective child firmware nodes will be + * associated with the correspondingly created child devices. + * + * The controller used by the parent device will be used to instantiate the new + * devices. See ssam_device_add() for details. + * + * Return: Returns zero on success, nonzero on failure. + */ +static inline int ssam_device_register_clients(struct ssam_device *sdev) +{ + return ssam_register_clients(&sdev->dev, sdev->ctrl); +} + /* -- Helpers for client-device requests. ----------------------------------- */ -- cgit v1.2.3 From 504148fedb854299972d164b001357b888a9193e Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 30 Jun 2022 15:07:50 +0000 Subject: net: add skb_[inner_]tcp_all_headers helpers Most drivers use "skb_transport_offset(skb) + tcp_hdrlen(skb)" to compute headers length for a TCP packet, but others use more convoluted (but equivalent) ways. Add skb_tcp_all_headers() and skb_inner_tcp_all_headers() helpers to harmonize this a bit. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/infiniband/ulp/ipoib/ipoib_ib.c | 2 +- drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 6 ++--- drivers/net/ethernet/atheros/atl1c/atl1c_main.c | 9 ++++--- drivers/net/ethernet/atheros/atl1e/atl1e_main.c | 8 +++--- drivers/net/ethernet/atheros/atlx/atl1.c | 7 +++-- drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c | 17 +++++------- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 7 ++--- drivers/net/ethernet/broadcom/tg3.c | 2 +- drivers/net/ethernet/brocade/bna/bnad.c | 6 ++--- drivers/net/ethernet/cadence/macb_main.c | 2 +- drivers/net/ethernet/cavium/thunder/nicvf_queues.c | 4 +-- drivers/net/ethernet/chelsio/cxgb4/sge.c | 2 +- .../chelsio/inline_crypto/ch_ktls/chcr_ktls.c | 6 ++--- drivers/net/ethernet/cisco/enic/enic_main.c | 5 ++-- drivers/net/ethernet/emulex/benet/be_main.c | 6 ++--- drivers/net/ethernet/freescale/fec_main.c | 2 +- drivers/net/ethernet/fungible/funeth/funeth_tx.c | 2 +- drivers/net/ethernet/google/gve/gve_tx_dqo.c | 4 +-- drivers/net/ethernet/hisilicon/hns/hns_enet.c | 6 ++--- drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 4 +-- drivers/net/ethernet/hisilicon/hns3/hns3_trace.h | 3 +-- drivers/net/ethernet/ibm/ehea/ehea_main.c | 2 +- drivers/net/ethernet/intel/e1000/e1000_main.c | 4 +-- drivers/net/ethernet/intel/e1000e/netdev.c | 4 +-- drivers/net/ethernet/intel/ixgb/ixgb_main.c | 2 +- drivers/net/ethernet/marvell/mv643xx_eth.c | 2 +- drivers/net/ethernet/marvell/mvneta.c | 4 +-- .../net/ethernet/marvell/octeontx2/nic/otx2_txrx.c | 4 +-- drivers/net/ethernet/marvell/sky2.c | 2 +- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 4 +-- .../ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en_tx.c | 4 +-- drivers/net/ethernet/myricom/myri10ge/myri10ge.c | 2 +- drivers/net/ethernet/netronome/nfp/nfd3/dp.c | 5 ++-- drivers/net/ethernet/netronome/nfp/nfdk/dp.c | 5 ++-- .../net/ethernet/netronome/nfp/nfp_net_common.c | 7 +++-- drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 5 ++-- .../net/ethernet/qlogic/netxen/netxen_nic_main.c | 2 +- drivers/net/ethernet/qlogic/qede/qede_fp.c | 8 +++--- drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c | 2 +- drivers/net/ethernet/qualcomm/emac/emac-mac.c | 4 +-- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 2 +- drivers/net/ethernet/synopsys/dwc-xlgmac-net.c | 2 +- drivers/net/wireless/ath/wil6210/txrx.c | 4 +-- drivers/net/xen-netback/netback.c | 4 +-- drivers/staging/qlge/qlge_main.c | 2 +- include/linux/tcp.h | 30 ++++++++++++++++++++++ net/tls/tls_device_fallback.c | 6 ++--- 48 files changed, 119 insertions(+), 115 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c index 2c3dca41d3bd..f7995519bbc8 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c @@ -573,7 +573,7 @@ int ipoib_send(struct net_device *dev, struct sk_buff *skb, unsigned int usable_sge = priv->max_send_sge - !!skb_headlen(skb); if (skb_is_gso(skb)) { - hlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + hlen = skb_tcp_all_headers(skb); phead = skb->data; if (unlikely(!skb_pull(skb, hlen))) { ipoib_warn(priv, "linear data too small\n"); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 4d46780fad13..f342bb853189 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -1673,12 +1673,10 @@ static int xgbe_prep_tso(struct sk_buff *skb, struct xgbe_packet_data *packet) return ret; if (XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, VXLAN)) { - packet->header_len = skb_inner_transport_offset(skb) + - inner_tcp_hdrlen(skb); + packet->header_len = skb_inner_tcp_all_headers(skb); packet->tcp_header_len = inner_tcp_hdrlen(skb); } else { - packet->header_len = skb_transport_offset(skb) + - tcp_hdrlen(skb); + packet->header_len = skb_tcp_all_headers(skb); packet->tcp_header_len = tcp_hdrlen(skb); } packet->tcp_payload_len = skb->len - packet->header_len; diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c index 24fe967c18cd..948584761e66 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c @@ -2072,7 +2072,7 @@ static u16 atl1c_cal_tpd_req(const struct sk_buff *skb) tpd_req = skb_shinfo(skb)->nr_frags + 1; if (skb_is_gso(skb)) { - proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + proto_hdr_len = skb_tcp_all_headers(skb); if (proto_hdr_len < skb_headlen(skb)) tpd_req++; if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) @@ -2107,7 +2107,7 @@ static int atl1c_tso_csum(struct atl1c_adapter *adapter, if (real_len < skb->len) pskb_trim(skb, real_len); - hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb)); + hdr_len = skb_tcp_all_headers(skb); if (unlikely(skb->len == hdr_len)) { /* only xsum need */ if (netif_msg_tx_queued(adapter)) @@ -2132,7 +2132,7 @@ static int atl1c_tso_csum(struct atl1c_adapter *adapter, *tpd = atl1c_get_tpd(adapter, queue); ipv6_hdr(skb)->payload_len = 0; /* check payload == 0 byte ? */ - hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb)); + hdr_len = skb_tcp_all_headers(skb); if (unlikely(skb->len == hdr_len)) { /* only xsum need */ if (netif_msg_tx_queued(adapter)) @@ -2219,7 +2219,8 @@ static int atl1c_tx_map(struct atl1c_adapter *adapter, tso = (tpd->word1 >> TPD_LSO_EN_SHIFT) & TPD_LSO_EN_MASK; if (tso) { /* TSO */ - map_len = hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); + map_len = hdr_len; use_tpd = tpd; buffer_info = atl1c_get_tx_buffer(adapter, use_tpd); diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c index 4fc9e6350c49..57a51fb7746c 100644 --- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c @@ -1609,8 +1609,7 @@ static u16 atl1e_cal_tdp_req(const struct sk_buff *skb) if (skb_is_gso(skb)) { if (skb->protocol == htons(ETH_P_IP) || (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6)) { - proto_hdr_len = skb_transport_offset(skb) + - tcp_hdrlen(skb); + proto_hdr_len = skb_tcp_all_headers(skb); if (proto_hdr_len < skb_headlen(skb)) { tpd_req += ((skb_headlen(skb) - proto_hdr_len + MAX_TX_BUF_LEN - 1) >> @@ -1645,7 +1644,7 @@ static int atl1e_tso_csum(struct atl1e_adapter *adapter, if (real_len < skb->len) pskb_trim(skb, real_len); - hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb)); + hdr_len = skb_tcp_all_headers(skb); if (unlikely(skb->len == hdr_len)) { /* only xsum need */ netdev_warn(adapter->netdev, @@ -1713,7 +1712,8 @@ static int atl1e_tx_map(struct atl1e_adapter *adapter, segment = (tpd->word3 >> TPD_SEGMENT_EN_SHIFT) & TPD_SEGMENT_EN_MASK; if (segment) { /* TSO */ - map_len = hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); + map_len = hdr_len; use_tpd = tpd; tx_buffer = atl1e_get_tx_buffer(adapter, use_tpd); diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index 6a969969d221..ff1fe09abf9f 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -2115,7 +2115,7 @@ static int atl1_tso(struct atl1_adapter *adapter, struct sk_buff *skb, ntohs(iph->tot_len)); if (real_len < skb->len) pskb_trim(skb, real_len); - hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb)); + hdr_len = skb_tcp_all_headers(skb); if (skb->len == hdr_len) { iph->check = 0; tcp_hdr(skb)->check = @@ -2206,7 +2206,7 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, retval = (ptpd->word3 >> TPD_SEGMENT_EN_SHIFT) & TPD_SEGMENT_EN_MASK; if (retval) { /* TSO */ - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); buffer_info->length = hdr_len; page = virt_to_page(skb->data); offset = offset_in_page(skb->data); @@ -2367,8 +2367,7 @@ static netdev_tx_t atl1_xmit_frame(struct sk_buff *skb, mss = skb_shinfo(skb)->gso_size; if (mss) { if (skb->protocol == htons(ETH_P_IP)) { - proto_hdr_len = (skb_transport_offset(skb) + - tcp_hdrlen(skb)); + proto_hdr_len = skb_tcp_all_headers(skb); if (unlikely(proto_hdr_len > len)) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index 5729a5ab059d..712b5595bc39 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -3421,12 +3421,9 @@ static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb, /* Headers length */ if (xmit_type & XMIT_GSO_ENC) - hlen = (int)(skb_inner_transport_header(skb) - - skb->data) + - inner_tcp_hdrlen(skb); + hlen = skb_inner_tcp_all_headers(skb); else - hlen = (int)(skb_transport_header(skb) - - skb->data) + tcp_hdrlen(skb); + hlen = skb_tcp_all_headers(skb); /* Amount of data (w/o headers) on linear part of SKB*/ first_bd_sz = skb_headlen(skb) - hlen; @@ -3534,15 +3531,13 @@ static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb, ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; - return skb_inner_transport_header(skb) + - inner_tcp_hdrlen(skb) - skb->data; + return skb_inner_tcp_all_headers(skb); } /* We support checksum offload for TCP and UDP only. * No need to pass the UDP header length - it's a constant. */ - return skb_inner_transport_header(skb) + - sizeof(struct udphdr) - skb->data; + return skb_inner_transport_offset(skb) + sizeof(struct udphdr); } /** @@ -3568,12 +3563,12 @@ static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb, ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; - return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data; + return skb_tcp_all_headers(skb); } /* We support checksum offload for TCP and UDP only. * No need to pass the UDP header length - it's a constant. */ - return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data; + return skb_transport_offset(skb) + sizeof(struct udphdr); } /* set FW indication according to inner or outer protocols if tunneled */ diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index b474a4fe4039..11d35da61921 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -535,12 +535,9 @@ normal_tx: u32 hdr_len; if (skb->encapsulation) - hdr_len = skb_inner_network_offset(skb) + - skb_inner_network_header_len(skb) + - inner_tcp_hdrlen(skb); + hdr_len = skb_inner_tcp_all_headers(skb); else - hdr_len = skb_transport_offset(skb) + - tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | TX_BD_FLAGS_T_IPID | diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index c28f8cc00d1c..db1e9d810b41 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -7944,7 +7944,7 @@ static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, struct net_device *dev) iph = ip_hdr(skb); tcp_opt_len = tcp_optlen(skb); - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb) - ETH_HLEN; + hdr_len = skb_tcp_all_headers(skb) - ETH_HLEN; /* HW/FW can not correctly segment packets that have been * vlan encapsulated. diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index f6fe08df568b..29dd0f93d6c0 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -2823,8 +2823,7 @@ bnad_txq_wi_prepare(struct bnad *bnad, struct bna_tcb *tcb, BNAD_UPDATE_CTR(bnad, tx_skb_mss_too_long); return -EINVAL; } - if (unlikely((gso_size + skb_transport_offset(skb) + - tcp_hdrlen(skb)) >= skb->len)) { + if (unlikely((gso_size + skb_tcp_all_headers(skb)) >= skb->len)) { txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND); txqent->hdr.wi.lso_mss = 0; BNAD_UPDATE_CTR(bnad, tx_skb_tso_too_short); @@ -2872,8 +2871,7 @@ bnad_txq_wi_prepare(struct bnad *bnad, struct bna_tcb *tcb, BNAD_UPDATE_CTR(bnad, tcpcsum_offload); if (unlikely(skb_headlen(skb) < - skb_transport_offset(skb) + - tcp_hdrlen(skb))) { + skb_tcp_all_headers(skb))) { BNAD_UPDATE_CTR(bnad, tx_skb_tcp_hdr); return -EINVAL; } diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index d0ea8dbfa213..90a9798424af 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -2267,7 +2267,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) /* only queue eth + ip headers separately for UDP */ hdrlen = skb_transport_offset(skb); else - hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); if (skb_headlen(skb) < hdrlen) { netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); /* if this is required, would need to copy to single buffer */ diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c index 4367edbdd579..06397cc8bb36 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c @@ -1261,7 +1261,7 @@ int nicvf_xdp_sq_append_pkt(struct nicvf *nic, struct snd_queue *sq, static int nicvf_tso_count_subdescs(struct sk_buff *skb) { struct skb_shared_info *sh = skb_shinfo(skb); - unsigned int sh_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + unsigned int sh_len = skb_tcp_all_headers(skb); unsigned int data_len = skb->len - sh_len; unsigned int p_len = sh->gso_size; long f_id = -1; /* id of the current fragment */ @@ -1382,7 +1382,7 @@ nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry, if (nic->hw_tso && skb_shinfo(skb)->gso_size) { hdr->tso = 1; - hdr->tso_start = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr->tso_start = skb_tcp_all_headers(skb); hdr->tso_max_paysize = skb_shinfo(skb)->gso_size; /* For non-tunneled pkts, point this to L2 ethertype */ hdr->inner_l3_offset = skb_network_offset(skb) - 2; diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index f889f404305c..ee52e3b1d74f 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -1531,7 +1531,7 @@ static netdev_tx_t cxgb4_eth_xmit(struct sk_buff *skb, struct net_device *dev) #if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) if (cxgb4_is_ktls_skb(skb) && - (skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)))) + (skb->len - skb_tcp_all_headers(skb))) return adap->uld[CXGB4_ULD_KTLS].tx_handler(skb, dev); #endif /* CHELSIO_TLS_DEVICE */ diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c index 60b648b46f75..bfee0e4e54b1 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c @@ -1012,7 +1012,7 @@ chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb, /* packet length = eth hdr len + ip hdr len + tcp hdr len * (including options). */ - pktlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + pktlen = skb_tcp_all_headers(skb); ctrl = sizeof(*cpl) + pktlen; len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16); @@ -1907,7 +1907,7 @@ static int chcr_ktls_sw_fallback(struct sk_buff *skb, return 0; th = tcp_hdr(nskb); - skb_offset = skb_transport_offset(nskb) + tcp_hdrlen(nskb); + skb_offset = skb_tcp_all_headers(nskb); data_len = nskb->len - skb_offset; skb_tx_timestamp(nskb); @@ -1938,7 +1938,7 @@ static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) unsigned long flags; tcp_seq = ntohl(th->seq); - skb_offset = skb_transport_offset(skb) + tcp_hdrlen(skb); + skb_offset = skb_tcp_all_headers(skb); skb_data_len = skb->len - skb_offset; data_len = skb_data_len; diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index 1c81b161de52..372fb7b3a282 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -680,11 +680,10 @@ static int enic_queue_wq_skb_tso(struct enic *enic, struct vnic_wq *wq, skb_frag_t *frag; if (skb->encapsulation) { - hdr_len = skb_inner_transport_header(skb) - skb->data; - hdr_len += inner_tcp_hdrlen(skb); + hdr_len = skb_inner_tcp_all_headers(skb); enic_preload_tcp_csum_encap(skb); } else { - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); enic_preload_tcp_csum(skb); } diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index 41acd18a3fd2..414362febbb9 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -737,9 +737,9 @@ void be_link_status_update(struct be_adapter *adapter, u8 link_status) static int be_gso_hdr_len(struct sk_buff *skb) { if (skb->encapsulation) - return skb_inner_transport_offset(skb) + - inner_tcp_hdrlen(skb); - return skb_transport_offset(skb) + tcp_hdrlen(skb); + return skb_inner_tcp_all_headers(skb); + + return skb_tcp_all_headers(skb); } static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb) diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index a90275143d87..e8e2aa1e7f01 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -691,7 +691,7 @@ fec_enet_txq_put_hdr_tso(struct fec_enet_priv_tx_q *txq, struct bufdesc *bdp, int index) { struct fec_enet_private *fep = netdev_priv(ndev); - int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + int hdr_len = skb_tcp_all_headers(skb); struct bufdesc_ex *ebdp = container_of(bdp, struct bufdesc_ex, desc); void *bufaddr; unsigned long dmabuf; diff --git a/drivers/net/ethernet/fungible/funeth/funeth_tx.c b/drivers/net/ethernet/fungible/funeth/funeth_tx.c index 0a4a590218ba..a97e3af00cb9 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_tx.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_tx.c @@ -83,7 +83,7 @@ static struct sk_buff *fun_tls_tx(struct sk_buff *skb, struct funeth_txq *q, const struct fun_ktls_tx_ctx *tls_ctx; u32 datalen, seq; - datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + datalen = skb->len - skb_tcp_all_headers(skb); if (!datalen) return skb; diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index f7ba616195f3..588d64819ed5 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -386,7 +386,7 @@ static int gve_prep_tso(struct sk_buff *skb) (__force __wsum)htonl(paylen)); /* Compute length of segmentation header. */ - header_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + header_len = skb_tcp_all_headers(skb); break; default: return -EINVAL; @@ -598,9 +598,9 @@ static int gve_num_buffer_descs_needed(const struct sk_buff *skb) */ static bool gve_can_send_tso(const struct sk_buff *skb) { - const int header_len = skb_checksum_start_offset(skb) + tcp_hdrlen(skb); const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1; const struct skb_shared_info *shinfo = skb_shinfo(skb); + const int header_len = skb_tcp_all_headers(skb); const int gso_size = shinfo->gso_size; int cur_seg_num_bufs; int cur_seg_size; diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c index 2f0bd21a9082..d94cc8c6681f 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c @@ -31,8 +31,6 @@ #define HNS_BUFFER_SIZE_2048 2048 #define BD_MAX_SEND_SIZE 8191 -#define SKB_TMP_LEN(SKB) \ - (((SKB)->transport_header - (SKB)->mac_header) + tcp_hdrlen(SKB)) static void fill_v2_desc_hw(struct hnae_ring *ring, void *priv, int size, int send_sz, dma_addr_t dma, int frag_end, @@ -94,7 +92,7 @@ static void fill_v2_desc_hw(struct hnae_ring *ring, void *priv, int size, HNSV2_TXD_TSE_B, 1); l4_len = tcp_hdrlen(skb); mss = skb_shinfo(skb)->gso_size; - paylen = skb->len - SKB_TMP_LEN(skb); + paylen = skb->len - skb_tcp_all_headers(skb); } } else if (skb->protocol == htons(ETH_P_IPV6)) { hnae_set_bit(tvsvsn, HNSV2_TXD_IPV6_B, 1); @@ -108,7 +106,7 @@ static void fill_v2_desc_hw(struct hnae_ring *ring, void *priv, int size, HNSV2_TXD_TSE_B, 1); l4_len = tcp_hdrlen(skb); mss = skb_shinfo(skb)->gso_size; - paylen = skb->len - SKB_TMP_LEN(skb); + paylen = skb->len - skb_tcp_all_headers(skb); } } desc->tx.ip_offset = ip_offset; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index ae56306400b8..35d70041b9e8 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -1838,9 +1838,9 @@ static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size, static unsigned int hns3_gso_hdr_len(struct sk_buff *skb) { if (!skb->encapsulation) - return skb_transport_offset(skb) + tcp_hdrlen(skb); + return skb_tcp_all_headers(skb); - return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); + return skb_inner_tcp_all_headers(skb); } /* HW need every continuous max_non_tso_bd_num buffer data to be larger diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h b/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h index 5153e5d41bbd..b8a1ecb4b8fb 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h @@ -37,8 +37,7 @@ DECLARE_EVENT_CLASS(hns3_skb_template, __entry->gso_segs = skb_shinfo(skb)->gso_segs; __entry->gso_type = skb_shinfo(skb)->gso_type; __entry->hdr_len = skb->encapsulation ? - skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb) : - skb_transport_offset(skb) + tcp_hdrlen(skb); + skb_inner_tcp_all_headers(skb) : skb_tcp_all_headers(skb); __entry->ip_summed = skb->ip_summed; __entry->fraglist = skb_has_frag_list(skb); hns3_shinfo_pack(skb_shinfo(skb), __entry->size); diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c index 8ce3348edf08..5dc302880f5f 100644 --- a/drivers/net/ethernet/ibm/ehea/ehea_main.c +++ b/drivers/net/ethernet/ibm/ehea/ehea_main.c @@ -1617,7 +1617,7 @@ static void write_swqe2_immediate(struct sk_buff *skb, struct ehea_swqe *swqe, * For TSO packets we only copy the headers into the * immediate area. */ - immediate_len = ETH_HLEN + ip_hdrlen(skb) + tcp_hdrlen(skb); + immediate_len = skb_tcp_all_headers(skb); } if (skb_is_gso(skb) || skb_data_size >= SWQE2_MAX_IMM) { diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 3f5feb55cfba..23299fc56199 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -2708,7 +2708,7 @@ static int e1000_tso(struct e1000_adapter *adapter, if (err < 0) return err; - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); mss = skb_shinfo(skb)->gso_size; if (protocol == htons(ETH_P_IP)) { struct iphdr *iph = ip_hdr(skb); @@ -3139,7 +3139,7 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, max_per_txd = min(mss << 2, max_per_txd); max_txd_pwr = fls(max_per_txd) - 1; - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); if (skb->data_len && hdr_len == len) { switch (hw->mac_type) { case e1000_82544: { diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index fa06f68c8c80..38e60def8520 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -5474,7 +5474,7 @@ static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb, if (err < 0) return err; - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); mss = skb_shinfo(skb)->gso_size; if (protocol == htons(ETH_P_IP)) { struct iphdr *iph = ip_hdr(skb); @@ -5846,7 +5846,7 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, * points to just header, pull a few bytes of payload from * frags into skb->data */ - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); /* we do this workaround for ES2LAN, but it is un-necessary, * avoiding it could save a lot of cycles */ diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_main.c b/drivers/net/ethernet/intel/ixgb/ixgb_main.c index bca53625da33..45be9a1ab6af 100644 --- a/drivers/net/ethernet/intel/ixgb/ixgb_main.c +++ b/drivers/net/ethernet/intel/ixgb/ixgb_main.c @@ -1187,7 +1187,7 @@ ixgb_tso(struct ixgb_adapter *adapter, struct sk_buff *skb) if (err < 0) return err; - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); mss = skb_shinfo(skb)->gso_size; iph = ip_hdr(skb); iph->tot_len = 0; diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index 57eff4e9e6de..b6be0552a6c1 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -775,7 +775,7 @@ txq_put_hdr_tso(struct sk_buff *skb, struct tx_queue *txq, int length, u32 *first_cmd_sts, bool first_desc) { struct mv643xx_eth_private *mp = txq_to_mp(txq); - int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + int hdr_len = skb_tcp_all_headers(skb); int tx_index; struct tx_desc *desc; int ret; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 384f5a16753d..0caa2df87c04 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -2664,8 +2664,8 @@ err_drop_frame: static inline void mvneta_tso_put_hdr(struct sk_buff *skb, struct mvneta_tx_queue *txq) { - int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index]; + int hdr_len = skb_tcp_all_headers(skb); struct mvneta_tx_desc *tx_desc; tx_desc = mvneta_txq_next_desc_get(txq); @@ -2727,7 +2727,7 @@ static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev, if ((txq->count + tso_count_descs(skb)) >= txq->size) return 0; - if (skb_headlen(skb) < (skb_transport_offset(skb) + tcp_hdrlen(skb))) { + if (skb_headlen(skb) < skb_tcp_all_headers(skb)) { pr_info("*** Is this even possible?\n"); return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 3baeafc40807..a18e8efd0f1e 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -624,7 +624,7 @@ static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, ext->subdc = NIX_SUBDC_EXT; if (skb_shinfo(skb)->gso_size) { ext->lso = 1; - ext->lso_sb = skb_transport_offset(skb) + tcp_hdrlen(skb); + ext->lso_sb = skb_tcp_all_headers(skb); ext->lso_mps = skb_shinfo(skb)->gso_size; /* Only TSOv4 and TSOv6 GSO offloads are supported */ @@ -931,7 +931,7 @@ static bool is_hw_tso_supported(struct otx2_nic *pfvf, * be correctly modified, hence don't offload such TSO segments. */ - payload_len = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + payload_len = skb->len - skb_tcp_all_headers(skb); last_seg_size = payload_len % skb_shinfo(skb)->gso_size; if (last_seg_size && last_seg_size < 16) return false; diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index 9f1a7ec0491a..bbea5458000b 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -1863,7 +1863,7 @@ static netdev_tx_t sky2_xmit_frame(struct sk_buff *skb, if (mss != 0) { if (!(hw->flags & SKY2_HW_NEW_LE)) - mss += ETH_HLEN + ip_hdrlen(skb) + tcp_hdrlen(skb); + mss += skb_tcp_all_headers(skb); if (mss != sky2->tx_last_mss) { le = get_tx_le(sky2, &slot); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index af3b2b59a2a6..43a4102e9c09 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -645,7 +645,7 @@ static int get_real_size(const struct sk_buff *skb, *inline_ok = false; *hopbyhop = 0; if (skb->encapsulation) { - *lso_header_size = (skb_inner_transport_header(skb) - skb->data) + inner_tcp_hdrlen(skb); + *lso_header_size = skb_inner_tcp_all_headers(skb); } else { /* Detects large IPV6 TCP packets and prepares for removal of * HBH header that has been pushed by ip6_xmit(), @@ -653,7 +653,7 @@ static int get_real_size(const struct sk_buff *skb, */ if (ipv6_has_hopopt_jumbo(skb)) *hopbyhop = sizeof(struct hop_jumbo_hdr); - *lso_header_size = skb_transport_offset(skb) + tcp_hdrlen(skb); + *lso_header_size = skb_tcp_all_headers(skb); } real_size = CTRL_SIZE + shinfo->nr_frags * DS_SIZE + ALIGN(*lso_header_size - *hopbyhop + 4, DS_SIZE); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 4b6f0d1ea59a..cc5cb3010e64 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -458,7 +458,7 @@ bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, int datalen; u32 seq; - datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + datalen = skb->len - skb_tcp_all_headers(skb); if (!datalen) return true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 50d14cec4894..64d78fd99c6e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -152,14 +152,14 @@ mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop) *hopbyhop = 0; if (skb->encapsulation) { - ihs = skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); + ihs = skb_tcp_all_headers(skb); stats->tso_inner_packets++; stats->tso_inner_bytes += skb->len - ihs; } else { if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { ihs = skb_transport_offset(skb) + sizeof(struct udphdr); } else { - ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); + ihs = skb_tcp_all_headers(skb); if (ipv6_has_hopopt_jumbo(skb)) { *hopbyhop = sizeof(struct hop_jumbo_hdr); ihs -= sizeof(struct hop_jumbo_hdr); diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 61497c3e4cfb..971dde8c3286 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -2692,7 +2692,7 @@ again: * send loop that we are still in the * header portion of the TSO packet. * TSO header can be at most 1KB long */ - cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); + cum_len = -skb_tcp_all_headers(skb); /* for IPv6 TSO, the checksum offset stores the * TCP header length, to save the firmware from diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/dp.c b/drivers/net/ethernet/netronome/nfp/nfd3/dp.c index f9410d59146d..8a9d36619659 100644 --- a/drivers/net/ethernet/netronome/nfp/nfd3/dp.c +++ b/drivers/net/ethernet/netronome/nfp/nfd3/dp.c @@ -81,12 +81,11 @@ nfp_nfd3_tx_tso(struct nfp_net_r_vector *r_vec, struct nfp_nfd3_tx_buf *txbuf, if (!skb->encapsulation) { l3_offset = skb_network_offset(skb); l4_offset = skb_transport_offset(skb); - hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); } else { l3_offset = skb_inner_network_offset(skb); l4_offset = skb_inner_transport_offset(skb); - hdrlen = skb_inner_transport_header(skb) - skb->data + - inner_tcp_hdrlen(skb); + hdrlen = skb_inner_tcp_all_headers(skb); } txbuf->pkt_cnt = skb_shinfo(skb)->gso_segs; diff --git a/drivers/net/ethernet/netronome/nfp/nfdk/dp.c b/drivers/net/ethernet/netronome/nfp/nfdk/dp.c index 300637e576a8..85dd376a6adc 100644 --- a/drivers/net/ethernet/netronome/nfp/nfdk/dp.c +++ b/drivers/net/ethernet/netronome/nfp/nfdk/dp.c @@ -46,12 +46,11 @@ nfp_nfdk_tx_tso(struct nfp_net_r_vector *r_vec, struct nfp_nfdk_tx_buf *txbuf, if (!skb->encapsulation) { l3_offset = skb_network_offset(skb); l4_offset = skb_transport_offset(skb); - hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); } else { l3_offset = skb_inner_network_offset(skb); l4_offset = skb_inner_transport_offset(skb); - hdrlen = skb_inner_transport_header(skb) - skb->data + - inner_tcp_hdrlen(skb); + hdrlen = skb_inner_tcp_all_headers(skb); } segs = skb_shinfo(skb)->gso_segs; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index bcdd5ab0da5a..3e2ebe42e679 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -598,7 +598,7 @@ nfp_net_tls_tx(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk)) return skb; - datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + datalen = skb->len - skb_tcp_all_headers(skb); seq = ntohl(tcp_hdr(skb)->seq); ntls = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX); resync_pending = tls_offload_tx_resync_pending(skb->sk); @@ -666,7 +666,7 @@ void nfp_net_tls_tx_undo(struct sk_buff *skb, u64 tls_handle) if (WARN_ON_ONCE(!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))) return; - datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + datalen = skb->len - skb_tcp_all_headers(skb); seq = ntohl(tcp_hdr(skb)->seq); ntls = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX); @@ -1758,8 +1758,7 @@ nfp_net_features_check(struct sk_buff *skb, struct net_device *dev, if (skb_is_gso(skb)) { u32 hdrlen; - hdrlen = skb_inner_transport_header(skb) - skb->data + - inner_tcp_hdrlen(skb); + hdrlen = skb_inner_tcp_all_headers(skb); /* Assume worst case scenario of having longest possible * metadata prepend - 8B diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index f54035455ad6..c03986bf2628 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -947,10 +947,9 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) } if (encap) - hdrlen = skb_inner_transport_header(skb) - skb->data + - inner_tcp_hdrlen(skb); + hdrlen = skb_inner_tcp_all_headers(skb); else - hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); tso_rem = len; seg_rem = min(tso_rem, hdrlen + mss); diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c index 07dd3c3b1771..4e6f00af17d9 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c @@ -1877,7 +1877,7 @@ netxen_tso_check(struct net_device *netdev, if ((netdev->features & (NETIF_F_TSO | NETIF_F_TSO6)) && skb_shinfo(skb)->gso_size > 0) { - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); first_desc->total_hdr_length = hdr_len; diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index b7cc36589f59..7c2af482192d 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -260,11 +260,9 @@ static int map_frag_to_bd(struct qede_tx_queue *txq, static u16 qede_get_skb_hlen(struct sk_buff *skb, bool is_encap_pkt) { if (is_encap_pkt) - return (skb_inner_transport_header(skb) + - inner_tcp_hdrlen(skb) - skb->data); - else - return (skb_transport_header(skb) + - tcp_hdrlen(skb) - skb->data); + return skb_inner_tcp_all_headers(skb); + + return skb_tcp_all_headers(skb); } /* +2 for 1st BD for headers and 2nd BD for headlen (if required) */ diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c index 8d43ca282956..9da5e97f8a0a 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c @@ -497,7 +497,7 @@ set_flags: } opcode = QLCNIC_TX_ETHER_PKT; if (skb_is_gso(skb)) { - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); first_desc->hdr_length = hdr_len; opcode = (protocol == ETH_P_IPV6) ? QLCNIC_TX_TCP_LSO6 : diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.c b/drivers/net/ethernet/qualcomm/emac/emac-mac.c index 80c95c331c82..0d80447d4d3b 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac-mac.c +++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.c @@ -1264,7 +1264,7 @@ static int emac_tso_csum(struct emac_adapter *adpt, pskb_trim(skb, pkt_len); } - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); if (unlikely(skb->len == hdr_len)) { /* we only need to do csum */ netif_warn(adpt, tx_err, adpt->netdev, @@ -1339,7 +1339,7 @@ static void emac_tx_fill_tpd(struct emac_adapter *adpt, /* if Large Segment Offload is (in TCP Segmentation Offload struct) */ if (TPD_LSO(tpd)) { - mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + mapped_len = skb_tcp_all_headers(skb); tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); tpbuf->length = mapped_len; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index fe263cad8248..6f14b00c0b14 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -3961,7 +3961,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); hdr = sizeof(struct udphdr); } else { - proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + proto_hdr_len = skb_tcp_all_headers(skb); hdr = tcp_hdrlen(skb); } diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c index d435519236e4..e54ce73396ee 100644 --- a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c +++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c @@ -81,7 +81,7 @@ static int xlgmac_prep_tso(struct sk_buff *skb, if (ret) return ret; - pkt_info->header_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + pkt_info->header_len = skb_tcp_all_headers(skb); pkt_info->tcp_header_len = tcp_hdrlen(skb); pkt_info->tcp_payload_len = skb->len - pkt_info->header_len; pkt_info->mss = skb_shinfo(skb)->gso_size; diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c index 5704defd7be1..237cbd5c5060 100644 --- a/drivers/net/wireless/ath/wil6210/txrx.c +++ b/drivers/net/wireless/ath/wil6210/txrx.c @@ -1782,9 +1782,7 @@ static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif, } /* Header Length = MAC header len + IP header len + TCP header len*/ - hdrlen = ETH_HLEN + - (int)skb_network_header_len(skb) + - tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4); switch (gso_type) { diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index fc61a4418737..a256695fc89e 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -1201,9 +1201,7 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) } mss = skb_shinfo(skb)->gso_size; - hdrlen = skb_transport_header(skb) - - skb_mac_header(skb) + - tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len - hdrlen, mss); diff --git a/drivers/staging/qlge/qlge_main.c b/drivers/staging/qlge/qlge_main.c index 113a3efd12e9..6cd7fc9589c3 100644 --- a/drivers/staging/qlge/qlge_main.c +++ b/drivers/staging/qlge/qlge_main.c @@ -2461,7 +2461,7 @@ static int qlge_tso(struct sk_buff *skb, struct qlge_ob_mac_tso_iocb_req *mac_io mac_iocb_ptr->flags3 |= OB_MAC_TSO_IOCB_IC; mac_iocb_ptr->frame_len = cpu_to_le32((u32)skb->len); mac_iocb_ptr->total_hdrs_len = - cpu_to_le16(skb_transport_offset(skb) + tcp_hdrlen(skb)); + cpu_to_le16(skb_tcp_all_headers(skb)); mac_iocb_ptr->net_trans_offset = cpu_to_le16(skb_network_offset(skb) | skb_transport_offset(skb) diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 1168302b7927..a9fbe22732c3 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -46,6 +46,36 @@ static inline unsigned int inner_tcp_hdrlen(const struct sk_buff *skb) return inner_tcp_hdr(skb)->doff * 4; } +/** + * skb_tcp_all_headers - Returns size of all headers for a TCP packet + * @skb: buffer + * + * Used in TX path, for a packet known to be a TCP one. + * + * if (skb_is_gso(skb)) { + * int hlen = skb_tcp_all_headers(skb); + * ... + */ +static inline int skb_tcp_all_headers(const struct sk_buff *skb) +{ + return skb_transport_offset(skb) + tcp_hdrlen(skb); +} + +/** + * skb_inner_tcp_all_headers - Returns size of all headers for an encap TCP packet + * @skb: buffer + * + * Used in TX path, for a packet known to be a TCP one. + * + * if (skb_is_gso(skb) && skb->encapsulation) { + * int hlen = skb_inner_tcp_all_headers(skb); + * ... + */ +static inline int skb_inner_tcp_all_headers(const struct sk_buff *skb) +{ + return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); +} + static inline unsigned int tcp_optlen(const struct sk_buff *skb) { return (tcp_hdr(skb)->doff - 5) * 4; diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c index e40bedd112b6..3bae29ae57ca 100644 --- a/net/tls/tls_device_fallback.c +++ b/net/tls/tls_device_fallback.c @@ -232,7 +232,7 @@ static int fill_sg_in(struct scatterlist *sg_in, s32 *sync_size, int *resync_sgs) { - int tcp_payload_offset = skb_transport_offset(skb) + tcp_hdrlen(skb); + int tcp_payload_offset = skb_tcp_all_headers(skb); int payload_len = skb->len - tcp_payload_offset; u32 tcp_seq = ntohl(tcp_hdr(skb)->seq); struct tls_record_info *record; @@ -310,8 +310,8 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx, struct sk_buff *skb, s32 sync_size, u64 rcd_sn) { - int tcp_payload_offset = skb_transport_offset(skb) + tcp_hdrlen(skb); struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx); + int tcp_payload_offset = skb_tcp_all_headers(skb); int payload_len = skb->len - tcp_payload_offset; void *buf, *iv, *aad, *dummy_buf; struct aead_request *aead_req; @@ -372,7 +372,7 @@ free_nskb: static struct sk_buff *tls_sw_fallback(struct sock *sk, struct sk_buff *skb) { - int tcp_payload_offset = skb_transport_offset(skb) + tcp_hdrlen(skb); + int tcp_payload_offset = skb_tcp_all_headers(skb); struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx); int payload_len = skb->len - tcp_payload_offset; -- cgit v1.2.3 From f019679ea5f2ab650c3348a79e7d9c3625f62899 Mon Sep 17 00:00:00 2001 From: Chris Mi Date: Mon, 30 May 2022 06:07:57 +0300 Subject: net/mlx5: E-switch, Remove dependency between sriov and eswitch mode Currently, there are three eswitch modes, none, legacy and switchdev. None is the default mode. Remove redundant none mode as eswitch mode should always be either legacy mode or switchdev mode. With this patch, there are two behavior changes: 1. Legacy becomes the default mode. When querying eswitch mode using devlink, a valid mode is always returned. 2. When disabling sriov, the eswitch mode will not change, only vfs are unloaded. Signed-off-by: Chris Mi Reviewed-by: Maor Dickman Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/eswitch.c | 87 ++++++++++++++-------- drivers/net/ethernet/mellanox/mlx5/core/eswitch.h | 9 ++- .../ethernet/mellanox/mlx5/core/eswitch_offloads.c | 52 ++++--------- drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c | 10 +-- drivers/net/ethernet/mellanox/mlx5/core/main.c | 1 + .../net/ethernet/mellanox/mlx5/core/sf/devlink.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/sriov.c | 3 +- include/linux/mlx5/eswitch.h | 8 +- 8 files changed, 89 insertions(+), 83 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index b9a3473f5672..82b62ab1d1d9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1152,8 +1152,6 @@ mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs) { const u32 *out; - WARN_ON_ONCE(esw->mode != MLX5_ESWITCH_NONE); - if (num_vfs < 0) return; @@ -1287,7 +1285,7 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) return 0; abort: - esw->mode = MLX5_ESWITCH_NONE; + esw->mode = MLX5_ESWITCH_LEGACY; if (mode == MLX5_ESWITCH_OFFLOADS) mlx5_rescan_drivers(esw->dev); @@ -1312,13 +1310,13 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) if (!mlx5_esw_allowed(esw)) return 0; - toggle_lag = esw->mode == MLX5_ESWITCH_NONE; + toggle_lag = !mlx5_esw_is_fdb_created(esw); if (toggle_lag) mlx5_lag_disable_change(esw->dev); down_write(&esw->mode_lock); - if (esw->mode == MLX5_ESWITCH_NONE) { + if (!mlx5_esw_is_fdb_created(esw)) { ret = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, num_vfs); } else { enum mlx5_eswitch_vport_event vport_events; @@ -1337,56 +1335,79 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) return ret; } -void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) +/* When disabling sriov, free driver level resources. */ +void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) { - struct devlink *devlink = priv_to_devlink(esw->dev); - int old_mode; - - lockdep_assert_held_write(&esw->mode_lock); - - if (esw->mode == MLX5_ESWITCH_NONE) + if (!mlx5_esw_allowed(esw)) return; - esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n", + down_write(&esw->mode_lock); + /* If driver is unloaded, this function is called twice by remove_one() + * and mlx5_unload(). Prevent the second call. + */ + if (!esw->esw_funcs.num_vfs && !clear_vf) + goto unlock; + + esw_info(esw->dev, "Unload vfs: mode(%s), nvfs(%d), active vports(%d)\n", esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); + mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); + if (clear_vf) + mlx5_eswitch_clear_vf_vports_info(esw); + /* If disabling sriov in switchdev mode, free meta rules here + * because it depends on num_vfs. + */ + if (esw->mode == MLX5_ESWITCH_OFFLOADS) { + struct devlink *devlink = priv_to_devlink(esw->dev); + + esw_offloads_del_send_to_vport_meta_rules(esw); + devlink_rate_nodes_destroy(devlink); + } + + esw->esw_funcs.num_vfs = 0; + +unlock: + up_write(&esw->mode_lock); +} + +/* Free resources for corresponding eswitch mode. It is called by devlink + * when changing eswitch mode or modprobe when unloading driver. + */ +void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw) +{ + struct devlink *devlink = priv_to_devlink(esw->dev); + /* Notify eswitch users that it is exiting from current mode. * So that it can do necessary cleanup before the eswitch is disabled. */ - mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_NONE); + mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_LEGACY); mlx5_eswitch_event_handlers_unregister(esw); + esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n", + esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", + esw->esw_funcs.num_vfs, esw->enabled_vports); + esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED; - if (esw->mode == MLX5_ESWITCH_LEGACY) - esw_legacy_disable(esw); - else if (esw->mode == MLX5_ESWITCH_OFFLOADS) + if (esw->mode == MLX5_ESWITCH_OFFLOADS) esw_offloads_disable(esw); - - old_mode = esw->mode; - esw->mode = MLX5_ESWITCH_NONE; - - if (old_mode == MLX5_ESWITCH_OFFLOADS) - mlx5_rescan_drivers(esw->dev); - - devlink_rate_nodes_destroy(devlink); - + else if (esw->mode == MLX5_ESWITCH_LEGACY) + esw_legacy_disable(esw); mlx5_esw_acls_ns_cleanup(esw); - if (clear_vf) - mlx5_eswitch_clear_vf_vports_info(esw); + if (esw->mode == MLX5_ESWITCH_OFFLOADS) + devlink_rate_nodes_destroy(devlink); } -void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) +void mlx5_eswitch_disable(struct mlx5_eswitch *esw) { if (!mlx5_esw_allowed(esw)) return; mlx5_lag_disable_change(esw->dev); down_write(&esw->mode_lock); - mlx5_eswitch_disable_locked(esw, clear_vf); - esw->esw_funcs.num_vfs = 0; + mlx5_eswitch_disable_locked(esw); up_write(&esw->mode_lock); mlx5_lag_enable_change(esw->dev); } @@ -1581,7 +1602,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) refcount_set(&esw->qos.refcnt, 0); esw->enabled_vports = 0; - esw->mode = MLX5_ESWITCH_NONE; + esw->mode = MLX5_ESWITCH_LEGACY; esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE; if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) && MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap)) @@ -1883,7 +1904,7 @@ u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev) { struct mlx5_eswitch *esw = dev->priv.eswitch; - return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_NONE; + return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_LEGACY; } EXPORT_SYMBOL_GPL(mlx5_eswitch_mode); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index a9ba0e324834..5452437e531f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -342,6 +342,7 @@ void esw_offloads_disable(struct mlx5_eswitch *esw); int esw_offloads_enable(struct mlx5_eswitch *esw); void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw); int esw_offloads_init_reps(struct mlx5_eswitch *esw); +void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw); bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw); int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable); @@ -357,8 +358,9 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw); #define MLX5_ESWITCH_IGNORE_NUM_VFS (-1) int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs); int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs); -void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf); -void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf); +void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf); +void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw); +void mlx5_eswitch_disable(struct mlx5_eswitch *esw); int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, u16 vport, const u8 *mac); int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, @@ -729,7 +731,8 @@ int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw); static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {} static inline int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { return 0; } -static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) {} +static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) {} +static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw) {} static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; } static inline int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 2ce3728576d1..7c51011aed2b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -1040,6 +1040,15 @@ static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) mlx5_del_flow_rules(flows[i]); kvfree(flows); + /* If changing eswitch mode from switchdev to legacy, but num_vfs is not 0, + * meta rules could be freed again. So set it to NULL. + */ + esw->fdb_table.offloads.send_to_vport_meta_rules = NULL; +} + +void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) +{ + mlx5_eswitch_del_send_to_vport_meta_rules(esw); } static int @@ -2034,7 +2043,7 @@ static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) if (!MLX5_CAP_GEN(dev, vport_group_manager)) return -EOPNOTSUPP; - if (esw->mode == MLX5_ESWITCH_NONE) + if (!mlx5_esw_is_fdb_created(esw)) return -EOPNOTSUPP; switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { @@ -2170,7 +2179,6 @@ static int esw_offloads_start(struct mlx5_eswitch *esw, { int err, err1; - mlx5_eswitch_disable_locked(esw, false); err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, esw->dev->priv.sriov.num_vfs); if (err) { @@ -2894,7 +2902,7 @@ int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable) int err = 0; down_write(&esw->mode_lock); - if (esw->mode != MLX5_ESWITCH_NONE) { + if (mlx5_esw_is_fdb_created(esw)) { err = -EBUSY; goto done; } @@ -3229,7 +3237,6 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw, { int err, err1; - mlx5_eswitch_disable_locked(esw, false); err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, MLX5_ESWITCH_IGNORE_NUM_VFS); if (err) { @@ -3334,15 +3341,6 @@ static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) return 0; } -static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) -{ - /* devlink commands in NONE eswitch mode are currently supported only - * on ECPF. - */ - return (esw->mode == MLX5_ESWITCH_NONE && - !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0; -} - /* FIXME: devl_unlock() followed by devl_lock() inside driver callback * is never correct and prone to races. It's a transitional workaround, * never repeat this pattern. @@ -3399,6 +3397,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (cur_mlx5_mode == mlx5_mode) goto unlock; + mlx5_eswitch_disable_locked(esw); if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { NL_SET_ERR_MSG_MOD(extack, @@ -3409,6 +3408,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, err = esw_offloads_start(esw, extack); } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { err = esw_offloads_stop(esw, extack); + mlx5_rescan_drivers(esw->dev); } else { err = -EINVAL; } @@ -3431,12 +3431,7 @@ int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) return PTR_ERR(esw); mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - err = esw_mode_to_devlink(esw->mode, mode); -unlock: mlx5_eswtich_mode_callback_exit(devlink, esw); return err; } @@ -3485,9 +3480,6 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, return PTR_ERR(esw); mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto out; switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: @@ -3539,12 +3531,7 @@ int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) return PTR_ERR(esw); mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); -unlock: mlx5_eswtich_mode_callback_exit(devlink, esw); return err; } @@ -3555,16 +3542,13 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, { struct mlx5_core_dev *dev = devlink_priv(devlink); struct mlx5_eswitch *esw; - int err; + int err = 0; esw = mlx5_devlink_eswitch_get(devlink); if (IS_ERR(esw)) return PTR_ERR(esw); mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || @@ -3615,21 +3599,15 @@ int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, enum devlink_eswitch_encap_mode *encap) { struct mlx5_eswitch *esw; - int err; esw = mlx5_devlink_eswitch_get(devlink); if (IS_ERR(esw)) return PTR_ERR(esw); mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - *encap = esw->offloads.encap; -unlock: mlx5_eswtich_mode_callback_exit(devlink, esw); - return err; + return 0; } static bool diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 2a8fc547eb37..641505d2c0c2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -632,6 +632,7 @@ static int mlx5_deactivate_lag(struct mlx5_lag *ldev) static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) { #ifdef CONFIG_MLX5_ESWITCH + struct mlx5_core_dev *dev; u8 mode; #endif int i; @@ -641,11 +642,11 @@ static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) return false; #ifdef CONFIG_MLX5_ESWITCH - mode = mlx5_eswitch_mode(ldev->pf[MLX5_LAG_P1].dev); - - if (mode != MLX5_ESWITCH_NONE && mode != MLX5_ESWITCH_OFFLOADS) + dev = ldev->pf[MLX5_LAG_P1].dev; + if ((mlx5_sriov_is_enabled(dev)) && !is_mdev_switchdev_mode(dev)) return false; + mode = mlx5_eswitch_mode(dev); for (i = 0; i < ldev->ports; i++) if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode) return false; @@ -760,8 +761,7 @@ static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev) #ifdef CONFIG_MLX5_ESWITCH for (i = 0; i < ldev->ports; i++) - roce_lag = roce_lag && - ldev->pf[i].dev->priv.eswitch->mode == MLX5_ESWITCH_NONE; + roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev); #endif return roce_lag; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 2078d9f03a5f..a9e51c1b7738 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1250,6 +1250,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev) { mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); + mlx5_eswitch_disable(dev->priv.eswitch); mlx5_lag_remove_mdev(dev); mlx5_ec_cleanup(dev); mlx5_sf_hw_table_destroy(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index 3be659cd91f1..7d955a4d9f14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -501,7 +501,7 @@ static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, voi case MLX5_ESWITCH_OFFLOADS: mlx5_sf_table_enable(table); break; - case MLX5_ESWITCH_NONE: + case MLX5_ESWITCH_LEGACY: mlx5_sf_table_disable(table); break; default: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index 2935614f6fa9..5757cd6e1819 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -145,8 +145,7 @@ mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf) sriov->vfs_ctx[vf].enabled = 0; } - if (MLX5_ESWITCH_MANAGER(dev)) - mlx5_eswitch_disable(dev->priv.eswitch, clear_vf); + mlx5_eswitch_disable_sriov(dev->priv.eswitch, clear_vf); if (mlx5_wait_for_pages(dev, &dev->priv.vfs_pages)) mlx5_core_warn(dev, "timeout reclaiming VFs pages\n"); diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index 8b18fe9771f9..e2701ed0200e 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -12,7 +12,6 @@ #define MLX5_ESWITCH_MANAGER(mdev) MLX5_CAP_GEN(mdev, eswitch_manager) enum { - MLX5_ESWITCH_NONE, MLX5_ESWITCH_LEGACY, MLX5_ESWITCH_OFFLOADS }; @@ -153,7 +152,7 @@ struct mlx5_core_dev *mlx5_eswitch_get_core_dev(struct mlx5_eswitch *esw); static inline u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev) { - return MLX5_ESWITCH_NONE; + return MLX5_ESWITCH_LEGACY; } static inline enum devlink_eswitch_encap_mode @@ -198,6 +197,11 @@ static inline struct mlx5_core_dev *mlx5_eswitch_get_core_dev(struct mlx5_eswitc #endif /* CONFIG_MLX5_ESWITCH */ +static inline bool is_mdev_legacy_mode(struct mlx5_core_dev *dev) +{ + return mlx5_eswitch_mode(dev) == MLX5_ESWITCH_LEGACY; +} + static inline bool is_mdev_switchdev_mode(struct mlx5_core_dev *dev) { return mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS; -- cgit v1.2.3 From 036bff2800cbcf8217dd0bc93d8421b5b8f72476 Mon Sep 17 00:00:00 2001 From: Dario Binacchi Date: Tue, 28 Jun 2022 18:31:28 +0200 Subject: can: netlink: dump bitrate 0 if can_priv::bittiming.bitrate is -1U Upcoming changes on slcan driver will require you to specify a bitrate of value -1 to prevent the open_candev() from failing but at the same time highlighting that it is a fake value. In this case the command `ip --details -s -s link show' would print 4294967295 as the bitrate value. The patch change this value in 0. Link: https://lore.kernel.org/all/20220628163137.413025-5-dario.binacchi@amarulasolutions.com Suggested-by: Marc Kleine-Budde Signed-off-by: Dario Binacchi Tested-by: Jeroen Hofstee Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/netlink.c | 3 ++- include/linux/can/bittiming.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/can/dev/netlink.c b/drivers/net/can/dev/netlink.c index 037824011266..8efa22d9f214 100644 --- a/drivers/net/can/dev/netlink.c +++ b/drivers/net/can/dev/netlink.c @@ -511,7 +511,8 @@ static int can_fill_info(struct sk_buff *skb, const struct net_device *dev) if (priv->do_get_state) priv->do_get_state(dev, &state); - if ((priv->bittiming.bitrate && + if ((priv->bittiming.bitrate != CAN_BITRATE_UNSET && + priv->bittiming.bitrate != CAN_BITRATE_UNKNOWN && nla_put(skb, IFLA_CAN_BITTIMING, sizeof(priv->bittiming), &priv->bittiming)) || diff --git a/include/linux/can/bittiming.h b/include/linux/can/bittiming.h index 7ae21c0f7f23..ef0a77173e3c 100644 --- a/include/linux/can/bittiming.h +++ b/include/linux/can/bittiming.h @@ -11,6 +11,8 @@ #define CAN_SYNC_SEG 1 +#define CAN_BITRATE_UNSET 0 +#define CAN_BITRATE_UNKNOWN (-1U) #define CAN_CTRLMODE_TDC_MASK \ (CAN_CTRLMODE_TDC_AUTO | CAN_CTRLMODE_TDC_MANUAL) -- cgit v1.2.3 From 507db7927cd181d409dd495c8384b8e14c21c600 Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Sun, 3 Jul 2022 18:08:36 -0700 Subject: mm: rmap: use the correct parameter name for DEFINE_PAGE_VMA_WALK The parameter used by DEFINE_PAGE_VMA_WALK is _page not page, fix the parameter name. It didn't cause any build error, it is probably because the only caller is write_protect_page() from ksm.c, which pass in page. Link: https://lkml.kernel.org/r/20220512174551.81279-1-shy828301@gmail.com Fixes: 2aff7a4755be ("mm: Convert page_vma_mapped_walk to work on PFNs") Signed-off-by: Yang Shi Reviewed-by: Muchun Song Reviewed-by: Matthew Wilcox (Oracle) Cc: Muchun Song Signed-off-by: Andrew Morton --- include/linux/rmap.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 9ec23138e410..bf80adca980b 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -325,8 +325,8 @@ struct page_vma_mapped_walk { #define DEFINE_PAGE_VMA_WALK(name, _page, _vma, _address, _flags) \ struct page_vma_mapped_walk name = { \ .pfn = page_to_pfn(_page), \ - .nr_pages = compound_nr(page), \ - .pgoff = page_to_pgoff(page), \ + .nr_pages = compound_nr(_page), \ + .pgoff = page_to_pgoff(_page), \ .vma = _vma, \ .address = _address, \ .flags = _flags, \ -- cgit v1.2.3 From c453d8c7d1384d7e1d7f26d3ec0d527092edf801 Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Fri, 13 May 2022 12:17:05 -0700 Subject: mm/page_vma_mapped.c: check possible huge PMD map with transhuge_vma_suitable() IIUC page_vma_mapped_walk() checks if the vma is possibly huge PMD mapped with transparent_hugepage_active() and "pvmw->nr_pages >= HPAGE_PMD_NR". Actually pvmw->nr_pages is returned by compound_nr() or folio_nr_pages(), so the page should be THP as long as "pvmw->nr_pages >= HPAGE_PMD_NR". And it is guaranteed THP is allocated for valid VMA in the first place. But it may be not PMD mapped if the VMA is file VMA and it is not properly aligned. The transhuge_vma_suitable() is used to do such check, so replace transparent_hugepage_active() to it, which is too heavy and overkilling. Link: https://lkml.kernel.org/r/20220513191705.457775-1-shy828301@gmail.com Signed-off-by: Yang Shi Reviewed-by: Muchun Song Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 8 ++++++-- mm/page_vma_mapped.c | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index de29821231c9..648cb3ce7099 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -117,8 +117,10 @@ extern struct kobj_attribute shmem_enabled_attr; extern unsigned long transparent_hugepage_flags; static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, - unsigned long haddr) + unsigned long addr) { + unsigned long haddr; + /* Don't have to check pgoff for anonymous vma */ if (!vma_is_anonymous(vma)) { if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, @@ -126,6 +128,8 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, return false; } + haddr = addr & HPAGE_PMD_MASK; + if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) return false; return true; @@ -342,7 +346,7 @@ static inline bool transparent_hugepage_active(struct vm_area_struct *vma) } static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, - unsigned long haddr) + unsigned long addr) { return false; } diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index c10f839fc410..e971a467fcdf 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -243,7 +243,7 @@ restart: * cleared *pmd but not decremented compound_mapcount(). */ if ((pvmw->flags & PVMW_SYNC) && - transparent_hugepage_active(vma) && + transhuge_vma_suitable(vma, pvmw->address) && (pvmw->nr_pages >= HPAGE_PMD_NR)) { spinlock_t *ptl = pmd_lock(mm, pvmw->pmd); -- cgit v1.2.3 From 7ce82f4c3f3ead13a9d9498768e3b1a79975c4d8 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Mon, 30 May 2022 19:30:15 +0800 Subject: mm/migration: return errno when isolate_huge_page failed We might fail to isolate huge page due to e.g. the page is under migration which cleared HPageMigratable. We should return errno in this case rather than always return 1 which could confuse the user, i.e. the caller might think all of the memory is migrated while the hugetlb page is left behind. We make the prototype of isolate_huge_page consistent with isolate_lru_page as suggested by Huang Ying and rename isolate_huge_page to isolate_hugetlb as suggested by Muchun to improve the readability. Link: https://lkml.kernel.org/r/20220530113016.16663-4-linmiaohe@huawei.com Fixes: e8db67eb0ded ("mm: migrate: move_pages() supports thp migration") Signed-off-by: Miaohe Lin Suggested-by: Huang Ying Reported-by: kernel test robot (build error) Cc: Alistair Popple Cc: Christoph Hellwig Cc: Christoph Lameter Cc: David Hildenbrand Cc: David Howells Cc: Mike Kravetz Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Xu Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 6 +++--- mm/gup.c | 2 +- mm/hugetlb.c | 11 +++++------ mm/memory-failure.c | 2 +- mm/memory_hotplug.c | 2 +- mm/mempolicy.c | 2 +- mm/migrate.c | 7 ++++--- 7 files changed, 16 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index e4cff27d1198..756b66ff025e 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -170,7 +170,7 @@ bool hugetlb_reserve_pages(struct inode *inode, long from, long to, vm_flags_t vm_flags); long hugetlb_unreserve_pages(struct inode *inode, long start, long end, long freed); -bool isolate_huge_page(struct page *page, struct list_head *list); +int isolate_hugetlb(struct page *page, struct list_head *list); int get_hwpoison_huge_page(struct page *page, bool *hugetlb); int get_huge_page_for_hwpoison(unsigned long pfn, int flags); void putback_active_hugepage(struct page *page); @@ -376,9 +376,9 @@ static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, return NULL; } -static inline bool isolate_huge_page(struct page *page, struct list_head *list) +static inline int isolate_hugetlb(struct page *page, struct list_head *list) { - return false; + return -EBUSY; } static inline int get_hwpoison_huge_page(struct page *page, bool *hugetlb) diff --git a/mm/gup.c b/mm/gup.c index 407a81d5ca03..3129b754ade3 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1930,7 +1930,7 @@ static long check_and_migrate_movable_pages(unsigned long nr_pages, * Try to move out any movable page before pinning the range. */ if (folio_test_hugetlb(folio)) { - if (!isolate_huge_page(&folio->page, + if (isolate_hugetlb(&folio->page, &movable_page_list)) isolation_error_count++; continue; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b36a4ef87a2e..dd9a46ccb79c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2766,8 +2766,7 @@ retry: * Fail with -EBUSY if not possible. */ spin_unlock_irq(&hugetlb_lock); - if (!isolate_huge_page(old_page, list)) - ret = -EBUSY; + ret = isolate_hugetlb(old_page, list); spin_lock_irq(&hugetlb_lock); goto free_new; } else if (!HPageFreed(old_page)) { @@ -2843,7 +2842,7 @@ int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list) if (hstate_is_gigantic(h)) return -ENOMEM; - if (page_count(head) && isolate_huge_page(head, list)) + if (page_count(head) && !isolate_hugetlb(head, list)) ret = 0; else if (!page_count(head)) ret = alloc_and_dissolve_huge_page(h, head, list); @@ -6960,15 +6959,15 @@ follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int fla return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT); } -bool isolate_huge_page(struct page *page, struct list_head *list) +int isolate_hugetlb(struct page *page, struct list_head *list) { - bool ret = true; + int ret = 0; spin_lock_irq(&hugetlb_lock); if (!PageHeadHuge(page) || !HPageMigratable(page) || !get_page_unless_zero(page)) { - ret = false; + ret = -EBUSY; goto unlock; } ClearHPageMigratable(page); diff --git a/mm/memory-failure.c b/mm/memory-failure.c index da39ec8afca8..845369f839e1 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -2178,7 +2178,7 @@ static bool isolate_page(struct page *page, struct list_head *pagelist) bool lru = PageLRU(page); if (PageHuge(page)) { - isolated = isolate_huge_page(page, pagelist); + isolated = !isolate_hugetlb(page, pagelist); } else { if (lru) isolated = !isolate_lru_page(page); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 1f1a730c4499..84990a14d51a 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1641,7 +1641,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) if (PageHuge(page)) { pfn = page_to_pfn(head) + compound_nr(head) - 1; - isolate_huge_page(head, &source); + isolate_hugetlb(head, &source); continue; } else if (PageTransHuge(page)) pfn = page_to_pfn(head) + thp_nr_pages(page) - 1; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index d39b01fd52fe..9689919a2829 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -602,7 +602,7 @@ static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask, /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */ if (flags & (MPOL_MF_MOVE_ALL) || (flags & MPOL_MF_MOVE && page_mapcount(page) == 1)) { - if (!isolate_huge_page(page, qp->pagelist) && + if (isolate_hugetlb(page, qp->pagelist) && (flags & MPOL_MF_STRICT)) /* * Failed to isolate page but allow migrating pages diff --git a/mm/migrate.c b/mm/migrate.c index c83b3ae2e285..1d036dec1328 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -133,7 +133,7 @@ static void putback_movable_page(struct page *page) * * This function shall be used whenever the isolated pageset has been * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() - * and isolate_huge_page(). + * and isolate_hugetlb(). */ void putback_movable_pages(struct list_head *l) { @@ -1628,8 +1628,9 @@ static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, if (PageHuge(page)) { if (PageHead(page)) { - isolate_huge_page(page, pagelist); - err = 1; + err = isolate_hugetlb(page, pagelist); + if (!err) + err = 1; } } else { struct page *head; -- cgit v1.2.3 From ad1ac596e8a8c4b06715dfbd89853eb73c9886b2 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Mon, 30 May 2022 19:30:16 +0800 Subject: mm/migration: fix potential pte_unmap on an not mapped pte __migration_entry_wait and migration_entry_wait_on_locked assume pte is always mapped from caller. But this is not the case when it's called from migration_entry_wait_huge and follow_huge_pmd. Add a hugetlbfs variant that calls hugetlb_migration_entry_wait(ptep == NULL) to fix this issue. Link: https://lkml.kernel.org/r/20220530113016.16663-5-linmiaohe@huawei.com Fixes: 30dad30922cc ("mm: migration: add migrate_entry_wait_huge()") Signed-off-by: Miaohe Lin Suggested-by: David Hildenbrand Reviewed-by: David Hildenbrand Cc: Alistair Popple Cc: Christoph Hellwig Cc: Christoph Lameter Cc: David Howells Cc: Huang Ying Cc: kernel test robot Cc: Mike Kravetz Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Xu Signed-off-by: Andrew Morton --- include/linux/swapops.h | 12 ++++++++---- mm/hugetlb.c | 4 ++-- mm/migrate.c | 23 +++++++++++++++++++---- 3 files changed, 29 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swapops.h b/include/linux/swapops.h index f24775b41880..bb7afd03a324 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -244,8 +244,10 @@ extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl); extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address); -extern void migration_entry_wait_huge(struct vm_area_struct *vma, - struct mm_struct *mm, pte_t *pte); +#ifdef CONFIG_HUGETLB_PAGE +extern void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl); +extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); +#endif #else static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) { @@ -271,8 +273,10 @@ static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl) { } static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address) { } -static inline void migration_entry_wait_huge(struct vm_area_struct *vma, - struct mm_struct *mm, pte_t *pte) { } +#ifdef CONFIG_HUGETLB_PAGE +static inline void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { } +static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { } +#endif static inline int is_writable_migration_entry(swp_entry_t entry) { return 0; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index dd9a46ccb79c..ed202d29ca46 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5702,7 +5702,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, */ entry = huge_ptep_get(ptep); if (unlikely(is_hugetlb_entry_migration(entry))) { - migration_entry_wait_huge(vma, mm, ptep); + migration_entry_wait_huge(vma, ptep); return 0; } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) return VM_FAULT_HWPOISON_LARGE | @@ -6927,7 +6927,7 @@ retry: } else { if (is_hugetlb_entry_migration(pte)) { spin_unlock(ptl); - __migration_entry_wait(mm, (pte_t *)pmd, ptl); + __migration_entry_wait_huge((pte_t *)pmd, ptl); goto retry; } /* diff --git a/mm/migrate.c b/mm/migrate.c index 1d036dec1328..7934eebf1689 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -315,13 +315,28 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, __migration_entry_wait(mm, ptep, ptl); } -void migration_entry_wait_huge(struct vm_area_struct *vma, - struct mm_struct *mm, pte_t *pte) +#ifdef CONFIG_HUGETLB_PAGE +void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { - spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); - __migration_entry_wait(mm, pte, ptl); + pte_t pte; + + spin_lock(ptl); + pte = huge_ptep_get(ptep); + + if (unlikely(!is_hugetlb_entry_migration(pte))) + spin_unlock(ptl); + else + migration_entry_wait_on_locked(pte_to_swp_entry(pte), NULL, ptl); } +void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) +{ + spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), vma->vm_mm, pte); + + __migration_entry_wait_huge(pte, ptl); +} +#endif + #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { -- cgit v1.2.3 From c9e124e0382d83d458db204f929002ea98daa6a8 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 6 Jun 2022 18:23:06 +0000 Subject: mm/damon/{dbgfs,sysfs}: move target_has_pid() from dbgfs to damon.h The function for knowing if given monitoring context's targets will have pid or not is defined and used in dbgfs only. However, the logic is also needed for sysfs. This commit moves the code to damon.h and makes both dbgfs and sysfs to use it. Link: https://lkml.kernel.org/r/20220606182310.48781-3-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- include/linux/damon.h | 6 ++++++ mm/damon/dbgfs.c | 15 +++++---------- mm/damon/sysfs.c | 8 +++----- 3 files changed, 14 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 2765c7d99beb..b9aae19fab3e 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -525,6 +525,12 @@ bool damon_is_registered_ops(enum damon_ops_id id); int damon_register_ops(struct damon_operations *ops); int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id); +static inline bool damon_target_has_pid(const struct damon_ctx *ctx) +{ + return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR; +} + + int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index a0dab8b5e45f..5ae810927309 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -275,11 +275,6 @@ out: return ret; } -static inline bool target_has_pid(const struct damon_ctx *ctx) -{ - return ctx->ops.id == DAMON_OPS_VADDR; -} - static ssize_t sprint_target_ids(struct damon_ctx *ctx, char *buf, ssize_t len) { struct damon_target *t; @@ -288,7 +283,7 @@ static ssize_t sprint_target_ids(struct damon_ctx *ctx, char *buf, ssize_t len) int rc; damon_for_each_target(t, ctx) { - if (target_has_pid(ctx)) + if (damon_target_has_pid(ctx)) /* Show pid numbers to debugfs users */ id = pid_vnr(t->pid); else @@ -415,7 +410,7 @@ static int dbgfs_set_targets(struct damon_ctx *ctx, ssize_t nr_targets, struct damon_target *t, *next; damon_for_each_target_safe(t, next, ctx) { - if (target_has_pid(ctx)) + if (damon_target_has_pid(ctx)) put_pid(t->pid); damon_destroy_target(t); } @@ -425,11 +420,11 @@ static int dbgfs_set_targets(struct damon_ctx *ctx, ssize_t nr_targets, if (!t) { damon_for_each_target_safe(t, next, ctx) damon_destroy_target(t); - if (target_has_pid(ctx)) + if (damon_target_has_pid(ctx)) dbgfs_put_pids(pids, nr_targets); return -ENOMEM; } - if (target_has_pid(ctx)) + if (damon_target_has_pid(ctx)) t->pid = pids[i]; damon_add_target(ctx, t); } @@ -722,7 +717,7 @@ static void dbgfs_before_terminate(struct damon_ctx *ctx) { struct damon_target *t, *next; - if (!target_has_pid(ctx)) + if (!damon_target_has_pid(ctx)) return; mutex_lock(&ctx->kdamond_lock); diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 09f9e8ca3d1f..8810e6abdb06 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2136,8 +2136,7 @@ static void damon_sysfs_destroy_targets(struct damon_ctx *ctx) struct damon_target *t, *next; damon_for_each_target_safe(t, next, ctx) { - if (ctx->ops.id == DAMON_OPS_VADDR || - ctx->ops.id == DAMON_OPS_FVADDR) + if (damon_target_has_pid(ctx)) put_pid(t->pid); damon_destroy_target(t); } @@ -2181,8 +2180,7 @@ static int damon_sysfs_add_target(struct damon_sysfs_target *sys_target, if (!t) return -ENOMEM; - if (ctx->ops.id == DAMON_OPS_VADDR || - ctx->ops.id == DAMON_OPS_FVADDR) { + if (damon_target_has_pid(ctx)) { t->pid = find_get_pid(sys_target->pid); if (!t->pid) goto destroy_targets_out; @@ -2210,7 +2208,7 @@ static struct damon_target *damon_sysfs_existing_target( struct pid *pid; struct damon_target *t; - if (ctx->ops.id == DAMON_OPS_PADDR) { + if (!damon_target_has_pid(ctx)) { /* Up to only one target for paddr could exist */ damon_for_each_target(t, ctx) return t; -- cgit v1.2.3 From d9da8f6cf55eeca642c021912af1890002464c64 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Thu, 9 Jun 2022 20:18:46 +0200 Subject: mm: introduce clear_highpage_kasan_tagged Add a clear_highpage_kasan_tagged() helper that does clear_highpage() on a page potentially tagged by KASAN. This helper is used by the following patch. Link: https://lkml.kernel.org/r/4471979b46b2c487787ddcd08b9dc5fedd1b6ffd.1654798516.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Marco Elver Signed-off-by: Andrew Morton --- include/linux/highmem.h | 10 ++++++++++ mm/page_alloc.c | 8 ++------ 2 files changed, 12 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index fee9835e3793..22379a63e293 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -243,6 +243,16 @@ static inline void clear_highpage(struct page *page) kunmap_local(kaddr); } +static inline void clear_highpage_kasan_tagged(struct page *page) +{ + u8 tag; + + tag = page_kasan_tag(page); + page_kasan_tag_reset(page); + clear_highpage(page); + page_kasan_tag_set(page, tag); +} + #ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGE static inline void tag_clear_highpage(struct page *page) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 9234863f2488..248469134962 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1302,12 +1302,8 @@ static void kernel_init_pages(struct page *page, int numpages) /* s390's use of memset() could override KASAN redzones. */ kasan_disable_current(); - for (i = 0; i < numpages; i++) { - u8 tag = page_kasan_tag(page + i); - page_kasan_tag_reset(page + i); - clear_highpage(page + i); - page_kasan_tag_set(page + i, tag); - } + for (i = 0; i < numpages; i++) + clear_highpage_kasan_tagged(page + i); kasan_enable_current(); } -- cgit v1.2.3 From c15187a4a2d660bf490f7873afd0de5288f65c8f Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Tue, 31 May 2022 20:22:22 -0700 Subject: mm: memcontrol: introduce mem_cgroup_ino() and mem_cgroup_get_from_ino() Patch series "mm: introduce shrinker debugfs interface", v5. The only existing debugging mechanism is a couple of tracepoints in do_shrink_slab(): mm_shrink_slab_start and mm_shrink_slab_end. They aren't covering everything though: shrinkers which report 0 objects will never show up, there is no support for memcg-aware shrinkers. Shrinkers are identified by their scan function, which is not always enough (e.g. hard to guess which super block's shrinker it is having only "super_cache_scan"). To provide a better visibility and debug options for memory shrinkers this patchset introduces a /sys/kernel/debug/shrinker interface, to some extent similar to /sys/kernel/slab. For each shrinker registered in the system a directory is created. As now, the directory will contain only a "scan" file, which allows to get the number of managed objects for each memory cgroup (for memcg-aware shrinkers) and each numa node (for numa-aware shrinkers on a numa machine). Other interfaces might be added in the future. To make debugging more pleasant, the patchset also names all shrinkers, so that debugfs entries can have meaningful names. This patch (of 5): Shrinker debugfs requires a way to represent memory cgroups without using full paths, both for displaying information and getting input from a user. Cgroup inode number is a perfect way, already used by bpf. This commit adds a couple of helper functions which will be used to handle memcg-aware shrinkers. Link: https://lkml.kernel.org/r/20220601032227.4076670-1-roman.gushchin@linux.dev Link: https://lkml.kernel.org/r/20220601032227.4076670-2-roman.gushchin@linux.dev Signed-off-by: Roman Gushchin Acked-by: Muchun Song Cc: Dave Chinner Cc: Kent Overstreet Cc: Hillf Danton Cc: Christophe JAILLET Cc: Roman Gushchin Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 21 +++++++++++++++++++++ mm/memcontrol.c | 23 +++++++++++++++++++++++ 2 files changed, 44 insertions(+) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 04f2f33607e9..4d31ce55b1c0 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -837,6 +837,15 @@ static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) } struct mem_cgroup *mem_cgroup_from_id(unsigned short id); +#ifdef CONFIG_SHRINKER_DEBUG +static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg) +{ + return memcg ? cgroup_ino(memcg->css.cgroup) : 0; +} + +struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino); +#endif + static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) { return mem_cgroup_from_css(seq_css(m)); @@ -1343,6 +1352,18 @@ static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id) return NULL; } +#ifdef CONFIG_SHRINKER_DEBUG +static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg) +{ + return 0; +} + +static inline struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) +{ + return NULL; +} +#endif + static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) { return NULL; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 655c09393ad5..1497affe08c4 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5088,6 +5088,29 @@ struct mem_cgroup *mem_cgroup_from_id(unsigned short id) return idr_find(&mem_cgroup_idr, id); } +#ifdef CONFIG_SHRINKER_DEBUG +struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) +{ + struct cgroup *cgrp; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg; + + cgrp = cgroup_get_from_id(ino); + if (!cgrp) + return ERR_PTR(-ENOENT); + + css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys); + if (css) + memcg = container_of(css, struct mem_cgroup, css); + else + memcg = ERR_PTR(-ENOENT); + + cgroup_put(cgrp); + + return memcg; +} +#endif + static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) { struct mem_cgroup_per_node *pn; -- cgit v1.2.3 From 5035ebc644aec92d55d1bbfe042f35341e4bffb5 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Tue, 31 May 2022 20:22:23 -0700 Subject: mm: shrinkers: introduce debugfs interface for memory shrinkers This commit introduces the /sys/kernel/debug/shrinker debugfs interface which provides an ability to observe the state of individual kernel memory shrinkers. Because the feature adds some memory overhead (which shouldn't be large unless there is a huge amount of registered shrinkers), it's guarded by a config option (enabled by default). This commit introduces the "count" interface for each shrinker registered in the system. The output is in the following format: ... ... ... To reduce the size of output on machines with many thousands cgroups, if the total number of objects on all nodes is 0, the line is omitted. If the shrinker is not memcg-aware or CONFIG_MEMCG is off, 0 is printed as cgroup inode id. If the shrinker is not numa-aware, 0's are printed for all nodes except the first one. This commit gives debugfs entries simple numeric names, which are not very convenient. The following commit in the series will provide shrinkers with more meaningful names. [akpm@linux-foundation.org: remove WARN_ON_ONCE(), per Roman] Reported-by: syzbot+300d27c79fe6d4cbcc39@syzkaller.appspotmail.com Link: https://lkml.kernel.org/r/20220601032227.4076670-3-roman.gushchin@linux.dev Signed-off-by: Roman Gushchin Reviewed-by: Kent Overstreet Acked-by: Muchun Song Cc: Christophe JAILLET Cc: Dave Chinner Cc: Hillf Danton Signed-off-by: Andrew Morton --- include/linux/shrinker.h | 19 +++++- lib/Kconfig.debug | 9 +++ mm/Makefile | 1 + mm/shrinker_debug.c | 168 +++++++++++++++++++++++++++++++++++++++++++++++ mm/vmscan.c | 6 +- 5 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 mm/shrinker_debug.c (limited to 'include/linux') diff --git a/include/linux/shrinker.h b/include/linux/shrinker.h index 76fbf92b04d9..2ced8149c513 100644 --- a/include/linux/shrinker.h +++ b/include/linux/shrinker.h @@ -72,6 +72,10 @@ struct shrinker { #ifdef CONFIG_MEMCG /* ID in shrinker_idr */ int id; +#endif +#ifdef CONFIG_SHRINKER_DEBUG + int debugfs_id; + struct dentry *debugfs_entry; #endif /* objs pending delete, per node */ atomic_long_t *nr_deferred; @@ -94,4 +98,17 @@ extern int register_shrinker(struct shrinker *shrinker); extern void unregister_shrinker(struct shrinker *shrinker); extern void free_prealloced_shrinker(struct shrinker *shrinker); extern void synchronize_shrinkers(void); -#endif + +#ifdef CONFIG_SHRINKER_DEBUG +extern int shrinker_debugfs_add(struct shrinker *shrinker); +extern void shrinker_debugfs_remove(struct shrinker *shrinker); +#else /* CONFIG_SHRINKER_DEBUG */ +static inline int shrinker_debugfs_add(struct shrinker *shrinker) +{ + return 0; +} +static inline void shrinker_debugfs_remove(struct shrinker *shrinker) +{ +} +#endif /* CONFIG_SHRINKER_DEBUG */ +#endif /* _LINUX_SHRINKER_H */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2e24db4bff19..0b483a8da409 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -699,6 +699,15 @@ config DEBUG_OBJECTS_ENABLE_DEFAULT help Debug objects boot parameter default value +config SHRINKER_DEBUG + default y + bool "Enable shrinker debugging support" + depends on DEBUG_FS + help + Say Y to enable the shrinker debugfs interface which provides + visibility into the kernel memory shrinkers subsystem. + Disable it to avoid an extra memory footprint. + config HAVE_DEBUG_KMEMLEAK bool diff --git a/mm/Makefile b/mm/Makefile index 6f9ffa968a1a..9a564f836403 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -133,3 +133,4 @@ obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o obj-$(CONFIG_IO_MAPPING) += io-mapping.o obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o +obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o diff --git a/mm/shrinker_debug.c b/mm/shrinker_debug.c new file mode 100644 index 000000000000..1a70556bd46c --- /dev/null +++ b/mm/shrinker_debug.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +/* defined in vmscan.c */ +extern struct rw_semaphore shrinker_rwsem; +extern struct list_head shrinker_list; + +static DEFINE_IDA(shrinker_debugfs_ida); +static struct dentry *shrinker_debugfs_root; + +static unsigned long shrinker_count_objects(struct shrinker *shrinker, + struct mem_cgroup *memcg, + unsigned long *count_per_node) +{ + unsigned long nr, total = 0; + int nid; + + for_each_node(nid) { + if (nid == 0 || (shrinker->flags & SHRINKER_NUMA_AWARE)) { + struct shrink_control sc = { + .gfp_mask = GFP_KERNEL, + .nid = nid, + .memcg = memcg, + }; + + nr = shrinker->count_objects(shrinker, &sc); + if (nr == SHRINK_EMPTY) + nr = 0; + } else { + nr = 0; + } + + count_per_node[nid] = nr; + total += nr; + } + + return total; +} + +static int shrinker_debugfs_count_show(struct seq_file *m, void *v) +{ + struct shrinker *shrinker = m->private; + unsigned long *count_per_node; + struct mem_cgroup *memcg; + unsigned long total; + bool memcg_aware; + int ret, nid; + + count_per_node = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL); + if (!count_per_node) + return -ENOMEM; + + ret = down_read_killable(&shrinker_rwsem); + if (ret) { + kfree(count_per_node); + return ret; + } + rcu_read_lock(); + + memcg_aware = shrinker->flags & SHRINKER_MEMCG_AWARE; + + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + if (memcg && !mem_cgroup_online(memcg)) + continue; + + total = shrinker_count_objects(shrinker, + memcg_aware ? memcg : NULL, + count_per_node); + if (total) { + seq_printf(m, "%lu", mem_cgroup_ino(memcg)); + for_each_node(nid) + seq_printf(m, " %lu", count_per_node[nid]); + seq_putc(m, '\n'); + } + + if (!memcg_aware) { + mem_cgroup_iter_break(NULL, memcg); + break; + } + + if (signal_pending(current)) { + mem_cgroup_iter_break(NULL, memcg); + ret = -EINTR; + break; + } + } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); + + rcu_read_unlock(); + up_read(&shrinker_rwsem); + + kfree(count_per_node); + return ret; +} +DEFINE_SHOW_ATTRIBUTE(shrinker_debugfs_count); + +int shrinker_debugfs_add(struct shrinker *shrinker) +{ + struct dentry *entry; + char buf[16]; + int id; + + lockdep_assert_held(&shrinker_rwsem); + + /* debugfs isn't initialized yet, add debugfs entries later. */ + if (!shrinker_debugfs_root) + return 0; + + id = ida_alloc(&shrinker_debugfs_ida, GFP_KERNEL); + if (id < 0) + return id; + shrinker->debugfs_id = id; + + snprintf(buf, sizeof(buf), "%d", id); + + /* create debugfs entry */ + entry = debugfs_create_dir(buf, shrinker_debugfs_root); + if (IS_ERR(entry)) { + ida_free(&shrinker_debugfs_ida, id); + return PTR_ERR(entry); + } + shrinker->debugfs_entry = entry; + + debugfs_create_file("count", 0220, entry, shrinker, + &shrinker_debugfs_count_fops); + return 0; +} + +void shrinker_debugfs_remove(struct shrinker *shrinker) +{ + lockdep_assert_held(&shrinker_rwsem); + + if (!shrinker->debugfs_entry) + return; + + debugfs_remove_recursive(shrinker->debugfs_entry); + ida_free(&shrinker_debugfs_ida, shrinker->debugfs_id); +} + +static int __init shrinker_debugfs_init(void) +{ + struct shrinker *shrinker; + struct dentry *dentry; + int ret = 0; + + dentry = debugfs_create_dir("shrinker", NULL); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + shrinker_debugfs_root = dentry; + + /* Create debugfs entries for shrinkers registered at boot */ + down_write(&shrinker_rwsem); + list_for_each_entry(shrinker, &shrinker_list, list) + if (!shrinker->debugfs_entry) { + ret = shrinker_debugfs_add(shrinker); + if (ret) + break; + } + up_write(&shrinker_rwsem); + + return ret; +} +late_initcall(shrinker_debugfs_init); diff --git a/mm/vmscan.c b/mm/vmscan.c index f7d9a683e3a7..35dedff79eb4 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -190,8 +190,8 @@ static void set_task_reclaim_state(struct task_struct *task, task->reclaim_state = rs; } -static LIST_HEAD(shrinker_list); -static DECLARE_RWSEM(shrinker_rwsem); +LIST_HEAD(shrinker_list); +DECLARE_RWSEM(shrinker_rwsem); #ifdef CONFIG_MEMCG static int shrinker_nr_max; @@ -650,6 +650,7 @@ void register_shrinker_prepared(struct shrinker *shrinker) down_write(&shrinker_rwsem); list_add_tail(&shrinker->list, &shrinker_list); shrinker->flags |= SHRINKER_REGISTERED; + shrinker_debugfs_add(shrinker); up_write(&shrinker_rwsem); } @@ -677,6 +678,7 @@ void unregister_shrinker(struct shrinker *shrinker) shrinker->flags &= ~SHRINKER_REGISTERED; if (shrinker->flags & SHRINKER_MEMCG_AWARE) unregister_memcg_shrinker(shrinker); + shrinker_debugfs_remove(shrinker); up_write(&shrinker_rwsem); kfree(shrinker->nr_deferred); -- cgit v1.2.3 From e33c267ab70de4249d22d7eab1cc7d68a889bac2 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Tue, 31 May 2022 20:22:24 -0700 Subject: mm: shrinkers: provide shrinkers with names Currently shrinkers are anonymous objects. For debugging purposes they can be identified by count/scan function names, but it's not always useful: e.g. for superblock's shrinkers it's nice to have at least an idea of to which superblock the shrinker belongs. This commit adds names to shrinkers. register_shrinker() and prealloc_shrinker() functions are extended to take a format and arguments to master a name. In some cases it's not possible to determine a good name at the time when a shrinker is allocated. For such cases shrinker_debugfs_rename() is provided. The expected format is: -[:]- For some shrinkers an instance can be encoded as (MAJOR:MINOR) pair. After this change the shrinker debugfs directory looks like: $ cd /sys/kernel/debug/shrinker/ $ ls dquota-cache-16 sb-devpts-28 sb-proc-47 sb-tmpfs-42 mm-shadow-18 sb-devtmpfs-5 sb-proc-48 sb-tmpfs-43 mm-zspool:zram0-34 sb-hugetlbfs-17 sb-pstore-31 sb-tmpfs-44 rcu-kfree-0 sb-hugetlbfs-33 sb-rootfs-2 sb-tmpfs-49 sb-aio-20 sb-iomem-12 sb-securityfs-6 sb-tracefs-13 sb-anon_inodefs-15 sb-mqueue-21 sb-selinuxfs-22 sb-xfs:vda1-36 sb-bdev-3 sb-nsfs-4 sb-sockfs-8 sb-zsmalloc-19 sb-bpf-32 sb-pipefs-14 sb-sysfs-26 thp-deferred_split-10 sb-btrfs:vda2-24 sb-proc-25 sb-tmpfs-1 thp-zero-9 sb-cgroup2-30 sb-proc-39 sb-tmpfs-27 xfs-buf:vda1-37 sb-configfs-23 sb-proc-41 sb-tmpfs-29 xfs-inodegc:vda1-38 sb-dax-11 sb-proc-45 sb-tmpfs-35 sb-debugfs-7 sb-proc-46 sb-tmpfs-40 [roman.gushchin@linux.dev: fix build warnings] Link: https://lkml.kernel.org/r/Yr+ZTnLb9lJk6fJO@castle Reported-by: kernel test robot Link: https://lkml.kernel.org/r/20220601032227.4076670-4-roman.gushchin@linux.dev Signed-off-by: Roman Gushchin Cc: Christophe JAILLET Cc: Dave Chinner Cc: Hillf Danton Cc: Kent Overstreet Cc: Muchun Song Signed-off-by: Andrew Morton --- arch/x86/kvm/mmu/mmu.c | 2 +- drivers/android/binder_alloc.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_shrinker.c | 3 +- drivers/gpu/drm/msm/msm_gem_shrinker.c | 2 +- drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c | 2 +- drivers/gpu/drm/ttm/ttm_pool.c | 2 +- drivers/md/bcache/btree.c | 2 +- drivers/md/dm-bufio.c | 3 +- drivers/md/dm-zoned-metadata.c | 4 +- drivers/md/raid5.c | 2 +- drivers/misc/vmw_balloon.c | 2 +- drivers/virtio/virtio_balloon.c | 2 +- drivers/xen/xenbus/xenbus_probe_backend.c | 2 +- fs/btrfs/super.c | 2 + fs/erofs/utils.c | 2 +- fs/ext4/extents_status.c | 3 +- fs/f2fs/super.c | 2 +- fs/gfs2/glock.c | 2 +- fs/gfs2/main.c | 2 +- fs/jbd2/journal.c | 3 +- fs/mbcache.c | 2 +- fs/nfs/nfs42xattr.c | 7 +-- fs/nfs/super.c | 2 +- fs/nfsd/filecache.c | 2 +- fs/nfsd/nfscache.c | 3 +- fs/quota/dquot.c | 2 +- fs/super.c | 6 ++- fs/ubifs/super.c | 2 +- fs/xfs/xfs_buf.c | 3 +- fs/xfs/xfs_icache.c | 2 +- fs/xfs/xfs_qm.c | 3 +- include/linux/shrinker.h | 14 +++++- kernel/rcu/tree.c | 2 +- mm/huge_memory.c | 4 +- mm/shrinker_debug.c | 47 ++++++++++++++++++- mm/vmscan.c | 58 ++++++++++++++++++++++-- mm/workingset.c | 2 +- mm/zsmalloc.c | 3 +- net/sunrpc/auth.c | 2 +- 39 files changed, 167 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 17252f39bd7c..797d3286ecc1 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6317,7 +6317,7 @@ int kvm_mmu_vendor_module_init(void) if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL)) goto out; - ret = register_shrinker(&mmu_shrinker); + ret = register_shrinker(&mmu_shrinker, "x86-mmu"); if (ret) goto out; diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 5649a0371a1f..51b502217d00 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -1084,7 +1084,7 @@ int binder_alloc_shrinker_init(void) int ret = list_lru_init(&binder_alloc_lru); if (ret == 0) { - ret = register_shrinker(&binder_shrinker); + ret = register_shrinker(&binder_shrinker, "android-binder"); if (ret) list_lru_destroy(&binder_alloc_lru); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index 6a6ff98a8746..e43577e03067 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -426,7 +426,8 @@ void i915_gem_driver_register__shrinker(struct drm_i915_private *i915) i915->mm.shrinker.count_objects = i915_gem_shrinker_count; i915->mm.shrinker.seeks = DEFAULT_SEEKS; i915->mm.shrinker.batch = 4096; - drm_WARN_ON(&i915->drm, register_shrinker(&i915->mm.shrinker)); + drm_WARN_ON(&i915->drm, register_shrinker(&i915->mm.shrinker, + "drm-i915_gem")); i915->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; drm_WARN_ON(&i915->drm, register_oom_notifier(&i915->mm.oom_notifier)); diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c index 086dacf2f26a..26e84d2ea6ae 100644 --- a/drivers/gpu/drm/msm/msm_gem_shrinker.c +++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c @@ -221,7 +221,7 @@ void msm_gem_shrinker_init(struct drm_device *dev) priv->shrinker.count_objects = msm_gem_shrinker_count; priv->shrinker.scan_objects = msm_gem_shrinker_scan; priv->shrinker.seeks = DEFAULT_SEEKS; - WARN_ON(register_shrinker(&priv->shrinker)); + WARN_ON(register_shrinker(&priv->shrinker, "drm-msm_gem")); priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap; WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier)); diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c index 77e7cb6d1ae3..bf0170782f25 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c @@ -103,7 +103,7 @@ void panfrost_gem_shrinker_init(struct drm_device *dev) pfdev->shrinker.count_objects = panfrost_gem_shrinker_count; pfdev->shrinker.scan_objects = panfrost_gem_shrinker_scan; pfdev->shrinker.seeks = DEFAULT_SEEKS; - WARN_ON(register_shrinker(&pfdev->shrinker)); + WARN_ON(register_shrinker(&pfdev->shrinker, "drm-panfrost")); } /** diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index 1bba0a0ed3f9..21b61631f73a 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -722,7 +722,7 @@ int ttm_pool_mgr_init(unsigned long num_pages) mm_shrinker.count_objects = ttm_pool_shrinker_count; mm_shrinker.scan_objects = ttm_pool_shrinker_scan; mm_shrinker.seeks = 1; - return register_shrinker(&mm_shrinker); + return register_shrinker(&mm_shrinker, "drm-ttm_pool"); } /** diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index e136d6edc1ed..147c493a989a 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -812,7 +812,7 @@ int bch_btree_cache_alloc(struct cache_set *c) c->shrink.seeks = 4; c->shrink.batch = c->btree_pages * 2; - if (register_shrinker(&c->shrink)) + if (register_shrinker(&c->shrink, "md-bcache:%pU", c->set_uuid)) pr_warn("bcache: %s: could not register shrinker\n", __func__); diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 5ffa1dcf84cf..3ff571b20f14 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1806,7 +1806,8 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign c->shrinker.scan_objects = dm_bufio_shrink_scan; c->shrinker.seeks = 1; c->shrinker.batch = 0; - r = register_shrinker(&c->shrinker); + r = register_shrinker(&c->shrinker, "md-%s:(%u:%u)", slab_name, + MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev)); if (r) goto bad; diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c index d1ea66114d14..46648f6100fb 100644 --- a/drivers/md/dm-zoned-metadata.c +++ b/drivers/md/dm-zoned-metadata.c @@ -2944,7 +2944,9 @@ int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev, zmd->mblk_shrinker.seeks = DEFAULT_SEEKS; /* Metadata cache shrinker */ - ret = register_shrinker(&zmd->mblk_shrinker); + ret = register_shrinker(&zmd->mblk_shrinker, "md-meta:(%u:%u)", + MAJOR(dev->bdev->bd_dev), + MINOR(dev->bdev->bd_dev)); if (ret) { dmz_zmd_err(zmd, "Register metadata cache shrinker failed"); goto err; diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 5d09256d7f81..780ae66840b7 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -7414,7 +7414,7 @@ static struct r5conf *setup_conf(struct mddev *mddev) conf->shrinker.count_objects = raid5_cache_count; conf->shrinker.batch = 128; conf->shrinker.flags = 0; - ret = register_shrinker(&conf->shrinker); + ret = register_shrinker(&conf->shrinker, "md-raid5:%s", mdname(mddev)); if (ret) { pr_warn("md/raid:%s: couldn't register shrinker.\n", mdname(mddev)); diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c index 086ce77d9074..c2d2fa114e65 100644 --- a/drivers/misc/vmw_balloon.c +++ b/drivers/misc/vmw_balloon.c @@ -1587,7 +1587,7 @@ static int vmballoon_register_shrinker(struct vmballoon *b) b->shrinker.count_objects = vmballoon_shrinker_count; b->shrinker.seeks = DEFAULT_SEEKS; - r = register_shrinker(&b->shrinker); + r = register_shrinker(&b->shrinker, "vmw-balloon"); if (r == 0) b->shrinker_registered = true; diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index b9737da6c4dd..cba57b1f382f 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -875,7 +875,7 @@ static int virtio_balloon_register_shrinker(struct virtio_balloon *vb) vb->shrinker.count_objects = virtio_balloon_shrinker_count; vb->shrinker.seeks = DEFAULT_SEEKS; - return register_shrinker(&vb->shrinker); + return register_shrinker(&vb->shrinker, "virtio-balloon"); } static int virtballoon_probe(struct virtio_device *vdev) diff --git a/drivers/xen/xenbus/xenbus_probe_backend.c b/drivers/xen/xenbus/xenbus_probe_backend.c index 5abded97e1a7..9c09f89d8278 100644 --- a/drivers/xen/xenbus/xenbus_probe_backend.c +++ b/drivers/xen/xenbus/xenbus_probe_backend.c @@ -305,7 +305,7 @@ static int __init xenbus_probe_backend_init(void) register_xenstore_notifier(&xenstore_notifier); - if (register_shrinker(&backend_memory_shrinker)) + if (register_shrinker(&backend_memory_shrinker, "xen-backend")) pr_warn("shrinker registration failed\n"); return 0; diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 6627dd7875ee..eee3e96d877f 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -1815,6 +1815,8 @@ static struct dentry *btrfs_mount_root(struct file_system_type *fs_type, error = -EBUSY; } else { snprintf(s->s_id, sizeof(s->s_id), "%pg", bdev); + shrinker_debugfs_rename(&s->s_shrink, "sb-%s:%s", fs_type->name, + s->s_id); btrfs_sb(s)->bdev_holder = fs_type; if (!strstr(crc32c_impl(), "generic")) set_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags); diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c index ec9a1d780dc1..46627cb69abe 100644 --- a/fs/erofs/utils.c +++ b/fs/erofs/utils.c @@ -282,7 +282,7 @@ static struct shrinker erofs_shrinker_info = { int __init erofs_init_shrinker(void) { - return register_shrinker(&erofs_shrinker_info); + return register_shrinker(&erofs_shrinker_info, "erofs-shrinker"); } void erofs_exit_shrinker(void) diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c index 9a3a8996aacf..23167efda95e 100644 --- a/fs/ext4/extents_status.c +++ b/fs/ext4/extents_status.c @@ -1654,7 +1654,8 @@ int ext4_es_register_shrinker(struct ext4_sb_info *sbi) sbi->s_es_shrinker.scan_objects = ext4_es_scan; sbi->s_es_shrinker.count_objects = ext4_es_count; sbi->s_es_shrinker.seeks = DEFAULT_SEEKS; - err = register_shrinker(&sbi->s_es_shrinker); + err = register_shrinker(&sbi->s_es_shrinker, "ext4-es:%s", + sbi->s_sb->s_id); if (err) goto err4; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 37221e94e5ef..bce02306f7a0 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -4579,7 +4579,7 @@ static int __init init_f2fs_fs(void) err = f2fs_init_sysfs(); if (err) goto free_garbage_collection_cache; - err = register_shrinker(&f2fs_shrinker_info); + err = register_shrinker(&f2fs_shrinker_info, "f2fs-shrinker"); if (err) goto free_sysfs; err = register_filesystem(&f2fs_fs_type); diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index c992d53013d3..dca842379cab 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -2533,7 +2533,7 @@ int __init gfs2_glock_init(void) return -ENOMEM; } - ret = register_shrinker(&glock_shrinker); + ret = register_shrinker(&glock_shrinker, "gfs2-glock"); if (ret) { destroy_workqueue(gfs2_delete_workqueue); destroy_workqueue(glock_workqueue); diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c index 244187e3e70f..b66a3e1ec152 100644 --- a/fs/gfs2/main.c +++ b/fs/gfs2/main.c @@ -148,7 +148,7 @@ static int __init init_gfs2_fs(void) if (!gfs2_trans_cachep) goto fail_cachep8; - error = register_shrinker(&gfs2_qd_shrinker); + error = register_shrinker(&gfs2_qd_shrinker, "gfs2-qd"); if (error) goto fail_shrinker; diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index c0cbeeaec2d1..45e4655c8033 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -1418,7 +1418,8 @@ static journal_t *journal_init_common(struct block_device *bdev, if (percpu_counter_init(&journal->j_checkpoint_jh_count, 0, GFP_KERNEL)) goto err_cleanup; - if (register_shrinker(&journal->j_shrinker)) { + if (register_shrinker(&journal->j_shrinker, "jbd2-journal:(%u:%u)", + MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev))) { percpu_counter_destroy(&journal->j_checkpoint_jh_count); goto err_cleanup; } diff --git a/fs/mbcache.c b/fs/mbcache.c index 97c54d3a2227..0b833da0a9a5 100644 --- a/fs/mbcache.c +++ b/fs/mbcache.c @@ -367,7 +367,7 @@ struct mb_cache *mb_cache_create(int bucket_bits) cache->c_shrink.count_objects = mb_cache_count; cache->c_shrink.scan_objects = mb_cache_scan; cache->c_shrink.seeks = DEFAULT_SEEKS; - if (register_shrinker(&cache->c_shrink)) { + if (register_shrinker(&cache->c_shrink, "mbcache-shrinker")) { kfree(cache->c_hash); kfree(cache); goto err_out; diff --git a/fs/nfs/nfs42xattr.c b/fs/nfs/nfs42xattr.c index e7b34f7e0614..a9bf09fdf2c3 100644 --- a/fs/nfs/nfs42xattr.c +++ b/fs/nfs/nfs42xattr.c @@ -1017,15 +1017,16 @@ int __init nfs4_xattr_cache_init(void) if (ret) goto out2; - ret = register_shrinker(&nfs4_xattr_cache_shrinker); + ret = register_shrinker(&nfs4_xattr_cache_shrinker, "nfs-xattr_cache"); if (ret) goto out1; - ret = register_shrinker(&nfs4_xattr_entry_shrinker); + ret = register_shrinker(&nfs4_xattr_entry_shrinker, "nfs-xattr_entry"); if (ret) goto out; - ret = register_shrinker(&nfs4_xattr_large_entry_shrinker); + ret = register_shrinker(&nfs4_xattr_large_entry_shrinker, + "nfs-xattr_large_entry"); if (!ret) return 0; diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 6ab5eeb000dc..82944e14fcea 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -149,7 +149,7 @@ int __init register_nfs_fs(void) ret = nfs_register_sysctl(); if (ret < 0) goto error_2; - ret = register_shrinker(&acl_shrinker); + ret = register_shrinker(&acl_shrinker, "nfs-acl"); if (ret < 0) goto error_3; #ifdef CONFIG_NFS_V4_2 diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 9cb2d590c036..a605c0e39b09 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -670,7 +670,7 @@ nfsd_file_cache_init(void) goto out_err; } - ret = register_shrinker(&nfsd_file_shrinker); + ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache"); if (ret) { pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret); goto out_lru; diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index 7da88bdc0d6c..9b31e1103e7b 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -176,7 +176,8 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) nn->nfsd_reply_cache_shrinker.scan_objects = nfsd_reply_cache_scan; nn->nfsd_reply_cache_shrinker.count_objects = nfsd_reply_cache_count; nn->nfsd_reply_cache_shrinker.seeks = 1; - status = register_shrinker(&nn->nfsd_reply_cache_shrinker); + status = register_shrinker(&nn->nfsd_reply_cache_shrinker, + "nfsd-reply:%s", nn->nfsd_name); if (status) goto out_stats_destroy; diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 09d1307959d0..e0b659900e70 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -2995,7 +2995,7 @@ static int __init dquot_init(void) pr_info("VFS: Dquot-cache hash table entries: %ld (order %ld," " %ld bytes)\n", nr_hash, order, (PAGE_SIZE << order)); - if (register_shrinker(&dqcache_shrinker)) + if (register_shrinker(&dqcache_shrinker, "dquota-cache")) panic("Cannot register dquot shrinker"); return 0; diff --git a/fs/super.c b/fs/super.c index 60f57c7bc0a6..4fca6657f442 100644 --- a/fs/super.c +++ b/fs/super.c @@ -265,7 +265,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, s->s_shrink.count_objects = super_cache_count; s->s_shrink.batch = 1024; s->s_shrink.flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE; - if (prealloc_shrinker(&s->s_shrink)) + if (prealloc_shrinker(&s->s_shrink, "sb-%s", type->name)) goto fail; if (list_lru_init_memcg(&s->s_dentry_lru, &s->s_shrink)) goto fail; @@ -1288,6 +1288,8 @@ int get_tree_bdev(struct fs_context *fc, } else { s->s_mode = mode; snprintf(s->s_id, sizeof(s->s_id), "%pg", bdev); + shrinker_debugfs_rename(&s->s_shrink, "sb-%s:%s", + fc->fs_type->name, s->s_id); sb_set_blocksize(s, block_size(bdev)); error = fill_super(s, fc); if (error) { @@ -1363,6 +1365,8 @@ struct dentry *mount_bdev(struct file_system_type *fs_type, } else { s->s_mode = mode; snprintf(s->s_id, sizeof(s->s_id), "%pg", bdev); + shrinker_debugfs_rename(&s->s_shrink, "sb-%s:%s", + fs_type->name, s->s_id); sb_set_blocksize(s, block_size(bdev)); error = fill_super(s, data, flags & SB_SILENT ? 1 : 0); if (error) { diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index 0978d01b0ea4..d0c9a09988bc 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c @@ -2430,7 +2430,7 @@ static int __init ubifs_init(void) if (!ubifs_inode_slab) return -ENOMEM; - err = register_shrinker(&ubifs_shrinker_info); + err = register_shrinker(&ubifs_shrinker_info, "ubifs-slab"); if (err) goto out_slab; diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index bf4e60871068..4aa9c9cf5b6e 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1986,7 +1986,8 @@ xfs_alloc_buftarg( btp->bt_shrinker.scan_objects = xfs_buftarg_shrink_scan; btp->bt_shrinker.seeks = DEFAULT_SEEKS; btp->bt_shrinker.flags = SHRINKER_NUMA_AWARE; - if (register_shrinker(&btp->bt_shrinker)) + if (register_shrinker(&btp->bt_shrinker, "xfs-buf:%s", + mp->m_super->s_id)) goto error_pcpu; return btp; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 5269354b1b69..a1941c8b8630 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -2201,5 +2201,5 @@ xfs_inodegc_register_shrinker( shrink->flags = SHRINKER_NONSLAB; shrink->batch = XFS_INODEGC_SHRINKER_BATCH; - return register_shrinker(shrink); + return register_shrinker(shrink, "xfs-inodegc:%s", mp->m_super->s_id); } diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index abf08bbf34a9..c31d57453ceb 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -677,7 +677,8 @@ xfs_qm_init_quotainfo( qinf->qi_shrinker.seeks = DEFAULT_SEEKS; qinf->qi_shrinker.flags = SHRINKER_NUMA_AWARE; - error = register_shrinker(&qinf->qi_shrinker); + error = register_shrinker(&qinf->qi_shrinker, "xfs-qm:%s", + mp->m_super->s_id); if (error) goto out_free_inos; diff --git a/include/linux/shrinker.h b/include/linux/shrinker.h index 2ced8149c513..08e6054e061f 100644 --- a/include/linux/shrinker.h +++ b/include/linux/shrinker.h @@ -75,6 +75,7 @@ struct shrinker { #endif #ifdef CONFIG_SHRINKER_DEBUG int debugfs_id; + const char *name; struct dentry *debugfs_entry; #endif /* objs pending delete, per node */ @@ -92,9 +93,11 @@ struct shrinker { */ #define SHRINKER_NONSLAB (1 << 3) -extern int prealloc_shrinker(struct shrinker *shrinker); +extern int __printf(2, 3) prealloc_shrinker(struct shrinker *shrinker, + const char *fmt, ...); extern void register_shrinker_prepared(struct shrinker *shrinker); -extern int register_shrinker(struct shrinker *shrinker); +extern int __printf(2, 3) register_shrinker(struct shrinker *shrinker, + const char *fmt, ...); extern void unregister_shrinker(struct shrinker *shrinker); extern void free_prealloced_shrinker(struct shrinker *shrinker); extern void synchronize_shrinkers(void); @@ -102,6 +105,8 @@ extern void synchronize_shrinkers(void); #ifdef CONFIG_SHRINKER_DEBUG extern int shrinker_debugfs_add(struct shrinker *shrinker); extern void shrinker_debugfs_remove(struct shrinker *shrinker); +extern int __printf(2, 3) shrinker_debugfs_rename(struct shrinker *shrinker, + const char *fmt, ...); #else /* CONFIG_SHRINKER_DEBUG */ static inline int shrinker_debugfs_add(struct shrinker *shrinker) { @@ -110,5 +115,10 @@ static inline int shrinker_debugfs_add(struct shrinker *shrinker) static inline void shrinker_debugfs_remove(struct shrinker *shrinker) { } +static inline __printf(2, 3) +int shrinker_debugfs_rename(struct shrinker *shrinker, const char *fmt, ...) +{ + return 0; +} #endif /* CONFIG_SHRINKER_DEBUG */ #endif /* _LINUX_SHRINKER_H */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c25ba442044a..4b3bf6ebb1eb 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4884,7 +4884,7 @@ static void __init kfree_rcu_batch_init(void) INIT_DELAYED_WORK(&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true; } - if (register_shrinker(&kfree_rcu_shrinker)) + if (register_shrinker(&kfree_rcu_shrinker, "rcu-kfree")) pr_err("Failed to register kfree_rcu() shrinker!\n"); } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index f9b90a8d7dfa..60d742c33de3 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -423,10 +423,10 @@ static int __init hugepage_init(void) if (err) goto err_slab; - err = register_shrinker(&huge_zero_page_shrinker); + err = register_shrinker(&huge_zero_page_shrinker, "thp-zero"); if (err) goto err_hzp_shrinker; - err = register_shrinker(&deferred_split_shrinker); + err = register_shrinker(&deferred_split_shrinker, "thp-deferred_split"); if (err) goto err_split_shrinker; diff --git a/mm/shrinker_debug.c b/mm/shrinker_debug.c index 1a70556bd46c..781ecbd3d608 100644 --- a/mm/shrinker_debug.c +++ b/mm/shrinker_debug.c @@ -102,7 +102,7 @@ DEFINE_SHOW_ATTRIBUTE(shrinker_debugfs_count); int shrinker_debugfs_add(struct shrinker *shrinker) { struct dentry *entry; - char buf[16]; + char buf[128]; int id; lockdep_assert_held(&shrinker_rwsem); @@ -116,7 +116,7 @@ int shrinker_debugfs_add(struct shrinker *shrinker) return id; shrinker->debugfs_id = id; - snprintf(buf, sizeof(buf), "%d", id); + snprintf(buf, sizeof(buf), "%s-%d", shrinker->name, id); /* create debugfs entry */ entry = debugfs_create_dir(buf, shrinker_debugfs_root); @@ -131,10 +131,53 @@ int shrinker_debugfs_add(struct shrinker *shrinker) return 0; } +int shrinker_debugfs_rename(struct shrinker *shrinker, const char *fmt, ...) +{ + struct dentry *entry; + char buf[128]; + const char *new, *old; + va_list ap; + int ret = 0; + + va_start(ap, fmt); + new = kvasprintf_const(GFP_KERNEL, fmt, ap); + va_end(ap); + + if (!new) + return -ENOMEM; + + down_write(&shrinker_rwsem); + + old = shrinker->name; + shrinker->name = new; + + if (shrinker->debugfs_entry) { + snprintf(buf, sizeof(buf), "%s-%d", shrinker->name, + shrinker->debugfs_id); + + entry = debugfs_rename(shrinker_debugfs_root, + shrinker->debugfs_entry, + shrinker_debugfs_root, buf); + if (IS_ERR(entry)) + ret = PTR_ERR(entry); + else + shrinker->debugfs_entry = entry; + } + + up_write(&shrinker_rwsem); + + kfree_const(old); + + return ret; +} +EXPORT_SYMBOL(shrinker_debugfs_rename); + void shrinker_debugfs_remove(struct shrinker *shrinker) { lockdep_assert_held(&shrinker_rwsem); + kfree_const(shrinker->name); + if (!shrinker->debugfs_entry) return; diff --git a/mm/vmscan.c b/mm/vmscan.c index 35dedff79eb4..97ac6c6c026d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -608,7 +608,7 @@ static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, /* * Add a shrinker callback to be called from the vm. */ -int prealloc_shrinker(struct shrinker *shrinker) +static int __prealloc_shrinker(struct shrinker *shrinker) { unsigned int size; int err; @@ -632,8 +632,36 @@ int prealloc_shrinker(struct shrinker *shrinker) return 0; } +#ifdef CONFIG_SHRINKER_DEBUG +int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) +{ + va_list ap; + int err; + + va_start(ap, fmt); + shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); + va_end(ap); + if (!shrinker->name) + return -ENOMEM; + + err = __prealloc_shrinker(shrinker); + if (err) + kfree_const(shrinker->name); + + return err; +} +#else +int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) +{ + return __prealloc_shrinker(shrinker); +} +#endif + void free_prealloced_shrinker(struct shrinker *shrinker) { +#ifdef CONFIG_SHRINKER_DEBUG + kfree_const(shrinker->name); +#endif if (shrinker->flags & SHRINKER_MEMCG_AWARE) { down_write(&shrinker_rwsem); unregister_memcg_shrinker(shrinker); @@ -654,15 +682,39 @@ void register_shrinker_prepared(struct shrinker *shrinker) up_write(&shrinker_rwsem); } -int register_shrinker(struct shrinker *shrinker) +static int __register_shrinker(struct shrinker *shrinker) { - int err = prealloc_shrinker(shrinker); + int err = __prealloc_shrinker(shrinker); if (err) return err; register_shrinker_prepared(shrinker); return 0; } + +#ifdef CONFIG_SHRINKER_DEBUG +int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) +{ + va_list ap; + int err; + + va_start(ap, fmt); + shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); + va_end(ap); + if (!shrinker->name) + return -ENOMEM; + + err = __register_shrinker(shrinker); + if (err) + kfree_const(shrinker->name); + return err; +} +#else +int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) +{ + return __register_shrinker(shrinker); +} +#endif EXPORT_SYMBOL(register_shrinker); /* diff --git a/mm/workingset.c b/mm/workingset.c index 592569a8974c..a5e84862fc86 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -625,7 +625,7 @@ static int __init workingset_init(void) pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n", timestamp_bits, max_order, bucket_order); - ret = prealloc_shrinker(&workingset_shadow_shrinker); + ret = prealloc_shrinker(&workingset_shadow_shrinker, "mm-shadow"); if (ret) goto err; ret = __list_lru_init(&shadow_nodes, true, &shadow_nodes_key, diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 5d5fc04385b8..f24b71568e83 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -2217,7 +2217,8 @@ static int zs_register_shrinker(struct zs_pool *pool) pool->shrinker.batch = 0; pool->shrinker.seeks = DEFAULT_SEEKS; - return register_shrinker(&pool->shrinker); + return register_shrinker(&pool->shrinker, "mm-zspool:%s", + pool->name); } /** diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 682fcd24bf43..04e7b55fe0d9 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -874,7 +874,7 @@ int __init rpcauth_init_module(void) err = rpc_init_authunix(); if (err < 0) goto out1; - err = register_shrinker(&rpc_cred_shrinker); + err = register_shrinker(&rpc_cred_shrinker, "sunrpc_cred"); if (err < 0) goto out2; return 0; -- cgit v1.2.3 From 8cdcc532268df0893d9756f537cbce479f4c4831 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 13 Jun 2022 19:22:56 +0000 Subject: mm/damon/schemes: add 'LRU_PRIO' DAMOS action This commit adds a new DAMOS action called 'LRU_PRIO' for the physical address space. The action prioritizes pages in the memory regions of the user-specified target access pattern on their LRU lists. This is hence supposed to be used for frequently accessed (hot) memory regions so that hot pages could be more likely protected under memory pressure. Internally, it simply calls 'mark_page_accessed()'. Link: https://lkml.kernel.org/r/20220613192301.8817-5-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 ++ mm/damon/ops-common.c | 42 ++++++++++++++++++++++++++++++++++++++++++ mm/damon/ops-common.h | 2 ++ mm/damon/paddr.c | 20 ++++++++++++++++++++ mm/damon/sysfs.c | 1 + 5 files changed, 67 insertions(+) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index b9aae19fab3e..4c64e03e94d8 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -86,6 +86,7 @@ struct damon_target { * @DAMOS_PAGEOUT: Call ``madvise()`` for the region with MADV_PAGEOUT. * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. + * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. * @DAMOS_STAT: Do nothing but count the stat. * @NR_DAMOS_ACTIONS: Total number of DAMOS actions */ @@ -95,6 +96,7 @@ enum damos_action { DAMOS_PAGEOUT, DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, + DAMOS_LRU_PRIO, DAMOS_STAT, /* Do nothing but only record the stat */ NR_DAMOS_ACTIONS, }; diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 10ef20b2003f..b1335de200e7 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -130,3 +130,45 @@ int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, /* Return coldness of the region */ return DAMOS_MAX_SCORE - hotness; } + +int damon_hot_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s) +{ + unsigned int max_nr_accesses; + int freq_subscore; + unsigned int age_in_sec; + int age_in_log, age_subscore; + unsigned int freq_weight = s->quota.weight_nr_accesses; + unsigned int age_weight = s->quota.weight_age; + int hotness; + + max_nr_accesses = c->aggr_interval / c->sample_interval; + freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses; + + age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000; + for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; + age_in_log++, age_in_sec >>= 1) + ; + + /* If frequency is 0, higher age means it's colder */ + if (freq_subscore == 0) + age_in_log *= -1; + + /* + * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. + * Scale it to be in [0, 100] and set it as age subscore. + */ + age_in_log += DAMON_MAX_AGE_IN_LOG; + age_subscore = age_in_log * DAMON_MAX_SUBSCORE / + DAMON_MAX_AGE_IN_LOG / 2; + + hotness = (freq_weight * freq_subscore + age_weight * age_subscore); + if (freq_weight + age_weight) + hotness /= freq_weight + age_weight; + /* + * Transform it to fit in [0, DAMOS_MAX_SCORE] + */ + hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; + + return hotness; +} diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index e790cb5f8fe0..52329ff361cd 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -14,3 +14,5 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr); int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, struct damos *s); +int damon_hot_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 7bcd48066b43..f145b1d51e13 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -233,6 +233,22 @@ static unsigned long damon_pa_pageout(struct damon_region *r) return applied * PAGE_SIZE; } +static unsigned long damon_pa_mark_accessed(struct damon_region *r) +{ + unsigned long addr, applied = 0; + + for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { + struct page *page = damon_get_page(PHYS_PFN(addr)); + + if (!page) + continue; + mark_page_accessed(page); + put_page(page); + applied++; + } + return applied * PAGE_SIZE; +} + static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme) @@ -240,6 +256,8 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, switch (scheme->action) { case DAMOS_PAGEOUT: return damon_pa_pageout(r); + case DAMOS_LRU_PRIO: + return damon_pa_mark_accessed(r); default: break; } @@ -253,6 +271,8 @@ static int damon_pa_scheme_score(struct damon_ctx *context, switch (scheme->action) { case DAMOS_PAGEOUT: return damon_pageout_score(context, r, scheme); + case DAMOS_LRU_PRIO: + return damon_hot_score(context, r, scheme); default: break; } diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index c35809c6087c..86c69f980927 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -762,6 +762,7 @@ static const char * const damon_sysfs_damos_action_strs[] = { "pageout", "hugepage", "nohugepage", + "lru_prio", "stat", }; -- cgit v1.2.3 From 99cdc2cd180a7adc87badc9ca92f8af803d8bf3b Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 13 Jun 2022 19:22:58 +0000 Subject: mm/damon/schemes: add 'LRU_DEPRIO' action This commit adds a new DAMON-based operation scheme action called 'LRU_DEPRIO' for physical address space. The action deprioritizes pages in the memory area of the target access pattern on their LRU lists. This is hence supposed to be used for rarely accessed (cold) memory regions so that cold pages could be more likely reclaimed first under memory pressure. Internally, it simply calls 'lru_deactivate()'. Using this with 'LRU_PRIO' action for hot pages, users can proactively sort LRU lists based on the access pattern. That is, it can make the LRU lists somewhat more trustworthy source of access temperature. As a result, efficiency of LRU-lists based mechanisms including the reclamation target selection could be improved. Link: https://lkml.kernel.org/r/20220613192301.8817-7-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 ++ mm/damon/paddr.c | 20 ++++++++++++++++++++ mm/damon/sysfs.c | 1 + 3 files changed, 23 insertions(+) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 4c64e03e94d8..7b1f4a488230 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -87,6 +87,7 @@ struct damon_target { * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. + * @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists. * @DAMOS_STAT: Do nothing but count the stat. * @NR_DAMOS_ACTIONS: Total number of DAMOS actions */ @@ -97,6 +98,7 @@ enum damos_action { DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, DAMOS_LRU_PRIO, + DAMOS_LRU_DEPRIO, DAMOS_STAT, /* Do nothing but only record the stat */ NR_DAMOS_ACTIONS, }; diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index f145b1d51e13..dc131c6a5403 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -249,6 +249,22 @@ static unsigned long damon_pa_mark_accessed(struct damon_region *r) return applied * PAGE_SIZE; } +static unsigned long damon_pa_deactivate_pages(struct damon_region *r) +{ + unsigned long addr, applied = 0; + + for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { + struct page *page = damon_get_page(PHYS_PFN(addr)); + + if (!page) + continue; + deactivate_page(page); + put_page(page); + applied++; + } + return applied * PAGE_SIZE; +} + static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme) @@ -258,6 +274,8 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, return damon_pa_pageout(r); case DAMOS_LRU_PRIO: return damon_pa_mark_accessed(r); + case DAMOS_LRU_DEPRIO: + return damon_pa_deactivate_pages(r); default: break; } @@ -273,6 +291,8 @@ static int damon_pa_scheme_score(struct damon_ctx *context, return damon_pageout_score(context, r, scheme); case DAMOS_LRU_PRIO: return damon_hot_score(context, r, scheme); + case DAMOS_LRU_DEPRIO: + return damon_pageout_score(context, r, scheme); default: break; } diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 86c69f980927..7488e27c87c3 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -763,6 +763,7 @@ static const char * const damon_sysfs_damos_action_strs[] = { "hugepage", "nohugepage", "lru_prio", + "lru_deprio", "stat", }; -- cgit v1.2.3 From 64fe24a3e05e5f3ac56fcd45afd2fd1d9cc8fcb6 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 14 Jun 2022 11:36:29 +0200 Subject: mm/mprotect: try avoiding write faults for exclusive anonymous pages when changing protection Similar to our MM_CP_DIRTY_ACCT handling for shared, writable mappings, we can try mapping anonymous pages in a private writable mapping writable if they are exclusive, the PTE is already dirty, and no special handling applies. Mapping the anonymous page writable is essentially the same thing the write fault handler would do in this case. Special handling is required for uffd-wp and softdirty tracking, so take care of that properly. Also, leave PROT_NONE handling alone for now; in the future, we could similarly extend the logic in do_numa_page() or use pte_mk_savedwrite() here. While this improves mprotect(PROT_READ)+mprotect(PROT_READ|PROT_WRITE) performance, it should also be a valuable optimization for uffd-wp, when un-protecting. This has been previously suggested by Peter Collingbourne in [1], relevant in the context of the Scudo memory allocator, before we had PageAnonExclusive. This commit doesn't add the same handling for PMDs (i.e., anonymous THP, anonymous hugetlb); benchmark results from Andrea indicate that there are minor performance gains, so it's might still be valuable to streamline that logic for all anonymous pages in the future. As we now also set MM_CP_DIRTY_ACCT for private mappings, let's rename it to MM_CP_TRY_CHANGE_WRITABLE, to make it clearer what's actually happening. Micro-benchmark courtesy of Andrea: === #define _GNU_SOURCE #include #include #include #include #include #define SIZE (1024*1024*1024) int main(int argc, char *argv[]) { char *p; if (posix_memalign((void **)&p, sysconf(_SC_PAGESIZE)*512, SIZE)) perror("posix_memalign"), exit(1); if (madvise(p, SIZE, argc > 1 ? MADV_HUGEPAGE : MADV_NOHUGEPAGE)) perror("madvise"); explicit_bzero(p, SIZE); for (int loops = 0; loops < 40; loops++) { if (mprotect(p, SIZE, PROT_READ)) perror("mprotect"), exit(1); if (mprotect(p, SIZE, PROT_READ|PROT_WRITE)) perror("mprotect"), exit(1); explicit_bzero(p, SIZE); } } === Results on my Ryzen 9 3900X: Stock 10 runs (lower is better): AVG 6.398s, STDEV 0.043 Patched 10 runs (lower is better): AVG 3.780s, STDEV 0.026 === [1] https://lkml.kernel.org/r/20210429214801.2583336-1-pcc@google.com Link: https://lkml.kernel.org/r/20220614093629.76309-1-david@redhat.com Signed-off-by: David Hildenbrand Suggested-by: Peter Collingbourne Acked-by: Peter Xu Cc: Nadav Amit Cc: Dave Hansen Cc: Andrea Arcangeli Cc: Yang Shi Cc: Hugh Dickins Cc: Mel Gorman Signed-off-by: Andrew Morton --- include/linux/mm.h | 8 ++++-- mm/mprotect.c | 77 +++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 68 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index cf3d0d673f6b..09ea26056e2f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1962,8 +1962,12 @@ extern unsigned long move_page_tables(struct vm_area_struct *vma, * for now all the callers are only use one of the flags at the same * time. */ -/* Whether we should allow dirty bit accounting */ -#define MM_CP_DIRTY_ACCT (1UL << 0) +/* + * Whether we should manually check if we can map individual PTEs writable, + * because something (e.g., COW, uffd-wp) blocks that from happening for all + * PTEs automatically in a writable mapping. + */ +#define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0) /* Whether this protection change is for NUMA hints */ #define MM_CP_PROT_NUMA (1UL << 1) /* Whether this change is for write protecting */ diff --git a/mm/mprotect.c b/mm/mprotect.c index ba5592655ee3..996a97e213ad 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -38,6 +38,39 @@ #include "internal.h" +static inline bool can_change_pte_writable(struct vm_area_struct *vma, + unsigned long addr, pte_t pte) +{ + struct page *page; + + VM_BUG_ON(!(vma->vm_flags & VM_WRITE) || pte_write(pte)); + + if (pte_protnone(pte) || !pte_dirty(pte)) + return false; + + /* Do we need write faults for softdirty tracking? */ + if ((vma->vm_flags & VM_SOFTDIRTY) && !pte_soft_dirty(pte)) + return false; + + /* Do we need write faults for uffd-wp tracking? */ + if (userfaultfd_pte_wp(vma, pte)) + return false; + + if (!(vma->vm_flags & VM_SHARED)) { + /* + * We can only special-case on exclusive anonymous pages, + * because we know that our write-fault handler similarly would + * map them writable without any additional checks while holding + * the PT lock. + */ + page = vm_normal_page(vma, addr, pte); + if (!page || !PageAnon(page) || !PageAnonExclusive(page)) + return false; + } + + return true; +} + static unsigned long change_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, unsigned long cp_flags) @@ -46,7 +79,6 @@ static unsigned long change_pte_range(struct mmu_gather *tlb, spinlock_t *ptl; unsigned long pages = 0; int target_node = NUMA_NO_NODE; - bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; @@ -137,21 +169,27 @@ static unsigned long change_pte_range(struct mmu_gather *tlb, ptent = pte_wrprotect(ptent); ptent = pte_mkuffd_wp(ptent); } else if (uffd_wp_resolve) { - /* - * Leave the write bit to be handled - * by PF interrupt handler, then - * things like COW could be properly - * handled. - */ ptent = pte_clear_uffd_wp(ptent); } - /* Avoid taking write faults for known dirty pages */ - if (dirty_accountable && pte_dirty(ptent) && - (pte_soft_dirty(ptent) || - !(vma->vm_flags & VM_SOFTDIRTY))) { + /* + * In some writable, shared mappings, we might want + * to catch actual write access -- see + * vma_wants_writenotify(). + * + * In all writable, private mappings, we have to + * properly handle COW. + * + * In both cases, we can sometimes still change PTEs + * writable and avoid the write-fault handler, for + * example, if a PTE is already dirty and no other + * COW or special handling is required. + */ + if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && + !pte_write(ptent) && + can_change_pte_writable(vma, addr, ptent)) ptent = pte_mkwrite(ptent); - } + ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent); if (pte_needs_flush(oldpte, ptent)) tlb_flush_pte_range(tlb, addr, PAGE_SIZE); @@ -505,9 +543,9 @@ mprotect_fixup(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long oldflags = vma->vm_flags; long nrpages = (end - start) >> PAGE_SHIFT; unsigned long charged = 0; + bool try_change_writable; pgoff_t pgoff; int error; - int dirty_accountable = 0; if (newflags == oldflags) { *pprev = vma; @@ -583,11 +621,20 @@ success: * held in write mode. */ vma->vm_flags = newflags; - dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); + /* + * We want to check manually if we can change individual PTEs writable + * if we can't do that automatically for all PTEs in a mapping. For + * private mappings, that's always the case when we have write + * permissions as we properly have to handle COW. + */ + if (vma->vm_flags & VM_SHARED) + try_change_writable = vma_wants_writenotify(vma, vma->vm_page_prot); + else + try_change_writable = !!(vma->vm_flags & VM_WRITE); vma_set_page_prot(vma); change_protection(tlb, vma, start, end, vma->vm_page_prot, - dirty_accountable ? MM_CP_DIRTY_ACCT : 0); + try_change_writable ? MM_CP_TRY_CHANGE_WRITABLE : 0); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major -- cgit v1.2.3 From b8cecb9376b9d3031cf62b476a0db087b6b01072 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jun 2022 16:42:44 +0100 Subject: mm/vmscan: convert reclaim_clean_pages_from_list() to folios Patch series "nvert much of vmscan to folios" vmscan always operates on folios since it puts the pages on the LRU list. Switching all of these functions from pages to folios saves 1483 bytes of text from removing all the baggage around calling compound_page() and similar functions. This patch (of 5): This is a straightforward conversion which removes several hidden calls to compound_head, saving 330 bytes of kernel text. Link: https://lkml.kernel.org/r/20220617154248.700416-1-willy@infradead.org Link: https://lkml.kernel.org/r/20220617154248.700416-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 6 ++++++ mm/vmscan.c | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index e66f7aa3191d..f32aade2a6e0 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -670,6 +670,12 @@ static __always_inline bool PageAnon(struct page *page) return folio_test_anon(page_folio(page)); } +static __always_inline bool __folio_test_movable(const struct folio *folio) +{ + return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) == + PAGE_MAPPING_MOVABLE; +} + static __always_inline int __PageMovable(struct page *page) { return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == diff --git a/mm/vmscan.c b/mm/vmscan.c index 97ac6c6c026d..2ecca45672e2 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2041,7 +2041,7 @@ keep: } unsigned int reclaim_clean_pages_from_list(struct zone *zone, - struct list_head *page_list) + struct list_head *folio_list) { struct scan_control sc = { .gfp_mask = GFP_KERNEL, @@ -2049,16 +2049,16 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, }; struct reclaim_stat stat; unsigned int nr_reclaimed; - struct page *page, *next; - LIST_HEAD(clean_pages); + struct folio *folio, *next; + LIST_HEAD(clean_folios); unsigned int noreclaim_flag; - list_for_each_entry_safe(page, next, page_list, lru) { - if (!PageHuge(page) && page_is_file_lru(page) && - !PageDirty(page) && !__PageMovable(page) && - !PageUnevictable(page)) { - ClearPageActive(page); - list_move(&page->lru, &clean_pages); + list_for_each_entry_safe(folio, next, folio_list, lru) { + if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && + !folio_test_dirty(folio) && !__folio_test_movable(folio) && + !folio_test_unevictable(folio)) { + folio_clear_active(folio); + list_move(&folio->lru, &clean_folios); } } @@ -2069,11 +2069,11 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, * change in the future. */ noreclaim_flag = memalloc_noreclaim_save(); - nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc, + nr_reclaimed = shrink_page_list(&clean_folios, zone->zone_pgdat, &sc, &stat, true); memalloc_noreclaim_restore(noreclaim_flag); - list_splice(&clean_pages, page_list); + list_splice(&clean_folios, folio_list); mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -(long)nr_reclaimed); /* -- cgit v1.2.3 From e3c4cebf3f9db8c9150eb1982da7e353d9938bed Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jun 2022 18:49:59 +0100 Subject: mm: add folios_put() Patch series "Convert the swap code to be more folio-based". There's still more to do with the swap code, but this reaps a lot of the folio benefit. More than 4kB of kernel text saved (with the UEK7 kernel config). I don't know how much that's going to translate into CPU savings, but some of those compound_head() calls are on every page free, so it should be noticable. It might even be noticable just from an I-cache consumption perspective. This patch (of 22): This is just a wrapper around release_pages() for now. Place the prototype in mm.h along with folio_put() and folio_put_refs(). Link: https://lkml.kernel.org/r/20220617175020.717127-1-willy@infradead.org Link: https://lkml.kernel.org/r/20220617175020.717127-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 19 +++++++++++++++++++ include/linux/pagemap.h | 2 -- 2 files changed, 19 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 09ea26056e2f..09670ccb94e7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1220,6 +1220,25 @@ static inline void folio_put_refs(struct folio *folio, int refs) __put_page(&folio->page); } +void release_pages(struct page **pages, int nr); + +/** + * folios_put - Decrement the reference count on an array of folios. + * @folios: The folios. + * @nr: How many folios there are. + * + * Like folio_put(), but for an array of folios. This is more efficient + * than writing the loop yourself as it will optimise the locks which + * need to be taken if the folios are freed. + * + * Context: May be called in process or interrupt context, but not in NMI + * context. May be called while holding a spinlock. + */ +static inline void folios_put(struct folio **folios, unsigned int nr) +{ + release_pages((struct page **)folios, nr); +} + static inline void put_page(struct page *page) { struct folio *folio = page_folio(page); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ce96866fbec4..c399a9c5da7d 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -345,8 +345,6 @@ static inline void filemap_nr_thps_dec(struct address_space *mapping) #endif } -void release_pages(struct page **pages, int nr); - struct address_space *page_mapping(struct page *); struct address_space *folio_mapping(struct folio *); struct address_space *swapcache_mapping(struct folio *); -- cgit v1.2.3 From 7d80dd096f8f889128f67a2d452e4dadeed71e63 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jun 2022 18:50:01 +0100 Subject: mm/swap: make __pagevec_lru_add static __pagevec_lru_add has no callers outside swap.c, so make it static, and move it to a more logical position in the file. Link: https://lkml.kernel.org/r/20220617175020.717127-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/pagevec.h | 1 - mm/swap.c | 126 ++++++++++++++++++++++++------------------------ 2 files changed, 63 insertions(+), 64 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index 67b1246f136b..b0e3540f3a4c 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -26,7 +26,6 @@ struct pagevec { }; void __pagevec_release(struct pagevec *pvec); -void __pagevec_lru_add(struct pagevec *pvec); unsigned pagevec_lookup_range(struct pagevec *pvec, struct address_space *mapping, pgoff_t *start, pgoff_t end); diff --git a/mm/swap.c b/mm/swap.c index 4265bee41bbd..cab77a5c64c7 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -228,6 +228,69 @@ static bool pagevec_add_and_need_flush(struct pagevec *pvec, struct page *page) typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio); +static void __pagevec_lru_add_fn(struct folio *folio, struct lruvec *lruvec) +{ + int was_unevictable = folio_test_clear_unevictable(folio); + long nr_pages = folio_nr_pages(folio); + + VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + + folio_set_lru(folio); + /* + * Is an smp_mb__after_atomic() still required here, before + * folio_evictable() tests PageMlocked, to rule out the possibility + * of stranding an evictable folio on an unevictable LRU? I think + * not, because __munlock_page() only clears PageMlocked while the LRU + * lock is held. + * + * (That is not true of __page_cache_release(), and not necessarily + * true of release_pages(): but those only clear PageMlocked after + * put_page_testzero() has excluded any other users of the page.) + */ + if (folio_evictable(folio)) { + if (was_unevictable) + __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); + } else { + folio_clear_active(folio); + folio_set_unevictable(folio); + /* + * folio->mlock_count = !!folio_test_mlocked(folio)? + * But that leaves __mlock_page() in doubt whether another + * actor has already counted the mlock or not. Err on the + * safe side, underestimate, let page reclaim fix it, rather + * than leaving a page on the unevictable LRU indefinitely. + */ + folio->mlock_count = 0; + if (!was_unevictable) + __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); + } + + lruvec_add_folio(lruvec, folio); + trace_mm_lru_insertion(folio); +} + +/* + * Add the passed pages to the LRU, then drop the caller's refcount + * on them. Reinitialises the caller's pagevec. + */ +static void __pagevec_lru_add(struct pagevec *pvec) +{ + int i; + struct lruvec *lruvec = NULL; + unsigned long flags = 0; + + for (i = 0; i < pagevec_count(pvec); i++) { + struct folio *folio = page_folio(pvec->pages[i]); + + lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags); + __pagevec_lru_add_fn(folio, lruvec); + } + if (lruvec) + unlock_page_lruvec_irqrestore(lruvec, flags); + release_pages(pvec->pages, pvec->nr); + pagevec_reinit(pvec); +} + static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) { int i; @@ -1036,69 +1099,6 @@ void __pagevec_release(struct pagevec *pvec) } EXPORT_SYMBOL(__pagevec_release); -static void __pagevec_lru_add_fn(struct folio *folio, struct lruvec *lruvec) -{ - int was_unevictable = folio_test_clear_unevictable(folio); - long nr_pages = folio_nr_pages(folio); - - VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); - - folio_set_lru(folio); - /* - * Is an smp_mb__after_atomic() still required here, before - * folio_evictable() tests PageMlocked, to rule out the possibility - * of stranding an evictable folio on an unevictable LRU? I think - * not, because __munlock_page() only clears PageMlocked while the LRU - * lock is held. - * - * (That is not true of __page_cache_release(), and not necessarily - * true of release_pages(): but those only clear PageMlocked after - * put_page_testzero() has excluded any other users of the page.) - */ - if (folio_evictable(folio)) { - if (was_unevictable) - __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); - } else { - folio_clear_active(folio); - folio_set_unevictable(folio); - /* - * folio->mlock_count = !!folio_test_mlocked(folio)? - * But that leaves __mlock_page() in doubt whether another - * actor has already counted the mlock or not. Err on the - * safe side, underestimate, let page reclaim fix it, rather - * than leaving a page on the unevictable LRU indefinitely. - */ - folio->mlock_count = 0; - if (!was_unevictable) - __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); - } - - lruvec_add_folio(lruvec, folio); - trace_mm_lru_insertion(folio); -} - -/* - * Add the passed pages to the LRU, then drop the caller's refcount - * on them. Reinitialises the caller's pagevec. - */ -void __pagevec_lru_add(struct pagevec *pvec) -{ - int i; - struct lruvec *lruvec = NULL; - unsigned long flags = 0; - - for (i = 0; i < pagevec_count(pvec); i++) { - struct folio *folio = page_folio(pvec->pages[i]); - - lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags); - __pagevec_lru_add_fn(folio, lruvec); - } - if (lruvec) - unlock_page_lruvec_irqrestore(lruvec, flags); - release_pages(pvec->pages, pvec->nr); - pagevec_reinit(pvec); -} - /** * folio_batch_remove_exceptionals() - Prune non-folios from a batch. * @fbatch: The batch to prune -- cgit v1.2.3 From 8d29c7036f5ff360ea1f51b9fed5d909be7c8094 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jun 2022 18:50:13 +0100 Subject: mm/swap: convert __put_page() to __folio_put() Saves 11 bytes of text by removing a check of PageTail. Link: https://lkml.kernel.org/r/20220617175020.717127-16-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 6 +++--- mm/swap.c | 14 +++++++------- net/core/page_pool.c | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 09670ccb94e7..3fb49aec13fd 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -855,7 +855,7 @@ static inline struct folio *virt_to_folio(const void *x) return page_folio(page); } -void __put_page(struct page *page); +void __folio_put(struct folio *folio); void put_pages_list(struct list_head *pages); @@ -1197,7 +1197,7 @@ static inline __must_check bool try_get_page(struct page *page) static inline void folio_put(struct folio *folio) { if (folio_put_testzero(folio)) - __put_page(&folio->page); + __folio_put(folio); } /** @@ -1217,7 +1217,7 @@ static inline void folio_put(struct folio *folio) static inline void folio_put_refs(struct folio *folio, int refs) { if (folio_ref_sub_and_test(folio, refs)) - __put_page(&folio->page); + __folio_put(folio); } void release_pages(struct page **pages, int nr); diff --git a/mm/swap.c b/mm/swap.c index a5a91aec83da..d09e9ac53809 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -119,16 +119,16 @@ static void __put_compound_page(struct page *page) destroy_compound_page(page); } -void __put_page(struct page *page) +void __folio_put(struct folio *folio) { - if (unlikely(is_zone_device_page(page))) - free_zone_device_page(page); - else if (unlikely(PageCompound(page))) - __put_compound_page(page); + if (unlikely(folio_is_zone_device(folio))) + free_zone_device_page(&folio->page); + else if (unlikely(folio_test_large(folio))) + __put_compound_page(&folio->page); else - __put_single_page(page); + __put_single_page(&folio->page); } -EXPORT_SYMBOL(__put_page); +EXPORT_SYMBOL(__folio_put); /** * put_pages_list() - release a list of pages diff --git a/net/core/page_pool.c b/net/core/page_pool.c index f18e6e771993..db70e94c8df2 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -16,7 +16,7 @@ #include #include #include -#include /* for __put_page() */ +#include /* for put_page() */ #include #include -- cgit v1.2.3 From 5375336c8c42a343c3b440b6f1e21c65e7b174b9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jun 2022 18:50:17 +0100 Subject: mm: convert destroy_compound_page() to destroy_large_folio() All callers now have a folio, so push the folio->page conversion down to this function. [akpm@linux-foundation.org: uninline destroy_large_folio() to fix build issue] Link: https://lkml.kernel.org/r/20220617175020.717127-20-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 6 +----- mm/page_alloc.c | 8 ++++++++ mm/swap.c | 2 +- mm/vmscan.c | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 3fb49aec13fd..9cc02a7e503b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -892,11 +892,7 @@ static inline void set_compound_page_dtor(struct page *page, page[1].compound_dtor = compound_dtor; } -static inline void destroy_compound_page(struct page *page) -{ - VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page); - compound_page_dtors[page[1].compound_dtor](page); -} +void destroy_large_folio(struct folio *folio); static inline int head_compound_pincount(struct page *head) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 248469134962..52fd92b2c1fe 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -744,6 +744,14 @@ void prep_compound_page(struct page *page, unsigned int order) prep_compound_head(page, order); } +void destroy_large_folio(struct folio *folio) +{ + enum compound_dtor_id dtor = folio_page(folio, 1)->compound_dtor; + + VM_BUG_ON_FOLIO(dtor >= NR_COMPOUND_DTORS, folio); + compound_page_dtors[dtor](&folio->page); +} + #ifdef CONFIG_DEBUG_PAGEALLOC unsigned int _debug_guardpage_minorder; diff --git a/mm/swap.c b/mm/swap.c index 5f6caa651599..1f563d857768 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -115,7 +115,7 @@ static void __folio_put_large(struct folio *folio) */ if (!folio_test_hugetlb(folio)) __page_cache_release(folio); - destroy_compound_page(&folio->page); + destroy_large_folio(folio); } void __folio_put(struct folio *folio) diff --git a/mm/vmscan.c b/mm/vmscan.c index e7d3db64a4e0..e660d7205f47 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1979,7 +1979,7 @@ free_it: * appear not as the counts should be low */ if (unlikely(folio_test_large(folio))) - destroy_compound_page(&folio->page); + destroy_large_folio(folio); else list_add(&folio->lru, &free_pages); continue; @@ -2348,7 +2348,7 @@ static unsigned int move_pages_to_lru(struct lruvec *lruvec, if (unlikely(folio_test_large(folio))) { spin_unlock_irq(&lruvec->lru_lock); - destroy_compound_page(&folio->page); + destroy_large_folio(folio); spin_lock_irq(&lruvec->lru_lock); } else list_add(&folio->lru, &folios_to_free); -- cgit v1.2.3 From ed7802dd48f7a507213cbb95bb4c6f1fe134eb5d Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 17 Jun 2022 21:56:49 +0800 Subject: mm: memory_hotplug: enumerate all supported section flags Patch series "make hugetlb_optimize_vmemmap compatible with memmap_on_memory", v3. This series makes hugetlb_optimize_vmemmap compatible with memmap_on_memory. This patch (of 2): We are almost running out of section flags, only one bit is available in the worst case (powerpc with 256k pages). However, there are still some free bits (in ->section_mem_map) on other architectures (e.g. x86_64 has 10 bits available, arm64 has 8 bits available with worst case of 64K pages). We have hard coded those numbers in code, it is inconvenient to use those bits on other architectures except powerpc. So transfer those section flags to enumeration to make it easy to add new section flags in the future. Also, move SECTION_TAINT_ZONE_DEVICE into the scope of CONFIG_ZONE_DEVICE to save a bit on non-zone-device case. [songmuchun@bytedance.com: replace enum with defines per David] Link: https://lkml.kernel.org/r/20220620110616.12056-2-songmuchun@bytedance.com Link: https://lkml.kernel.org/r/20220617135650.74901-1-songmuchun@bytedance.com Link: https://lkml.kernel.org/r/20220617135650.74901-2-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: David Hildenbrand Cc: Jonathan Corbet Cc: Mike Kravetz Cc: Oscar Salvador Cc: Paul E. McKenney Cc: Xiongchun Duan Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 41 ++++++++++++++++++++++++++++++++--------- mm/memory_hotplug.c | 6 ++++++ mm/sparse.c | 2 +- 3 files changed, 39 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index aab70355d64f..2b5757752333 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1418,16 +1418,32 @@ extern size_t mem_section_usage_size(void); * (equal SECTION_SIZE_BITS - PAGE_SHIFT), and the * worst combination is powerpc with 256k pages, * which results in PFN_SECTION_SHIFT equal 6. - * To sum it up, at least 6 bits are available. + * To sum it up, at least 6 bits are available on all architectures. + * However, we can exceed 6 bits on some other architectures except + * powerpc (e.g. 15 bits are available on x86_64, 13 bits are available + * with the worst case of 64K pages on arm64) if we make sure the + * exceeded bit is not applicable to powerpc. */ -#define SECTION_MARKED_PRESENT (1UL<<0) -#define SECTION_HAS_MEM_MAP (1UL<<1) -#define SECTION_IS_ONLINE (1UL<<2) -#define SECTION_IS_EARLY (1UL<<3) -#define SECTION_TAINT_ZONE_DEVICE (1UL<<4) -#define SECTION_MAP_LAST_BIT (1UL<<5) -#define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1)) -#define SECTION_NID_SHIFT 6 +enum { + SECTION_MARKED_PRESENT_BIT, + SECTION_HAS_MEM_MAP_BIT, + SECTION_IS_ONLINE_BIT, + SECTION_IS_EARLY_BIT, +#ifdef CONFIG_ZONE_DEVICE + SECTION_TAINT_ZONE_DEVICE_BIT, +#endif + SECTION_MAP_LAST_BIT, +}; + +#define SECTION_MARKED_PRESENT BIT(SECTION_MARKED_PRESENT_BIT) +#define SECTION_HAS_MEM_MAP BIT(SECTION_HAS_MEM_MAP_BIT) +#define SECTION_IS_ONLINE BIT(SECTION_IS_ONLINE_BIT) +#define SECTION_IS_EARLY BIT(SECTION_IS_EARLY_BIT) +#ifdef CONFIG_ZONE_DEVICE +#define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT) +#endif +#define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1)) +#define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT static inline struct page *__section_mem_map_addr(struct mem_section *section) { @@ -1466,12 +1482,19 @@ static inline int online_section(struct mem_section *section) return (section && (section->section_mem_map & SECTION_IS_ONLINE)); } +#ifdef CONFIG_ZONE_DEVICE static inline int online_device_section(struct mem_section *section) { unsigned long flags = SECTION_IS_ONLINE | SECTION_TAINT_ZONE_DEVICE; return section && ((section->section_mem_map & flags) == flags); } +#else +static inline int online_device_section(struct mem_section *section) +{ + return 0; +} +#endif static inline int online_section_nr(unsigned long nr) { diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 84990a14d51a..a2a6d280054f 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -670,12 +670,18 @@ static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned lon } +#ifdef CONFIG_ZONE_DEVICE static void section_taint_zone_device(unsigned long pfn) { struct mem_section *ms = __pfn_to_section(pfn); ms->section_mem_map |= SECTION_TAINT_ZONE_DEVICE; } +#else +static inline void section_taint_zone_device(unsigned long pfn) +{ +} +#endif /* * Associate the pfn range with the given zone, initializing the memmaps diff --git a/mm/sparse.c b/mm/sparse.c index cb3bfae64036..e5a8a3a0edd7 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -281,7 +281,7 @@ static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long p { unsigned long coded_mem_map = (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); - BUILD_BUG_ON(SECTION_MAP_LAST_BIT > (1UL< PFN_SECTION_SHIFT); BUG_ON(coded_mem_map & ~SECTION_MAP_MASK); return coded_mem_map; } -- cgit v1.2.3 From 66361095129b3b5d065e6c09cf0c085ef4a8c40f Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 17 Jun 2022 21:56:50 +0800 Subject: mm: memory_hotplug: make hugetlb_optimize_vmemmap compatible with memmap_on_memory For now, the feature of hugetlb_free_vmemmap is not compatible with the feature of memory_hotplug.memmap_on_memory, and hugetlb_free_vmemmap takes precedence over memory_hotplug.memmap_on_memory. However, someone wants to make memory_hotplug.memmap_on_memory takes precedence over hugetlb_free_vmemmap since memmap_on_memory makes it more likely to succeed memory hotplug in close-to-OOM situations. So the decision of making hugetlb_free_vmemmap take precedence is not wise and elegant. The proper approach is to have hugetlb_vmemmap.c do the check whether the section which the HugeTLB pages belong to can be optimized. If the section's vmemmap pages are allocated from the added memory block itself, hugetlb_free_vmemmap should refuse to optimize the vmemmap, otherwise, do the optimization. Then both kernel parameters are compatible. So this patch introduces VmemmapSelfHosted to mask any non-optimizable vmemmap pages. The hugetlb_vmemmap can use this flag to detect if a vmemmap page can be optimized. [songmuchun@bytedance.com: walk vmemmap page tables to avoid false-positive] Link: https://lkml.kernel.org/r/20220620110616.12056-3-songmuchun@bytedance.com Link: https://lkml.kernel.org/r/20220617135650.74901-3-songmuchun@bytedance.com Signed-off-by: Muchun Song Co-developed-by: Oscar Salvador Signed-off-by: Oscar Salvador Acked-by: David Hildenbrand Cc: Jonathan Corbet Cc: Mike Kravetz Cc: Paul E. McKenney Cc: Xiongchun Duan Signed-off-by: Andrew Morton --- Documentation/admin-guide/kernel-parameters.txt | 22 ++++----- Documentation/admin-guide/sysctl/vm.rst | 5 +- include/linux/memory_hotplug.h | 9 ---- include/linux/page-flags.h | 11 +++++ mm/hugetlb_vmemmap.c | 66 ++++++++++++++++++++++--- mm/memory_hotplug.c | 27 +++++----- 6 files changed, 93 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 8c0ea6b6c6a9..2cacd4f8deb7 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1722,9 +1722,11 @@ Built with CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON=y, the default is on. - This is not compatible with memory_hotplug.memmap_on_memory. - If both parameters are enabled, hugetlb_free_vmemmap takes - precedence over memory_hotplug.memmap_on_memory. + Note that the vmemmap pages may be allocated from the added + memory block itself when memory_hotplug.memmap_on_memory is + enabled, those vmemmap pages cannot be optimized even if this + feature is enabled. Other vmemmap pages not allocated from + the added memory block itself do not be affected. hung_task_panic= [KNL] Should the hung task detector generate panics. @@ -3068,10 +3070,12 @@ [KNL,X86,ARM] Boolean flag to enable this feature. Format: {on | off (default)} When enabled, runtime hotplugged memory will - allocate its internal metadata (struct pages) - from the hotadded memory which will allow to - hotadd a lot of memory without requiring - additional memory to do so. + allocate its internal metadata (struct pages, + those vmemmap pages cannot be optimized even + if hugetlb_free_vmemmap is enabled) from the + hotadded memory which will allow to hotadd a + lot of memory without requiring additional + memory to do so. This feature is disabled by default because it has some implication on large (e.g. GB) allocations in some configurations (e.g. small @@ -3081,10 +3085,6 @@ Note that even when enabled, there are a few cases where the feature is not effective. - This is not compatible with hugetlb_free_vmemmap. If - both parameters are enabled, hugetlb_free_vmemmap takes - precedence over memory_hotplug.memmap_on_memory. - memtest= [KNL,X86,ARM,M68K,PPC,RISCV] Enable memtest Format: default : 0 diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 4a440a7cfeb0..f74f722ad702 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -565,9 +565,8 @@ See Documentation/admin-guide/mm/hugetlbpage.rst hugetlb_optimize_vmemmap ======================== -This knob is not available when memory_hotplug.memmap_on_memory (kernel parameter) -is configured or the size of 'struct page' (a structure defined in -include/linux/mm_types.h) is not power of two (an unusual system config could +This knob is not available when the size of 'struct page' (a structure defined +in include/linux/mm_types.h) is not power of two (an unusual system config could result in this). Enable (set to 1) or disable (set to 0) the feature of optimizing vmemmap pages diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 20d7edf62a6a..e0b2209ab71c 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -351,13 +351,4 @@ void arch_remove_linear_mapping(u64 start, u64 size); extern bool mhp_supports_memmap_on_memory(unsigned long size); #endif /* CONFIG_MEMORY_HOTPLUG */ -#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY -bool mhp_memmap_on_memory(void); -#else -static inline bool mhp_memmap_on_memory(void) -{ - return false; -} -#endif - #endif /* __LINUX_MEMORY_HOTPLUG_H */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index f32aade2a6e0..82719d33c0f1 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -193,6 +193,11 @@ enum pageflags { /* Only valid for buddy pages. Used to track pages that are reported */ PG_reported = PG_uptodate, + +#ifdef CONFIG_MEMORY_HOTPLUG + /* For self-hosted memmap pages */ + PG_vmemmap_self_hosted = PG_owner_priv_1, +#endif }; #define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1) @@ -628,6 +633,12 @@ PAGEFLAG_FALSE(SkipKASanPoison, skip_kasan_poison) */ __PAGEFLAG(Reported, reported, PF_NO_COMPOUND) +#ifdef CONFIG_MEMORY_HOTPLUG +PAGEFLAG(VmemmapSelfHosted, vmemmap_self_hosted, PF_ANY) +#else +PAGEFLAG_FALSE(VmemmapSelfHosted, vmemmap_self_hosted) +#endif + /* * On an anonymous page mapped into a user virtual memory area, * page->mapping points to its anon_vma, not to a struct address_space; diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index ba29c15c53d6..1362feb3c6c9 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -10,7 +10,7 @@ */ #define pr_fmt(fmt) "HugeTLB: " fmt -#include +#include #include "hugetlb_vmemmap.h" /* @@ -97,18 +97,68 @@ int hugetlb_vmemmap_alloc(struct hstate *h, struct page *head) return ret; } +static unsigned int vmemmap_optimizable_pages(struct hstate *h, + struct page *head) +{ + if (READ_ONCE(vmemmap_optimize_mode) == VMEMMAP_OPTIMIZE_OFF) + return 0; + + if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) { + pmd_t *pmdp, pmd; + struct page *vmemmap_page; + unsigned long vaddr = (unsigned long)head; + + /* + * Only the vmemmap page's vmemmap page can be self-hosted. + * Walking the page tables to find the backing page of the + * vmemmap page. + */ + pmdp = pmd_off_k(vaddr); + /* + * The READ_ONCE() is used to stabilize *pmdp in a register or + * on the stack so that it will stop changing under the code. + * The only concurrent operation where it can be changed is + * split_vmemmap_huge_pmd() (*pmdp will be stable after this + * operation). + */ + pmd = READ_ONCE(*pmdp); + if (pmd_leaf(pmd)) + vmemmap_page = pmd_page(pmd) + pte_index(vaddr); + else + vmemmap_page = pte_page(*pte_offset_kernel(pmdp, vaddr)); + /* + * Due to HugeTLB alignment requirements and the vmemmap pages + * being at the start of the hotplugged memory region in + * memory_hotplug.memmap_on_memory case. Checking any vmemmap + * page's vmemmap page if it is marked as VmemmapSelfHosted is + * sufficient. + * + * [ hotplugged memory ] + * [ section ][...][ section ] + * [ vmemmap ][ usable memory ] + * ^ | | | + * +---+ | | + * ^ | | + * +-------+ | + * ^ | + * +-------------------------------------------+ + */ + if (PageVmemmapSelfHosted(vmemmap_page)) + return 0; + } + + return hugetlb_optimize_vmemmap_pages(h); +} + void hugetlb_vmemmap_free(struct hstate *h, struct page *head) { unsigned long vmemmap_addr = (unsigned long)head; unsigned long vmemmap_end, vmemmap_reuse, vmemmap_pages; - vmemmap_pages = hugetlb_optimize_vmemmap_pages(h); + vmemmap_pages = vmemmap_optimizable_pages(h, head); if (!vmemmap_pages) return; - if (READ_ONCE(vmemmap_optimize_mode) == VMEMMAP_OPTIMIZE_OFF) - return; - static_branch_inc(&hugetlb_optimize_vmemmap_key); vmemmap_addr += RESERVE_VMEMMAP_SIZE; @@ -199,10 +249,10 @@ static struct ctl_table hugetlb_vmemmap_sysctls[] = { static __init int hugetlb_vmemmap_sysctls_init(void) { /* - * If "memory_hotplug.memmap_on_memory" is enabled or "struct page" - * crosses page boundaries, the vmemmap pages cannot be optimized. + * If "struct page" crosses page boundaries, the vmemmap pages cannot + * be optimized. */ - if (!mhp_memmap_on_memory() && is_power_of_2(sizeof(struct page))) + if (is_power_of_2(sizeof(struct page))) register_sysctl_init("vm", hugetlb_vmemmap_sysctls); return 0; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index a2a6d280054f..99ecb2b3ff53 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -43,30 +43,22 @@ #include "shuffle.h" #ifdef CONFIG_MHP_MEMMAP_ON_MEMORY -static int memmap_on_memory_set(const char *val, const struct kernel_param *kp) -{ - if (hugetlb_optimize_vmemmap_enabled()) - return 0; - return param_set_bool(val, kp); -} - -static const struct kernel_param_ops memmap_on_memory_ops = { - .flags = KERNEL_PARAM_OPS_FL_NOARG, - .set = memmap_on_memory_set, - .get = param_get_bool, -}; - /* * memory_hotplug.memmap_on_memory parameter */ static bool memmap_on_memory __ro_after_init; -module_param_cb(memmap_on_memory, &memmap_on_memory_ops, &memmap_on_memory, 0444); +module_param(memmap_on_memory, bool, 0444); MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug"); -bool mhp_memmap_on_memory(void) +static inline bool mhp_memmap_on_memory(void) { return memmap_on_memory; } +#else +static inline bool mhp_memmap_on_memory(void) +{ + return false; +} #endif enum { @@ -1035,7 +1027,7 @@ int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, struct zone *zone) { unsigned long end_pfn = pfn + nr_pages; - int ret; + int ret, i; ret = kasan_add_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages)); if (ret) @@ -1043,6 +1035,9 @@ int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE); + for (i = 0; i < nr_pages; i++) + SetPageVmemmapSelfHosted(pfn_to_page(pfn + i)); + /* * It might be that the vmemmap_pages fully span sections. If that is * the case, mark those sections online here as otherwise they will be -- cgit v1.2.3 From e8da368a1e42a8056d1a6b419e1b91b6cf11d77e Mon Sep 17 00:00:00 2001 From: Yun-Ze Li Date: Mon, 20 Jun 2022 07:15:16 +0000 Subject: mm, docs: fix comments that mention mem_hotplug_end() Comments that mention mem_hotplug_end() are confusing as there is no function called mem_hotplug_end(). Fix them by replacing all the occurences of mem_hotplug_end() in the comments with mem_hotplug_done(). [akpm@linux-foundation.org: grammatical fixes] Link: https://lkml.kernel.org/r/20220620071516.1286101-1-p76091292@gs.ncku.edu.tw Signed-off-by: Yun-Ze Li Cc: Souptick Joarder Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 6 +++--- mm/compaction.c | 2 +- mm/vmscan.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 2b5757752333..735bf5b37949 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -591,8 +591,8 @@ struct zone { * give them a chance of being in the same cacheline. * * Write access to present_pages at runtime should be protected by - * mem_hotplug_begin/end(). Any reader who can't tolerant drift of - * present_pages should get_online_mems() to get a stable value. + * mem_hotplug_begin/done(). Any reader who can't tolerant drift of + * present_pages should use get_online_mems() to get a stable value. */ atomic_long_t managed_pages; unsigned long spanned_pages; @@ -870,7 +870,7 @@ typedef struct pglist_data { unsigned long nr_reclaim_start; /* nr pages written while throttled * when throttling started. */ struct task_struct *kswapd; /* Protected by - mem_hotplug_begin/end() */ + mem_hotplug_begin/done() */ int kswapd_order; enum zone_type kswapd_highest_zoneidx; diff --git a/mm/compaction.c b/mm/compaction.c index 1f89b969c12b..cd029ab03d0e 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -3011,7 +3011,7 @@ void kcompactd_run(int nid) /* * Called by memory hotplug when all memory in a node is offlined. Caller must - * hold mem_hotplug_begin/end(). + * be holding mem_hotplug_begin/done(). */ void kcompactd_stop(int nid) { diff --git a/mm/vmscan.c b/mm/vmscan.c index 161096d9311a..f58761cea0a0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4645,7 +4645,7 @@ void kswapd_run(int nid) /* * Called by memory hotplug when all memory in a node is offlined. Caller must - * hold mem_hotplug_begin/end(). + * be holding mem_hotplug_begin/done(). */ void kswapd_stop(int nid) { -- cgit v1.2.3 From 18f3962953e40401b7ed98e8524167282c3e626e Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Sun, 26 Jun 2022 22:57:17 +0800 Subject: mm: hugetlb: kill set_huge_swap_pte_at() Commit e5251fd43007 ("mm/hugetlb: introduce set_huge_swap_pte_at() helper") add set_huge_swap_pte_at() to handle swap entries on architectures that support hugepages consisting of contiguous ptes. And currently the set_huge_swap_pte_at() is only overridden by arm64. set_huge_swap_pte_at() provide a sz parameter to help determine the number of entries to be updated. But in fact, all hugetlb swap entries contain pfn information, so we can find the corresponding folio through the pfn recorded in the swap entry, then the folio_size() is the number of entries that need to be updated. And considering that users will easily cause bugs by ignoring the difference between set_huge_swap_pte_at() and set_huge_pte_at(). Let's handle swap entries in set_huge_pte_at() and remove the set_huge_swap_pte_at(), then we can call set_huge_pte_at() anywhere, which simplifies our coding. Link: https://lkml.kernel.org/r/20220626145717.53572-1-zhengqi.arch@bytedance.com Signed-off-by: Qi Zheng Acked-by: Muchun Song Cc: Mike Kravetz Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/include/asm/hugetlb.h | 3 --- arch/arm64/mm/hugetlbpage.c | 34 +++++++++++++++++----------------- include/linux/hugetlb.h | 13 ------------- mm/hugetlb.c | 8 +++----- mm/rmap.c | 11 +++-------- 5 files changed, 23 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h index 1fd2846dbefe..d20f5da2d76f 100644 --- a/arch/arm64/include/asm/hugetlb.h +++ b/arch/arm64/include/asm/hugetlb.h @@ -46,9 +46,6 @@ extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long sz); #define __HAVE_ARCH_HUGE_PTEP_GET extern pte_t huge_ptep_get(pte_t *ptep); -extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte, unsigned long sz); -#define set_huge_swap_pte_at set_huge_swap_pte_at void __init arm64_hugetlb_cma_reserve(void); diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index e2a5ec9fdc0d..3be8f25aa5be 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -238,6 +238,13 @@ static void clear_flush(struct mm_struct *mm, flush_tlb_range(&vma, saddr, addr); } +static inline struct folio *hugetlb_swap_entry_to_folio(swp_entry_t entry) +{ + VM_BUG_ON(!is_migration_entry(entry) && !is_hwpoison_entry(entry)); + + return page_folio(pfn_to_page(swp_offset(entry))); +} + void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { @@ -247,11 +254,16 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, unsigned long pfn, dpfn; pgprot_t hugeprot; - /* - * Code needs to be expanded to handle huge swap and migration - * entries. Needed for HUGETLB and MEMORY_FAILURE. - */ - WARN_ON(!pte_present(pte)); + if (!pte_present(pte)) { + struct folio *folio; + + folio = hugetlb_swap_entry_to_folio(pte_to_swp_entry(pte)); + ncontig = num_contig_ptes(folio_size(folio), &pgsize); + + for (i = 0; i < ncontig; i++, ptep++) + set_pte_at(mm, addr, ptep, pte); + return; + } if (!pte_cont(pte)) { set_pte_at(mm, addr, ptep, pte); @@ -269,18 +281,6 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); } -void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte, unsigned long sz) -{ - int i, ncontig; - size_t pgsize; - - ncontig = num_contig_ptes(sz, &pgsize); - - for (i = 0; i < ncontig; i++, ptep++) - set_pte(ptep, pte); -} - pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, unsigned long sz) { diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 756b66ff025e..c6cccfaf8708 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -903,14 +903,6 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) atomic_long_sub(l, &mm->hugetlb_usage); } -#ifndef set_huge_swap_pte_at -static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte, unsigned long sz) -{ - set_huge_pte_at(mm, addr, ptep, pte); -} -#endif - #ifndef huge_ptep_modify_prot_start #define huge_ptep_modify_prot_start huge_ptep_modify_prot_start static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, @@ -1094,11 +1086,6 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) { } -static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte, unsigned long sz) -{ -} - static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 65454896f174..064da8ffbac6 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4798,12 +4798,11 @@ again: entry = swp_entry_to_pte(swp_entry); if (userfaultfd_wp(src_vma) && uffd_wp) entry = huge_pte_mkuffd_wp(entry); - set_huge_swap_pte_at(src, addr, src_pte, - entry, sz); + set_huge_pte_at(src, addr, src_pte, entry); } if (!userfaultfd_wp(dst_vma) && uffd_wp) entry = huge_pte_clear_uffd_wp(entry); - set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz); + set_huge_pte_at(dst, addr, dst_pte, entry); } else if (unlikely(is_pte_marker(entry))) { /* * We copy the pte marker only if the dst vma has @@ -6344,8 +6343,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) newpte = pte_swp_clear_uffd_wp(newpte); - set_huge_swap_pte_at(mm, address, ptep, - newpte, psize); + set_huge_pte_at(mm, address, ptep, newpte); pages++; } spin_unlock(ptl); diff --git a/mm/rmap.c b/mm/rmap.c index 56134cdc5ca3..83172ee0ea35 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1618,9 +1618,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); if (folio_test_hugetlb(folio)) { hugetlb_count_sub(folio_nr_pages(folio), mm); - set_huge_swap_pte_at(mm, address, - pvmw.pte, pteval, - vma_mmu_pagesize(vma)); + set_huge_pte_at(mm, address, pvmw.pte, pteval); } else { dec_mm_counter(mm, mm_counter(&folio->page)); set_pte_at(mm, address, pvmw.pte, pteval); @@ -2004,9 +2002,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); if (folio_test_hugetlb(folio)) { hugetlb_count_sub(folio_nr_pages(folio), mm); - set_huge_swap_pte_at(mm, address, - pvmw.pte, pteval, - vma_mmu_pagesize(vma)); + set_huge_pte_at(mm, address, pvmw.pte, pteval); } else { dec_mm_counter(mm, mm_counter(&folio->page)); set_pte_at(mm, address, pvmw.pte, pteval); @@ -2074,8 +2070,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (pte_uffd_wp(pteval)) swp_pte = pte_swp_mkuffd_wp(swp_pte); if (folio_test_hugetlb(folio)) - set_huge_swap_pte_at(mm, address, pvmw.pte, - swp_pte, vma_mmu_pagesize(vma)); + set_huge_pte_at(mm, address, pvmw.pte, swp_pte); else set_pte_at(mm, address, pvmw.pte, swp_pte); trace_set_migration_pte(address, pte_val(swp_pte), -- cgit v1.2.3 From 1baec203b77cafa24610b5c9ae7a2aa380d74ef6 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Sat, 25 Jun 2022 17:28:16 +0800 Subject: mm/khugepaged: try to free transhuge swapcache when possible Transhuge swapcaches won't be freed in __collapse_huge_page_copy(). It's because release_pte_page() is not called for these pages and thus free_page_and_swap_cache can't grab the page lock. These pages won't be freed from swap cache even if we are the only user until next time reclaim. It shouldn't hurt indeed, but we could try to free these pages to save more memory for system. Link: https://lkml.kernel.org/r/20220625092816.4856-8-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Alistair Popple Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: David Howells Cc: Matthew Wilcox (Oracle) Cc: NeilBrown Cc: Peter Xu Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yang Shi Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/swap.h | 5 +++++ mm/khugepaged.c | 7 ++++++- mm/swap.h | 5 ----- 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 95a5b7aa1ae9..6d11c51b2b62 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -455,6 +455,7 @@ static inline unsigned long total_swapcache_pages(void) return global_node_page_state(NR_SWAPCACHE); } +extern void free_swap_cache(struct page *page); extern void free_page_and_swap_cache(struct page *); extern void free_pages_and_swap_cache(struct page **, int); /* linux/mm/swapfile.c */ @@ -539,6 +540,10 @@ static inline void put_swap_device(struct swap_info_struct *si) /* used to sanity check ptes in zap_pte_range when CONFIG_SWAP=0 */ #define free_swap_and_cache(e) is_pfn_swap_entry(e) +static inline void free_swap_cache(struct page *page) +{ +} + static inline int add_swap_count_continuation(swp_entry_t swp, gfp_t gfp_mask) { return 0; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 08e885f28def..01e0d6336754 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -755,7 +755,12 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, list_for_each_entry_safe(src_page, tmp, compound_pagelist, lru) { list_del(&src_page->lru); - release_pte_page(src_page); + mod_node_page_state(page_pgdat(src_page), + NR_ISOLATED_ANON + page_is_file_lru(src_page), + -compound_nr(src_page)); + unlock_page(src_page); + free_swap_cache(src_page); + putback_lru_page(src_page); } } diff --git a/mm/swap.h b/mm/swap.h index fa0816af4712..17936e068c1c 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -41,7 +41,6 @@ void __delete_from_swap_cache(struct folio *folio, void delete_from_swap_cache(struct folio *folio); void clear_shadow_from_swap_cache(int type, unsigned long begin, unsigned long end); -void free_swap_cache(struct page *page); struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma, unsigned long addr); @@ -81,10 +80,6 @@ static inline struct address_space *swap_address_space(swp_entry_t entry) return NULL; } -static inline void free_swap_cache(struct page *page) -{ -} - static inline void show_swap_cache_info(void) { } -- cgit v1.2.3 From 39bfb3c12d792181de0cf3306be1ea03664a1b05 Mon Sep 17 00:00:00 2001 From: Kurt Kanzenbach Date: Fri, 1 Jul 2022 19:56:06 +0200 Subject: net: phy: broadcom: Add support for BCM53128 internal PHYs Add support for BCM53128 internal PHYs. These support interrupts as well as statistics. Therefore, enable the Broadcom PHY driver for them. Tested on BCM53128 switch using the mainline b53 DSA driver. Signed-off-by: Kurt Kanzenbach Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/phy/broadcom.c | 15 +++++++++++++++ include/linux/brcmphy.h | 1 + 2 files changed, 16 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index 876bc45ede60..31fbcdddc9ad 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -1066,6 +1066,20 @@ static struct phy_driver broadcom_drivers[] = { .config_intr = bcm_phy_config_intr, .handle_interrupt = bcm_phy_handle_interrupt, .link_change_notify = bcm54xx_link_change_notify, +}, { + .phy_id = PHY_ID_BCM53128, + .phy_id_mask = 0xfffffff0, + .name = "Broadcom BCM53128", + .flags = PHY_IS_INTERNAL, + /* PHY_GBIT_FEATURES */ + .get_sset_count = bcm_phy_get_sset_count, + .get_strings = bcm_phy_get_strings, + .get_stats = bcm54xx_get_stats, + .probe = bcm54xx_phy_probe, + .config_init = bcm54xx_config_init, + .config_intr = bcm_phy_config_intr, + .handle_interrupt = bcm_phy_handle_interrupt, + .link_change_notify = bcm54xx_link_change_notify, }, { .phy_id = PHY_ID_BCM89610, .phy_id_mask = 0xfffffff0, @@ -1102,6 +1116,7 @@ static struct mdio_device_id __maybe_unused broadcom_tbl[] = { { PHY_ID_BCM5241, 0xfffffff0 }, { PHY_ID_BCM5395, 0xfffffff0 }, { PHY_ID_BCM53125, 0xfffffff0 }, + { PHY_ID_BCM53128, 0xfffffff0 }, { PHY_ID_BCM89610, 0xfffffff0 }, { } }; diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 747fad264033..6ff567ece34a 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -16,6 +16,7 @@ #define PHY_ID_BCM5481 0x0143bca0 #define PHY_ID_BCM5395 0x0143bcf0 #define PHY_ID_BCM53125 0x03625f20 +#define PHY_ID_BCM53128 0x03625e10 #define PHY_ID_BCM54810 0x03625d00 #define PHY_ID_BCM54811 0x03625cc0 #define PHY_ID_BCM5482 0x0143bcb0 -- cgit v1.2.3 From df76234276e22136b2468825c18407fdfbb2076a Mon Sep 17 00:00:00 2001 From: Stefan Wahren Date: Sat, 25 Jun 2022 13:36:15 +0200 Subject: mfd: bcm2835-pm: Add support for BCM2711 In BCM2711 the new RPiVid ASB took over V3D. The old ASB is still present with the ISP and H264 bits, and V3D is in the same place in the new ASB as the old one. As per the devicetree bindings, BCM2711 will provide both the old and new ASB resources, so get both of them and pass them into 'bcm2835-power,' which will take care of selecting which one to use accordingly. Since the RPiVid ASB's resources were being provided prior to formalizing the bindings[1], also support the old DT files that didn't use 'reg-names.' [1] See: 7dbe8c62ceeb ("ARM: dts: Add minimal Raspberry Pi 4 support") Signed-off-by: Stefan Wahren Reviewed-by: Peter Robinson Acked-by: Florian Fainelli Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220625113619.15944-8-stefan.wahren@i2se.com --- drivers/mfd/bcm2835-pm.c | 13 +++++++++++++ include/linux/mfd/bcm2835-pm.h | 1 + 2 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/drivers/mfd/bcm2835-pm.c b/drivers/mfd/bcm2835-pm.c index 418c8a16427d..49cd1f03884a 100644 --- a/drivers/mfd/bcm2835-pm.c +++ b/drivers/mfd/bcm2835-pm.c @@ -42,6 +42,14 @@ static int bcm2835_pm_get_pdata(struct platform_device *pdev, pm->asb = NULL; } + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "rpivid_asb"); + if (res) { + pm->rpivid_asb = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pm->rpivid_asb)) + pm->rpivid_asb = NULL; + } + return 0; } @@ -54,6 +62,10 @@ static int bcm2835_pm_get_pdata(struct platform_device *pdev, if (IS_ERR(pm->asb)) pm->asb = NULL; + pm->rpivid_asb = devm_platform_ioremap_resource(pdev, 2); + if (IS_ERR(pm->rpivid_asb)) + pm->rpivid_asb = NULL; + return 0; } @@ -95,6 +107,7 @@ static int bcm2835_pm_probe(struct platform_device *pdev) static const struct of_device_id bcm2835_pm_of_match[] = { { .compatible = "brcm,bcm2835-pm-wdt", }, { .compatible = "brcm,bcm2835-pm", }, + { .compatible = "brcm,bcm2711-pm", }, {}, }; MODULE_DEVICE_TABLE(of, bcm2835_pm_of_match); diff --git a/include/linux/mfd/bcm2835-pm.h b/include/linux/mfd/bcm2835-pm.h index ed37dc40e82a..f70a810c55f7 100644 --- a/include/linux/mfd/bcm2835-pm.h +++ b/include/linux/mfd/bcm2835-pm.h @@ -9,6 +9,7 @@ struct bcm2835_pm { struct device *dev; void __iomem *base; void __iomem *asb; + void __iomem *rpivid_asb; }; #endif /* BCM2835_MFD_PM_H */ -- cgit v1.2.3 From 2fcfa72fc13f0203bb676bd1ec30ec85e17855be Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Sun, 3 Jul 2022 17:11:25 +0800 Subject: interconnect: add device managed bulk API Add device managed bulk API to simplify driver. Signed-off-by: Peng Fan Link: https://lore.kernel.org/r/20220703091132.1412063-4-peng.fan@oss.nxp.com Signed-off-by: Georgi Djakov --- drivers/interconnect/bulk.c | 42 ++++++++++++++++++++++++++++++++++++++++++ include/linux/interconnect.h | 7 +++++++ 2 files changed, 49 insertions(+) (limited to 'include/linux') diff --git a/drivers/interconnect/bulk.c b/drivers/interconnect/bulk.c index 448cc536aa79..8b1d8a412464 100644 --- a/drivers/interconnect/bulk.c +++ b/drivers/interconnect/bulk.c @@ -115,3 +115,45 @@ void icc_bulk_disable(int num_paths, const struct icc_bulk_data *paths) icc_disable(paths[num_paths].path); } EXPORT_SYMBOL_GPL(icc_bulk_disable); + +struct icc_bulk_devres { + struct icc_bulk_data *paths; + int num_paths; +}; + +static void devm_icc_bulk_release(struct device *dev, void *res) +{ + struct icc_bulk_devres *devres = res; + + icc_bulk_put(devres->num_paths, devres->paths); +} + +/** + * devm_of_icc_bulk_get() - resource managed of_icc_bulk_get + * @dev: the device requesting the path + * @num_paths: the number of icc_bulk_data + * @paths: the table with the paths we want to get + * + * Returns 0 on success or negative errno otherwise. + */ +int devm_of_icc_bulk_get(struct device *dev, int num_paths, struct icc_bulk_data *paths) +{ + struct icc_bulk_devres *devres; + int ret; + + devres = devres_alloc(devm_icc_bulk_release, sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; + + ret = of_icc_bulk_get(dev, num_paths, paths); + if (!ret) { + devres->paths = paths; + devres->num_paths = num_paths; + devres_add(dev, devres); + } else { + devres_free(devres); + } + + return ret; +} +EXPORT_SYMBOL_GPL(devm_of_icc_bulk_get); diff --git a/include/linux/interconnect.h b/include/linux/interconnect.h index f685777b875e..2b0e784ba771 100644 --- a/include/linux/interconnect.h +++ b/include/linux/interconnect.h @@ -44,6 +44,7 @@ struct icc_path *icc_get(struct device *dev, const int src_id, const int dst_id); struct icc_path *of_icc_get(struct device *dev, const char *name); struct icc_path *devm_of_icc_get(struct device *dev, const char *name); +int devm_of_icc_bulk_get(struct device *dev, int num_paths, struct icc_bulk_data *paths); struct icc_path *of_icc_get_by_index(struct device *dev, int idx); void icc_put(struct icc_path *path); int icc_enable(struct icc_path *path); @@ -116,6 +117,12 @@ static inline int of_icc_bulk_get(struct device *dev, int num_paths, struct icc_ return 0; } +static inline int devm_of_icc_bulk_get(struct device *dev, int num_paths, + struct icc_bulk_data *paths) +{ + return 0; +} + static inline void icc_bulk_put(int num_paths, struct icc_bulk_data *paths) { } -- cgit v1.2.3 From 7097f29819bb70374dfae9f705e548a720f16f94 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 4 Jul 2022 11:19:31 +0100 Subject: firmware: arm_scmi: Add SCMI v3.1 System Power extensions Add support for SCMIv3.1 System Power optional timeout field while dispatching SYSTEM_POWER_STATE_NOTIFIER notification. Link: https://lore.kernel.org/r/20220704101933.2981635-3-cristian.marussi@arm.com Signed-off-by: Cristian Marussi Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/system.c | 17 ++++++++++++++++- include/linux/scmi_protocol.h | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/system.c b/drivers/firmware/arm_scmi/system.c index 220e399118ad..9383d7584539 100644 --- a/drivers/firmware/arm_scmi/system.c +++ b/drivers/firmware/arm_scmi/system.c @@ -27,10 +27,12 @@ struct scmi_system_power_state_notifier_payld { __le32 agent_id; __le32 flags; __le32 system_state; + __le32 timeout; }; struct scmi_system_info { u32 version; + bool graceful_timeout_supported; }; static int scmi_system_request_notify(const struct scmi_protocol_handle *ph, @@ -72,17 +74,27 @@ scmi_system_fill_custom_report(const struct scmi_protocol_handle *ph, const void *payld, size_t payld_sz, void *report, u32 *src_id) { + size_t expected_sz; const struct scmi_system_power_state_notifier_payld *p = payld; struct scmi_system_power_state_notifier_report *r = report; + struct scmi_system_info *pinfo = ph->get_priv(ph); + expected_sz = pinfo->graceful_timeout_supported ? + sizeof(*p) : sizeof(*p) - sizeof(__le32); if (evt_id != SCMI_EVENT_SYSTEM_POWER_STATE_NOTIFIER || - sizeof(*p) != payld_sz) + payld_sz != expected_sz) return NULL; r->timestamp = timestamp; r->agent_id = le32_to_cpu(p->agent_id); r->flags = le32_to_cpu(p->flags); r->system_state = le32_to_cpu(p->system_state); + if (pinfo->graceful_timeout_supported && + r->system_state == SCMI_SYSTEM_SHUTDOWN && + SCMI_SYSPOWER_IS_REQUEST_GRACEFUL(r->flags)) + r->timeout = le32_to_cpu(p->timeout); + else + r->timeout = 0x00; *src_id = 0; return r; @@ -129,6 +141,9 @@ static int scmi_system_protocol_init(const struct scmi_protocol_handle *ph) return -ENOMEM; pinfo->version = version; + if (PROTOCOL_REV_MAJOR(pinfo->version) >= 0x2) + pinfo->graceful_timeout_supported = true; + return ph->set_priv(ph, pinfo); } diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 704111f63993..311aa3c73aac 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -781,8 +781,10 @@ struct scmi_clock_rate_notif_report { struct scmi_system_power_state_notifier_report { ktime_t timestamp; unsigned int agent_id; +#define SCMI_SYSPOWER_IS_REQUEST_GRACEFUL(flags) ((flags) & BIT(0)) unsigned int flags; unsigned int system_state; + unsigned int timeout; }; struct scmi_perf_limits_report { -- cgit v1.2.3 From d91079995fa62720e11a39c08b932f2f9a8cbfae Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 4 Jul 2022 11:19:32 +0100 Subject: firmware: arm_scmi: Add devm_protocol_acquire helper Add a method to get hold of a protocol, causing it to be initialized and its resource accounting updated, without getting access to its operations and handle. Some protocols, like SCMI SystemPower, do not expose any protocol ops to the kernel OSPM agent but still need to be at least initialized. This helper avoids the need to invoke a full devm_get_protocol() only to get the protocol initialized while throwing away unused the protocol ops and handle. Link: https://lore.kernel.org/r/20220704101933.2981635-4-cristian.marussi@arm.com Signed-off-by: Cristian Marussi Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/driver.c | 70 ++++++++++++++++++++++++++++++-------- include/linux/scmi_protocol.h | 5 +++ 2 files changed, 60 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 0f73d743a7c2..1125a1f167a6 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -1523,6 +1523,30 @@ static void scmi_devm_release_protocol(struct device *dev, void *res) scmi_protocol_release(dres->handle, dres->protocol_id); } +static struct scmi_protocol_instance __must_check * +scmi_devres_protocol_instance_get(struct scmi_device *sdev, u8 protocol_id) +{ + struct scmi_protocol_instance *pi; + struct scmi_protocol_devres *dres; + + dres = devres_alloc(scmi_devm_release_protocol, + sizeof(*dres), GFP_KERNEL); + if (!dres) + return ERR_PTR(-ENOMEM); + + pi = scmi_get_protocol_instance(sdev->handle, protocol_id); + if (IS_ERR(pi)) { + devres_free(dres); + return pi; + } + + dres->handle = sdev->handle; + dres->protocol_id = protocol_id; + devres_add(&sdev->dev, dres); + + return pi; +} + /** * scmi_devm_protocol_get - Devres managed get protocol operations and handle * @sdev: A reference to an scmi_device whose embedded struct device is to @@ -1546,32 +1570,47 @@ scmi_devm_protocol_get(struct scmi_device *sdev, u8 protocol_id, struct scmi_protocol_handle **ph) { struct scmi_protocol_instance *pi; - struct scmi_protocol_devres *dres; - struct scmi_handle *handle = sdev->handle; if (!ph) return ERR_PTR(-EINVAL); - dres = devres_alloc(scmi_devm_release_protocol, - sizeof(*dres), GFP_KERNEL); - if (!dres) - return ERR_PTR(-ENOMEM); - - pi = scmi_get_protocol_instance(handle, protocol_id); - if (IS_ERR(pi)) { - devres_free(dres); + pi = scmi_devres_protocol_instance_get(sdev, protocol_id); + if (IS_ERR(pi)) return pi; - } - - dres->handle = handle; - dres->protocol_id = protocol_id; - devres_add(&sdev->dev, dres); *ph = &pi->ph; return pi->proto->ops; } +/** + * scmi_devm_protocol_acquire - Devres managed helper to get hold of a protocol + * @sdev: A reference to an scmi_device whose embedded struct device is to + * be used for devres accounting. + * @protocol_id: The protocol being requested. + * + * Get hold of a protocol accounting for its usage, possibly triggering its + * initialization but without getting access to its protocol specific operations + * and handle. + * + * Being a devres based managed method, protocol hold will be automatically + * released, and possibly de-initialized on last user, once the SCMI driver + * owning the scmi_device is unbound from it. + * + * Return: 0 on SUCCESS + */ +static int __must_check scmi_devm_protocol_acquire(struct scmi_device *sdev, + u8 protocol_id) +{ + struct scmi_protocol_instance *pi; + + pi = scmi_devres_protocol_instance_get(sdev, protocol_id); + if (IS_ERR(pi)) + return PTR_ERR(pi); + + return 0; +} + static int scmi_devm_protocol_match(struct device *dev, void *res, void *data) { struct scmi_protocol_devres *dres = res; @@ -2176,6 +2215,7 @@ static int scmi_probe(struct platform_device *pdev) handle = &info->handle; handle->dev = info->dev; handle->version = &info->version; + handle->devm_protocol_acquire = scmi_devm_protocol_acquire; handle->devm_protocol_get = scmi_devm_protocol_get; handle->devm_protocol_put = scmi_devm_protocol_put; diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 311aa3c73aac..898488c7f185 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -624,6 +624,9 @@ struct scmi_notify_ops { * * @dev: pointer to the SCMI device * @version: pointer to the structure containing SCMI version information + * @devm_protocol_acquire: devres managed method to get hold of a protocol, + * causing its initialization and related resource + * accounting * @devm_protocol_get: devres managed method to acquire a protocol and get specific * operations and a dedicated protocol handler * @devm_protocol_put: devres managed method to release a protocol @@ -642,6 +645,8 @@ struct scmi_handle { struct device *dev; struct scmi_revision_info *version; + int __must_check (*devm_protocol_acquire)(struct scmi_device *sdev, + u8 proto); const void __must_check * (*devm_protocol_get)(struct scmi_device *sdev, u8 proto, struct scmi_protocol_handle **ph); -- cgit v1.2.3 From 0316f99c4780b0a5fd60b7f136c64cb1af8d5fc3 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 4 Jul 2022 11:22:36 +0100 Subject: firmware: arm_scmi: Add SCMI v3.1 powercap protocol basic support Add support for SCMI v3.1 powercap protocol, with the exception of powercap fast channels, exposing all the new related powercap protocol operations as usual in include/linux/scmi_protocol.h. Link: https://lore.kernel.org/r/20220704102241.2988447-3-cristian.marussi@arm.com Signed-off-by: Cristian Marussi Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/Makefile | 2 +- drivers/firmware/arm_scmi/driver.c | 2 + drivers/firmware/arm_scmi/powercap.c | 753 ++++++++++++++++++++++++++++++++++ drivers/firmware/arm_scmi/protocols.h | 1 + include/linux/scmi_protocol.h | 125 ++++++ 5 files changed, 882 insertions(+), 1 deletion(-) create mode 100644 drivers/firmware/arm_scmi/powercap.c (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/Makefile b/drivers/firmware/arm_scmi/Makefile index faacd0ec7b81..9ea86f8cc8f7 100644 --- a/drivers/firmware/arm_scmi/Makefile +++ b/drivers/firmware/arm_scmi/Makefile @@ -7,7 +7,7 @@ scmi-transport-$(CONFIG_ARM_SCMI_TRANSPORT_SMC) += smc.o scmi-transport-$(CONFIG_ARM_SCMI_HAVE_MSG) += msg.o scmi-transport-$(CONFIG_ARM_SCMI_TRANSPORT_VIRTIO) += virtio.o scmi-transport-$(CONFIG_ARM_SCMI_TRANSPORT_OPTEE) += optee.o -scmi-protocols-y = base.o clock.o perf.o power.o reset.o sensors.o system.o voltage.o +scmi-protocols-y = base.o clock.o perf.o power.o reset.o sensors.o system.o voltage.o powercap.o scmi-module-objs := $(scmi-bus-y) $(scmi-driver-y) $(scmi-protocols-y) \ $(scmi-transport-y) obj-$(CONFIG_ARM_SCMI_PROTOCOL) += scmi-module.o diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 1125a1f167a6..92a71e02b4cb 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -2485,6 +2485,7 @@ static int __init scmi_driver_init(void) scmi_sensors_register(); scmi_voltage_register(); scmi_system_register(); + scmi_powercap_register(); return platform_driver_register(&scmi_driver); } @@ -2501,6 +2502,7 @@ static void __exit scmi_driver_exit(void) scmi_sensors_unregister(); scmi_voltage_unregister(); scmi_system_unregister(); + scmi_powercap_unregister(); scmi_bus_exit(); diff --git a/drivers/firmware/arm_scmi/powercap.c b/drivers/firmware/arm_scmi/powercap.c new file mode 100644 index 000000000000..d71a1f6c2d8a --- /dev/null +++ b/drivers/firmware/arm_scmi/powercap.c @@ -0,0 +1,753 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * System Control and Management Interface (SCMI) Powercap Protocol + * + * Copyright (C) 2022 ARM Ltd. + */ + +#define pr_fmt(fmt) "SCMI Notifications POWERCAP - " fmt + +#include +#include +#include + +#include "protocols.h" +#include "notify.h" + +enum scmi_powercap_protocol_cmd { + POWERCAP_DOMAIN_ATTRIBUTES = 0x3, + POWERCAP_CAP_GET = 0x4, + POWERCAP_CAP_SET = 0x5, + POWERCAP_PAI_GET = 0x6, + POWERCAP_PAI_SET = 0x7, + POWERCAP_DOMAIN_NAME_GET = 0x8, + POWERCAP_MEASUREMENTS_GET = 0x9, + POWERCAP_CAP_NOTIFY = 0xa, + POWERCAP_MEASUREMENTS_NOTIFY = 0xb, + POWERCAP_DESCRIBE_FASTCHANNEL = 0xc, +}; + +struct scmi_msg_resp_powercap_domain_attributes { + __le32 attributes; +#define SUPPORTS_POWERCAP_CAP_CHANGE_NOTIFY(x) ((x) & BIT(31)) +#define SUPPORTS_POWERCAP_MEASUREMENTS_CHANGE_NOTIFY(x) ((x) & BIT(30)) +#define SUPPORTS_ASYNC_POWERCAP_CAP_SET(x) ((x) & BIT(29)) +#define SUPPORTS_EXTENDED_NAMES(x) ((x) & BIT(28)) +#define SUPPORTS_POWERCAP_CAP_CONFIGURATION(x) ((x) & BIT(27)) +#define SUPPORTS_POWERCAP_MONITORING(x) ((x) & BIT(26)) +#define SUPPORTS_POWERCAP_PAI_CONFIGURATION(x) ((x) & BIT(25)) +#define POWERCAP_POWER_UNIT(x) \ + (FIELD_GET(GENMASK(24, 23), (x))) +#define SUPPORTS_POWER_UNITS_MW(x) \ + (POWERCAP_POWER_UNIT(x) == 0x2) +#define SUPPORTS_POWER_UNITS_UW(x) \ + (POWERCAP_POWER_UNIT(x) == 0x1) + u8 name[SCMI_SHORT_NAME_MAX_SIZE]; + __le32 min_pai; + __le32 max_pai; + __le32 pai_step; + __le32 min_power_cap; + __le32 max_power_cap; + __le32 power_cap_step; + __le32 sustainable_power; + __le32 accuracy; + __le32 parent_id; +}; + +struct scmi_msg_powercap_set_cap_or_pai { + __le32 domain; + __le32 flags; +#define CAP_SET_ASYNC BIT(1) +#define CAP_SET_IGNORE_DRESP BIT(0) + __le32 value; +}; + +struct scmi_msg_resp_powercap_cap_set_complete { + __le32 domain; + __le32 power_cap; +}; + +struct scmi_msg_resp_powercap_meas_get { + __le32 power; + __le32 pai; +}; + +struct scmi_msg_powercap_notify_cap { + __le32 domain; + __le32 notify_enable; +}; + +struct scmi_msg_powercap_notify_thresh { + __le32 domain; + __le32 notify_enable; + __le32 power_thresh_low; + __le32 power_thresh_high; +}; + +struct scmi_powercap_cap_changed_notify_payld { + __le32 agent_id; + __le32 domain_id; + __le32 power_cap; + __le32 pai; +}; + +struct scmi_powercap_meas_changed_notify_payld { + __le32 agent_id; + __le32 domain_id; + __le32 power; +}; + +struct scmi_powercap_state { + bool meas_notif_enabled; + u64 thresholds; +#define THRESH_LOW(p, id) \ + (lower_32_bits((p)->states[(id)].thresholds)) +#define THRESH_HIGH(p, id) \ + (upper_32_bits((p)->states[(id)].thresholds)) +}; + +struct powercap_info { + u32 version; + int num_domains; + struct scmi_powercap_state *states; + struct scmi_powercap_info *powercaps; +}; + +static enum scmi_powercap_protocol_cmd evt_2_cmd[] = { + POWERCAP_CAP_NOTIFY, + POWERCAP_MEASUREMENTS_NOTIFY, +}; + +static int scmi_powercap_notify(const struct scmi_protocol_handle *ph, + u32 domain, int message_id, bool enable); + +static int +scmi_powercap_attributes_get(const struct scmi_protocol_handle *ph, + struct powercap_info *pi) +{ + int ret; + struct scmi_xfer *t; + + ret = ph->xops->xfer_get_init(ph, PROTOCOL_ATTRIBUTES, 0, + sizeof(u32), &t); + if (ret) + return ret; + + ret = ph->xops->do_xfer(ph, t); + if (!ret) { + u32 attributes; + + attributes = get_unaligned_le32(t->rx.buf); + pi->num_domains = FIELD_GET(GENMASK(15, 0), attributes); + } + + ph->xops->xfer_put(ph, t); + return ret; +} + +static inline int +scmi_powercap_validate(unsigned int min_val, unsigned int max_val, + unsigned int step_val, bool configurable) +{ + if (!min_val || !max_val) + return -EPROTO; + + if ((configurable && min_val == max_val) || + (!configurable && min_val != max_val)) + return -EPROTO; + + if (min_val != max_val && !step_val) + return -EPROTO; + + return 0; +} + +static int +scmi_powercap_domain_attributes_get(const struct scmi_protocol_handle *ph, + struct powercap_info *pinfo, u32 domain) +{ + int ret; + u32 flags; + struct scmi_xfer *t; + struct scmi_powercap_info *dom_info = pinfo->powercaps + domain; + struct scmi_msg_resp_powercap_domain_attributes *resp; + + ret = ph->xops->xfer_get_init(ph, POWERCAP_DOMAIN_ATTRIBUTES, + sizeof(domain), sizeof(*resp), &t); + if (ret) + return ret; + + put_unaligned_le32(domain, t->tx.buf); + resp = t->rx.buf; + + ret = ph->xops->do_xfer(ph, t); + if (!ret) { + flags = le32_to_cpu(resp->attributes); + + dom_info->id = domain; + dom_info->notify_powercap_cap_change = + SUPPORTS_POWERCAP_CAP_CHANGE_NOTIFY(flags); + dom_info->notify_powercap_measurement_change = + SUPPORTS_POWERCAP_MEASUREMENTS_CHANGE_NOTIFY(flags); + dom_info->async_powercap_cap_set = + SUPPORTS_ASYNC_POWERCAP_CAP_SET(flags); + dom_info->powercap_cap_config = + SUPPORTS_POWERCAP_CAP_CONFIGURATION(flags); + dom_info->powercap_monitoring = + SUPPORTS_POWERCAP_MONITORING(flags); + dom_info->powercap_pai_config = + SUPPORTS_POWERCAP_PAI_CONFIGURATION(flags); + dom_info->powercap_scale_mw = + SUPPORTS_POWER_UNITS_MW(flags); + dom_info->powercap_scale_uw = + SUPPORTS_POWER_UNITS_UW(flags); + + strscpy(dom_info->name, resp->name, SCMI_SHORT_NAME_MAX_SIZE); + + dom_info->min_pai = le32_to_cpu(resp->min_pai); + dom_info->max_pai = le32_to_cpu(resp->max_pai); + dom_info->pai_step = le32_to_cpu(resp->pai_step); + ret = scmi_powercap_validate(dom_info->min_pai, + dom_info->max_pai, + dom_info->pai_step, + dom_info->powercap_pai_config); + if (ret) { + dev_err(ph->dev, + "Platform reported inconsistent PAI config for domain %d - %s\n", + dom_info->id, dom_info->name); + goto clean; + } + + dom_info->min_power_cap = le32_to_cpu(resp->min_power_cap); + dom_info->max_power_cap = le32_to_cpu(resp->max_power_cap); + dom_info->power_cap_step = le32_to_cpu(resp->power_cap_step); + ret = scmi_powercap_validate(dom_info->min_power_cap, + dom_info->max_power_cap, + dom_info->power_cap_step, + dom_info->powercap_cap_config); + if (ret) { + dev_err(ph->dev, + "Platform reported inconsistent CAP config for domain %d - %s\n", + dom_info->id, dom_info->name); + goto clean; + } + + dom_info->sustainable_power = + le32_to_cpu(resp->sustainable_power); + dom_info->accuracy = le32_to_cpu(resp->accuracy); + + dom_info->parent_id = le32_to_cpu(resp->parent_id); + if (dom_info->parent_id != SCMI_POWERCAP_ROOT_ZONE_ID && + (dom_info->parent_id >= pinfo->num_domains || + dom_info->parent_id == dom_info->id)) { + dev_err(ph->dev, + "Platform reported inconsistent parent ID for domain %d - %s\n", + dom_info->id, dom_info->name); + ret = -ENODEV; + } + } + +clean: + ph->xops->xfer_put(ph, t); + + /* + * If supported overwrite short name with the extended one; + * on error just carry on and use already provided short name. + */ + if (!ret && SUPPORTS_EXTENDED_NAMES(flags)) + ph->hops->extended_name_get(ph, POWERCAP_DOMAIN_NAME_GET, + domain, dom_info->name, + SCMI_MAX_STR_SIZE); + + return ret; +} + +static int scmi_powercap_num_domains_get(const struct scmi_protocol_handle *ph) +{ + struct powercap_info *pi = ph->get_priv(ph); + + return pi->num_domains; +} + +static const struct scmi_powercap_info * +scmi_powercap_dom_info_get(const struct scmi_protocol_handle *ph, u32 domain_id) +{ + struct powercap_info *pi = ph->get_priv(ph); + + if (domain_id >= pi->num_domains) + return NULL; + + return pi->powercaps + domain_id; +} + +static int scmi_powercap_cap_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *power_cap) +{ + int ret; + struct scmi_xfer *t; + struct powercap_info *pi = ph->get_priv(ph); + + if (!power_cap || domain_id >= pi->num_domains) + return -EINVAL; + + ret = ph->xops->xfer_get_init(ph, POWERCAP_CAP_GET, sizeof(u32), + sizeof(u32), &t); + if (ret) + return ret; + + put_unaligned_le32(domain_id, t->tx.buf); + ret = ph->xops->do_xfer(ph, t); + if (!ret) + *power_cap = get_unaligned_le32(t->rx.buf); + + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_powercap_cap_set(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 power_cap, + bool ignore_dresp) +{ + int ret; + struct scmi_xfer *t; + struct scmi_msg_powercap_set_cap_or_pai *msg; + const struct scmi_powercap_info *pc; + + pc = scmi_powercap_dom_info_get(ph, domain_id); + if (!pc || !pc->powercap_cap_config || !power_cap || + power_cap < pc->min_power_cap || + power_cap > pc->max_power_cap) + return -EINVAL; + + ret = ph->xops->xfer_get_init(ph, POWERCAP_CAP_SET, + sizeof(*msg), 0, &t); + if (ret) + return ret; + + msg = t->tx.buf; + msg->domain = cpu_to_le32(domain_id); + msg->flags = + cpu_to_le32(FIELD_PREP(CAP_SET_ASYNC, !!pc->async_powercap_cap_set) | + FIELD_PREP(CAP_SET_IGNORE_DRESP, !!ignore_dresp)); + msg->value = cpu_to_le32(power_cap); + + if (!pc->async_powercap_cap_set || ignore_dresp) { + ret = ph->xops->do_xfer(ph, t); + } else { + ret = ph->xops->do_xfer_with_response(ph, t); + if (!ret) { + struct scmi_msg_resp_powercap_cap_set_complete *resp; + + resp = t->rx.buf; + if (le32_to_cpu(resp->domain) == domain_id) + dev_dbg(ph->dev, + "Powercap ID %d CAP set async to %u\n", + domain_id, + get_unaligned_le32(&resp->power_cap)); + else + ret = -EPROTO; + } + } + + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_powercap_pai_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *pai) +{ + int ret; + struct scmi_xfer *t; + struct powercap_info *pi = ph->get_priv(ph); + + if (!pai || domain_id >= pi->num_domains) + return -EINVAL; + + ret = ph->xops->xfer_get_init(ph, POWERCAP_PAI_GET, sizeof(u32), + sizeof(u32), &t); + if (ret) + return ret; + + put_unaligned_le32(domain_id, t->tx.buf); + ret = ph->xops->do_xfer(ph, t); + if (!ret) + *pai = get_unaligned_le32(t->rx.buf); + + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_powercap_pai_set(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 pai) +{ + int ret; + struct scmi_xfer *t; + struct scmi_msg_powercap_set_cap_or_pai *msg; + const struct scmi_powercap_info *pc; + + pc = scmi_powercap_dom_info_get(ph, domain_id); + if (!pc || !pc->powercap_pai_config || !pai || + pai < pc->min_pai || pai > pc->max_pai) + return -EINVAL; + + ret = ph->xops->xfer_get_init(ph, POWERCAP_PAI_SET, + sizeof(*msg), 0, &t); + if (ret) + return ret; + + msg = t->tx.buf; + msg->domain = cpu_to_le32(domain_id); + msg->flags = cpu_to_le32(0); + msg->value = cpu_to_le32(pai); + + ret = ph->xops->do_xfer(ph, t); + + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_powercap_measurements_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *average_power, + u32 *pai) +{ + int ret; + struct scmi_xfer *t; + struct scmi_msg_resp_powercap_meas_get *resp; + const struct scmi_powercap_info *pc; + + pc = scmi_powercap_dom_info_get(ph, domain_id); + if (!pc || !pc->powercap_monitoring || !pai || !average_power) + return -EINVAL; + + ret = ph->xops->xfer_get_init(ph, POWERCAP_MEASUREMENTS_GET, + sizeof(u32), sizeof(*resp), &t); + if (ret) + return ret; + + resp = t->rx.buf; + put_unaligned_le32(domain_id, t->tx.buf); + ret = ph->xops->do_xfer(ph, t); + if (!ret) { + *average_power = le32_to_cpu(resp->power); + *pai = le32_to_cpu(resp->pai); + } + + ph->xops->xfer_put(ph, t); + return ret; +} + +static int +scmi_powercap_measurements_threshold_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *power_thresh_low, + u32 *power_thresh_high) +{ + struct powercap_info *pi = ph->get_priv(ph); + + if (!power_thresh_low || !power_thresh_high || + domain_id >= pi->num_domains) + return -EINVAL; + + *power_thresh_low = THRESH_LOW(pi, domain_id); + *power_thresh_high = THRESH_HIGH(pi, domain_id); + + return 0; +} + +static int +scmi_powercap_measurements_threshold_set(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 power_thresh_low, + u32 power_thresh_high) +{ + int ret = 0; + struct powercap_info *pi = ph->get_priv(ph); + + if (domain_id >= pi->num_domains || + power_thresh_low > power_thresh_high) + return -EINVAL; + + /* Anything to do ? */ + if (THRESH_LOW(pi, domain_id) == power_thresh_low && + THRESH_HIGH(pi, domain_id) == power_thresh_high) + return ret; + + pi->states[domain_id].thresholds = + (FIELD_PREP(GENMASK_ULL(31, 0), power_thresh_low) | + FIELD_PREP(GENMASK_ULL(63, 32), power_thresh_high)); + + /* Update thresholds if notification already enabled */ + if (pi->states[domain_id].meas_notif_enabled) + ret = scmi_powercap_notify(ph, domain_id, + POWERCAP_MEASUREMENTS_NOTIFY, + true); + + return ret; +} + +static const struct scmi_powercap_proto_ops powercap_proto_ops = { + .num_domains_get = scmi_powercap_num_domains_get, + .info_get = scmi_powercap_dom_info_get, + .cap_get = scmi_powercap_cap_get, + .cap_set = scmi_powercap_cap_set, + .pai_get = scmi_powercap_pai_get, + .pai_set = scmi_powercap_pai_set, + .measurements_get = scmi_powercap_measurements_get, + .measurements_threshold_set = scmi_powercap_measurements_threshold_set, + .measurements_threshold_get = scmi_powercap_measurements_threshold_get, +}; + +static int scmi_powercap_notify(const struct scmi_protocol_handle *ph, + u32 domain, int message_id, bool enable) +{ + int ret; + struct scmi_xfer *t; + + switch (message_id) { + case POWERCAP_CAP_NOTIFY: + { + struct scmi_msg_powercap_notify_cap *notify; + + ret = ph->xops->xfer_get_init(ph, message_id, + sizeof(*notify), 0, &t); + if (ret) + return ret; + + notify = t->tx.buf; + notify->domain = cpu_to_le32(domain); + notify->notify_enable = cpu_to_le32(enable ? BIT(0) : 0); + break; + } + case POWERCAP_MEASUREMENTS_NOTIFY: + { + u32 low, high; + struct scmi_msg_powercap_notify_thresh *notify; + + /* + * Note that we have to pick the most recently configured + * thresholds to build a proper POWERCAP_MEASUREMENTS_NOTIFY + * enable request and we fail, complaining, if no thresholds + * were ever set, since this is an indication the API has been + * used wrongly. + */ + ret = scmi_powercap_measurements_threshold_get(ph, domain, + &low, &high); + if (ret) + return ret; + + if (enable && !low && !high) { + dev_err(ph->dev, + "Invalid Measurements Notify thresholds: %u/%u\n", + low, high); + return -EINVAL; + } + + ret = ph->xops->xfer_get_init(ph, message_id, + sizeof(*notify), 0, &t); + if (ret) + return ret; + + notify = t->tx.buf; + notify->domain = cpu_to_le32(domain); + notify->notify_enable = cpu_to_le32(enable ? BIT(0) : 0); + notify->power_thresh_low = cpu_to_le32(low); + notify->power_thresh_high = cpu_to_le32(high); + break; + } + default: + return -EINVAL; + } + + ret = ph->xops->do_xfer(ph, t); + + ph->xops->xfer_put(ph, t); + return ret; +} + +static int +scmi_powercap_set_notify_enabled(const struct scmi_protocol_handle *ph, + u8 evt_id, u32 src_id, bool enable) +{ + int ret, cmd_id; + struct powercap_info *pi = ph->get_priv(ph); + + if (evt_id >= ARRAY_SIZE(evt_2_cmd) || src_id >= pi->num_domains) + return -EINVAL; + + cmd_id = evt_2_cmd[evt_id]; + ret = scmi_powercap_notify(ph, src_id, cmd_id, enable); + if (ret) + pr_debug("FAIL_ENABLED - evt[%X] dom[%d] - ret:%d\n", + evt_id, src_id, ret); + else if (cmd_id == POWERCAP_MEASUREMENTS_NOTIFY) + /* + * On success save the current notification enabled state, so + * as to be able to properly update the notification thresholds + * when they are modified on a domain for which measurement + * notifications were currently enabled. + * + * This is needed because the SCMI Notification core machinery + * and API does not support passing per-notification custom + * arguments at callback registration time. + * + * Note that this can be done here with a simple flag since the + * SCMI core Notifications code takes care of keeping proper + * per-domain enables refcounting, so that this helper function + * will be called only once (for enables) when the first user + * registers a callback on this domain and once more (disable) + * when the last user de-registers its callback. + */ + pi->states[src_id].meas_notif_enabled = enable; + + return ret; +} + +static void * +scmi_powercap_fill_custom_report(const struct scmi_protocol_handle *ph, + u8 evt_id, ktime_t timestamp, + const void *payld, size_t payld_sz, + void *report, u32 *src_id) +{ + void *rep = NULL; + + switch (evt_id) { + case SCMI_EVENT_POWERCAP_CAP_CHANGED: + { + const struct scmi_powercap_cap_changed_notify_payld *p = payld; + struct scmi_powercap_cap_changed_report *r = report; + + if (sizeof(*p) != payld_sz) + break; + + r->timestamp = timestamp; + r->agent_id = le32_to_cpu(p->agent_id); + r->domain_id = le32_to_cpu(p->domain_id); + r->power_cap = le32_to_cpu(p->power_cap); + r->pai = le32_to_cpu(p->pai); + *src_id = r->domain_id; + rep = r; + break; + } + case SCMI_EVENT_POWERCAP_MEASUREMENTS_CHANGED: + { + const struct scmi_powercap_meas_changed_notify_payld *p = payld; + struct scmi_powercap_meas_changed_report *r = report; + + if (sizeof(*p) != payld_sz) + break; + + r->timestamp = timestamp; + r->agent_id = le32_to_cpu(p->agent_id); + r->domain_id = le32_to_cpu(p->domain_id); + r->power = le32_to_cpu(p->power); + *src_id = r->domain_id; + rep = r; + break; + } + default: + break; + } + + return rep; +} + +static int +scmi_powercap_get_num_sources(const struct scmi_protocol_handle *ph) +{ + struct powercap_info *pi = ph->get_priv(ph); + + if (!pi) + return -EINVAL; + + return pi->num_domains; +} + +static const struct scmi_event powercap_events[] = { + { + .id = SCMI_EVENT_POWERCAP_CAP_CHANGED, + .max_payld_sz = + sizeof(struct scmi_powercap_cap_changed_notify_payld), + .max_report_sz = + sizeof(struct scmi_powercap_cap_changed_report), + }, + { + .id = SCMI_EVENT_POWERCAP_MEASUREMENTS_CHANGED, + .max_payld_sz = + sizeof(struct scmi_powercap_meas_changed_notify_payld), + .max_report_sz = + sizeof(struct scmi_powercap_meas_changed_report), + }, +}; + +static const struct scmi_event_ops powercap_event_ops = { + .get_num_sources = scmi_powercap_get_num_sources, + .set_notify_enabled = scmi_powercap_set_notify_enabled, + .fill_custom_report = scmi_powercap_fill_custom_report, +}; + +static const struct scmi_protocol_events powercap_protocol_events = { + .queue_sz = SCMI_PROTO_QUEUE_SZ, + .ops = &powercap_event_ops, + .evts = powercap_events, + .num_events = ARRAY_SIZE(powercap_events), +}; + +static int +scmi_powercap_protocol_init(const struct scmi_protocol_handle *ph) +{ + int domain, ret; + u32 version; + struct powercap_info *pinfo; + + ret = ph->xops->version_get(ph, &version); + if (ret) + return ret; + + dev_dbg(ph->dev, "Powercap Version %d.%d\n", + PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version)); + + pinfo = devm_kzalloc(ph->dev, sizeof(*pinfo), GFP_KERNEL); + if (!pinfo) + return -ENOMEM; + + ret = scmi_powercap_attributes_get(ph, pinfo); + if (ret) + return ret; + + pinfo->powercaps = devm_kcalloc(ph->dev, pinfo->num_domains, + sizeof(*pinfo->powercaps), + GFP_KERNEL); + if (!pinfo->powercaps) + return -ENOMEM; + + /* + * Note that any failure in retrieving any domain attribute leads to + * the whole Powercap protocol initialization failure: this way the + * reported Powercap domains are all assured, when accessed, to be well + * formed and correlated by sane parent-child relationship (if any). + */ + for (domain = 0; domain < pinfo->num_domains; domain++) { + ret = scmi_powercap_domain_attributes_get(ph, pinfo, domain); + if (ret) + return ret; + } + + pinfo->states = devm_kcalloc(ph->dev, pinfo->num_domains, + sizeof(*pinfo->states), GFP_KERNEL); + if (!pinfo->states) + return -ENOMEM; + + pinfo->version = version; + + return ph->set_priv(ph, pinfo); +} + +static const struct scmi_protocol scmi_powercap = { + .id = SCMI_PROTOCOL_POWERCAP, + .owner = THIS_MODULE, + .instance_init = &scmi_powercap_protocol_init, + .ops = &powercap_proto_ops, + .events = &powercap_protocol_events, +}; + +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(powercap, scmi_powercap) diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h index 51c31379f9b3..99d36d503d1e 100644 --- a/drivers/firmware/arm_scmi/protocols.h +++ b/drivers/firmware/arm_scmi/protocols.h @@ -315,5 +315,6 @@ DECLARE_SCMI_REGISTER_UNREGISTER(reset); DECLARE_SCMI_REGISTER_UNREGISTER(sensors); DECLARE_SCMI_REGISTER_UNREGISTER(voltage); DECLARE_SCMI_REGISTER_UNREGISTER(system); +DECLARE_SCMI_REGISTER_UNREGISTER(powercap); #endif /* _SCMI_PROTOCOLS_H */ diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 898488c7f185..95023dc8b3a3 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -560,6 +560,114 @@ struct scmi_voltage_proto_ops { s32 *volt_uV); }; +/** + * struct scmi_powercap_info - Describe one available Powercap domain + * + * @id: Domain ID as advertised by the platform. + * @notify_powercap_cap_change: CAP change notification support. + * @notify_powercap_measurement_change: MEASUREMENTS change notifications + * support. + * @async_powercap_cap_set: Asynchronous CAP set support. + * @powercap_cap_config: CAP configuration support. + * @powercap_monitoring: Monitoring (measurements) support. + * @powercap_pai_config: PAI configuration support. + * @powercap_scale_mw: Domain reports power data in milliwatt units. + * @powercap_scale_uw: Domain reports power data in microwatt units. + * Note that, when both @powercap_scale_mw and + * @powercap_scale_uw are set to false, the domain + * reports power data on an abstract linear scale. + * @name: name assigned to the Powercap Domain by platform. + * @min_pai: Minimum configurable PAI. + * @max_pai: Maximum configurable PAI. + * @pai_step: Step size between two consecutive PAI values. + * @min_power_cap: Minimum configurable CAP. + * @max_power_cap: Maximum configurable CAP. + * @power_cap_step: Step size between two consecutive CAP values. + * @sustainable_power: Maximum sustainable power consumption for this domain + * under normal conditions. + * @accuracy: The accuracy with which the power is measured and reported in + * integral multiples of 0.001 percent. + * @parent_id: Identifier of the containing parent power capping domain, or the + * value 0xFFFFFFFF if this powercap domain is a root domain not + * contained in any other domain. + */ +struct scmi_powercap_info { + unsigned int id; + bool notify_powercap_cap_change; + bool notify_powercap_measurement_change; + bool async_powercap_cap_set; + bool powercap_cap_config; + bool powercap_monitoring; + bool powercap_pai_config; + bool powercap_scale_mw; + bool powercap_scale_uw; + char name[SCMI_MAX_STR_SIZE]; + unsigned int min_pai; + unsigned int max_pai; + unsigned int pai_step; + unsigned int min_power_cap; + unsigned int max_power_cap; + unsigned int power_cap_step; + unsigned int sustainable_power; + unsigned int accuracy; +#define SCMI_POWERCAP_ROOT_ZONE_ID 0xFFFFFFFFUL + unsigned int parent_id; +}; + +/** + * struct scmi_powercap_proto_ops - represents the various operations provided + * by SCMI Powercap Protocol + * + * @num_domains_get: get the count of powercap domains provided by SCMI. + * @info_get: get the information for the specified domain. + * @cap_get: get the current CAP value for the specified domain. + * @cap_set: set the CAP value for the specified domain to the provided value; + * if the domain supports setting the CAP with an asynchronous command + * this request will finally trigger an asynchronous transfer, but, if + * @ignore_dresp here is set to true, this call will anyway return + * immediately without waiting for the related delayed response. + * @pai_get: get the current PAI value for the specified domain. + * @pai_set: set the PAI value for the specified domain to the provided value. + * @measurements_get: retrieve the current average power measurements for the + * specified domain and the related PAI upon which is + * calculated. + * @measurements_threshold_set: set the desired low and high power thresholds + * to be used when registering for notification + * of type POWERCAP_MEASUREMENTS_NOTIFY with this + * powercap domain. + * Note that this must be called at least once + * before registering any callback with the usual + * @scmi_notify_ops; moreover, in case this method + * is called with measurement notifications already + * enabled it will also trigger, transparently, a + * proper update of the power thresholds configured + * in the SCMI backend server. + * @measurements_threshold_get: get the currently configured low and high power + * thresholds used when registering callbacks for + * notification POWERCAP_MEASUREMENTS_NOTIFY. + */ +struct scmi_powercap_proto_ops { + int (*num_domains_get)(const struct scmi_protocol_handle *ph); + const struct scmi_powercap_info __must_check *(*info_get) + (const struct scmi_protocol_handle *ph, u32 domain_id); + int (*cap_get)(const struct scmi_protocol_handle *ph, u32 domain_id, + u32 *power_cap); + int (*cap_set)(const struct scmi_protocol_handle *ph, u32 domain_id, + u32 power_cap, bool ignore_dresp); + int (*pai_get)(const struct scmi_protocol_handle *ph, u32 domain_id, + u32 *pai); + int (*pai_set)(const struct scmi_protocol_handle *ph, u32 domain_id, + u32 pai); + int (*measurements_get)(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *average_power, u32 *pai); + int (*measurements_threshold_set)(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 power_thresh_low, + u32 power_thresh_high); + int (*measurements_threshold_get)(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *power_thresh_low, + u32 *power_thresh_high); +}; + /** * struct scmi_notify_ops - represents notifications' operations provided by * SCMI core @@ -666,6 +774,7 @@ enum scmi_std_protocol { SCMI_PROTOCOL_SENSOR = 0x15, SCMI_PROTOCOL_RESET = 0x16, SCMI_PROTOCOL_VOLTAGE = 0x17, + SCMI_PROTOCOL_POWERCAP = 0x18, }; enum scmi_system_events { @@ -767,6 +876,8 @@ enum scmi_notification_events { SCMI_EVENT_RESET_ISSUED = 0x0, SCMI_EVENT_BASE_ERROR_EVENT = 0x0, SCMI_EVENT_SYSTEM_POWER_STATE_NOTIFIER = 0x0, + SCMI_EVENT_POWERCAP_CAP_CHANGED = 0x0, + SCMI_EVENT_POWERCAP_MEASUREMENTS_CHANGED = 0x1, }; struct scmi_power_state_changed_report { @@ -837,4 +948,18 @@ struct scmi_base_error_report { unsigned long long reports[]; }; +struct scmi_powercap_cap_changed_report { + ktime_t timestamp; + unsigned int agent_id; + unsigned int domain_id; + unsigned int power_cap; + unsigned int pai; +}; + +struct scmi_powercap_meas_changed_report { + ktime_t timestamp; + unsigned int agent_id; + unsigned int domain_id; + unsigned int power; +}; #endif /* _LINUX_SCMI_PROTOCOL_H */ -- cgit v1.2.3 From 855aa26e5f56d415b71d3f8d86ef0cc51b2166a3 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 4 Jul 2022 11:22:38 +0100 Subject: firmware: arm_scmi: Add SCMI v3.1 powercap fast channels support Add SCMIv3.1 powercap protocol fast channel support using common helpers provided by the SCMI core with scmi_proto_helpers_ops operations. Link: https://lore.kernel.org/r/20220704102241.2988447-5-cristian.marussi@arm.com Signed-off-by: Cristian Marussi Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/powercap.c | 169 ++++++++++++++++++++++++++++------- include/linux/scmi_protocol.h | 2 + 2 files changed, 138 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/powercap.c b/drivers/firmware/arm_scmi/powercap.c index d71a1f6c2d8a..10b77c3401b5 100644 --- a/drivers/firmware/arm_scmi/powercap.c +++ b/drivers/firmware/arm_scmi/powercap.c @@ -8,6 +8,7 @@ #define pr_fmt(fmt) "SCMI Notifications POWERCAP - " fmt #include +#include #include #include @@ -27,6 +28,12 @@ enum scmi_powercap_protocol_cmd { POWERCAP_DESCRIBE_FASTCHANNEL = 0xc, }; +enum { + POWERCAP_FC_CAP, + POWERCAP_FC_PAI, + POWERCAP_FC_MAX, +}; + struct scmi_msg_resp_powercap_domain_attributes { __le32 attributes; #define SUPPORTS_POWERCAP_CAP_CHANGE_NOTIFY(x) ((x) & BIT(31)) @@ -36,6 +43,7 @@ struct scmi_msg_resp_powercap_domain_attributes { #define SUPPORTS_POWERCAP_CAP_CONFIGURATION(x) ((x) & BIT(27)) #define SUPPORTS_POWERCAP_MONITORING(x) ((x) & BIT(26)) #define SUPPORTS_POWERCAP_PAI_CONFIGURATION(x) ((x) & BIT(25)) +#define SUPPORTS_POWERCAP_FASTCHANNELS(x) ((x) & BIT(22)) #define POWERCAP_POWER_UNIT(x) \ (FIELD_GET(GENMASK(24, 23), (x))) #define SUPPORTS_POWER_UNITS_MW(x) \ @@ -201,6 +209,8 @@ scmi_powercap_domain_attributes_get(const struct scmi_protocol_handle *ph, SUPPORTS_POWER_UNITS_MW(flags); dom_info->powercap_scale_uw = SUPPORTS_POWER_UNITS_UW(flags); + dom_info->fastchannels = + SUPPORTS_POWERCAP_FASTCHANNELS(flags); strscpy(dom_info->name, resp->name, SCMI_SHORT_NAME_MAX_SIZE); @@ -280,15 +290,11 @@ scmi_powercap_dom_info_get(const struct scmi_protocol_handle *ph, u32 domain_id) return pi->powercaps + domain_id; } -static int scmi_powercap_cap_get(const struct scmi_protocol_handle *ph, - u32 domain_id, u32 *power_cap) +static int scmi_powercap_xfer_cap_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *power_cap) { int ret; struct scmi_xfer *t; - struct powercap_info *pi = ph->get_priv(ph); - - if (!power_cap || domain_id >= pi->num_domains) - return -EINVAL; ret = ph->xops->xfer_get_init(ph, POWERCAP_CAP_GET, sizeof(u32), sizeof(u32), &t); @@ -305,20 +311,31 @@ static int scmi_powercap_cap_get(const struct scmi_protocol_handle *ph, return ret; } -static int scmi_powercap_cap_set(const struct scmi_protocol_handle *ph, - u32 domain_id, u32 power_cap, - bool ignore_dresp) +static int scmi_powercap_cap_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *power_cap) +{ + struct scmi_powercap_info *dom; + struct powercap_info *pi = ph->get_priv(ph); + + if (!power_cap || domain_id >= pi->num_domains) + return -EINVAL; + + dom = pi->powercaps + domain_id; + if (dom->fc_info && dom->fc_info[POWERCAP_FC_CAP].get_addr) { + *power_cap = ioread32(dom->fc_info[POWERCAP_FC_CAP].get_addr); + return 0; + } + + return scmi_powercap_xfer_cap_get(ph, domain_id, power_cap); +} + +static int scmi_powercap_xfer_cap_set(const struct scmi_protocol_handle *ph, + const struct scmi_powercap_info *pc, + u32 power_cap, bool ignore_dresp) { int ret; struct scmi_xfer *t; struct scmi_msg_powercap_set_cap_or_pai *msg; - const struct scmi_powercap_info *pc; - - pc = scmi_powercap_dom_info_get(ph, domain_id); - if (!pc || !pc->powercap_cap_config || !power_cap || - power_cap < pc->min_power_cap || - power_cap > pc->max_power_cap) - return -EINVAL; ret = ph->xops->xfer_get_init(ph, POWERCAP_CAP_SET, sizeof(*msg), 0, &t); @@ -326,7 +343,7 @@ static int scmi_powercap_cap_set(const struct scmi_protocol_handle *ph, return ret; msg = t->tx.buf; - msg->domain = cpu_to_le32(domain_id); + msg->domain = cpu_to_le32(pc->id); msg->flags = cpu_to_le32(FIELD_PREP(CAP_SET_ASYNC, !!pc->async_powercap_cap_set) | FIELD_PREP(CAP_SET_IGNORE_DRESP, !!ignore_dresp)); @@ -340,10 +357,10 @@ static int scmi_powercap_cap_set(const struct scmi_protocol_handle *ph, struct scmi_msg_resp_powercap_cap_set_complete *resp; resp = t->rx.buf; - if (le32_to_cpu(resp->domain) == domain_id) + if (le32_to_cpu(resp->domain) == pc->id) dev_dbg(ph->dev, "Powercap ID %d CAP set async to %u\n", - domain_id, + pc->id, get_unaligned_le32(&resp->power_cap)); else ret = -EPROTO; @@ -354,16 +371,35 @@ static int scmi_powercap_cap_set(const struct scmi_protocol_handle *ph, return ret; } -static int scmi_powercap_pai_get(const struct scmi_protocol_handle *ph, - u32 domain_id, u32 *pai) +static int scmi_powercap_cap_set(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 power_cap, + bool ignore_dresp) { - int ret; - struct scmi_xfer *t; - struct powercap_info *pi = ph->get_priv(ph); + const struct scmi_powercap_info *pc; - if (!pai || domain_id >= pi->num_domains) + pc = scmi_powercap_dom_info_get(ph, domain_id); + if (!pc || !pc->powercap_cap_config || !power_cap || + power_cap < pc->min_power_cap || + power_cap > pc->max_power_cap) return -EINVAL; + if (pc->fc_info && pc->fc_info[POWERCAP_FC_CAP].set_addr) { + struct scmi_fc_info *fci = &pc->fc_info[POWERCAP_FC_CAP]; + + iowrite32(power_cap, fci->set_addr); + ph->hops->fastchannel_db_ring(fci->set_db); + return 0; + } + + return scmi_powercap_xfer_cap_set(ph, pc, power_cap, ignore_dresp); +} + +static int scmi_powercap_xfer_pai_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *pai) +{ + int ret; + struct scmi_xfer *t; + ret = ph->xops->xfer_get_init(ph, POWERCAP_PAI_GET, sizeof(u32), sizeof(u32), &t); if (ret) @@ -379,18 +415,30 @@ static int scmi_powercap_pai_get(const struct scmi_protocol_handle *ph, return ret; } -static int scmi_powercap_pai_set(const struct scmi_protocol_handle *ph, - u32 domain_id, u32 pai) +static int scmi_powercap_pai_get(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 *pai) +{ + struct scmi_powercap_info *dom; + struct powercap_info *pi = ph->get_priv(ph); + + if (!pai || domain_id >= pi->num_domains) + return -EINVAL; + + dom = pi->powercaps + domain_id; + if (dom->fc_info && dom->fc_info[POWERCAP_FC_PAI].get_addr) { + *pai = ioread32(dom->fc_info[POWERCAP_FC_PAI].get_addr); + return 0; + } + + return scmi_powercap_xfer_pai_get(ph, domain_id, pai); +} + +static int scmi_powercap_xfer_pai_set(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 pai) { int ret; struct scmi_xfer *t; struct scmi_msg_powercap_set_cap_or_pai *msg; - const struct scmi_powercap_info *pc; - - pc = scmi_powercap_dom_info_get(ph, domain_id); - if (!pc || !pc->powercap_pai_config || !pai || - pai < pc->min_pai || pai > pc->max_pai) - return -EINVAL; ret = ph->xops->xfer_get_init(ph, POWERCAP_PAI_SET, sizeof(*msg), 0, &t); @@ -408,6 +456,27 @@ static int scmi_powercap_pai_set(const struct scmi_protocol_handle *ph, return ret; } +static int scmi_powercap_pai_set(const struct scmi_protocol_handle *ph, + u32 domain_id, u32 pai) +{ + const struct scmi_powercap_info *pc; + + pc = scmi_powercap_dom_info_get(ph, domain_id); + if (!pc || !pc->powercap_pai_config || !pai || + pai < pc->min_pai || pai > pc->max_pai) + return -EINVAL; + + if (pc->fc_info && pc->fc_info[POWERCAP_FC_PAI].set_addr) { + struct scmi_fc_info *fci = &pc->fc_info[POWERCAP_FC_PAI]; + + iowrite32(pai, fci->set_addr); + ph->hops->fastchannel_db_ring(fci->set_db); + return 0; + } + + return scmi_powercap_xfer_pai_set(ph, domain_id, pai); +} + static int scmi_powercap_measurements_get(const struct scmi_protocol_handle *ph, u32 domain_id, u32 *average_power, u32 *pai) @@ -497,6 +566,36 @@ static const struct scmi_powercap_proto_ops powercap_proto_ops = { .measurements_threshold_get = scmi_powercap_measurements_threshold_get, }; +static void scmi_powercap_domain_init_fc(const struct scmi_protocol_handle *ph, + u32 domain, struct scmi_fc_info **p_fc) +{ + struct scmi_fc_info *fc; + + fc = devm_kcalloc(ph->dev, POWERCAP_FC_MAX, sizeof(*fc), GFP_KERNEL); + if (!fc) + return; + + ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, + POWERCAP_CAP_SET, 4, domain, + &fc[POWERCAP_FC_CAP].set_addr, + &fc[POWERCAP_FC_CAP].set_db); + + ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, + POWERCAP_CAP_GET, 4, domain, + &fc[POWERCAP_FC_CAP].get_addr, NULL); + + ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, + POWERCAP_PAI_SET, 4, domain, + &fc[POWERCAP_FC_PAI].set_addr, + &fc[POWERCAP_FC_PAI].set_db); + + ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, + POWERCAP_PAI_GET, 4, domain, + &fc[POWERCAP_FC_PAI].get_addr, NULL); + + *p_fc = fc; +} + static int scmi_powercap_notify(const struct scmi_protocol_handle *ph, u32 domain, int message_id, bool enable) { @@ -730,6 +829,10 @@ scmi_powercap_protocol_init(const struct scmi_protocol_handle *ph) ret = scmi_powercap_domain_attributes_get(ph, pinfo, domain); if (ret) return ret; + + if (pinfo->powercaps[domain].fastchannels) + scmi_powercap_domain_init_fc(ph, domain, + &pinfo->powercaps[domain].fc_info); } pinfo->states = devm_kcalloc(ph->dev, pinfo->num_domains, diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 95023dc8b3a3..7f4f9df1b20f 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -601,6 +601,7 @@ struct scmi_powercap_info { bool powercap_pai_config; bool powercap_scale_mw; bool powercap_scale_uw; + bool fastchannels; char name[SCMI_MAX_STR_SIZE]; unsigned int min_pai; unsigned int max_pai; @@ -612,6 +613,7 @@ struct scmi_powercap_info { unsigned int accuracy; #define SCMI_POWERCAP_ROOT_ZONE_ID 0xFFFFFFFFUL unsigned int parent_id; + struct scmi_fc_info *fc_info; }; /** -- cgit v1.2.3 From cc1cfc47ea47187a21ec1f079b3c53264157fe15 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Mon, 4 Jul 2022 11:15:49 +0100 Subject: cacheinfo: Add support to check if last level cache(LLC) is valid or shared It is useful to have helper to check if the given two CPUs share last level cache. We can do that check by comparing fw_token or by comparing the cache ID. Currently we check just for fw_token as the cache ID is optional. This helper can be used to build the llc_sibling during arch specific topology parsing and feeding information to the sched_domains. This also helps to get rid of llc_id in the CPU topology as it is sort of duplicate information. Also add helper to check if the llc information in cacheinfo is valid or not. Link: https://lore.kernel.org/r/20220704101605.1318280-6-sudeep.holla@arm.com Tested-by: Ionela Voinescu Tested-by: Conor Dooley Reviewed-by: Gavin Shan Signed-off-by: Sudeep Holla --- drivers/base/cacheinfo.c | 26 ++++++++++++++++++++++++++ include/linux/cacheinfo.h | 2 ++ 2 files changed, 28 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index 2cea9201f31c..fdc1baa342f1 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -47,6 +47,32 @@ static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf, return sib_leaf->fw_token == this_leaf->fw_token; } +bool last_level_cache_is_valid(unsigned int cpu) +{ + struct cacheinfo *llc; + + if (!cache_leaves(cpu)) + return false; + + llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); + + return !!llc->fw_token; +} + +bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y) +{ + struct cacheinfo *llc_x, *llc_y; + + if (!last_level_cache_is_valid(cpu_x) || + !last_level_cache_is_valid(cpu_y)) + return false; + + llc_x = per_cpu_cacheinfo_idx(cpu_x, cache_leaves(cpu_x) - 1); + llc_y = per_cpu_cacheinfo_idx(cpu_y, cache_leaves(cpu_y) - 1); + + return cache_leaves_are_shared(llc_x, llc_y); +} + #ifdef CONFIG_OF /* OF properties to query for a given cache type */ struct cache_type_info { diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index 4ff37cb763ae..7e429bc5c1a4 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -82,6 +82,8 @@ struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu); int init_cache_level(unsigned int cpu); int populate_cache_leaves(unsigned int cpu); int cache_setup_acpi(unsigned int cpu); +bool last_level_cache_is_valid(unsigned int cpu); +bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y); #ifndef CONFIG_ACPI_PPTT /* * acpi_find_last_cache_level is only called on ACPI enabled -- cgit v1.2.3 From 36bbc5b4ffab33ccac0f4db27f619a6ba7a4fd32 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Mon, 4 Jul 2022 11:15:50 +0100 Subject: cacheinfo: Allow early detection and population of cache attributes Some architecture/platforms may need to setup cache properties very early in the boot along with other cpu topologies so that all these information can be used to build sched_domains which is used by the scheduler. Allow detect_cache_attributes to be called quite early during the boot. Link: https://lore.kernel.org/r/20220704101605.1318280-7-sudeep.holla@arm.com Tested-by: Ionela Voinescu Tested-by: Conor Dooley Reviewed-by: Gavin Shan Signed-off-by: Sudeep Holla --- drivers/base/cacheinfo.c | 55 +++++++++++++++++++++++++++++++---------------- include/linux/cacheinfo.h | 1 + 2 files changed, 38 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index fdc1baa342f1..4d21a1022fa9 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -193,14 +193,8 @@ static int cache_setup_of_node(unsigned int cpu) { struct device_node *np; struct cacheinfo *this_leaf; - struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); unsigned int index = 0; - /* skip if fw_token is already populated */ - if (this_cpu_ci->info_list->fw_token) { - return 0; - } - np = of_cpu_device_node_get(cpu); if (!np) { pr_err("Failed to find cpu%d device node\n", cpu); @@ -236,6 +230,18 @@ int __weak cache_setup_acpi(unsigned int cpu) unsigned int coherency_max_size; +static int cache_setup_properties(unsigned int cpu) +{ + int ret = 0; + + if (of_have_populated_dt()) + ret = cache_setup_of_node(cpu); + else if (!acpi_disabled) + ret = cache_setup_acpi(cpu); + + return ret; +} + static int cache_shared_cpu_map_setup(unsigned int cpu) { struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); @@ -246,21 +252,21 @@ static int cache_shared_cpu_map_setup(unsigned int cpu) if (this_cpu_ci->cpu_map_populated) return 0; - if (of_have_populated_dt()) - ret = cache_setup_of_node(cpu); - else if (!acpi_disabled) - ret = cache_setup_acpi(cpu); - - if (ret) - return ret; + /* + * skip setting up cache properties if LLC is valid, just need + * to update the shared cpu_map if the cache attributes were + * populated early before all the cpus are brought online + */ + if (!last_level_cache_is_valid(cpu)) { + ret = cache_setup_properties(cpu); + if (ret) + return ret; + } for (index = 0; index < cache_leaves(cpu); index++) { unsigned int i; this_leaf = per_cpu_cacheinfo_idx(cpu, index); - /* skip if shared_cpu_map is already populated */ - if (!cpumask_empty(&this_leaf->shared_cpu_map)) - continue; cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); for_each_online_cpu(i) { @@ -330,17 +336,28 @@ int __weak populate_cache_leaves(unsigned int cpu) return -ENOENT; } -static int detect_cache_attributes(unsigned int cpu) +int detect_cache_attributes(unsigned int cpu) { int ret; + /* Since early detection of the cacheinfo is allowed via this + * function and this also gets called as CPU hotplug callbacks via + * cacheinfo_cpu_online, the initialisation can be skipped and only + * CPU maps can be updated as the CPU online status would be update + * if called via cacheinfo_cpu_online path. + */ + if (per_cpu_cacheinfo(cpu)) + goto update_cpu_map; + if (init_cache_level(cpu) || !cache_leaves(cpu)) return -ENOENT; per_cpu_cacheinfo(cpu) = kcalloc(cache_leaves(cpu), sizeof(struct cacheinfo), GFP_KERNEL); - if (per_cpu_cacheinfo(cpu) == NULL) + if (per_cpu_cacheinfo(cpu) == NULL) { + cache_leaves(cpu) = 0; return -ENOMEM; + } /* * populate_cache_leaves() may completely setup the cache leaves and @@ -349,6 +366,8 @@ static int detect_cache_attributes(unsigned int cpu) ret = populate_cache_leaves(cpu); if (ret) goto free_ci; + +update_cpu_map: /* * For systems using DT for cache hierarchy, fw_token * and shared_cpu_map will be set up here only if they are diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index 7e429bc5c1a4..00b7a6ae8617 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -84,6 +84,7 @@ int populate_cache_leaves(unsigned int cpu); int cache_setup_acpi(unsigned int cpu); bool last_level_cache_is_valid(unsigned int cpu); bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y); +int detect_cache_attributes(unsigned int cpu); #ifndef CONFIG_ACPI_PPTT /* * acpi_find_last_cache_level is only called on ACPI enabled -- cgit v1.2.3 From 5b8dc787ce4a45c87254d1b0b22f161347ab7f81 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Mon, 4 Jul 2022 11:15:56 +0100 Subject: arch_topology: Drop LLC identifier stash from the CPU topology Since the cacheinfo LLC information is used directly in arch_topology, there is no need to parse and store the LLC ID information only for ACPI systems in the CPU topology. Remove the redundant LLC ID from the generic CPU arch_topology information. Link: https://lore.kernel.org/r/20220704101605.1318280-13-sudeep.holla@arm.com Tested-by: Ionela Voinescu Tested-by: Conor Dooley Reviewed-by: Gavin Shan Signed-off-by: Sudeep Holla --- drivers/base/arch_topology.c | 1 - include/linux/arch_topology.h | 1 - 2 files changed, 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 4f936c984fb6..8206990c679f 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -752,7 +752,6 @@ void __init reset_cpu_topology(void) cpu_topo->core_id = -1; cpu_topo->cluster_id = -1; cpu_topo->package_id = -1; - cpu_topo->llc_id = -1; clear_cpu_topology(cpu); } diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 58cbe18d825c..a07b510e7dc5 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -68,7 +68,6 @@ struct cpu_topology { int core_id; int cluster_id; int package_id; - int llc_id; cpumask_t thread_sibling; cpumask_t core_sibling; cpumask_t cluster_sibling; -- cgit v1.2.3 From 7128af87c7f1c30cd6cebe0b012cc25872c689e2 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Mon, 4 Jul 2022 11:16:05 +0100 Subject: ACPI: Remove the unused find_acpi_cpu_cache_topology() The sole user of this find_acpi_cpu_cache_topology() was arm64 topology which is now consolidated into the generic arch_topology without the need of this function. Drop the unused function find_acpi_cpu_cache_topology(). Link: https://lore.kernel.org/r/20220704101605.1318280-22-sudeep.holla@arm.com Cc: Rafael J. Wysocki Reported-by: Ionela Voinescu Tested-by: Conor Dooley Acked-by: Rafael J. Wysocki Signed-off-by: Sudeep Holla --- drivers/acpi/pptt.c | 37 ------------------------------------- include/linux/acpi.h | 5 ----- 2 files changed, 42 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c index 763f021d45e6..dd3222a15c9c 100644 --- a/drivers/acpi/pptt.c +++ b/drivers/acpi/pptt.c @@ -691,43 +691,6 @@ int find_acpi_cpu_topology(unsigned int cpu, int level) return find_acpi_cpu_topology_tag(cpu, level, 0); } -/** - * find_acpi_cpu_cache_topology() - Determine a unique cache topology value - * @cpu: Kernel logical CPU number - * @level: The cache level for which we would like a unique ID - * - * Determine a unique ID for each unified cache in the system - * - * Return: -ENOENT if the PPTT doesn't exist, or the CPU cannot be found. - * Otherwise returns a value which represents a unique topological feature. - */ -int find_acpi_cpu_cache_topology(unsigned int cpu, int level) -{ - struct acpi_table_header *table; - struct acpi_pptt_cache *found_cache; - acpi_status status; - u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu); - struct acpi_pptt_processor *cpu_node = NULL; - int ret = -1; - - status = acpi_get_table(ACPI_SIG_PPTT, 0, &table); - if (ACPI_FAILURE(status)) { - acpi_pptt_warn_missing(); - return -ENOENT; - } - - found_cache = acpi_find_cache_node(table, acpi_cpu_id, - CACHE_TYPE_UNIFIED, - level, - &cpu_node); - if (found_cache) - ret = ACPI_PTR_DIFF(cpu_node, table); - - acpi_put_table(table); - - return ret; -} - /** * find_acpi_cpu_topology_package() - Determine a unique CPU package value * @cpu: Kernel logical CPU number diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 4f82a5bc6d98..7b96a8bff6d2 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1429,7 +1429,6 @@ int find_acpi_cpu_topology(unsigned int cpu, int level); int find_acpi_cpu_topology_cluster(unsigned int cpu); int find_acpi_cpu_topology_package(unsigned int cpu); int find_acpi_cpu_topology_hetero_id(unsigned int cpu); -int find_acpi_cpu_cache_topology(unsigned int cpu, int level); #else static inline int acpi_pptt_cpu_is_thread(unsigned int cpu) { @@ -1451,10 +1450,6 @@ static inline int find_acpi_cpu_topology_hetero_id(unsigned int cpu) { return -EINVAL; } -static inline int find_acpi_cpu_cache_topology(unsigned int cpu, int level) -{ - return -EINVAL; -} #endif #ifdef CONFIG_ACPI_PCC -- cgit v1.2.3 From 50d6281ce9b8412f7ef02d1bc9d23aa62ae0cf98 Mon Sep 17 00:00:00 2001 From: Ren Zhijie Date: Thu, 30 Jun 2022 20:35:28 +0800 Subject: dma-mapping: Fix build error unused-value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If CONFIG_DMA_DECLARE_COHERENT is not set, make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- will be failed, like this: drivers/remoteproc/remoteproc_core.c: In function ‘rproc_rvdev_release’: ./include/linux/dma-map-ops.h:182:42: error: statement with no effect [-Werror=unused-value] #define dma_release_coherent_memory(dev) (0) ^ drivers/remoteproc/remoteproc_core.c:464:2: note: in expansion of macro ‘dma_release_coherent_memory’ dma_release_coherent_memory(dev); ^~~~~~~~~~~~~~~~~~~~~~~~~~~ cc1: all warnings being treated as errors The return type of function dma_release_coherent_memory in CONFIG_DMA_DECLARE_COHERENT area is void, so in !CONFIG_DMA_DECLARE_COHERENT area it should neither return any value nor be defined as zero. Reported-by: Hulk Robot Fixes: e61c451476e6 ("dma-mapping: Add dma_release_coherent_memory to DMA API") Signed-off-by: Ren Zhijie Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220630123528.251181-1-renzhijie2@huawei.com Signed-off-by: Mathieu Poirier --- include/linux/dma-map-ops.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 53db9655efe9..bfffe494356a 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -179,10 +179,10 @@ static inline int dma_declare_coherent_memory(struct device *dev, return -ENOSYS; } -#define dma_release_coherent_memory(dev) (0) #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0) #define dma_release_from_dev_coherent(dev, order, vaddr) (0) #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0) +static inline void dma_release_coherent_memory(struct device *dev) { } #endif /* CONFIG_DMA_DECLARE_COHERENT */ #ifdef CONFIG_DMA_GLOBAL_POOL -- cgit v1.2.3 From 3dcb861dbc6ab101838a1548b1efddd00ca3c3ec Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Thu, 30 Jun 2022 11:40:59 +0200 Subject: ACPI: VIOT: Fix ACS setup Currently acpi_viot_init() gets called after the pci device has been scanned and pci_enable_acs() has been called. So pci_request_acs() fails to be taken into account leading to wrong single iommu group topologies when dealing with multi-function root ports for instance. We cannot simply move the acpi_viot_init() earlier, similarly as the IORT init because the VIOT parsing relies on the pci scan. However we can detect VIOT is present earlier and in such a case, request ACS. Introduce a new acpi_viot_early_init() routine that allows to call pci_request_acs() before the scan. While at it, guard the call to pci_request_acs() with #ifdef CONFIG_PCI. Fixes: 3cf485540e7b ("ACPI: Add driver for the VIOT table") Signed-off-by: Eric Auger Reported-by: Jin Liu Reviewed-by: Jean-Philippe Brucker Tested-by: Jean-Philippe Brucker Signed-off-by: Rafael J. Wysocki --- drivers/acpi/bus.c | 1 + drivers/acpi/viot.c | 26 ++++++++++++++++++++------ include/linux/acpi_viot.h | 2 ++ 3 files changed, 23 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 86fa61a21826..906ad8153fd9 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -1400,6 +1400,7 @@ static int __init acpi_init(void) pci_mmcfg_late_init(); acpi_iort_init(); + acpi_viot_early_init(); acpi_hest_init(); acpi_ghes_init(); acpi_scan_init(); diff --git a/drivers/acpi/viot.c b/drivers/acpi/viot.c index d2256326c73a..647f11cf165d 100644 --- a/drivers/acpi/viot.c +++ b/drivers/acpi/viot.c @@ -248,6 +248,26 @@ err_free: return ret; } +/** + * acpi_viot_early_init - Test the presence of VIOT and enable ACS + * + * If the VIOT does exist, ACS must be enabled. This cannot be + * done in acpi_viot_init() which is called after the bus scan + */ +void __init acpi_viot_early_init(void) +{ +#ifdef CONFIG_PCI + acpi_status status; + struct acpi_table_header *hdr; + + status = acpi_get_table(ACPI_SIG_VIOT, 0, &hdr); + if (ACPI_FAILURE(status)) + return; + pci_request_acs(); + acpi_put_table(hdr); +#endif +} + /** * acpi_viot_init - Parse the VIOT table * @@ -319,12 +339,6 @@ static int viot_pci_dev_iommu_init(struct pci_dev *pdev, u16 dev_id, void *data) epid = ((domain_nr - ep->segment_start) << 16) + dev_id - ep->bdf_start + ep->endpoint_id; - /* - * If we found a PCI range managed by the viommu, we're - * the one that has to request ACS. - */ - pci_request_acs(); - return viot_dev_iommu_init(&pdev->dev, ep->viommu, epid); } diff --git a/include/linux/acpi_viot.h b/include/linux/acpi_viot.h index 1eb8ee5b0e5f..a5a122431563 100644 --- a/include/linux/acpi_viot.h +++ b/include/linux/acpi_viot.h @@ -6,9 +6,11 @@ #include #ifdef CONFIG_ACPI_VIOT +void __init acpi_viot_early_init(void); void __init acpi_viot_init(void); int viot_iommu_configure(struct device *dev); #else +static inline void acpi_viot_early_init(void) {} static inline void acpi_viot_init(void) {} static inline int viot_iommu_configure(struct device *dev) { -- cgit v1.2.3 From 09d3154a6f0f0bb5b604832095804780f3684b96 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 6 Jun 2022 22:51:58 -0500 Subject: PM: wakeup: Unify device_init_wakeup() for PM_SLEEP and !PM_SLEEP Previously the CONFIG_PM_SLEEP and !CONFIG_PM_SLEEP device_init_wakeup() implementations differed in confusing ways: - The PM_SLEEP version checked for a NULL device pointer and returned -EINVAL, while the !PM_SLEEP version did not and would simply dereference a NULL pointer. - When called with "false", the !PM_SLEEP version cleared "capable" and "enable" in the opposite order of the PM_SLEEP version. That was harmless because for !PM_SLEEP they're simple assignments, but it's unnecessary confusion. Use a simplified version of the PM_SLEEP implementation for both cases. Signed-off-by: Bjorn Helgaas Signed-off-by: Rafael J. Wysocki --- drivers/base/power/wakeup.c | 30 ------------------------------ include/linux/pm_wakeup.h | 31 +++++++++++++++++++++++-------- 2 files changed, 23 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 11a4ffe91367..e3befa2c1b66 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -500,36 +500,6 @@ void device_set_wakeup_capable(struct device *dev, bool capable) } EXPORT_SYMBOL_GPL(device_set_wakeup_capable); -/** - * device_init_wakeup - Device wakeup initialization. - * @dev: Device to handle. - * @enable: Whether or not to enable @dev as a wakeup device. - * - * By default, most devices should leave wakeup disabled. The exceptions are - * devices that everyone expects to be wakeup sources: keyboards, power buttons, - * possibly network interfaces, etc. Also, devices that don't generate their - * own wakeup requests but merely forward requests from one bus to another - * (like PCI bridges) should have wakeup enabled by default. - */ -int device_init_wakeup(struct device *dev, bool enable) -{ - int ret = 0; - - if (!dev) - return -EINVAL; - - if (enable) { - device_set_wakeup_capable(dev, true); - ret = device_wakeup_enable(dev); - } else { - device_wakeup_disable(dev); - device_set_wakeup_capable(dev, false); - } - - return ret; -} -EXPORT_SYMBOL_GPL(device_init_wakeup); - /** * device_set_wakeup_enable - Enable or disable a device to wake up the system. * @dev: Device to handle. diff --git a/include/linux/pm_wakeup.h b/include/linux/pm_wakeup.h index 196a157456aa..77f4849e3418 100644 --- a/include/linux/pm_wakeup.h +++ b/include/linux/pm_wakeup.h @@ -109,7 +109,6 @@ extern struct wakeup_source *wakeup_sources_walk_next(struct wakeup_source *ws); extern int device_wakeup_enable(struct device *dev); extern int device_wakeup_disable(struct device *dev); extern void device_set_wakeup_capable(struct device *dev, bool capable); -extern int device_init_wakeup(struct device *dev, bool val); extern int device_set_wakeup_enable(struct device *dev, bool enable); extern void __pm_stay_awake(struct wakeup_source *ws); extern void pm_stay_awake(struct device *dev); @@ -167,13 +166,6 @@ static inline int device_set_wakeup_enable(struct device *dev, bool enable) return 0; } -static inline int device_init_wakeup(struct device *dev, bool val) -{ - device_set_wakeup_capable(dev, val); - device_set_wakeup_enable(dev, val); - return 0; -} - static inline bool device_may_wakeup(struct device *dev) { return dev->power.can_wakeup && dev->power.should_wakeup; @@ -217,4 +209,27 @@ static inline void pm_wakeup_hard_event(struct device *dev) return pm_wakeup_dev_event(dev, 0, true); } +/** + * device_init_wakeup - Device wakeup initialization. + * @dev: Device to handle. + * @enable: Whether or not to enable @dev as a wakeup device. + * + * By default, most devices should leave wakeup disabled. The exceptions are + * devices that everyone expects to be wakeup sources: keyboards, power buttons, + * possibly network interfaces, etc. Also, devices that don't generate their + * own wakeup requests but merely forward requests from one bus to another + * (like PCI bridges) should have wakeup enabled by default. + */ +static inline int device_init_wakeup(struct device *dev, bool enable) +{ + if (enable) { + device_set_wakeup_capable(dev, true); + return device_wakeup_enable(dev); + } else { + device_wakeup_disable(dev); + device_set_wakeup_capable(dev, false); + return 0; + } +} + #endif /* _LINUX_PM_WAKEUP_H */ -- cgit v1.2.3 From e67198cc05b8ecbb7b8e2d8ef9fb5c8d26821873 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:25 +0200 Subject: context_tracking: Take idle eqs entrypoints over RCU The RCU dynticks counter is going to be merged into the context tracking subsystem. Start with moving the idle extended quiescent states entrypoints to context tracking. For now those are dumb redirections to existing RCU calls. [ paulmck: Apply kernel test robot feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- Documentation/RCU/stallwarn.rst | 4 ++-- arch/arm/mach-imx/cpuidle-imx6q.c | 5 +++-- drivers/acpi/processor_idle.c | 5 +++-- drivers/cpuidle/cpuidle.c | 9 +++++---- include/linux/context_tracking.h | 8 ++++++++ include/linux/rcupdate.h | 2 +- kernel/context_tracking.c | 15 +++++++++++++++ kernel/locking/lockdep.c | 2 +- kernel/rcu/Kconfig | 2 ++ kernel/rcu/tree.c | 2 -- kernel/rcu/update.c | 2 +- kernel/sched/idle.c | 10 +++++----- kernel/sched/sched.h | 1 + kernel/time/Kconfig | 6 ++++++ 14 files changed, 53 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index 794837eb519b..b95bda7755fa 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -97,8 +97,8 @@ warnings: which will include additional debugging information. - A low-level kernel issue that either fails to invoke one of the - variants of rcu_user_enter(), rcu_user_exit(), rcu_idle_enter(), - rcu_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one + variants of rcu_user_enter(), rcu_user_exit(), ct_idle_enter(), + ct_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one hand, or that invokes one of them too many times on the other. Historically, the most frequent issue has been an omission of either irq_enter() or irq_exit(), which in turn invoke diff --git a/arch/arm/mach-imx/cpuidle-imx6q.c b/arch/arm/mach-imx/cpuidle-imx6q.c index 094337dc1bc7..d086cbae09c3 100644 --- a/arch/arm/mach-imx/cpuidle-imx6q.c +++ b/arch/arm/mach-imx/cpuidle-imx6q.c @@ -3,6 +3,7 @@ * Copyright (C) 2012 Freescale Semiconductor, Inc. */ +#include #include #include #include @@ -24,9 +25,9 @@ static int imx6q_enter_wait(struct cpuidle_device *dev, imx6_set_lpm(WAIT_UNCLOCKED); raw_spin_unlock(&cpuidle_lock); - rcu_idle_enter(); + ct_idle_enter(); cpu_do_idle(); - rcu_idle_exit(); + ct_idle_exit(); raw_spin_lock(&cpuidle_lock); if (num_idle_cpus-- == num_online_cpus()) diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 6a5572a1a80c..1401d193a2df 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -23,6 +23,7 @@ #include #include #include +#include /* * Include the apic definitions for x86 to have the APIC timer related defines @@ -647,11 +648,11 @@ static int acpi_idle_enter_bm(struct cpuidle_driver *drv, raw_spin_unlock(&c3_lock); } - rcu_idle_enter(); + ct_idle_enter(); acpi_idle_do_entry(cx); - rcu_idle_exit(); + ct_idle_exit(); /* Re-enable bus master arbitration */ if (dis_bm) { diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index ef2ea1b12cd8..62dd956025f3 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "cpuidle.h" @@ -150,12 +151,12 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, */ stop_critical_timings(); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_enter(); + ct_idle_enter(); target_state->enter_s2idle(dev, drv, index); if (WARN_ON_ONCE(!irqs_disabled())) local_irq_disable(); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_exit(); + ct_idle_exit(); tick_unfreeze(); start_critical_timings(); @@ -233,10 +234,10 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, stop_critical_timings(); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_enter(); + ct_idle_enter(); entered_state = target_state->enter(dev, drv, index); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_exit(); + ct_idle_exit(); start_critical_timings(); sched_clock_idle_wakeup_event(); diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index e35ae66b4794..01abadb2f993 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -119,4 +119,12 @@ extern void context_tracking_init(void); static inline void context_tracking_init(void) { } #endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */ +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +extern void ct_idle_enter(void); +extern void ct_idle_exit(void); +#else +static inline void ct_idle_enter(void) { } +static inline void ct_idle_exit(void) { } +#endif /* !CONFIG_CONTEXT_TRACKING_IDLE */ + #endif diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1a32036c918c..6ebe754501c3 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -128,7 +128,7 @@ static inline void rcu_nocb_flush_deferred_wakeup(void) { } * @a: Code that RCU needs to pay attention to. * * RCU read-side critical sections are forbidden in the inner idle loop, - * that is, between the rcu_idle_enter() and the rcu_idle_exit() -- RCU + * that is, between the ct_idle_enter() and the ct_idle_exit() -- RCU * will happily ignore any such read-side critical sections. However, * things like powertop need tracepoints in the inner idle loop. * diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index f3dec1be2bf6..c0b3798d4e94 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -22,6 +22,21 @@ #include #include + +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +noinstr void ct_idle_enter(void) +{ + rcu_idle_enter(); +} +EXPORT_SYMBOL_GPL(ct_idle_enter); + +void ct_idle_exit(void) +{ + rcu_idle_exit(); +} +EXPORT_SYMBOL_GPL(ct_idle_exit); +#endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ + #ifdef CONFIG_CONTEXT_TRACKING_USER #define CREATE_TRACE_POINTS diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index f06b91ca6482..5ea690cb4f7a 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -6570,7 +6570,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s) /* * If a CPU is in the RCU-free window in idle (ie: in the section - * between rcu_idle_enter() and rcu_idle_exit(), then RCU + * between ct_idle_enter() and ct_idle_exit(), then RCU * considers that CPU to be in an "extended quiescent state", * which means that RCU will be completely ignoring that CPU. * Therefore, rcu_read_lock() and friends have absolutely no diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 1c630e573548..3fa24e63d6f9 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -8,6 +8,8 @@ menu "RCU Subsystem" config TREE_RCU bool default y if SMP + # Dynticks-idle tracking + select CONTEXT_TRACKING_IDLE help This option selects the RCU implementation that is designed for very large SMP system with hundreds or diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 9a5edab5558c..051fed0844b6 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -664,7 +664,6 @@ void noinstr rcu_idle_enter(void) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rcu_eqs_enter(false); } -EXPORT_SYMBOL_GPL(rcu_idle_enter); #ifdef CONFIG_NO_HZ_FULL @@ -904,7 +903,6 @@ void noinstr rcu_idle_exit(void) rcu_eqs_exit(false); raw_local_irq_restore(flags); } -EXPORT_SYMBOL_GPL(rcu_idle_exit); #ifdef CONFIG_NO_HZ_FULL /** diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index fc7fef575606..147214b2cd68 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -85,7 +85,7 @@ module_param(rcu_normal_after_boot, int, 0444); * and while lockdep is disabled. * * Note that if the CPU is in the idle loop from an RCU point of view (ie: - * that we are in the section between rcu_idle_enter() and rcu_idle_exit()) + * that we are in the section between ct_idle_enter() and ct_idle_exit()) * then rcu_read_lock_held() sets ``*ret`` to false even if the CPU did an * rcu_read_lock(). The reason for this is that RCU ignores CPUs that are * in such a section, considering these as in extended quiescent state, diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 328cccbee444..f26ab2675f7d 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -53,14 +53,14 @@ static noinline int __cpuidle cpu_idle_poll(void) { trace_cpu_idle(0, smp_processor_id()); stop_critical_timings(); - rcu_idle_enter(); + ct_idle_enter(); local_irq_enable(); while (!tif_need_resched() && (cpu_idle_force_poll || tick_check_broadcast_expired())) cpu_relax(); - rcu_idle_exit(); + ct_idle_exit(); start_critical_timings(); trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); @@ -98,12 +98,12 @@ void __cpuidle default_idle_call(void) * * Trace IRQs enable here, then switch off RCU, and have * arch_cpu_idle() use raw_local_irq_enable(). Note that - * rcu_idle_enter() relies on lockdep IRQ state, so switch that + * ct_idle_enter() relies on lockdep IRQ state, so switch that * last -- this is very similar to the entry code. */ trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); - rcu_idle_enter(); + ct_idle_enter(); lockdep_hardirqs_on(_THIS_IP_); arch_cpu_idle(); @@ -116,7 +116,7 @@ void __cpuidle default_idle_call(void) */ raw_local_irq_disable(); lockdep_hardirqs_off(_THIS_IP_); - rcu_idle_exit(); + ct_idle_exit(); lockdep_hardirqs_on(_THIS_IP_); raw_local_irq_enable(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 47b89a0fc6e5..0cfe2d0af294 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index 41f99bcfe9e6..a41753be1a2b 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -76,6 +76,12 @@ config TIME_KUNIT_TEST config CONTEXT_TRACKING bool +config CONTEXT_TRACKING_IDLE + bool + select CONTEXT_TRACKING + help + Tracks idle state on behalf of RCU. + if GENERIC_CLOCKEVENTS menu "Timers subsystem" -- cgit v1.2.3 From 6f0e6c1598b1a3d19fc30db86b6e26d6f881b43d Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:26 +0200 Subject: context_tracking: Take IRQ eqs entrypoints over RCU The RCU dynticks counter is going to be merged into the context tracking subsystem. Prepare with moving the IRQ extended quiescent states entrypoints to context tracking. For now those are dumb redirection to existing RCU calls. [ paulmck: Apply Stephen Rothwell feedback from -next. ] [ paulmck: Apply Nathan Chancellor feedback. ] Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- .../RCU/Design/Requirements/Requirements.rst | 10 ++++----- Documentation/RCU/stallwarn.rst | 4 ++-- arch/Kconfig | 2 +- arch/arm64/kernel/entry-common.c | 6 +++--- arch/x86/mm/fault.c | 2 +- drivers/cpuidle/cpuidle-psci.c | 8 ++++---- drivers/cpuidle/cpuidle-riscv-sbi.c | 8 ++++---- include/linux/context_tracking_irq.h | 17 +++++++++++++++ include/linux/context_tracking_state.h | 1 + include/linux/entry-common.h | 10 ++++----- include/linux/rcupdate.h | 5 +++-- include/linux/tracepoint.h | 4 ++-- kernel/cfi.c | 4 ++-- kernel/context_tracking.c | 24 ++++++++++++++++++++-- kernel/cpu_pm.c | 8 ++++---- kernel/entry/common.c | 12 +++++------ kernel/softirq.c | 4 ++-- kernel/trace/trace.c | 6 +++--- 18 files changed, 87 insertions(+), 48 deletions(-) create mode 100644 include/linux/context_tracking_irq.h (limited to 'include/linux') diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst index 04ed8bf27a0e..074810c73936 100644 --- a/Documentation/RCU/Design/Requirements/Requirements.rst +++ b/Documentation/RCU/Design/Requirements/Requirements.rst @@ -1844,10 +1844,10 @@ that meets this requirement. Furthermore, NMI handlers can be interrupted by what appear to RCU to be normal interrupts. One way that this can happen is for code that -directly invokes rcu_irq_enter() and rcu_irq_exit() to be called +directly invokes ct_irq_enter() and ct_irq_exit() to be called from an NMI handler. This astonishing fact of life prompted the current -code structure, which has rcu_irq_enter() invoking -rcu_nmi_enter() and rcu_irq_exit() invoking rcu_nmi_exit(). +code structure, which has ct_irq_enter() invoking +rcu_nmi_enter() and ct_irq_exit() invoking rcu_nmi_exit(). And yes, I also learned of this requirement the hard way. Loadable Modules @@ -2195,7 +2195,7 @@ scheduling-clock interrupt be enabled when RCU needs it to be: sections, and RCU believes this CPU to be idle, no problem. This sort of thing is used by some architectures for light-weight exception handlers, which can then avoid the overhead of - rcu_irq_enter() and rcu_irq_exit() at exception entry and + ct_irq_enter() and ct_irq_exit() at exception entry and exit, respectively. Some go further and avoid the entireties of irq_enter() and irq_exit(). Just make very sure you are running some of your tests with @@ -2226,7 +2226,7 @@ scheduling-clock interrupt be enabled when RCU needs it to be: +-----------------------------------------------------------------------+ | **Answer**: | +-----------------------------------------------------------------------+ -| One approach is to do ``rcu_irq_exit();rcu_irq_enter();`` every so | +| One approach is to do ``ct_irq_exit();ct_irq_enter();`` every so | | often. But given that long-running interrupt handlers can cause other | | problems, not least for response time, shouldn't you work to keep | | your interrupt handler's runtime within reasonable bounds? | diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index b95bda7755fa..ce1f58a9d954 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -98,11 +98,11 @@ warnings: - A low-level kernel issue that either fails to invoke one of the variants of rcu_user_enter(), rcu_user_exit(), ct_idle_enter(), - ct_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one + ct_idle_exit(), ct_irq_enter(), or ct_irq_exit() on the one hand, or that invokes one of them too many times on the other. Historically, the most frequent issue has been an omission of either irq_enter() or irq_exit(), which in turn invoke - rcu_irq_enter() or rcu_irq_exit(), respectively. Building your + ct_irq_enter() or ct_irq_exit(), respectively. Building your kernel with CONFIG_RCU_EQS_DEBUG=y can help track down these types of issues, which sometimes arise in architecture-specific code. diff --git a/arch/Kconfig b/arch/Kconfig index 154b7b78da09..342642be105f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -782,7 +782,7 @@ config HAVE_CONTEXT_TRACKING_USER Syscalls need to be wrapped inside user_exit()-user_enter(), either optimized behind static key or through the slow path using TIF_NOHZ flag. Exceptions handlers must be wrapped as well. Irqs are already - protected inside rcu_irq_enter/rcu_irq_exit() but preemption or signal + protected inside ct_irq_enter/ct_irq_exit() but preemption or signal handling on irq exit still need to be protected. config HAVE_CONTEXT_TRACKING_USER_OFFSTACK diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 56cefd33eb8e..8dabe9ec10f1 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -41,7 +41,7 @@ static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs) if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { lockdep_hardirqs_off(CALLER_ADDR0); - rcu_irq_enter(); + ct_irq_enter(); trace_hardirqs_off_finish(); regs->exit_rcu = true; @@ -76,7 +76,7 @@ static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) if (regs->exit_rcu) { trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); - rcu_irq_exit(); + ct_irq_exit(); lockdep_hardirqs_on(CALLER_ADDR0); return; } @@ -84,7 +84,7 @@ static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) trace_hardirqs_on(); } else { if (regs->exit_rcu) - rcu_irq_exit(); + ct_irq_exit(); } } diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index fad8faa29d04..971977c438fc 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1526,7 +1526,7 @@ DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) /* * Entry handling for valid #PF from kernel mode is slightly - * different: RCU is already watching and rcu_irq_enter() must not + * different: RCU is already watching and ct_irq_enter() must not * be invoked because a kernel fault on a user space address might * sleep. * diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c index 540105ca0781..57bc3e3ae391 100644 --- a/drivers/cpuidle/cpuidle-psci.c +++ b/drivers/cpuidle/cpuidle-psci.c @@ -69,12 +69,12 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, return -1; /* Do runtime PM to manage a hierarchical CPU toplogy. */ - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_suspend(pd_dev); else pm_runtime_put_sync_suspend(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); state = psci_get_domain_state(); if (!state) @@ -82,12 +82,12 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, ret = psci_cpu_suspend_enter(state) ? -1 : idx; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_resume(pd_dev); else pm_runtime_get_sync(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); cpu_pm_exit(); diff --git a/drivers/cpuidle/cpuidle-riscv-sbi.c b/drivers/cpuidle/cpuidle-riscv-sbi.c index 1151e5e2ba82..862a2876f1c9 100644 --- a/drivers/cpuidle/cpuidle-riscv-sbi.c +++ b/drivers/cpuidle/cpuidle-riscv-sbi.c @@ -116,12 +116,12 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, return -1; /* Do runtime PM to manage a hierarchical CPU toplogy. */ - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_suspend(pd_dev); else pm_runtime_put_sync_suspend(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); if (sbi_is_domain_state_available()) state = sbi_get_domain_state(); @@ -130,12 +130,12 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, ret = sbi_suspend(state) ? -1 : idx; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_resume(pd_dev); else pm_runtime_get_sync(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); cpu_pm_exit(); diff --git a/include/linux/context_tracking_irq.h b/include/linux/context_tracking_irq.h new file mode 100644 index 000000000000..62f62bbd1a50 --- /dev/null +++ b/include/linux/context_tracking_irq.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_CONTEXT_TRACKING_IRQ_H +#define _LINUX_CONTEXT_TRACKING_IRQ_H + +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +void ct_irq_enter(void); +void ct_irq_exit(void); +void ct_irq_enter_irqson(void); +void ct_irq_exit_irqson(void); +#else +static inline void ct_irq_enter(void) { } +static inline void ct_irq_exit(void) { } +static inline void ct_irq_enter_irqson(void) { } +static inline void ct_irq_exit_irqson(void) { } +#endif + +#endif diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 2b46afe105a9..9c16a8b2c194 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -4,6 +4,7 @@ #include #include +#include struct context_tracking { /* diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index c92ac75d6556..84a466b176cf 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -357,7 +357,7 @@ void irqentry_exit_to_user_mode(struct pt_regs *regs); /** * struct irqentry_state - Opaque object for exception state storage * @exit_rcu: Used exclusively in the irqentry_*() calls; signals whether the - * exit path has to invoke rcu_irq_exit(). + * exit path has to invoke ct_irq_exit(). * @lockdep: Used exclusively in the irqentry_nmi_*() calls; ensures that * lockdep state is restored correctly on exit from nmi. * @@ -395,12 +395,12 @@ typedef struct irqentry_state { * * For kernel mode entries RCU handling is done conditional. If RCU is * watching then the only RCU requirement is to check whether the tick has - * to be restarted. If RCU is not watching then rcu_irq_enter() has to be - * invoked on entry and rcu_irq_exit() on exit. + * to be restarted. If RCU is not watching then ct_irq_enter() has to be + * invoked on entry and ct_irq_exit() on exit. * - * Avoiding the rcu_irq_enter/exit() calls is an optimization but also + * Avoiding the ct_irq_enter/exit() calls is an optimization but also * solves the problem of kernel mode pagefaults which can schedule, which - * is not possible after invoking rcu_irq_enter() without undoing it. + * is not possible after invoking ct_irq_enter() without undoing it. * * For user mode entries irqentry_enter_from_user_mode() is invoked to * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 6ebe754501c3..f1562d91c67d 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -29,6 +29,7 @@ #include #include #include +#include #define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b)) #define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b)) @@ -143,9 +144,9 @@ static inline void rcu_nocb_flush_deferred_wakeup(void) { } */ #define RCU_NONIDLE(a) \ do { \ - rcu_irq_enter_irqson(); \ + ct_irq_enter_irqson(); \ do { a; } while (0); \ - rcu_irq_exit_irqson(); \ + ct_irq_exit_irqson(); \ } while (0) /* diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 28031b15f878..55717a2eda08 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -200,13 +200,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) */ \ if (rcuidle) { \ __idx = srcu_read_lock_notrace(&tracepoint_srcu);\ - rcu_irq_enter_irqson(); \ + ct_irq_enter_irqson(); \ } \ \ __DO_TRACE_CALL(name, TP_ARGS(args)); \ \ if (rcuidle) { \ - rcu_irq_exit_irqson(); \ + ct_irq_exit_irqson(); \ srcu_read_unlock_notrace(&tracepoint_srcu, __idx);\ } \ \ diff --git a/kernel/cfi.c b/kernel/cfi.c index 08102d19ec15..2046276ee234 100644 --- a/kernel/cfi.c +++ b/kernel/cfi.c @@ -295,7 +295,7 @@ static inline cfi_check_fn find_check_fn(unsigned long ptr) rcu_idle = !rcu_is_watching(); if (rcu_idle) { local_irq_save(flags); - rcu_irq_enter(); + ct_irq_enter(); } if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW)) @@ -304,7 +304,7 @@ static inline cfi_check_fn find_check_fn(unsigned long ptr) fn = find_module_check_fn(ptr); if (rcu_idle) { - rcu_irq_exit(); + ct_irq_exit(); local_irq_restore(flags); } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index c0b3798d4e94..72bd71a02c44 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -35,6 +35,26 @@ void ct_idle_exit(void) rcu_idle_exit(); } EXPORT_SYMBOL_GPL(ct_idle_exit); + +noinstr void ct_irq_enter(void) +{ + rcu_irq_enter(); +} + +noinstr void ct_irq_exit(void) +{ + rcu_irq_exit(); +} + +void ct_irq_enter_irqson(void) +{ + rcu_irq_enter_irqson(); +} + +void ct_irq_exit_irqson(void) +{ + rcu_irq_exit_irqson(); +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER @@ -90,7 +110,7 @@ void noinstr __ct_user_enter(enum ctx_state state) * At this stage, only low level arch entry code remains and * then we'll run in userspace. We can assume there won't be * any RCU read-side critical section until the next call to - * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency + * user_exit() or ct_irq_enter(). Let's remove RCU's dependency * on the tick. */ if (state == CONTEXT_USER) { @@ -136,7 +156,7 @@ void ct_user_enter(enum ctx_state state) /* * Some contexts may involve an exception occuring in an irq, * leading to that nesting: - * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit() + * ct_irq_enter() rcu_user_exit() rcu_user_exit() ct_irq_exit() * This would mess up the dyntick_nesting count though. And rcu_irq_*() * helpers are enough to protect RCU uses inside the exception. So * just return immediately if we detect we are in an IRQ. diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c index 246efc74e3f3..ba4ba71facf9 100644 --- a/kernel/cpu_pm.c +++ b/kernel/cpu_pm.c @@ -35,11 +35,11 @@ static int cpu_pm_notify(enum cpu_pm_event event) * disfunctional in cpu idle. Copy RCU_NONIDLE code to let RCU know * this. */ - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); rcu_read_lock(); ret = raw_notifier_call_chain(&cpu_pm_notifier.chain, event, NULL); rcu_read_unlock(); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); return notifier_to_errno(ret); } @@ -49,11 +49,11 @@ static int cpu_pm_notify_robust(enum cpu_pm_event event_up, enum cpu_pm_event ev unsigned long flags; int ret; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); raw_spin_lock_irqsave(&cpu_pm_notifier.lock, flags); ret = raw_notifier_call_chain_robust(&cpu_pm_notifier.chain, event_up, event_down, NULL); raw_spin_unlock_irqrestore(&cpu_pm_notifier.lock, flags); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); return notifier_to_errno(ret); } diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 032f164abe7c..667ba5d581ff 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -321,7 +321,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) } /* - * If this entry hit the idle task invoke rcu_irq_enter() whether + * If this entry hit the idle task invoke ct_irq_enter() whether * RCU is watching or not. * * Interrupts can nest when the first interrupt invokes softirq @@ -332,12 +332,12 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) * not nested into another interrupt. * * Checking for rcu_is_watching() here would prevent the nesting - * interrupt to invoke rcu_irq_enter(). If that nested interrupt is + * interrupt to invoke ct_irq_enter(). If that nested interrupt is * the tick then rcu_flavor_sched_clock_irq() would wrongfully * assume that it is the first interrupt and eventually claim * quiescent state and end grace periods prematurely. * - * Unconditionally invoke rcu_irq_enter() so RCU state stays + * Unconditionally invoke ct_irq_enter() so RCU state stays * consistent. * * TINY_RCU does not support EQS, so let the compiler eliminate @@ -350,7 +350,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) * as in irqentry_enter_from_user_mode(). */ lockdep_hardirqs_off(CALLER_ADDR0); - rcu_irq_enter(); + ct_irq_enter(); instrumentation_begin(); trace_hardirqs_off_finish(); instrumentation_end(); @@ -418,7 +418,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); instrumentation_end(); - rcu_irq_exit(); + ct_irq_exit(); lockdep_hardirqs_on(CALLER_ADDR0); return; } @@ -436,7 +436,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) * was not watching on entry. */ if (state.exit_rcu) - rcu_irq_exit(); + ct_irq_exit(); } } diff --git a/kernel/softirq.c b/kernel/softirq.c index 9f0aef8aa9ff..c8a6913c067d 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -620,7 +620,7 @@ void irq_enter_rcu(void) */ void irq_enter(void) { - rcu_irq_enter(); + ct_irq_enter(); irq_enter_rcu(); } @@ -672,7 +672,7 @@ void irq_exit_rcu(void) void irq_exit(void) { __irq_exit_rcu(); - rcu_irq_exit(); + ct_irq_exit(); /* must be last! */ lockdep_hardirq_exit(); } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 2c95992e2c71..fe78a6818126 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -3107,15 +3107,15 @@ void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, /* * When an NMI triggers, RCU is enabled via rcu_nmi_enter(), * but if the above rcu_is_watching() failed, then the NMI - * triggered someplace critical, and rcu_irq_enter() should + * triggered someplace critical, and ct_irq_enter() should * not be called from NMI. */ if (unlikely(in_nmi())) return; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); __ftrace_trace_stack(buffer, trace_ctx, skip, NULL); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); } /** -- cgit v1.2.3 From 493c1822825f00025d6754ec0632990a27edc6f8 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:27 +0200 Subject: context_tracking: Take NMI eqs entrypoints over RCU The RCU dynticks counter is going to be merged into the context tracking subsystem. Prepare with moving the NMI extended quiescent states entrypoints to context tracking. For now those are dumb redirection to existing RCU calls. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- Documentation/RCU/Design/Requirements/Requirements.rst | 2 +- arch/Kconfig | 2 +- arch/arm64/kernel/entry-common.c | 8 ++++---- include/linux/context_tracking_irq.h | 4 ++++ include/linux/hardirq.h | 4 ++-- kernel/context_tracking.c | 10 ++++++++++ kernel/entry/common.c | 4 ++-- kernel/extable.c | 4 ++-- kernel/trace/trace.c | 2 +- 9 files changed, 27 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst index 074810c73936..a0f8164c8513 100644 --- a/Documentation/RCU/Design/Requirements/Requirements.rst +++ b/Documentation/RCU/Design/Requirements/Requirements.rst @@ -1847,7 +1847,7 @@ normal interrupts. One way that this can happen is for code that directly invokes ct_irq_enter() and ct_irq_exit() to be called from an NMI handler. This astonishing fact of life prompted the current code structure, which has ct_irq_enter() invoking -rcu_nmi_enter() and ct_irq_exit() invoking rcu_nmi_exit(). +ct_nmi_enter() and ct_irq_exit() invoking ct_nmi_exit(). And yes, I also learned of this requirement the hard way. Loadable Modules diff --git a/arch/Kconfig b/arch/Kconfig index 342642be105f..f56f7c0e924d 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -797,7 +797,7 @@ config HAVE_CONTEXT_TRACKING_USER_OFFSTACK - Critical entry code isn't preemptible (or better yet: not interruptible). - - No use of RCU read side critical sections, unless rcu_nmi_enter() + - No use of RCU read side critical sections, unless ct_nmi_enter() got called. - No use of instrumentation, unless instrumentation_begin() got called. diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 8dabe9ec10f1..c75ca36b4a49 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -161,7 +161,7 @@ static void noinstr arm64_enter_nmi(struct pt_regs *regs) __nmi_enter(); lockdep_hardirqs_off(CALLER_ADDR0); lockdep_hardirq_enter(); - rcu_nmi_enter(); + ct_nmi_enter(); trace_hardirqs_off_finish(); ftrace_nmi_enter(); @@ -182,7 +182,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs) lockdep_hardirqs_on_prepare(); } - rcu_nmi_exit(); + ct_nmi_exit(); lockdep_hardirq_exit(); if (restore) lockdep_hardirqs_on(CALLER_ADDR0); @@ -199,7 +199,7 @@ static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); lockdep_hardirqs_off(CALLER_ADDR0); - rcu_nmi_enter(); + ct_nmi_enter(); trace_hardirqs_off_finish(); } @@ -218,7 +218,7 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) lockdep_hardirqs_on_prepare(); } - rcu_nmi_exit(); + ct_nmi_exit(); if (restore) lockdep_hardirqs_on(CALLER_ADDR0); } diff --git a/include/linux/context_tracking_irq.h b/include/linux/context_tracking_irq.h index 62f62bbd1a50..c50b5670c4a5 100644 --- a/include/linux/context_tracking_irq.h +++ b/include/linux/context_tracking_irq.h @@ -7,11 +7,15 @@ void ct_irq_enter(void); void ct_irq_exit(void); void ct_irq_enter_irqson(void); void ct_irq_exit_irqson(void); +void ct_nmi_enter(void); +void ct_nmi_exit(void); #else static inline void ct_irq_enter(void) { } static inline void ct_irq_exit(void) { } static inline void ct_irq_enter_irqson(void) { } static inline void ct_irq_exit_irqson(void) { } +static inline void ct_nmi_enter(void) { } +static inline void ct_nmi_exit(void) { } #endif #endif diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 76878b357ffa..345cdbe9c1b7 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -124,7 +124,7 @@ extern void rcu_nmi_exit(void); do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - rcu_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -143,7 +143,7 @@ extern void rcu_nmi_exit(void); instrumentation_begin(); \ ftrace_nmi_exit(); \ instrumentation_end(); \ - rcu_nmi_exit(); \ + ct_nmi_exit(); \ lockdep_hardirq_exit(); \ __nmi_exit(); \ } while (0) diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 72bd71a02c44..b8a731f20778 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -55,6 +55,16 @@ void ct_irq_exit_irqson(void) { rcu_irq_exit_irqson(); } + +noinstr void ct_nmi_enter(void) +{ + rcu_nmi_enter(); +} + +noinstr void ct_nmi_exit(void) +{ + rcu_nmi_exit(); +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 667ba5d581ff..063068a9ea9b 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -449,7 +449,7 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) __nmi_enter(); lockdep_hardirqs_off(CALLER_ADDR0); lockdep_hardirq_enter(); - rcu_nmi_enter(); + ct_nmi_enter(); instrumentation_begin(); trace_hardirqs_off_finish(); @@ -469,7 +469,7 @@ void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) } instrumentation_end(); - rcu_nmi_exit(); + ct_nmi_exit(); lockdep_hardirq_exit(); if (irq_state.lockdep) lockdep_hardirqs_on(CALLER_ADDR0); diff --git a/kernel/extable.c b/kernel/extable.c index bda5e9761541..71f482581cab 100644 --- a/kernel/extable.c +++ b/kernel/extable.c @@ -114,7 +114,7 @@ int kernel_text_address(unsigned long addr) /* Treat this like an NMI as it can happen anywhere */ if (no_rcu) - rcu_nmi_enter(); + ct_nmi_enter(); if (is_module_text_address(addr)) goto out; @@ -127,7 +127,7 @@ int kernel_text_address(unsigned long addr) ret = 0; out: if (no_rcu) - rcu_nmi_exit(); + ct_nmi_exit(); return ret; } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index fe78a6818126..5fc7f17f5ec7 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -3105,7 +3105,7 @@ void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, } /* - * When an NMI triggers, RCU is enabled via rcu_nmi_enter(), + * When an NMI triggers, RCU is enabled via ct_nmi_enter(), * but if the above rcu_is_watching() failed, then the NMI * triggered someplace critical, and ct_irq_enter() should * not be called from NMI. -- cgit v1.2.3 From 3864caafe7c66f01b188ffccb6a4215f3bf56292 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:28 +0200 Subject: rcu/context-tracking: Remove rcu_irq_enter/exit() Now rcu_irq_enter/exit() is an unnecessary middle call between ct_irq_enter/exit() and nmi_irq_enter/exit(). Take this opportunity to remove the former functions and move the comments above them to the new entrypoints. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/rcutiny.h | 4 --- include/linux/rcutree.h | 4 --- kernel/context_tracking.c | 71 +++++++++++++++++++++++++++++++++++++--- kernel/rcu/tree.c | 83 ----------------------------------------------- 4 files changed, 67 insertions(+), 95 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 5fed476f977f..591119413cf1 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -78,10 +78,6 @@ static inline void rcu_cpu_stall_reset(void) { } static inline int rcu_jiffies_till_stall_check(void) { return 21 * HZ; } static inline void rcu_idle_enter(void) { } static inline void rcu_idle_exit(void) { } -static inline void rcu_irq_enter(void) { } -static inline void rcu_irq_exit_irqson(void) { } -static inline void rcu_irq_enter_irqson(void) { } -static inline void rcu_irq_exit(void) { } static inline void rcu_irq_exit_check_preempt(void) { } #define rcu_is_idle_cpu(cpu) \ (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq()) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9c6cfb742504..4522b6a7cc42 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -47,10 +47,6 @@ void cond_synchronize_rcu(unsigned long oldstate); void rcu_idle_enter(void); void rcu_idle_exit(void); -void rcu_irq_enter(void); -void rcu_irq_exit(void); -void rcu_irq_enter_irqson(void); -void rcu_irq_exit_irqson(void); bool rcu_is_idle_cpu(int cpu); #ifdef CONFIG_PROVE_RCU diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index b8a731f20778..c0d86dac98f1 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -36,24 +36,87 @@ void ct_idle_exit(void) } EXPORT_SYMBOL_GPL(ct_idle_exit); +/** + * ct_irq_enter - inform RCU that current CPU is entering irq away from idle + * + * Enter an interrupt handler, which might possibly result in exiting + * idle mode, in other words, entering the mode in which read-side critical + * sections can occur. The caller must have disabled interrupts. + * + * Note that the Linux kernel is fully capable of entering an interrupt + * handler that it never exits, for example when doing upcalls to user mode! + * This code assumes that the idle loop never does upcalls to user mode. + * If your architecture's idle loop does do upcalls to user mode (or does + * anything else that results in unbalanced calls to the irq_enter() and + * irq_exit() functions), RCU will give you what you deserve, good and hard. + * But very infrequently and irreproducibly. + * + * Use things like work queues to work around this limitation. + * + * You have been warned. + * + * If you add or remove a call to ct_irq_enter(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ noinstr void ct_irq_enter(void) { - rcu_irq_enter(); + lockdep_assert_irqs_disabled(); + ct_nmi_enter(); } +/** + * ct_irq_exit - inform RCU that current CPU is exiting irq towards idle + * + * Exit from an interrupt handler, which might possibly result in entering + * idle mode, in other words, leaving the mode in which read-side critical + * sections can occur. The caller must have disabled interrupts. + * + * This code assumes that the idle loop never does anything that might + * result in unbalanced calls to irq_enter() and irq_exit(). If your + * architecture's idle loop violates this assumption, RCU will give you what + * you deserve, good and hard. But very infrequently and irreproducibly. + * + * Use things like work queues to work around this limitation. + * + * You have been warned. + * + * If you add or remove a call to ct_irq_exit(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ noinstr void ct_irq_exit(void) { - rcu_irq_exit(); + lockdep_assert_irqs_disabled(); + ct_nmi_exit(); } +/* + * Wrapper for ct_irq_enter() where interrupts are enabled. + * + * If you add or remove a call to ct_irq_enter_irqson(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ void ct_irq_enter_irqson(void) { - rcu_irq_enter_irqson(); + unsigned long flags; + + local_irq_save(flags); + ct_irq_enter(); + local_irq_restore(flags); } +/* + * Wrapper for ct_irq_exit() where interrupts are enabled. + * + * If you add or remove a call to ct_irq_exit_irqson(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ void ct_irq_exit_irqson(void) { - rcu_irq_exit_irqson(); + unsigned long flags; + + local_irq_save(flags); + ct_irq_exit(); + local_irq_restore(flags); } noinstr void ct_nmi_enter(void) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 051fed0844b6..75b433dba427 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -789,31 +789,6 @@ noinstr void rcu_nmi_exit(void) rcu_dynticks_task_enter(); } -/** - * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle - * - * Exit from an interrupt handler, which might possibly result in entering - * idle mode, in other words, leaving the mode in which read-side critical - * sections can occur. The caller must have disabled interrupts. - * - * This code assumes that the idle loop never does anything that might - * result in unbalanced calls to irq_enter() and irq_exit(). If your - * architecture's idle loop violates this assumption, RCU will give you what - * you deserve, good and hard. But very infrequently and irreproducibly. - * - * Use things like work queues to work around this limitation. - * - * You have been warned. - * - * If you add or remove a call to rcu_irq_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_irq_exit(void) -{ - lockdep_assert_irqs_disabled(); - rcu_nmi_exit(); -} - #ifdef CONFIG_PROVE_RCU /** * rcu_irq_exit_check_preempt - Validate that scheduling is possible @@ -832,21 +807,6 @@ void rcu_irq_exit_check_preempt(void) } #endif /* #ifdef CONFIG_PROVE_RCU */ -/* - * Wrapper for rcu_irq_exit() where interrupts are enabled. - * - * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -void rcu_irq_exit_irqson(void) -{ - unsigned long flags; - - local_irq_save(flags); - rcu_irq_exit(); - local_irq_restore(flags); -} - /* * Exit an RCU extended quiescent state, which can be either the * idle loop or adaptive-tickless usermode execution. @@ -1041,49 +1001,6 @@ noinstr void rcu_nmi_enter(void) barrier(); } -/** - * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle - * - * Enter an interrupt handler, which might possibly result in exiting - * idle mode, in other words, entering the mode in which read-side critical - * sections can occur. The caller must have disabled interrupts. - * - * Note that the Linux kernel is fully capable of entering an interrupt - * handler that it never exits, for example when doing upcalls to user mode! - * This code assumes that the idle loop never does upcalls to user mode. - * If your architecture's idle loop does do upcalls to user mode (or does - * anything else that results in unbalanced calls to the irq_enter() and - * irq_exit() functions), RCU will give you what you deserve, good and hard. - * But very infrequently and irreproducibly. - * - * Use things like work queues to work around this limitation. - * - * You have been warned. - * - * If you add or remove a call to rcu_irq_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_irq_enter(void) -{ - lockdep_assert_irqs_disabled(); - rcu_nmi_enter(); -} - -/* - * Wrapper for rcu_irq_enter() where interrupts are enabled. - * - * If you add or remove a call to rcu_irq_enter_irqson(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -void rcu_irq_enter_irqson(void) -{ - unsigned long flags; - - local_irq_save(flags); - rcu_irq_enter(); - local_irq_restore(flags); -} - /* * Check to see if any future non-offloaded RCU-related work will need * to be done by the current CPU, even if none need be done immediately, -- cgit v1.2.3 From 62e2412df4b90ae6706ce1f1a9649b789b2e44ef Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:29 +0200 Subject: rcu/context_tracking: Move dynticks counter to context tracking In order to prepare for merging RCU dynticks counter into the context tracking state, move the rcu_data's dynticks field to the context tracking structure. It will later be mixed within the context tracking state itself. [ paulmck: Move enum ctx_state into global scope. ] Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking_state.h | 45 ++++++++++++++++++++++----- kernel/context_tracking.c | 10 ++++-- kernel/rcu/tree.c | 56 ++++++++++++++++------------------ kernel/rcu/tree.h | 1 - kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_stall.h | 4 +-- 6 files changed, 75 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 9c16a8b2c194..5a8da2787287 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -6,7 +6,15 @@ #include #include +enum ctx_state { + CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ + CONTEXT_KERNEL = 0, + CONTEXT_USER, + CONTEXT_GUEST, +}; + struct context_tracking { +#ifdef CONFIG_CONTEXT_TRACKING_USER /* * When active is false, probes are unset in order * to minimize overhead: TIF flags are cleared @@ -15,17 +23,40 @@ struct context_tracking { */ bool active; int recursion; - enum ctx_state { - CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ - CONTEXT_KERNEL = 0, - CONTEXT_USER, - CONTEXT_GUEST, - } state; + enum ctx_state state; +#endif +#ifdef CONFIG_CONTEXT_TRACKING_IDLE + atomic_t dynticks; /* Even value for idle, else odd. */ +#endif }; +#ifdef CONFIG_CONTEXT_TRACKING +DECLARE_PER_CPU(struct context_tracking, context_tracking); +#endif + +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +static __always_inline int ct_dynticks(void) +{ + return atomic_read(this_cpu_ptr(&context_tracking.dynticks)); +} + +static __always_inline int ct_dynticks_cpu(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return atomic_read(&ct->dynticks); +} + +static __always_inline int ct_dynticks_cpu_acquire(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return atomic_read_acquire(&ct->dynticks); +} +#endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ + #ifdef CONFIG_CONTEXT_TRACKING_USER extern struct static_key_false context_tracking_key; -DECLARE_PER_CPU(struct context_tracking, context_tracking); static __always_inline bool context_tracking_enabled(void) { diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index c0d86dac98f1..01abbcec52f7 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -23,6 +23,13 @@ #include +DEFINE_PER_CPU(struct context_tracking, context_tracking) = { +#ifdef CONFIG_CONTEXT_TRACKING_IDLE + .dynticks = ATOMIC_INIT(1), +#endif +}; +EXPORT_SYMBOL_GPL(context_tracking); + #ifdef CONFIG_CONTEXT_TRACKING_IDLE noinstr void ct_idle_enter(void) { @@ -138,9 +145,6 @@ noinstr void ct_nmi_exit(void) DEFINE_STATIC_KEY_FALSE(context_tracking_key); EXPORT_SYMBOL_GPL(context_tracking_key); -DEFINE_PER_CPU(struct context_tracking, context_tracking); -EXPORT_SYMBOL_GPL(context_tracking); - static noinstr bool context_tracking_recursion_enter(void) { int recursion; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 75b433dba427..a471edc3d893 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -77,7 +77,6 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, - .dynticks = ATOMIC_INIT(1), #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_RCU_CORE, #endif @@ -268,7 +267,7 @@ void rcu_softirq_qs(void) */ static noinline noinstr unsigned long rcu_dynticks_inc(int incby) { - return arch_atomic_add_return(incby, this_cpu_ptr(&rcu_data.dynticks)); + return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); } /* @@ -324,9 +323,7 @@ static noinstr void rcu_dynticks_eqs_exit(void) */ static void rcu_dynticks_eqs_online(void) { - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); - - if (atomic_read(&rdp->dynticks) & 0x1) + if (ct_dynticks() & 0x1) return; rcu_dynticks_inc(1); } @@ -338,17 +335,17 @@ static void rcu_dynticks_eqs_online(void) */ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) { - return !(arch_atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1); + return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); } /* * Snapshot the ->dynticks counter with full ordering so as to allow * stable comparison of this counter with past and future snapshots. */ -static int rcu_dynticks_snap(struct rcu_data *rdp) +static int rcu_dynticks_snap(int cpu) { smp_mb(); // Fundamental RCU ordering guarantee. - return atomic_read_acquire(&rdp->dynticks); + return ct_dynticks_cpu_acquire(cpu); } /* @@ -363,9 +360,7 @@ static bool rcu_dynticks_in_eqs(int snap) /* Return true if the specified CPU is currently idle from an RCU viewpoint. */ bool rcu_is_idle_cpu(int cpu) { - struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); - - return rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)); + return rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu)); } /* @@ -375,7 +370,7 @@ bool rcu_is_idle_cpu(int cpu) */ static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) { - return snap != rcu_dynticks_snap(rdp); + return snap != rcu_dynticks_snap(rdp->cpu); } /* @@ -384,11 +379,10 @@ static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) */ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) { - struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int snap; // If not quiescent, force back to earlier extended quiescent state. - snap = atomic_read(&rdp->dynticks) & ~0x1; + snap = ct_dynticks_cpu(cpu) & ~0x1; smp_rmb(); // Order ->dynticks and *vp reads. if (READ_ONCE(*vp)) @@ -396,7 +390,7 @@ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) smp_rmb(); // Order *vp read and ->dynticks re-read. // If still in the same extended quiescent state, we are good! - return snap == atomic_read(&rdp->dynticks); + return snap == ct_dynticks_cpu(cpu); } /* @@ -620,6 +614,7 @@ EXPORT_SYMBOL_GPL(rcutorture_get_gp_data); static noinstr void rcu_eqs_enter(bool user) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + struct context_tracking *ct = this_cpu_ptr(&context_tracking); WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); @@ -633,12 +628,12 @@ static noinstr void rcu_eqs_enter(bool user) instrumentation_begin(); lockdep_assert_irqs_disabled(); - trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks)); + trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rcu_preempt_deferred_qs(current); // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); instrumentation_end(); WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ @@ -740,7 +735,7 @@ noinstr void rcu_user_enter(void) * rcu_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting + * RCU-idle period, update ct->dynticks and rdp->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * @@ -749,6 +744,7 @@ noinstr void rcu_user_enter(void) */ noinstr void rcu_nmi_exit(void) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); struct rcu_data *rdp = this_cpu_ptr(&rcu_data); instrumentation_begin(); @@ -766,7 +762,7 @@ noinstr void rcu_nmi_exit(void) */ if (rdp->dynticks_nmi_nesting != 1) { trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, - atomic_read(&rdp->dynticks)); + ct_dynticks()); WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ rdp->dynticks_nmi_nesting - 2); instrumentation_end(); @@ -774,11 +770,11 @@ noinstr void rcu_nmi_exit(void) } /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ - trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks)); + trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, ct_dynticks()); WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); instrumentation_end(); // RCU is watching here ... @@ -817,6 +813,7 @@ void rcu_irq_exit_check_preempt(void) */ static void noinstr rcu_eqs_exit(bool user) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); struct rcu_data *rdp; long oldval; @@ -836,9 +833,9 @@ static void noinstr rcu_eqs_exit(bool user) instrumentation_begin(); // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); + trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); WRITE_ONCE(rdp->dynticks_nesting, 1); WARN_ON_ONCE(rdp->dynticks_nmi_nesting); @@ -944,7 +941,7 @@ void __rcu_irq_enter_check_tick(void) /** * rcu_nmi_enter - inform RCU of entry to NMI context * - * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and + * If the CPU was idle from RCU's viewpoint, update ct->dynticks and * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know * that the CPU is active. This implementation permits nested NMIs, as * long as the nesting level does not overflow an int. (You will probably @@ -957,6 +954,7 @@ noinstr void rcu_nmi_enter(void) { long incby = 2; struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + struct context_tracking *ct = this_cpu_ptr(&context_tracking); /* Complain about underflow. */ WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0); @@ -980,9 +978,9 @@ noinstr void rcu_nmi_enter(void) instrumentation_begin(); // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() - instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); incby = 1; } else if (!in_nmi()) { @@ -994,7 +992,7 @@ noinstr void rcu_nmi_enter(void) trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), rdp->dynticks_nmi_nesting, - rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks)); + rdp->dynticks_nmi_nesting + incby, ct_dynticks()); instrumentation_end(); WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */ rdp->dynticks_nmi_nesting + incby); @@ -1138,7 +1136,7 @@ static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) */ static int dyntick_save_progress_counter(struct rcu_data *rdp) { - rdp->dynticks_snap = rcu_dynticks_snap(rdp); + rdp->dynticks_snap = rcu_dynticks_snap(rdp->cpu); if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) { trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); rcu_gpnum_ovf(rdp->mynode, rdp); @@ -4142,7 +4140,7 @@ rcu_boot_init_percpu_data(int cpu) rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); INIT_WORK(&rdp->strict_work, strict_work_handler); WARN_ON_ONCE(rdp->dynticks_nesting != 1); - WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp))); + WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu))); rdp->barrier_seq_snap = rcu_state.barrier_sequence; rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED; diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 2ccf5845957d..ebb973f5b190 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -189,7 +189,6 @@ struct rcu_data { int dynticks_snap; /* Per-GP tracking for dynticks. */ long dynticks_nesting; /* Track process nesting level. */ long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ - atomic_t dynticks; /* Even value for idle, else odd. */ bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ bool rcu_urgent_qs; /* GP old need light quiescent state. */ bool rcu_forced_tick; /* Forced tick to provide QS. */ diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 0f70f62039a9..75c22d1034c1 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -356,7 +356,7 @@ static void __sync_rcu_exp_select_node_cpus(struct rcu_exp_work *rewp) !(rnp->qsmaskinitnext & mask)) { mask_ofl_test |= mask; } else { - snap = rcu_dynticks_snap(rdp); + snap = rcu_dynticks_snap(cpu); if (rcu_dynticks_in_eqs(snap)) mask_ofl_test |= mask; else diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 3556637768fd..250fbf2e8522 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -465,7 +465,7 @@ static void print_cpu_stall_info(int cpu) } delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq); falsepositive = rcu_is_gp_kthread_starving(NULL) && - rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)); + rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu)); rcuc_starved = rcu_is_rcuc_kthread_starving(rdp, &j); if (rcuc_starved) sprintf(buf, " rcuc=%ld jiffies(starved)", j); @@ -478,7 +478,7 @@ static void print_cpu_stall_info(int cpu) rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' : "!."[!delta], ticks_value, ticks_title, - rcu_dynticks_snap(rdp) & 0xfff, + rcu_dynticks_snap(cpu) & 0xfff, rdp->dynticks_nesting, rdp->dynticks_nmi_nesting, rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, -- cgit v1.2.3 From 904e600e60f46f92eb4bcfb95788b1fedf7e8237 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:30 +0200 Subject: rcu/context_tracking: Move dynticks_nesting to context tracking The RCU eqs tracking is going to be performed by the context tracking subsystem. The related nesting counters thus need to be moved to the context tracking structure. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking_state.h | 13 +++++++++++++ kernel/context_tracking.c | 1 + kernel/rcu/tree.c | 31 ++++++++++++++++--------------- kernel/rcu/tree.h | 1 - kernel/rcu/tree_stall.h | 2 +- 5 files changed, 31 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 5a8da2787287..13a4a9d1ec7e 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -27,6 +27,7 @@ struct context_tracking { #endif #ifdef CONFIG_CONTEXT_TRACKING_IDLE atomic_t dynticks; /* Even value for idle, else odd. */ + long dynticks_nesting; /* Track process nesting level. */ #endif }; @@ -53,6 +54,18 @@ static __always_inline int ct_dynticks_cpu_acquire(int cpu) return atomic_read_acquire(&ct->dynticks); } + +static __always_inline long ct_dynticks_nesting(void) +{ + return __this_cpu_read(context_tracking.dynticks_nesting); +} + +static __always_inline long ct_dynticks_nesting_cpu(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return ct->dynticks_nesting; +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 01abbcec52f7..dfefe04400f8 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -25,6 +25,7 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { #ifdef CONFIG_CONTEXT_TRACKING_IDLE + .dynticks_nesting = 1, .dynticks = ATOMIC_INIT(1), #endif }; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index a471edc3d893..f6bf328bb9cf 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -75,7 +75,6 @@ /* Data structures. */ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { - .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_RCU_CORE, @@ -436,7 +435,7 @@ static int rcu_is_cpu_rrupt_from_idle(void) lockdep_assert_irqs_disabled(); /* Check for counter underflows */ - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0, + RCU_LOCKDEP_WARN(ct_dynticks_nesting() < 0, "RCU dynticks_nesting counter underflow!"); RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0, "RCU dynticks_nmi_nesting counter underflow/zero!"); @@ -452,7 +451,7 @@ static int rcu_is_cpu_rrupt_from_idle(void) WARN_ON_ONCE(!nesting && !is_idle_task(current)); /* Does CPU appear to be idle from an RCU standpoint? */ - return __this_cpu_read(rcu_data.dynticks_nesting) == 0; + return ct_dynticks_nesting() == 0; } #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10) @@ -619,16 +618,16 @@ static noinstr void rcu_eqs_enter(bool user) WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && - rdp->dynticks_nesting == 0); - if (rdp->dynticks_nesting != 1) { + ct_dynticks_nesting() == 0); + if (ct_dynticks_nesting() != 1) { // RCU will still be watching, so just do accounting and leave. - rdp->dynticks_nesting--; + ct->dynticks_nesting--; return; } instrumentation_begin(); lockdep_assert_irqs_disabled(); - trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, ct_dynticks()); + trace_rcu_dyntick(TPS("Start"), ct_dynticks_nesting(), 0, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rcu_preempt_deferred_qs(current); @@ -636,7 +635,7 @@ static noinstr void rcu_eqs_enter(bool user) instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); instrumentation_end(); - WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ + WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ // RCU is watching here ... rcu_dynticks_eqs_enter(); // ... but is no longer watching here. @@ -793,7 +792,7 @@ void rcu_irq_exit_check_preempt(void) { lockdep_assert_irqs_disabled(); - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0, + RCU_LOCKDEP_WARN(ct_dynticks_nesting() <= 0, "RCU dynticks_nesting counter underflow/zero!"); RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) != DYNTICK_IRQ_NONIDLE, @@ -819,11 +818,11 @@ static void noinstr rcu_eqs_exit(bool user) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rdp = this_cpu_ptr(&rcu_data); - oldval = rdp->dynticks_nesting; + oldval = ct_dynticks_nesting(); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); if (oldval) { // RCU was already watching, so just do accounting and leave. - rdp->dynticks_nesting++; + ct->dynticks_nesting++; return; } rcu_dynticks_task_exit(); @@ -835,9 +834,9 @@ static void noinstr rcu_eqs_exit(bool user) // instrumentation for the noinstr rcu_dynticks_eqs_exit() instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, ct_dynticks()); + trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); - WRITE_ONCE(rdp->dynticks_nesting, 1); + WRITE_ONCE(ct->dynticks_nesting, 1); WARN_ON_ONCE(rdp->dynticks_nmi_nesting); WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); instrumentation_end(); @@ -4134,12 +4133,13 @@ static void rcu_init_new_rnp(struct rcu_node *rnp_leaf) static void __init rcu_boot_init_percpu_data(int cpu) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); /* Set up local state, ensuring consistent view of global state. */ rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); INIT_WORK(&rdp->strict_work, strict_work_handler); - WARN_ON_ONCE(rdp->dynticks_nesting != 1); + WARN_ON_ONCE(ct->dynticks_nesting != 1); WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu))); rdp->barrier_seq_snap = rcu_state.barrier_sequence; rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; @@ -4164,6 +4164,7 @@ rcu_boot_init_percpu_data(int cpu) int rcutree_prepare_cpu(unsigned int cpu) { unsigned long flags; + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); struct rcu_node *rnp = rcu_get_root(); @@ -4172,7 +4173,7 @@ int rcutree_prepare_cpu(unsigned int cpu) rdp->qlen_last_fqs_check = 0; rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); rdp->blimit = blimit; - rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */ + ct->dynticks_nesting = 1; /* CPU not up, no tearing. */ raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ /* diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index ebb973f5b190..650ff3cf0121 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -187,7 +187,6 @@ struct rcu_data { /* 3) dynticks interface. */ int dynticks_snap; /* Per-GP tracking for dynticks. */ - long dynticks_nesting; /* Track process nesting level. */ long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ bool rcu_urgent_qs; /* GP old need light quiescent state. */ diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 250fbf2e8522..a9c82254b6c6 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -479,7 +479,7 @@ static void print_cpu_stall_info(int cpu) "!."[!delta], ticks_value, ticks_title, rcu_dynticks_snap(cpu) & 0xfff, - rdp->dynticks_nesting, rdp->dynticks_nmi_nesting, + ct_dynticks_nesting_cpu(cpu), rdp->dynticks_nmi_nesting, rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, rcuc_starved ? buf : "", -- cgit v1.2.3 From 95e04f48ec0a634e2f221081f5fa1a904755f326 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:31 +0200 Subject: rcu/context_tracking: Move dynticks_nmi_nesting to context tracking The RCU eqs tracking is going to be performed by the context tracking subsystem. The related nesting counters thus need to be moved to the context tracking structure. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking_state.h | 16 ++++++++++++ kernel/context_tracking.c | 1 + kernel/rcu/rcu.h | 4 --- kernel/rcu/tree.c | 48 +++++++++++++++------------------- kernel/rcu/tree.h | 1 - kernel/rcu/tree_stall.h | 2 +- 6 files changed, 39 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 13a4a9d1ec7e..5f11e3d2d85a 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -13,6 +13,9 @@ enum ctx_state { CONTEXT_GUEST, }; +/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ +#define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) + struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING_USER /* @@ -28,6 +31,7 @@ struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING_IDLE atomic_t dynticks; /* Even value for idle, else odd. */ long dynticks_nesting; /* Track process nesting level. */ + long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ #endif }; @@ -66,6 +70,18 @@ static __always_inline long ct_dynticks_nesting_cpu(int cpu) return ct->dynticks_nesting; } + +static __always_inline long ct_dynticks_nmi_nesting(void) +{ + return __this_cpu_read(context_tracking.dynticks_nmi_nesting); +} + +static __always_inline long ct_dynticks_nmi_nesting_cpu(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return ct->dynticks_nmi_nesting; +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index dfefe04400f8..7c3033e9a518 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -26,6 +26,7 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { #ifdef CONFIG_CONTEXT_TRACKING_IDLE .dynticks_nesting = 1, + .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, .dynticks = ATOMIC_INIT(1), #endif }; diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 4916077119f3..7b4a88deff9a 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -12,10 +12,6 @@ #include -/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ -#define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) - - /* * Grace-period counter management. */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index f6bf328bb9cf..006939b29e82 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -75,7 +75,6 @@ /* Data structures. */ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { - .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_RCU_CORE, #endif @@ -437,11 +436,11 @@ static int rcu_is_cpu_rrupt_from_idle(void) /* Check for counter underflows */ RCU_LOCKDEP_WARN(ct_dynticks_nesting() < 0, "RCU dynticks_nesting counter underflow!"); - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0, + RCU_LOCKDEP_WARN(ct_dynticks_nmi_nesting() <= 0, "RCU dynticks_nmi_nesting counter underflow/zero!"); /* Are we at first interrupt nesting level? */ - nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting); + nesting = ct_dynticks_nmi_nesting(); if (nesting > 1) return false; @@ -612,11 +611,10 @@ EXPORT_SYMBOL_GPL(rcutorture_get_gp_data); */ static noinstr void rcu_eqs_enter(bool user) { - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct context_tracking *ct = this_cpu_ptr(&context_tracking); - WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); - WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); + WARN_ON_ONCE(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && ct_dynticks_nesting() == 0); if (ct_dynticks_nesting() != 1) { @@ -734,7 +732,7 @@ noinstr void rcu_user_enter(void) * rcu_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update ct->dynticks and rdp->dynticks_nmi_nesting + * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * @@ -744,7 +742,6 @@ noinstr void rcu_user_enter(void) noinstr void rcu_nmi_exit(void) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); instrumentation_begin(); /* @@ -752,25 +749,25 @@ noinstr void rcu_nmi_exit(void) * (We are exiting an NMI handler, so RCU better be paying attention * to us!) */ - WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0); + WARN_ON_ONCE(ct_dynticks_nmi_nesting() <= 0); WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); /* * If the nesting level is not 1, the CPU wasn't RCU-idle, so * leave it in non-RCU-idle state. */ - if (rdp->dynticks_nmi_nesting != 1) { - trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, + if (ct_dynticks_nmi_nesting() != 1) { + trace_rcu_dyntick(TPS("--="), ct_dynticks_nmi_nesting(), ct_dynticks_nmi_nesting() - 2, ct_dynticks()); - WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ - rdp->dynticks_nmi_nesting - 2); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* No store tearing. */ + ct_dynticks_nmi_nesting() - 2); instrumentation_end(); return; } /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ - trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, ct_dynticks()); - WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ + trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ // instrumentation for the noinstr rcu_dynticks_eqs_enter() instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); @@ -794,7 +791,7 @@ void rcu_irq_exit_check_preempt(void) RCU_LOCKDEP_WARN(ct_dynticks_nesting() <= 0, "RCU dynticks_nesting counter underflow/zero!"); - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) != + RCU_LOCKDEP_WARN(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE, "Bad RCU dynticks_nmi_nesting counter\n"); RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(), @@ -813,11 +810,9 @@ void rcu_irq_exit_check_preempt(void) static void noinstr rcu_eqs_exit(bool user) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); - struct rcu_data *rdp; long oldval; WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - rdp = this_cpu_ptr(&rcu_data); oldval = ct_dynticks_nesting(); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); if (oldval) { @@ -837,8 +832,8 @@ static void noinstr rcu_eqs_exit(bool user) trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); WRITE_ONCE(ct->dynticks_nesting, 1); - WARN_ON_ONCE(rdp->dynticks_nmi_nesting); - WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); + WARN_ON_ONCE(ct_dynticks_nmi_nesting()); + WRITE_ONCE(ct->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); instrumentation_end(); } @@ -941,7 +936,7 @@ void __rcu_irq_enter_check_tick(void) * rcu_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle from RCU's viewpoint, update ct->dynticks and - * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know + * ct->dynticks_nmi_nesting to let the RCU grace-period handling know * that the CPU is active. This implementation permits nested NMIs, as * long as the nesting level does not overflow an int. (You will probably * run out of stack space first.) @@ -952,11 +947,10 @@ void __rcu_irq_enter_check_tick(void) noinstr void rcu_nmi_enter(void) { long incby = 2; - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct context_tracking *ct = this_cpu_ptr(&context_tracking); /* Complain about underflow. */ - WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0); + WARN_ON_ONCE(ct_dynticks_nmi_nesting() < 0); /* * If idle from RCU viewpoint, atomically increment ->dynticks @@ -990,11 +984,11 @@ noinstr void rcu_nmi_enter(void) } trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), - rdp->dynticks_nmi_nesting, - rdp->dynticks_nmi_nesting + incby, ct_dynticks()); + ct_dynticks_nmi_nesting(), + ct_dynticks_nmi_nesting() + incby, ct_dynticks()); instrumentation_end(); - WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */ - rdp->dynticks_nmi_nesting + incby); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* Prevent store tearing. */ + ct_dynticks_nmi_nesting() + incby); barrier(); } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 650ff3cf0121..72dbf8512ce7 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -187,7 +187,6 @@ struct rcu_data { /* 3) dynticks interface. */ int dynticks_snap; /* Per-GP tracking for dynticks. */ - long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ bool rcu_urgent_qs; /* GP old need light quiescent state. */ bool rcu_forced_tick; /* Forced tick to provide QS. */ diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index a9c82254b6c6..2683ce0a7c72 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -479,7 +479,7 @@ static void print_cpu_stall_info(int cpu) "!."[!delta], ticks_value, ticks_title, rcu_dynticks_snap(cpu) & 0xfff, - ct_dynticks_nesting_cpu(cpu), rdp->dynticks_nmi_nesting, + ct_dynticks_nesting_cpu(cpu), ct_dynticks_nmi_nesting_cpu(cpu), rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, rcuc_starved ? buf : "", -- cgit v1.2.3 From 564506495ca96a6e66d077d3d5b9f02d4b9b0f45 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:32 +0200 Subject: rcu/context-tracking: Move deferred nocb resched to context tracking To prepare for migrating the RCU eqs accounting code to context tracking, split the last-resort deferred nocb resched from rcu_user_enter() and move it into a separate call from context tracking. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/rcupdate.h | 6 ++++++ kernel/context_tracking.c | 8 ++++++++ kernel/rcu/tree.c | 15 ++------------- 3 files changed, 16 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index f1562d91c67d..3717cad983a6 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -112,6 +112,12 @@ static inline void rcu_user_enter(void) { } static inline void rcu_user_exit(void) { } #endif /* CONFIG_NO_HZ_FULL */ +#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) +void rcu_irq_work_resched(void); +#else +static inline void rcu_irq_work_resched(void) { } +#endif + #ifdef CONFIG_RCU_NOCB_CPU void rcu_init_nohz(void); int rcu_nocb_cpu_offload(int cpu); diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 7c3033e9a518..8cf59d8a6af6 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -177,6 +177,8 @@ static __always_inline void context_tracking_recursion_exit(void) */ void noinstr __ct_user_enter(enum ctx_state state) { + lockdep_assert_irqs_disabled(); + /* Kernel threads aren't supposed to go to userspace */ WARN_ON_ONCE(!current->mm); @@ -198,6 +200,12 @@ void noinstr __ct_user_enter(enum ctx_state state) vtime_user_enter(current); instrumentation_end(); } + /* + * Other than generic entry implementation, we may be past the last + * rescheduling opportunity in the entry code. Trigger a self IPI + * that will fire and reschedule once we resume in user/guest mode. + */ + rcu_irq_work_resched(); rcu_user_enter(); } /* diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 006939b29e82..8c0c3490532e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -681,7 +681,7 @@ static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = * last resort is to fire a local irq_work that will trigger a reschedule once IRQs * get re-enabled again. */ -noinstr static void rcu_irq_work_resched(void) +noinstr void rcu_irq_work_resched(void) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); @@ -697,10 +697,7 @@ noinstr static void rcu_irq_work_resched(void) } instrumentation_end(); } - -#else -static inline void rcu_irq_work_resched(void) { } -#endif +#endif /* #if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) */ /** * rcu_user_enter - inform RCU that we are resuming userspace. @@ -715,14 +712,6 @@ static inline void rcu_irq_work_resched(void) { } */ noinstr void rcu_user_enter(void) { - lockdep_assert_irqs_disabled(); - - /* - * Other than generic entry implementation, we may be past the last - * rescheduling opportunity in the entry code. Trigger a self IPI - * that will fire and reschedule once we resume in user/guest mode. - */ - rcu_irq_work_resched(); rcu_eqs_enter(true); } -- cgit v1.2.3 From 172114552701b85d5c3b1a089a73ee85d0d7786b Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:33 +0200 Subject: rcu/context-tracking: Move RCU-dynticks internal functions to context_tracking Move the core RCU eqs/dynticks functions to context tracking so that we can later merge all that code within context tracking. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 20 +++ include/linux/rcutree.h | 3 + kernel/context_tracking.c | 336 +++++++++++++++++++++++++++++++++++++++ kernel/rcu/tree.c | 324 +------------------------------------ kernel/rcu/tree.h | 5 - kernel/rcu/tree_plugin.h | 38 +---- 6 files changed, 364 insertions(+), 362 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 01abadb2f993..1f568676bc1d 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -122,6 +122,26 @@ static inline void context_tracking_init(void) { } #ifdef CONFIG_CONTEXT_TRACKING_IDLE extern void ct_idle_enter(void); extern void ct_idle_exit(void); + +/* + * Is the current CPU in an extended quiescent state? + * + * No ordering, as we are sampling CPU-local information. + */ +static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) +{ + return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); +} + +/* + * Increment the current CPU's context_tracking structure's ->dynticks field + * with ordering. Return the new value. + */ +static __always_inline unsigned long rcu_dynticks_inc(int incby) +{ + return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); +} + #else static inline void ct_idle_enter(void) { } static inline void ct_idle_exit(void) { } diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 4522b6a7cc42..24db1e41695c 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -55,6 +55,9 @@ void rcu_irq_exit_check_preempt(void); static inline void rcu_irq_exit_check_preempt(void) { } #endif +struct task_struct; +void rcu_preempt_deferred_qs(struct task_struct *t); + void exit_rcu(void); void rcu_scheduler_starting(void); diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 8cf59d8a6af6..072c4b6044b3 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -21,6 +21,7 @@ #include #include #include +#include DEFINE_PER_CPU(struct context_tracking, context_tracking) = { @@ -33,6 +34,309 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { EXPORT_SYMBOL_GPL(context_tracking); #ifdef CONFIG_CONTEXT_TRACKING_IDLE +#define TPS(x) tracepoint_string(x) + +/* Record the current task on dyntick-idle entry. */ +static __always_inline void rcu_dynticks_task_enter(void) +{ +#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) + WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id()); +#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ +} + +/* Record no current task on dyntick-idle exit. */ +static __always_inline void rcu_dynticks_task_exit(void) +{ +#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) + WRITE_ONCE(current->rcu_tasks_idle_cpu, -1); +#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ +} + +/* Turn on heavyweight RCU tasks trace readers on idle/user entry. */ +static __always_inline void rcu_dynticks_task_trace_enter(void) +{ +#ifdef CONFIG_TASKS_TRACE_RCU + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) + current->trc_reader_special.b.need_mb = true; +#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ +} + +/* Turn off heavyweight RCU tasks trace readers on idle/user exit. */ +static __always_inline void rcu_dynticks_task_trace_exit(void) +{ +#ifdef CONFIG_TASKS_TRACE_RCU + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) + current->trc_reader_special.b.need_mb = false; +#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ +} + +/* + * Record entry into an extended quiescent state. This is only to be + * called when not already in an extended quiescent state, that is, + * RCU is watching prior to the call to this function and is no longer + * watching upon return. + */ +static noinstr void rcu_dynticks_eqs_enter(void) +{ + int seq; + + /* + * CPUs seeing atomic_add_return() must see prior RCU read-side + * critical sections, and we also must force ordering with the + * next idle sojourn. + */ + rcu_dynticks_task_trace_enter(); // Before ->dynticks update! + seq = rcu_dynticks_inc(1); + // RCU is no longer watching. Better be in extended quiescent state! + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & 0x1)); +} + +/* + * Record exit from an extended quiescent state. This is only to be + * called from an extended quiescent state, that is, RCU is not watching + * prior to the call to this function and is watching upon return. + */ +static noinstr void rcu_dynticks_eqs_exit(void) +{ + int seq; + + /* + * CPUs seeing atomic_add_return() must see prior idle sojourns, + * and we also must force ordering with the next RCU read-side + * critical section. + */ + seq = rcu_dynticks_inc(1); + // RCU is now watching. Better not be in an extended quiescent state! + rcu_dynticks_task_trace_exit(); // After ->dynticks update! + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & 0x1)); +} + +/* + * Enter an RCU extended quiescent state, which can be either the + * idle loop or adaptive-tickless usermode execution. + * + * We crowbar the ->dynticks_nmi_nesting field to zero to allow for + * the possibility of usermode upcalls having messed up our count + * of interrupt nesting level during the prior busy period. + */ +static void noinstr rcu_eqs_enter(bool user) +{ + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + + WARN_ON_ONCE(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && + ct_dynticks_nesting() == 0); + if (ct_dynticks_nesting() != 1) { + // RCU will still be watching, so just do accounting and leave. + ct->dynticks_nesting--; + return; + } + + instrumentation_begin(); + lockdep_assert_irqs_disabled(); + trace_rcu_dyntick(TPS("Start"), ct_dynticks_nesting(), 0, ct_dynticks()); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); + rcu_preempt_deferred_qs(current); + + // instrumentation for the noinstr rcu_dynticks_eqs_enter() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + + instrumentation_end(); + WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ + // RCU is watching here ... + rcu_dynticks_eqs_enter(); + // ... but is no longer watching here. + rcu_dynticks_task_enter(); +} + +/* + * Exit an RCU extended quiescent state, which can be either the + * idle loop or adaptive-tickless usermode execution. + * + * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to + * allow for the possibility of usermode upcalls messing up our count of + * interrupt nesting level during the busy period that is just now starting. + */ +static void noinstr rcu_eqs_exit(bool user) +{ + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + long oldval; + + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); + oldval = ct_dynticks_nesting(); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); + if (oldval) { + // RCU was already watching, so just do accounting and leave. + ct->dynticks_nesting++; + return; + } + rcu_dynticks_task_exit(); + // RCU is not watching here ... + rcu_dynticks_eqs_exit(); + // ... but is watching here. + instrumentation_begin(); + + // instrumentation for the noinstr rcu_dynticks_eqs_exit() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + + trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); + WRITE_ONCE(ct->dynticks_nesting, 1); + WARN_ON_ONCE(ct_dynticks_nmi_nesting()); + WRITE_ONCE(ct->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); + instrumentation_end(); +} + +/** + * rcu_nmi_exit - inform RCU of exit from NMI context + * + * If we are returning from the outermost NMI handler that interrupted an + * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting + * to let the RCU grace-period handling know that the CPU is back to + * being RCU-idle. + * + * If you add or remove a call to rcu_nmi_exit(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_nmi_exit(void) +{ + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + + instrumentation_begin(); + /* + * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. + * (We are exiting an NMI handler, so RCU better be paying attention + * to us!) + */ + WARN_ON_ONCE(ct_dynticks_nmi_nesting() <= 0); + WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); + + /* + * If the nesting level is not 1, the CPU wasn't RCU-idle, so + * leave it in non-RCU-idle state. + */ + if (ct_dynticks_nmi_nesting() != 1) { + trace_rcu_dyntick(TPS("--="), ct_dynticks_nmi_nesting(), ct_dynticks_nmi_nesting() - 2, + ct_dynticks()); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* No store tearing. */ + ct_dynticks_nmi_nesting() - 2); + instrumentation_end(); + return; + } + + /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ + trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ + + // instrumentation for the noinstr rcu_dynticks_eqs_enter() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + instrumentation_end(); + + // RCU is watching here ... + rcu_dynticks_eqs_enter(); + // ... but is no longer watching here. + + if (!in_nmi()) + rcu_dynticks_task_enter(); +} + +/** + * rcu_nmi_enter - inform RCU of entry to NMI context + * + * If the CPU was idle from RCU's viewpoint, update ct->dynticks and + * ct->dynticks_nmi_nesting to let the RCU grace-period handling know + * that the CPU is active. This implementation permits nested NMIs, as + * long as the nesting level does not overflow an int. (You will probably + * run out of stack space first.) + * + * If you add or remove a call to rcu_nmi_enter(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_nmi_enter(void) +{ + long incby = 2; + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + + /* Complain about underflow. */ + WARN_ON_ONCE(ct_dynticks_nmi_nesting() < 0); + + /* + * If idle from RCU viewpoint, atomically increment ->dynticks + * to mark non-idle and increment ->dynticks_nmi_nesting by one. + * Otherwise, increment ->dynticks_nmi_nesting by two. This means + * if ->dynticks_nmi_nesting is equal to one, we are guaranteed + * to be in the outermost NMI handler that interrupted an RCU-idle + * period (observation due to Andy Lutomirski). + */ + if (rcu_dynticks_curr_cpu_in_eqs()) { + + if (!in_nmi()) + rcu_dynticks_task_exit(); + + // RCU is not watching here ... + rcu_dynticks_eqs_exit(); + // ... but is watching here. + + instrumentation_begin(); + // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() + instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr rcu_dynticks_eqs_exit() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + + incby = 1; + } else if (!in_nmi()) { + instrumentation_begin(); + rcu_irq_enter_check_tick(); + } else { + instrumentation_begin(); + } + + trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), + ct_dynticks_nmi_nesting(), + ct_dynticks_nmi_nesting() + incby, ct_dynticks()); + instrumentation_end(); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* Prevent store tearing. */ + ct_dynticks_nmi_nesting() + incby); + barrier(); +} + +/** + * rcu_idle_enter - inform RCU that current CPU is entering idle + * + * Enter idle mode, in other words, -leave- the mode in which RCU + * read-side critical sections can occur. (Though RCU read-side + * critical sections can occur in irq handlers in idle, a possibility + * handled by irq_enter() and irq_exit().) + * + * If you add or remove a call to rcu_idle_enter(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_idle_enter(void) +{ + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); + rcu_eqs_enter(false); +} + +/** + * rcu_idle_exit - inform RCU that current CPU is leaving idle + * + * Exit idle mode, in other words, -enter- the mode in which RCU + * read-side critical sections can occur. + * + * If you add or remove a call to rcu_idle_exit(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_idle_exit(void) +{ + unsigned long flags; + + raw_local_irq_save(flags); + rcu_eqs_exit(false); + raw_local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(rcu_idle_exit); + noinstr void ct_idle_enter(void) { rcu_idle_enter(); @@ -139,6 +443,38 @@ noinstr void ct_nmi_exit(void) } #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ +#ifdef CONFIG_NO_HZ_FULL +/** + * rcu_user_enter - inform RCU that we are resuming userspace. + * + * Enter RCU idle mode right before resuming userspace. No use of RCU + * is permitted between this call and rcu_user_exit(). This way the + * CPU doesn't need to maintain the tick for RCU maintenance purposes + * when the CPU runs in userspace. + * + * If you add or remove a call to rcu_user_enter(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +noinstr void rcu_user_enter(void) +{ + rcu_eqs_enter(true); +} + +/** + * rcu_user_exit - inform RCU that we are exiting userspace. + * + * Exit RCU idle mode while entering the kernel because it can + * run a RCU read side critical section anytime. + * + * If you add or remove a call to rcu_user_exit(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_user_exit(void) +{ + rcu_eqs_exit(true); +} +#endif /* #ifdef CONFIG_NO_HZ_FULL */ + #ifdef CONFIG_CONTEXT_TRACKING_USER #define CREATE_TRACE_POINTS diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 8c0c3490532e..e2a2083079a2 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -62,6 +62,7 @@ #include #include #include +#include #include "../time/tick-internal.h" #include "tree.h" @@ -259,56 +260,6 @@ void rcu_softirq_qs(void) rcu_tasks_qs(current, false); } -/* - * Increment the current CPU's rcu_data structure's ->dynticks field - * with ordering. Return the new value. - */ -static noinline noinstr unsigned long rcu_dynticks_inc(int incby) -{ - return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); -} - -/* - * Record entry into an extended quiescent state. This is only to be - * called when not already in an extended quiescent state, that is, - * RCU is watching prior to the call to this function and is no longer - * watching upon return. - */ -static noinstr void rcu_dynticks_eqs_enter(void) -{ - int seq; - - /* - * CPUs seeing atomic_add_return() must see prior RCU read-side - * critical sections, and we also must force ordering with the - * next idle sojourn. - */ - rcu_dynticks_task_trace_enter(); // Before ->dynticks update! - seq = rcu_dynticks_inc(1); - // RCU is no longer watching. Better be in extended quiescent state! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & 0x1)); -} - -/* - * Record exit from an extended quiescent state. This is only to be - * called from an extended quiescent state, that is, RCU is not watching - * prior to the call to this function and is watching upon return. - */ -static noinstr void rcu_dynticks_eqs_exit(void) -{ - int seq; - - /* - * CPUs seeing atomic_add_return() must see prior idle sojourns, - * and we also must force ordering with the next RCU read-side - * critical section. - */ - seq = rcu_dynticks_inc(1); - // RCU is now watching. Better not be in an extended quiescent state! - rcu_dynticks_task_trace_exit(); // After ->dynticks update! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & 0x1)); -} - /* * Reset the current CPU's ->dynticks counter to indicate that the * newly onlined CPU is no longer in an extended quiescent state. @@ -326,16 +277,6 @@ static void rcu_dynticks_eqs_online(void) rcu_dynticks_inc(1); } -/* - * Is the current CPU in an extended quiescent state? - * - * No ordering, as we are sampling CPU-local information. - */ -static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) -{ - return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); -} - /* * Snapshot the ->dynticks counter with full ordering so as to allow * stable comparison of this counter with past and future snapshots. @@ -601,65 +542,7 @@ void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, } EXPORT_SYMBOL_GPL(rcutorture_get_gp_data); -/* - * Enter an RCU extended quiescent state, which can be either the - * idle loop or adaptive-tickless usermode execution. - * - * We crowbar the ->dynticks_nmi_nesting field to zero to allow for - * the possibility of usermode upcalls having messed up our count - * of interrupt nesting level during the prior busy period. - */ -static noinstr void rcu_eqs_enter(bool user) -{ - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - - WARN_ON_ONCE(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE); - WRITE_ONCE(ct->dynticks_nmi_nesting, 0); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && - ct_dynticks_nesting() == 0); - if (ct_dynticks_nesting() != 1) { - // RCU will still be watching, so just do accounting and leave. - ct->dynticks_nesting--; - return; - } - - instrumentation_begin(); - lockdep_assert_irqs_disabled(); - trace_rcu_dyntick(TPS("Start"), ct_dynticks_nesting(), 0, ct_dynticks()); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); - rcu_preempt_deferred_qs(current); - - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - - instrumentation_end(); - WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ - // RCU is watching here ... - rcu_dynticks_eqs_enter(); - // ... but is no longer watching here. - rcu_dynticks_task_enter(); -} - -/** - * rcu_idle_enter - inform RCU that current CPU is entering idle - * - * Enter idle mode, in other words, -leave- the mode in which RCU - * read-side critical sections can occur. (Though RCU read-side - * critical sections can occur in irq handlers in idle, a possibility - * handled by irq_enter() and irq_exit().) - * - * If you add or remove a call to rcu_idle_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_idle_enter(void) -{ - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - rcu_eqs_enter(false); -} - -#ifdef CONFIG_NO_HZ_FULL - -#if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) +#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) /* * An empty function that will trigger a reschedule on * IRQ tail once IRQs get re-enabled on userspace/guest resume. @@ -697,78 +580,7 @@ noinstr void rcu_irq_work_resched(void) } instrumentation_end(); } -#endif /* #if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) */ - -/** - * rcu_user_enter - inform RCU that we are resuming userspace. - * - * Enter RCU idle mode right before resuming userspace. No use of RCU - * is permitted between this call and rcu_user_exit(). This way the - * CPU doesn't need to maintain the tick for RCU maintenance purposes - * when the CPU runs in userspace. - * - * If you add or remove a call to rcu_user_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_user_enter(void) -{ - rcu_eqs_enter(true); -} - -#endif /* CONFIG_NO_HZ_FULL */ - -/** - * rcu_nmi_exit - inform RCU of exit from NMI context - * - * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting - * to let the RCU grace-period handling know that the CPU is back to - * being RCU-idle. - * - * If you add or remove a call to rcu_nmi_exit(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_nmi_exit(void) -{ - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - - instrumentation_begin(); - /* - * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. - * (We are exiting an NMI handler, so RCU better be paying attention - * to us!) - */ - WARN_ON_ONCE(ct_dynticks_nmi_nesting() <= 0); - WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); - - /* - * If the nesting level is not 1, the CPU wasn't RCU-idle, so - * leave it in non-RCU-idle state. - */ - if (ct_dynticks_nmi_nesting() != 1) { - trace_rcu_dyntick(TPS("--="), ct_dynticks_nmi_nesting(), ct_dynticks_nmi_nesting() - 2, - ct_dynticks()); - WRITE_ONCE(ct->dynticks_nmi_nesting, /* No store tearing. */ - ct_dynticks_nmi_nesting() - 2); - instrumentation_end(); - return; - } - - /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ - trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); - WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ - - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - instrumentation_end(); - - // RCU is watching here ... - rcu_dynticks_eqs_enter(); - // ... but is no longer watching here. - - if (!in_nmi()) - rcu_dynticks_task_enter(); -} +#endif /* #if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) */ #ifdef CONFIG_PROVE_RCU /** @@ -788,77 +600,7 @@ void rcu_irq_exit_check_preempt(void) } #endif /* #ifdef CONFIG_PROVE_RCU */ -/* - * Exit an RCU extended quiescent state, which can be either the - * idle loop or adaptive-tickless usermode execution. - * - * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to - * allow for the possibility of usermode upcalls messing up our count of - * interrupt nesting level during the busy period that is just now starting. - */ -static void noinstr rcu_eqs_exit(bool user) -{ - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - long oldval; - - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - oldval = ct_dynticks_nesting(); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); - if (oldval) { - // RCU was already watching, so just do accounting and leave. - ct->dynticks_nesting++; - return; - } - rcu_dynticks_task_exit(); - // RCU is not watching here ... - rcu_dynticks_eqs_exit(); - // ... but is watching here. - instrumentation_begin(); - - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - - trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); - WRITE_ONCE(ct->dynticks_nesting, 1); - WARN_ON_ONCE(ct_dynticks_nmi_nesting()); - WRITE_ONCE(ct->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); - instrumentation_end(); -} - -/** - * rcu_idle_exit - inform RCU that current CPU is leaving idle - * - * Exit idle mode, in other words, -enter- the mode in which RCU - * read-side critical sections can occur. - * - * If you add or remove a call to rcu_idle_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_idle_exit(void) -{ - unsigned long flags; - - raw_local_irq_save(flags); - rcu_eqs_exit(false); - raw_local_irq_restore(flags); -} - #ifdef CONFIG_NO_HZ_FULL -/** - * rcu_user_exit - inform RCU that we are exiting userspace. - * - * Exit RCU idle mode while entering the kernel because it can - * run a RCU read side critical section anytime. - * - * If you add or remove a call to rcu_user_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_user_exit(void) -{ - rcu_eqs_exit(true); -} - /** * __rcu_irq_enter_check_tick - Enable scheduler tick on CPU if RCU needs it. * @@ -921,66 +663,6 @@ void __rcu_irq_enter_check_tick(void) } #endif /* CONFIG_NO_HZ_FULL */ -/** - * rcu_nmi_enter - inform RCU of entry to NMI context - * - * If the CPU was idle from RCU's viewpoint, update ct->dynticks and - * ct->dynticks_nmi_nesting to let the RCU grace-period handling know - * that the CPU is active. This implementation permits nested NMIs, as - * long as the nesting level does not overflow an int. (You will probably - * run out of stack space first.) - * - * If you add or remove a call to rcu_nmi_enter(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_nmi_enter(void) -{ - long incby = 2; - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - - /* Complain about underflow. */ - WARN_ON_ONCE(ct_dynticks_nmi_nesting() < 0); - - /* - * If idle from RCU viewpoint, atomically increment ->dynticks - * to mark non-idle and increment ->dynticks_nmi_nesting by one. - * Otherwise, increment ->dynticks_nmi_nesting by two. This means - * if ->dynticks_nmi_nesting is equal to one, we are guaranteed - * to be in the outermost NMI handler that interrupted an RCU-idle - * period (observation due to Andy Lutomirski). - */ - if (rcu_dynticks_curr_cpu_in_eqs()) { - - if (!in_nmi()) - rcu_dynticks_task_exit(); - - // RCU is not watching here ... - rcu_dynticks_eqs_exit(); - // ... but is watching here. - - instrumentation_begin(); - // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() - instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - - incby = 1; - } else if (!in_nmi()) { - instrumentation_begin(); - rcu_irq_enter_check_tick(); - } else { - instrumentation_begin(); - } - - trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), - ct_dynticks_nmi_nesting(), - ct_dynticks_nmi_nesting() + incby, ct_dynticks()); - instrumentation_end(); - WRITE_ONCE(ct->dynticks_nmi_nesting, /* Prevent store tearing. */ - ct_dynticks_nmi_nesting() + incby); - barrier(); -} - /* * Check to see if any future non-offloaded RCU-related work will need * to be done by the current CPU, even if none need be done immediately, diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 72dbf8512ce7..0d5d1de327e4 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -427,7 +427,6 @@ static void rcu_cpu_kthread_setup(unsigned int cpu); static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp); static bool rcu_preempt_has_tasks(struct rcu_node *rnp); static bool rcu_preempt_need_deferred_qs(struct task_struct *t); -static void rcu_preempt_deferred_qs(struct task_struct *t); static void zero_cpu_stall_ticks(struct rcu_data *rdp); static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); @@ -467,10 +466,6 @@ do { \ static void rcu_bind_gp_kthread(void); static bool rcu_nohz_full_cpu(void); -static void rcu_dynticks_task_enter(void); -static void rcu_dynticks_task_exit(void); -static void rcu_dynticks_task_trace_enter(void); -static void rcu_dynticks_task_trace_exit(void); /* Forward declarations for tree_stall.h */ static void record_gp_stall_check_time(void); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c8ba0fe17267..4a53aa013f82 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -595,7 +595,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) * evaluate safety in terms of interrupt, softirq, and preemption * disabling. */ -static void rcu_preempt_deferred_qs(struct task_struct *t) +void rcu_preempt_deferred_qs(struct task_struct *t) { unsigned long flags; @@ -935,7 +935,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) // period for a quiescent state from this CPU. Note that requests from // tasks are handled when removing the task from the blocked-tasks list // below. -static void rcu_preempt_deferred_qs(struct task_struct *t) +void rcu_preempt_deferred_qs(struct task_struct *t) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); @@ -1290,37 +1290,3 @@ static void rcu_bind_gp_kthread(void) return; housekeeping_affine(current, HK_TYPE_RCU); } - -/* Record the current task on dyntick-idle entry. */ -static __always_inline void rcu_dynticks_task_enter(void) -{ -#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) - WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id()); -#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ -} - -/* Record no current task on dyntick-idle exit. */ -static __always_inline void rcu_dynticks_task_exit(void) -{ -#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) - WRITE_ONCE(current->rcu_tasks_idle_cpu, -1); -#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ -} - -/* Turn on heavyweight RCU tasks trace readers on idle/user entry. */ -static __always_inline void rcu_dynticks_task_trace_enter(void) -{ -#ifdef CONFIG_TASKS_TRACE_RCU - if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) - current->trc_reader_special.b.need_mb = true; -#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ -} - -/* Turn off heavyweight RCU tasks trace readers on idle/user exit. */ -static __always_inline void rcu_dynticks_task_trace_exit(void) -{ -#ifdef CONFIG_TASKS_TRACE_RCU - if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) - current->trc_reader_special.b.need_mb = false; -#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ -} -- cgit v1.2.3 From c33ef43a359001415032665dfcd433979c462b71 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:34 +0200 Subject: rcu/context-tracking: Remove unused and/or unecessary middle functions Some eqs functions are now only used internally by context tracking, so their public declarations can be removed. Also middle functions such as rcu_user_*() and rcu_idle_*() which now directly call to rcu_eqs_enter() and rcu_eqs_exit() can be wiped out as well. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- Documentation/RCU/stallwarn.rst | 2 +- include/linux/hardirq.h | 8 ---- include/linux/rcupdate.h | 8 ---- include/linux/rcutiny.h | 2 - include/linux/rcutree.h | 2 - kernel/context_tracking.c | 98 ++++++++++++----------------------------- 6 files changed, 28 insertions(+), 92 deletions(-) (limited to 'include/linux') diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index ce1f58a9d954..e38c587067fc 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -97,7 +97,7 @@ warnings: which will include additional debugging information. - A low-level kernel issue that either fails to invoke one of the - variants of rcu_user_enter(), rcu_user_exit(), ct_idle_enter(), + variants of rcu_eqs_enter(true), rcu_eqs_exit(true), ct_idle_enter(), ct_idle_exit(), ct_irq_enter(), or ct_irq_exit() on the one hand, or that invokes one of them too many times on the other. Historically, the most frequent issue has been an omission diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 345cdbe9c1b7..d57cab4d4c06 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -92,14 +92,6 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif -#ifdef CONFIG_TINY_RCU -static inline void rcu_nmi_enter(void) { } -static inline void rcu_nmi_exit(void) { } -#else -extern void rcu_nmi_enter(void); -extern void rcu_nmi_exit(void); -#endif - /* * NMI vs Tracing * -------------- diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 3717cad983a6..434da1eb88cd 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -104,14 +104,6 @@ static inline void rcu_sysrq_start(void) { } static inline void rcu_sysrq_end(void) { } #endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */ -#ifdef CONFIG_NO_HZ_FULL -void rcu_user_enter(void); -void rcu_user_exit(void); -#else -static inline void rcu_user_enter(void) { } -static inline void rcu_user_exit(void) { } -#endif /* CONFIG_NO_HZ_FULL */ - #if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) void rcu_irq_work_resched(void); #else diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 591119413cf1..900ba35c3582 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -76,8 +76,6 @@ static inline int rcu_needs_cpu(void) static inline void rcu_virt_note_context_switch(int cpu) { } static inline void rcu_cpu_stall_reset(void) { } static inline int rcu_jiffies_till_stall_check(void) { return 21 * HZ; } -static inline void rcu_idle_enter(void) { } -static inline void rcu_idle_exit(void) { } static inline void rcu_irq_exit_check_preempt(void) { } #define rcu_is_idle_cpu(cpu) \ (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq()) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 24db1e41695c..9cca00ed9bc9 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -45,8 +45,6 @@ unsigned long start_poll_synchronize_rcu(void); bool poll_state_synchronize_rcu(unsigned long oldstate); void cond_synchronize_rcu(unsigned long oldstate); -void rcu_idle_enter(void); -void rcu_idle_exit(void); bool rcu_is_idle_cpu(int cpu); #ifdef CONFIG_PROVE_RCU diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 072c4b6044b3..e485b6b01537 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -189,17 +189,17 @@ static void noinstr rcu_eqs_exit(bool user) } /** - * rcu_nmi_exit - inform RCU of exit from NMI context + * ct_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * - * If you add or remove a call to rcu_nmi_exit(), be sure to test + * If you add or remove a call to ct_nmi_exit(), be sure to test * with CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_nmi_exit(void) +void noinstr ct_nmi_exit(void) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); @@ -242,7 +242,7 @@ void noinstr rcu_nmi_exit(void) } /** - * rcu_nmi_enter - inform RCU of entry to NMI context + * ct_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle from RCU's viewpoint, update ct->dynticks and * ct->dynticks_nmi_nesting to let the RCU grace-period handling know @@ -250,10 +250,10 @@ void noinstr rcu_nmi_exit(void) * long as the nesting level does not overflow an int. (You will probably * run out of stack space first.) * - * If you add or remove a call to rcu_nmi_enter(), be sure to test + * If you add or remove a call to ct_nmi_enter(), be sure to test * with CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_nmi_enter(void) +void noinstr ct_nmi_enter(void) { long incby = 2; struct context_tracking *ct = this_cpu_ptr(&context_tracking); @@ -302,32 +302,33 @@ void noinstr rcu_nmi_enter(void) } /** - * rcu_idle_enter - inform RCU that current CPU is entering idle + * ct_idle_enter - inform RCU that current CPU is entering idle * * Enter idle mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in idle, a possibility * handled by irq_enter() and irq_exit().) * - * If you add or remove a call to rcu_idle_enter(), be sure to test with + * If you add or remove a call to ct_idle_enter(), be sure to test with * CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_idle_enter(void) +void noinstr ct_idle_enter(void) { WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rcu_eqs_enter(false); } +EXPORT_SYMBOL_GPL(ct_idle_enter); /** - * rcu_idle_exit - inform RCU that current CPU is leaving idle + * ct_idle_exit - inform RCU that current CPU is leaving idle * * Exit idle mode, in other words, -enter- the mode in which RCU * read-side critical sections can occur. * - * If you add or remove a call to rcu_idle_exit(), be sure to test with + * If you add or remove a call to ct_idle_exit(), be sure to test with * CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_idle_exit(void) +void noinstr ct_idle_exit(void) { unsigned long flags; @@ -335,18 +336,6 @@ void noinstr rcu_idle_exit(void) rcu_eqs_exit(false); raw_local_irq_restore(flags); } -EXPORT_SYMBOL_GPL(rcu_idle_exit); - -noinstr void ct_idle_enter(void) -{ - rcu_idle_enter(); -} -EXPORT_SYMBOL_GPL(ct_idle_enter); - -void ct_idle_exit(void) -{ - rcu_idle_exit(); -} EXPORT_SYMBOL_GPL(ct_idle_exit); /** @@ -431,50 +420,11 @@ void ct_irq_exit_irqson(void) ct_irq_exit(); local_irq_restore(flags); } - -noinstr void ct_nmi_enter(void) -{ - rcu_nmi_enter(); -} - -noinstr void ct_nmi_exit(void) -{ - rcu_nmi_exit(); -} +#else +static __always_inline void rcu_eqs_enter(bool user) { } +static __always_inline void rcu_eqs_exit(bool user) { } #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ -#ifdef CONFIG_NO_HZ_FULL -/** - * rcu_user_enter - inform RCU that we are resuming userspace. - * - * Enter RCU idle mode right before resuming userspace. No use of RCU - * is permitted between this call and rcu_user_exit(). This way the - * CPU doesn't need to maintain the tick for RCU maintenance purposes - * when the CPU runs in userspace. - * - * If you add or remove a call to rcu_user_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_user_enter(void) -{ - rcu_eqs_enter(true); -} - -/** - * rcu_user_exit - inform RCU that we are exiting userspace. - * - * Exit RCU idle mode while entering the kernel because it can - * run a RCU read side critical section anytime. - * - * If you add or remove a call to rcu_user_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_user_exit(void) -{ - rcu_eqs_exit(true); -} -#endif /* #ifdef CONFIG_NO_HZ_FULL */ - #ifdef CONFIG_CONTEXT_TRACKING_USER #define CREATE_TRACE_POINTS @@ -542,7 +492,13 @@ void noinstr __ct_user_enter(enum ctx_state state) * that will fire and reschedule once we resume in user/guest mode. */ rcu_irq_work_resched(); - rcu_user_enter(); + /* + * Enter RCU idle mode right before resuming userspace. No use of RCU + * is permitted between this call and rcu_eqs_exit(). This way the + * CPU doesn't need to maintain the tick for RCU maintenance purposes + * when the CPU runs in userspace. + */ + rcu_eqs_enter(true); } /* * Even if context tracking is disabled on this CPU, because it's outside @@ -579,7 +535,7 @@ void ct_user_enter(enum ctx_state state) /* * Some contexts may involve an exception occuring in an irq, * leading to that nesting: - * ct_irq_enter() rcu_user_exit() rcu_user_exit() ct_irq_exit() + * ct_irq_enter() rcu_eqs_exit(true) rcu_eqs_enter(true) ct_irq_exit() * This would mess up the dyntick_nesting count though. And rcu_irq_*() * helpers are enough to protect RCU uses inside the exception. So * just return immediately if we detect we are in an IRQ. @@ -631,10 +587,10 @@ void noinstr __ct_user_exit(enum ctx_state state) if (__this_cpu_read(context_tracking.state) == state) { if (__this_cpu_read(context_tracking.active)) { /* - * We are going to run code that may use RCU. Inform - * RCU core about that (ie: we may need the tick again). + * Exit RCU idle mode while entering the kernel because it can + * run a RCU read side critical section anytime. */ - rcu_user_exit(); + rcu_eqs_exit(true); if (state == CONTEXT_USER) { instrumentation_begin(); vtime_user_exit(current); -- cgit v1.2.3 From 171476775d32a40bfebf83250136c19b2e842672 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:35 +0200 Subject: context_tracking: Convert state to atomic_t Context tracking's state and dynticks counter are going to be merged in a single field so that both updates can happen atomically and at the same time. Prepare for that with converting the state into an atomic_t. [ paulmck: Apply kernel test robot feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 32 +++----- include/linux/context_tracking_state.h | 57 ++++++++++--- kernel/context_tracking.c | 143 ++++++++++++++++++++++----------- kernel/rcu/tree.c | 13 ++- kernel/rcu/tree_stall.h | 4 +- 5 files changed, 158 insertions(+), 91 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 1f568676bc1d..dcef4a9e4d63 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -56,7 +56,7 @@ static inline enum ctx_state exception_enter(void) !context_tracking_enabled()) return 0; - prev_ctx = this_cpu_read(context_tracking.state); + prev_ctx = __ct_state(); if (prev_ctx != CONTEXT_KERNEL) ct_user_exit(prev_ctx); @@ -86,33 +86,21 @@ static __always_inline void context_tracking_guest_exit(void) __ct_user_exit(CONTEXT_GUEST); } -/** - * ct_state() - return the current context tracking state if known - * - * Returns the current cpu's context tracking state if context tracking - * is enabled. If context tracking is disabled, returns - * CONTEXT_DISABLED. This should be used primarily for debugging. - */ -static __always_inline enum ctx_state ct_state(void) -{ - return context_tracking_enabled() ? - this_cpu_read(context_tracking.state) : CONTEXT_DISABLED; -} +#define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) + #else static inline void user_enter(void) { } static inline void user_exit(void) { } static inline void user_enter_irqoff(void) { } static inline void user_exit_irqoff(void) { } -static inline enum ctx_state exception_enter(void) { return 0; } +static inline int exception_enter(void) { return 0; } static inline void exception_exit(enum ctx_state prev_ctx) { } -static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; } +static inline int ct_state(void) { return -1; } static __always_inline bool context_tracking_guest_enter(void) { return false; } static inline void context_tracking_guest_exit(void) { } - +#define CT_WARN_ON(cond) do { } while (0) #endif /* !CONFIG_CONTEXT_TRACKING_USER */ -#define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) - #ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE extern void context_tracking_init(void); #else @@ -130,16 +118,16 @@ extern void ct_idle_exit(void); */ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) { - return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); + return !(arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX); } /* - * Increment the current CPU's context_tracking structure's ->dynticks field + * Increment the current CPU's context_tracking structure's ->state field * with ordering. Return the new value. */ -static __always_inline unsigned long rcu_dynticks_inc(int incby) +static __always_inline unsigned long ct_state_inc(int incby) { - return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); + return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state)); } #else diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 5f11e3d2d85a..e20a74bc0597 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -6,15 +6,23 @@ #include #include +/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ +#define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) + enum ctx_state { - CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ - CONTEXT_KERNEL = 0, - CONTEXT_USER, - CONTEXT_GUEST, + CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ + CONTEXT_KERNEL = 0, + CONTEXT_IDLE = 1, + CONTEXT_USER = 2, + CONTEXT_GUEST = 3, + CONTEXT_MAX = 4, }; -/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ -#define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) +/* Even value for idle, else odd. */ +#define RCU_DYNTICKS_IDX CONTEXT_MAX + +#define CT_STATE_MASK (CONTEXT_MAX - 1) +#define CT_DYNTICKS_MASK (~CT_STATE_MASK) struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING_USER @@ -26,10 +34,11 @@ struct context_tracking { */ bool active; int recursion; - enum ctx_state state; +#endif +#ifdef CONFIG_CONTEXT_TRACKING + atomic_t state; #endif #ifdef CONFIG_CONTEXT_TRACKING_IDLE - atomic_t dynticks; /* Even value for idle, else odd. */ long dynticks_nesting; /* Track process nesting level. */ long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ #endif @@ -37,26 +46,31 @@ struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING DECLARE_PER_CPU(struct context_tracking, context_tracking); + +static __always_inline int __ct_state(void) +{ + return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; +} #endif #ifdef CONFIG_CONTEXT_TRACKING_IDLE static __always_inline int ct_dynticks(void) { - return atomic_read(this_cpu_ptr(&context_tracking.dynticks)); + return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_DYNTICKS_MASK; } static __always_inline int ct_dynticks_cpu(int cpu) { struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); - return atomic_read(&ct->dynticks); + return atomic_read(&ct->state) & CT_DYNTICKS_MASK; } static __always_inline int ct_dynticks_cpu_acquire(int cpu) { struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); - return atomic_read_acquire(&ct->dynticks); + return atomic_read_acquire(&ct->state) & CT_DYNTICKS_MASK; } static __always_inline long ct_dynticks_nesting(void) @@ -102,6 +116,27 @@ static inline bool context_tracking_enabled_this_cpu(void) return context_tracking_enabled() && __this_cpu_read(context_tracking.active); } +/** + * ct_state() - return the current context tracking state if known + * + * Returns the current cpu's context tracking state if context tracking + * is enabled. If context tracking is disabled, returns + * CONTEXT_DISABLED. This should be used primarily for debugging. + */ +static __always_inline int ct_state(void) +{ + int ret; + + if (!context_tracking_enabled()) + return CONTEXT_DISABLED; + + preempt_disable(); + ret = __ct_state(); + preempt_enable(); + + return ret; +} + #else static __always_inline bool context_tracking_enabled(void) { return false; } static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index e485b6b01537..ca78ff27dc53 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -28,8 +28,8 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { #ifdef CONFIG_CONTEXT_TRACKING_IDLE .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, - .dynticks = ATOMIC_INIT(1), #endif + .state = ATOMIC_INIT(RCU_DYNTICKS_IDX), }; EXPORT_SYMBOL_GPL(context_tracking); @@ -76,7 +76,7 @@ static __always_inline void rcu_dynticks_task_trace_exit(void) * RCU is watching prior to the call to this function and is no longer * watching upon return. */ -static noinstr void rcu_dynticks_eqs_enter(void) +static noinstr void ct_kernel_exit_state(int offset) { int seq; @@ -86,9 +86,9 @@ static noinstr void rcu_dynticks_eqs_enter(void) * next idle sojourn. */ rcu_dynticks_task_trace_enter(); // Before ->dynticks update! - seq = rcu_dynticks_inc(1); + seq = ct_state_inc(offset); // RCU is no longer watching. Better be in extended quiescent state! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & 0x1)); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & RCU_DYNTICKS_IDX)); } /* @@ -96,7 +96,7 @@ static noinstr void rcu_dynticks_eqs_enter(void) * called from an extended quiescent state, that is, RCU is not watching * prior to the call to this function and is watching upon return. */ -static noinstr void rcu_dynticks_eqs_exit(void) +static noinstr void ct_kernel_enter_state(int offset) { int seq; @@ -105,10 +105,10 @@ static noinstr void rcu_dynticks_eqs_exit(void) * and we also must force ordering with the next RCU read-side * critical section. */ - seq = rcu_dynticks_inc(1); + seq = ct_state_inc(offset); // RCU is now watching. Better not be in an extended quiescent state! rcu_dynticks_task_trace_exit(); // After ->dynticks update! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & 0x1)); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & RCU_DYNTICKS_IDX)); } /* @@ -119,7 +119,7 @@ static noinstr void rcu_dynticks_eqs_exit(void) * the possibility of usermode upcalls having messed up our count * of interrupt nesting level during the prior busy period. */ -static void noinstr rcu_eqs_enter(bool user) +static void noinstr ct_kernel_exit(bool user, int offset) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); @@ -139,13 +139,13 @@ static void noinstr rcu_eqs_enter(bool user) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rcu_preempt_deferred_qs(current); - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr ct_kernel_exit_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); instrumentation_end(); WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ // RCU is watching here ... - rcu_dynticks_eqs_enter(); + ct_kernel_exit_state(offset); // ... but is no longer watching here. rcu_dynticks_task_enter(); } @@ -158,7 +158,7 @@ static void noinstr rcu_eqs_enter(bool user) * allow for the possibility of usermode upcalls messing up our count of * interrupt nesting level during the busy period that is just now starting. */ -static void noinstr rcu_eqs_exit(bool user) +static void noinstr ct_kernel_enter(bool user, int offset) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); long oldval; @@ -173,12 +173,12 @@ static void noinstr rcu_eqs_exit(bool user) } rcu_dynticks_task_exit(); // RCU is not watching here ... - rcu_dynticks_eqs_exit(); + ct_kernel_enter_state(offset); // ... but is watching here. instrumentation_begin(); - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr ct_kernel_enter_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); @@ -192,7 +192,7 @@ static void noinstr rcu_eqs_exit(bool user) * ct_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting + * RCU-idle period, update ct->state and ct->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * @@ -229,12 +229,12 @@ void noinstr ct_nmi_exit(void) trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr ct_kernel_exit_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); instrumentation_end(); // RCU is watching here ... - rcu_dynticks_eqs_enter(); + ct_kernel_exit_state(RCU_DYNTICKS_IDX); // ... but is no longer watching here. if (!in_nmi()) @@ -244,7 +244,7 @@ void noinstr ct_nmi_exit(void) /** * ct_nmi_enter - inform RCU of entry to NMI context * - * If the CPU was idle from RCU's viewpoint, update ct->dynticks and + * If the CPU was idle from RCU's viewpoint, update ct->state and * ct->dynticks_nmi_nesting to let the RCU grace-period handling know * that the CPU is active. This implementation permits nested NMIs, as * long as the nesting level does not overflow an int. (You will probably @@ -275,14 +275,14 @@ void noinstr ct_nmi_enter(void) rcu_dynticks_task_exit(); // RCU is not watching here ... - rcu_dynticks_eqs_exit(); + ct_kernel_enter_state(RCU_DYNTICKS_IDX); // ... but is watching here. instrumentation_begin(); // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() - instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + instrument_atomic_read(&ct->state, sizeof(ct->state)); + // instrumentation for the noinstr ct_kernel_enter_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); incby = 1; } else if (!in_nmi()) { @@ -315,7 +315,7 @@ void noinstr ct_nmi_enter(void) void noinstr ct_idle_enter(void) { WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - rcu_eqs_enter(false); + ct_kernel_exit(false, RCU_DYNTICKS_IDX + CONTEXT_IDLE); } EXPORT_SYMBOL_GPL(ct_idle_enter); @@ -333,7 +333,7 @@ void noinstr ct_idle_exit(void) unsigned long flags; raw_local_irq_save(flags); - rcu_eqs_exit(false); + ct_kernel_enter(false, RCU_DYNTICKS_IDX - CONTEXT_IDLE); raw_local_irq_restore(flags); } EXPORT_SYMBOL_GPL(ct_idle_exit); @@ -421,8 +421,8 @@ void ct_irq_exit_irqson(void) local_irq_restore(flags); } #else -static __always_inline void rcu_eqs_enter(bool user) { } -static __always_inline void rcu_eqs_exit(bool user) { } +static __always_inline void ct_kernel_exit(bool user, int offset) { } +static __always_inline void ct_kernel_enter(bool user, int offset) { } #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER @@ -463,6 +463,7 @@ static __always_inline void context_tracking_recursion_exit(void) */ void noinstr __ct_user_enter(enum ctx_state state) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); lockdep_assert_irqs_disabled(); /* Kernel threads aren't supposed to go to userspace */ @@ -471,8 +472,8 @@ void noinstr __ct_user_enter(enum ctx_state state) if (!context_tracking_recursion_enter()) return; - if ( __this_cpu_read(context_tracking.state) != state) { - if (__this_cpu_read(context_tracking.active)) { + if (__ct_state() != state) { + if (ct->active) { /* * At this stage, only low level arch entry code remains and * then we'll run in userspace. We can assume there won't be @@ -492,28 +493,49 @@ void noinstr __ct_user_enter(enum ctx_state state) * that will fire and reschedule once we resume in user/guest mode. */ rcu_irq_work_resched(); + /* * Enter RCU idle mode right before resuming userspace. No use of RCU * is permitted between this call and rcu_eqs_exit(). This way the * CPU doesn't need to maintain the tick for RCU maintenance purposes * when the CPU runs in userspace. */ - rcu_eqs_enter(true); + ct_kernel_exit(true, RCU_DYNTICKS_IDX + state); + + /* + * Special case if we only track user <-> kernel transitions for tickless + * cputime accounting but we don't support RCU extended quiescent state. + * In this we case we don't care about any concurrency/ordering. + */ + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) + atomic_set(&ct->state, state); + } else { + /* + * Even if context tracking is disabled on this CPU, because it's outside + * the full dynticks mask for example, we still have to keep track of the + * context transitions and states to prevent inconsistency on those of + * other CPUs. + * If a task triggers an exception in userspace, sleep on the exception + * handler and then migrate to another CPU, that new CPU must know where + * the exception returns by the time we call exception_exit(). + * This information can only be provided by the previous CPU when it called + * exception_enter(). + * OTOH we can spare the calls to vtime and RCU when context_tracking.active + * is false because we know that CPU is not tickless. + */ + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) { + /* Tracking for vtime only, no concurrent RCU EQS accounting */ + atomic_set(&ct->state, state); + } else { + /* + * Tracking for vtime and RCU EQS. Make sure we don't race + * with NMIs. OTOH we don't care about ordering here since + * RCU only requires RCU_DYNTICKS_IDX increments to be fully + * ordered. + */ + atomic_add(state, &ct->state); + } } - /* - * Even if context tracking is disabled on this CPU, because it's outside - * the full dynticks mask for example, we still have to keep track of the - * context transitions and states to prevent inconsistency on those of - * other CPUs. - * If a task triggers an exception in userspace, sleep on the exception - * handler and then migrate to another CPU, that new CPU must know where - * the exception returns by the time we call exception_exit(). - * This information can only be provided by the previous CPU when it called - * exception_enter(). - * OTOH we can spare the calls to vtime and RCU when context_tracking.active - * is false because we know that CPU is not tickless. - */ - __this_cpu_write(context_tracking.state, state); } context_tracking_recursion_exit(); } @@ -581,24 +603,47 @@ NOKPROBE_SYMBOL(user_enter_callable); */ void noinstr __ct_user_exit(enum ctx_state state) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + if (!context_tracking_recursion_enter()) return; - if (__this_cpu_read(context_tracking.state) == state) { - if (__this_cpu_read(context_tracking.active)) { + if (__ct_state() == state) { + if (ct->active) { /* * Exit RCU idle mode while entering the kernel because it can * run a RCU read side critical section anytime. */ - rcu_eqs_exit(true); + ct_kernel_enter(true, RCU_DYNTICKS_IDX - state); if (state == CONTEXT_USER) { instrumentation_begin(); vtime_user_exit(current); trace_user_exit(0); instrumentation_end(); } + + /* + * Special case if we only track user <-> kernel transitions for tickless + * cputime accounting but we don't support RCU extended quiescent state. + * In this we case we don't care about any concurrency/ordering. + */ + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) + atomic_set(&ct->state, CONTEXT_KERNEL); + + } else { + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) { + /* Tracking for vtime only, no concurrent RCU EQS accounting */ + atomic_set(&ct->state, CONTEXT_KERNEL); + } else { + /* + * Tracking for vtime and RCU EQS. Make sure we don't race + * with NMIs. OTOH we don't care about ordering here since + * RCU only requires RCU_DYNTICKS_IDX increments to be fully + * ordered. + */ + atomic_sub(state, &ct->state); + } } - __this_cpu_write(context_tracking.state, CONTEXT_KERNEL); } context_tracking_recursion_exit(); } diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e2a2083079a2..f9d20b40071f 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -272,9 +272,9 @@ void rcu_softirq_qs(void) */ static void rcu_dynticks_eqs_online(void) { - if (ct_dynticks() & 0x1) + if (ct_dynticks() & RCU_DYNTICKS_IDX) return; - rcu_dynticks_inc(1); + ct_state_inc(RCU_DYNTICKS_IDX); } /* @@ -293,7 +293,7 @@ static int rcu_dynticks_snap(int cpu) */ static bool rcu_dynticks_in_eqs(int snap) { - return !(snap & 0x1); + return !(snap & RCU_DYNTICKS_IDX); } /* Return true if the specified CPU is currently idle from an RCU viewpoint. */ @@ -321,8 +321,7 @@ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) int snap; // If not quiescent, force back to earlier extended quiescent state. - snap = ct_dynticks_cpu(cpu) & ~0x1; - + snap = ct_dynticks_cpu(cpu) & ~RCU_DYNTICKS_IDX; smp_rmb(); // Order ->dynticks and *vp reads. if (READ_ONCE(*vp)) return false; // Non-zero, so report failure; @@ -348,9 +347,9 @@ notrace void rcu_momentary_dyntick_idle(void) int seq; raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); - seq = rcu_dynticks_inc(2); + seq = ct_state_inc(2 * RCU_DYNTICKS_IDX); /* It is illegal to call this from idle state. */ - WARN_ON_ONCE(!(seq & 0x1)); + WARN_ON_ONCE(!(seq & RCU_DYNTICKS_IDX)); rcu_preempt_deferred_qs(current); } EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 2683ce0a7c72..195cad14742d 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -469,7 +469,7 @@ static void print_cpu_stall_info(int cpu) rcuc_starved = rcu_is_rcuc_kthread_starving(rdp, &j); if (rcuc_starved) sprintf(buf, " rcuc=%ld jiffies(starved)", j); - pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld%s%s\n", + pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%04x/%ld/%#lx softirq=%u/%u fqs=%ld%s%s\n", cpu, "O."[!!cpu_online(cpu)], "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)], @@ -478,7 +478,7 @@ static void print_cpu_stall_info(int cpu) rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' : "!."[!delta], ticks_value, ticks_title, - rcu_dynticks_snap(cpu) & 0xfff, + rcu_dynticks_snap(cpu) & 0xffff, ct_dynticks_nesting_cpu(cpu), ct_dynticks_nmi_nesting_cpu(cpu), rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, -- cgit v1.2.3 From 1dcaa3b462265f688613163a1562a65ee53a3311 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 16 Jun 2022 09:30:37 -0700 Subject: context_tracking: Use arch_atomic_read() in __ct_state for KASAN Context tracking's __ct_state() function can be invoked from noinstr state where RCU is not watching. This means that its use of atomic_read() causes KASAN to invoke the non-noinstr __kasan_check_read() function from the noinstr function __ct_state(). This is problematic because someone tracing the __kasan_check_read() function could get a nasty surprise because of RCU not watching. This commit therefore replaces the __ct_state() function's use of atomic_read() with arch_atomic_read(), which KASAN does not attempt to add instrumention to. Reported-by: kernel test robot Signed-off-by: Paul E. McKenney Cc: Frederic Weisbecker Cc: Marco Elver Reviewed-by: Marco Elver --- include/linux/context_tracking_state.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index e20a74bc0597..4a4d56f77180 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -49,7 +49,7 @@ DECLARE_PER_CPU(struct context_tracking, context_tracking); static __always_inline int __ct_state(void) { - return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; + return arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; } #endif -- cgit v1.2.3 From 6a4e9307cd3782f8e805fac970b9a240ab3078d6 Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sat, 21 May 2022 13:11:12 +0200 Subject: dmaengine: qcom: fix typo in comment Spelling mistake (triple letters) in comment. Detected with the help of Coccinelle. Signed-off-by: Julia Lawall Link: https://lore.kernel.org/r/20220521111145.81697-62-Julia.Lawall@inria.fr Signed-off-by: Vinod Koul --- include/linux/dma/qcom-gpi-dma.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/dma/qcom-gpi-dma.h b/include/linux/dma/qcom-gpi-dma.h index f46dc3372f11..6680dd1a43c6 100644 --- a/include/linux/dma/qcom-gpi-dma.h +++ b/include/linux/dma/qcom-gpi-dma.h @@ -26,7 +26,7 @@ enum spi_transfer_cmd { * @clk_div: source clock divider * @clk_src: serial clock * @cmd: spi cmd - * @fragmentation: keep CS assserted at end of sequence + * @fragmentation: keep CS asserted at end of sequence * @cs: chip select toggle * @set_config: set peripheral config * @rx_len: receive length for buffer -- cgit v1.2.3 From bd29c00edd0a5dac8b6e7332bb470cd50f92e893 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Tue, 21 Jun 2022 17:56:38 -0500 Subject: soundwire: revisit driver bind/unbind and callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the SoundWire probe, we store a pointer from the driver ops into the 'slave' structure. This can lead to kernel oopses when unbinding codec drivers, e.g. with the following sequence to remove machine driver and codec driver. /sbin/modprobe -r snd_soc_sof_sdw /sbin/modprobe -r snd_soc_rt711 The full details can be found in the BugLink below, for reference the two following examples show different cases of driver ops/callbacks being invoked after the driver .remove(). kernel: BUG: kernel NULL pointer dereference, address: 0000000000000150 kernel: Workqueue: events cdns_update_slave_status_work [soundwire_cadence] kernel: RIP: 0010:mutex_lock+0x19/0x30 kernel: Call Trace: kernel: ? sdw_handle_slave_status+0x426/0xe00 [soundwire_bus 94ff184bf398570c3f8ff7efe9e32529f532e4ae] kernel: ? newidle_balance+0x26a/0x400 kernel: ? cdns_update_slave_status_work+0x1e9/0x200 [soundwire_cadence 1bcf98eebe5ba9833cd433323769ac923c9c6f82] kernel: BUG: unable to handle page fault for address: ffffffffc07654c8 kernel: Workqueue: pm pm_runtime_work kernel: RIP: 0010:sdw_bus_prep_clk_stop+0x6f/0x160 [soundwire_bus] kernel: Call Trace: kernel: kernel: sdw_cdns_clock_stop+0xb5/0x1b0 [soundwire_cadence 1bcf98eebe5ba9833cd433323769ac923c9c6f82] kernel: intel_suspend_runtime+0x5f/0x120 [soundwire_intel aca858f7c87048d3152a4a41bb68abb9b663a1dd] kernel: ? dpm_sysfs_remove+0x60/0x60 This was not detected earlier in Intel tests since the tests first remove the parent PCI device and shut down the bus. The sequence above is a corner case which keeps the bus operational but without a driver bound. While trying to solve this kernel oopses, it became clear that the existing SoundWire bus does not deal well with the unbind case. Commit 528be501b7d4a ("soundwire: sdw_slave: add probe_complete structure and new fields") added a 'probed' status variable and a 'probe_complete' struct completion. This status is however not reset on remove and likewise the 'probe complete' is not re-initialized, so the bind/unbind/bind test cases would fail. The timeout used before the 'update_status' callback was also a bad idea in hindsight, there should really be no timing assumption as to if and when a driver is bound to a device. An initial draft was based on device_lock() and device_unlock() was tested. This proved too complicated, with deadlocks created during the suspend-resume sequences, which also use the same device_lock/unlock() as the bind/unbind sequences. On a CometLake device, a bad DSDT/BIOS caused spurious resumes and the use of device_lock() caused hangs during suspend. After multiple weeks or testing and painful reverse-engineering of deadlocks on different devices, we looked for alternatives that did not interfere with the device core. A bus notifier was used successfully to keep track of DRIVER_BOUND and DRIVER_UNBIND events. This solved the bind-unbind-bind case in tests, but it can still be defeated with a theoretical corner case where the memory is freed by a .remove while the callback is in use. The notifier only helps make sure the driver callbacks are valid, but not that the memory allocated in probe remains valid while the callbacks are invoked. This patch suggests the introduction of a new 'sdw_dev_lock' mutex protecting probe/remove and all driver callbacks. Since this mutex is 'local' to SoundWire only, it does not interfere with existing locks and does not create deadlocks. In addition, this patch removes the 'probe_complete' completion, instead we directly invoke the 'update_status' from the probe routine. That removes any sort of timing dependency and a much better support for the device/driver model, the driver could be bound before the bus started, or eons after the bus started and the hardware would be properly initialized in all cases. BugLink: https://github.com/thesofproject/linux/issues/3531 Fixes: 56d4fe31af77 ("soundwire: Add MIPI DisCo property helpers") Fixes: 528be501b7d4a ("soundwire: sdw_slave: add probe_complete structure and new fields") Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Reviewed-by: Ranjani Sridharan Reviewed-by: Bard Liao Reviewed-by: Péter Ujfalusi Link: https://lore.kernel.org/r/20220621225641.221170-2-pierre-louis.bossart@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 75 ++++++++++++++++++++++++------------------- drivers/soundwire/bus_type.c | 30 ++++++++++++++--- drivers/soundwire/slave.c | 3 +- drivers/soundwire/stream.c | 53 +++++++++++++++++++----------- include/linux/soundwire/sdw.h | 6 ++-- 5 files changed, 106 insertions(+), 61 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index a2bfb0434a67..8d4000664fa3 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "bus.h" #include "sysfs_local.h" @@ -842,15 +843,21 @@ static int sdw_slave_clk_stop_callback(struct sdw_slave *slave, enum sdw_clk_stop_mode mode, enum sdw_clk_stop_type type) { - int ret; + int ret = 0; - if (slave->ops && slave->ops->clk_stop) { - ret = slave->ops->clk_stop(slave, mode, type); - if (ret < 0) - return ret; + mutex_lock(&slave->sdw_dev_lock); + + if (slave->probed) { + struct device *dev = &slave->dev; + struct sdw_driver *drv = drv_to_sdw_driver(dev->driver); + + if (drv->ops && drv->ops->clk_stop) + ret = drv->ops->clk_stop(slave, mode, type); } - return 0; + mutex_unlock(&slave->sdw_dev_lock); + + return ret; } static int sdw_slave_clk_stop_prepare(struct sdw_slave *slave, @@ -1611,14 +1618,24 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } /* Update the Slave driver */ - if (slave_notify && slave->ops && - slave->ops->interrupt_callback) { - slave_intr.sdca_cascade = sdca_cascade; - slave_intr.control_port = clear; - memcpy(slave_intr.port, &port_status, - sizeof(slave_intr.port)); - - slave->ops->interrupt_callback(slave, &slave_intr); + if (slave_notify) { + mutex_lock(&slave->sdw_dev_lock); + + if (slave->probed) { + struct device *dev = &slave->dev; + struct sdw_driver *drv = drv_to_sdw_driver(dev->driver); + + if (drv->ops && drv->ops->interrupt_callback) { + slave_intr.sdca_cascade = sdca_cascade; + slave_intr.control_port = clear; + memcpy(slave_intr.port, &port_status, + sizeof(slave_intr.port)); + + drv->ops->interrupt_callback(slave, &slave_intr); + } + } + + mutex_unlock(&slave->sdw_dev_lock); } /* Ack interrupt */ @@ -1692,29 +1709,21 @@ io_err: static int sdw_update_slave_status(struct sdw_slave *slave, enum sdw_slave_status status) { - unsigned long time; + int ret = 0; - if (!slave->probed) { - /* - * the slave status update is typically handled in an - * interrupt thread, which can race with the driver - * probe, e.g. when a module needs to be loaded. - * - * make sure the probe is complete before updating - * status. - */ - time = wait_for_completion_timeout(&slave->probe_complete, - msecs_to_jiffies(DEFAULT_PROBE_TIMEOUT)); - if (!time) { - dev_err(&slave->dev, "Probe not complete, timed out\n"); - return -ETIMEDOUT; - } + mutex_lock(&slave->sdw_dev_lock); + + if (slave->probed) { + struct device *dev = &slave->dev; + struct sdw_driver *drv = drv_to_sdw_driver(dev->driver); + + if (drv->ops && drv->ops->update_status) + ret = drv->ops->update_status(slave, status); } - if (!slave->ops || !slave->ops->update_status) - return 0; + mutex_unlock(&slave->sdw_dev_lock); - return slave->ops->update_status(slave, status); + return ret; } /** diff --git a/drivers/soundwire/bus_type.c b/drivers/soundwire/bus_type.c index b81e04dd3a9f..04b3529f8929 100644 --- a/drivers/soundwire/bus_type.c +++ b/drivers/soundwire/bus_type.c @@ -98,8 +98,6 @@ static int sdw_drv_probe(struct device *dev) if (!id) return -ENODEV; - slave->ops = drv->ops; - /* * attach to power domain but don't turn on (last arg) */ @@ -107,19 +105,23 @@ static int sdw_drv_probe(struct device *dev) if (ret) return ret; + mutex_lock(&slave->sdw_dev_lock); + ret = drv->probe(slave, id); if (ret) { name = drv->name; if (!name) name = drv->driver.name; + mutex_unlock(&slave->sdw_dev_lock); + dev_err(dev, "Probe of %s failed: %d\n", name, ret); dev_pm_domain_detach(dev, false); return ret; } /* device is probed so let's read the properties now */ - if (slave->ops && slave->ops->read_prop) - slave->ops->read_prop(slave); + if (drv->ops && drv->ops->read_prop) + drv->ops->read_prop(slave); /* init the sysfs as we have properties now */ ret = sdw_slave_sysfs_init(slave); @@ -139,7 +141,19 @@ static int sdw_drv_probe(struct device *dev) slave->prop.clk_stop_timeout); slave->probed = true; - complete(&slave->probe_complete); + + /* + * if the probe happened after the bus was started, notify the codec driver + * of the current hardware status to e.g. start the initialization. + * Errors are only logged as warnings to avoid failing the probe. + */ + if (drv->ops && drv->ops->update_status) { + ret = drv->ops->update_status(slave, slave->status); + if (ret < 0) + dev_warn(dev, "%s: update_status failed with status %d\n", __func__, ret); + } + + mutex_unlock(&slave->sdw_dev_lock); dev_dbg(dev, "probe complete\n"); @@ -152,9 +166,15 @@ static int sdw_drv_remove(struct device *dev) struct sdw_driver *drv = drv_to_sdw_driver(dev->driver); int ret = 0; + mutex_lock(&slave->sdw_dev_lock); + + slave->probed = false; + if (drv->remove) ret = drv->remove(slave); + mutex_unlock(&slave->sdw_dev_lock); + dev_pm_domain_detach(dev, false); return ret; diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c index 669d7573320b..25e76b5d4a1a 100644 --- a/drivers/soundwire/slave.c +++ b/drivers/soundwire/slave.c @@ -12,6 +12,7 @@ static void sdw_slave_release(struct device *dev) { struct sdw_slave *slave = dev_to_sdw_dev(dev); + mutex_destroy(&slave->sdw_dev_lock); kfree(slave); } @@ -58,9 +59,9 @@ int sdw_slave_add(struct sdw_bus *bus, init_completion(&slave->enumeration_complete); init_completion(&slave->initialization_complete); slave->dev_num = 0; - init_completion(&slave->probe_complete); slave->probed = false; slave->first_interrupt_done = false; + mutex_init(&slave->sdw_dev_lock); for (i = 0; i < SDW_MAX_PORTS; i++) init_completion(&slave->port_ready[i]); diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c index d34150559142..bd502368339e 100644 --- a/drivers/soundwire/stream.c +++ b/drivers/soundwire/stream.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "bus.h" @@ -401,20 +402,26 @@ static int sdw_do_port_prep(struct sdw_slave_runtime *s_rt, struct sdw_prepare_ch prep_ch, enum sdw_port_prep_ops cmd) { - const struct sdw_slave_ops *ops = s_rt->slave->ops; - int ret; + int ret = 0; + struct sdw_slave *slave = s_rt->slave; - if (ops->port_prep) { - ret = ops->port_prep(s_rt->slave, &prep_ch, cmd); - if (ret < 0) { - dev_err(&s_rt->slave->dev, - "Slave Port Prep cmd %d failed: %d\n", - cmd, ret); - return ret; + mutex_lock(&slave->sdw_dev_lock); + + if (slave->probed) { + struct device *dev = &slave->dev; + struct sdw_driver *drv = drv_to_sdw_driver(dev->driver); + + if (drv->ops && drv->ops->port_prep) { + ret = drv->ops->port_prep(slave, &prep_ch, cmd); + if (ret < 0) + dev_err(dev, "Slave Port Prep cmd %d failed: %d\n", + cmd, ret); } } - return 0; + mutex_unlock(&slave->sdw_dev_lock); + + return ret; } static int sdw_prep_deprep_slave_ports(struct sdw_bus *bus, @@ -578,7 +585,7 @@ static int sdw_notify_config(struct sdw_master_runtime *m_rt) struct sdw_slave_runtime *s_rt; struct sdw_bus *bus = m_rt->bus; struct sdw_slave *slave; - int ret = 0; + int ret; if (bus->ops->set_bus_conf) { ret = bus->ops->set_bus_conf(bus, &bus->params); @@ -589,17 +596,27 @@ static int sdw_notify_config(struct sdw_master_runtime *m_rt) list_for_each_entry(s_rt, &m_rt->slave_rt_list, m_rt_node) { slave = s_rt->slave; - if (slave->ops->bus_config) { - ret = slave->ops->bus_config(slave, &bus->params); - if (ret < 0) { - dev_err(bus->dev, "Notify Slave: %d failed\n", - slave->dev_num); - return ret; + mutex_lock(&slave->sdw_dev_lock); + + if (slave->probed) { + struct device *dev = &slave->dev; + struct sdw_driver *drv = drv_to_sdw_driver(dev->driver); + + if (drv->ops && drv->ops->bus_config) { + ret = drv->ops->bus_config(slave, &bus->params); + if (ret < 0) { + dev_err(dev, "Notify Slave: %d failed\n", + slave->dev_num); + mutex_unlock(&slave->sdw_dev_lock); + return ret; + } } } + + mutex_unlock(&slave->sdw_dev_lock); } - return ret; + return 0; } /** diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index 76ce3f3ac0f2..bf6f0decb3f6 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -646,9 +646,6 @@ struct sdw_slave_ops { * @dev_num: Current Device Number, values can be 0 or dev_num_sticky * @dev_num_sticky: one-time static Device Number assigned by Bus * @probed: boolean tracking driver state - * @probe_complete: completion utility to control potential races - * on startup between driver probe/initialization and SoundWire - * Slave state changes/implementation-defined interrupts * @enumeration_complete: completion utility to control potential races * on startup between device enumeration and read/write access to the * Slave device @@ -663,6 +660,7 @@ struct sdw_slave_ops { * for a Slave happens for the first time after enumeration * @is_mockup_device: status flag used to squelch errors in the command/control * protocol for SoundWire mockup devices + * @sdw_dev_lock: mutex used to protect callbacks/remove races */ struct sdw_slave { struct sdw_slave_id id; @@ -680,12 +678,12 @@ struct sdw_slave { u16 dev_num; u16 dev_num_sticky; bool probed; - struct completion probe_complete; struct completion enumeration_complete; struct completion initialization_complete; u32 unattach_request; bool first_interrupt_done; bool is_mockup_device; + struct mutex sdw_dev_lock; /* protect callbacks/remove races */ }; #define dev_to_sdw_dev(_dev) container_of(_dev, struct sdw_slave, dev) -- cgit v1.2.3 From 9a24bb35b0d81edb826f174d7752c2a54bc00abd Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Tue, 21 Jun 2022 17:56:39 -0500 Subject: soundwire: peripheral: remove useless ops pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we are using the ops structure directly from the driver, there are no users left of this ops pointer. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Reviewed-by: Ranjani Sridharan Reviewed-by: Bard Liao Reviewed-by: Péter Ujfalusi Link: https://lore.kernel.org/r/20220621225641.221170-3-pierre-louis.bossart@linux.intel.com Signed-off-by: Vinod Koul --- include/linux/soundwire/sdw.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index bf6f0decb3f6..39058c841469 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -637,7 +637,6 @@ struct sdw_slave_ops { * @dev: Linux device * @status: Status reported by the Slave * @bus: Bus handle - * @ops: Slave callback ops * @prop: Slave properties * @debugfs: Slave debugfs * @node: node for bus list @@ -667,7 +666,6 @@ struct sdw_slave { struct device dev; enum sdw_slave_status status; struct sdw_bus *bus; - const struct sdw_slave_ops *ops; struct sdw_slave_prop prop; #ifdef CONFIG_DEBUG_FS struct dentry *debugfs; -- cgit v1.2.3 From 3b7e2482f9a3f2e99628048b945c9c6cc53bd38e Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Wed, 15 Jun 2022 11:10:36 +0100 Subject: iommu: Introduce a callback to struct iommu_resv_region MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A callback is introduced to struct iommu_resv_region to free memory allocations associated with the reserved region. This will be useful when we introduce support for IORT RMR based reserved regions. Reviewed-by: Christoph Hellwig Tested-by: Steven Price Tested-by: Laurentiu Tudor Tested-by: Hanjun Guo Signed-off-by: Shameer Kolothum Acked-by: Robin Murphy Link: https://lore.kernel.org/r/20220615101044.1972-2-shameerali.kolothum.thodi@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 16 +++++++++++----- include/linux/iommu.h | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index cdc86c39954e..f46431ac49e1 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2590,16 +2590,22 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) * @list: reserved region list for device * * IOMMU drivers can use this to implement their .put_resv_regions() callback - * for simple reservations. Memory allocated for each reserved region will be - * freed. If an IOMMU driver allocates additional resources per region, it is - * going to have to implement a custom callback. + * for simple reservations. If a per region callback is provided that will be + * used to free all memory allocations associated with the reserved region or + * else just free up the memory for the regions. If an IOMMU driver allocates + * additional resources per region, it is going to have to implement a custom + * callback. */ void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list) { struct iommu_resv_region *entry, *next; - list_for_each_entry_safe(entry, next, list, list) - kfree(entry); + list_for_each_entry_safe(entry, next, list, list) { + if (entry->free) + entry->free(dev, entry); + else + kfree(entry); + } } EXPORT_SYMBOL(generic_iommu_put_resv_regions); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 5e1afe169549..b22ffa6bc4a9 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -135,6 +135,7 @@ enum iommu_resv_type { * @length: Length of the region in bytes * @prot: IOMMU Protection flags (READ/WRITE/...) * @type: Type of the reserved region + * @free: Callback to free associated memory allocations */ struct iommu_resv_region { struct list_head list; @@ -142,6 +143,7 @@ struct iommu_resv_region { size_t length; int prot; enum iommu_resv_type type; + void (*free)(struct device *dev, struct iommu_resv_region *region); }; /** -- cgit v1.2.3 From 8778b1d48117055c710a01498f65fa730160fdfa Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Wed, 15 Jun 2022 11:10:37 +0100 Subject: ACPI/IORT: Make iort_iommu_msi_get_resv_regions() return void At present iort_iommu_msi_get_resv_regions() returns the number of MSI reserved regions on success and there are no users for this. The reserved region list will get populated anyway for platforms that require the HW MSI region reservation. Hence, change the function to return void instead. Reviewed-by: Christoph Hellwig Tested-by: Steven Price Tested-by: Laurentiu Tudor Reviewed-by: Hanjun Guo Signed-off-by: Shameer Kolothum Acked-by: Robin Murphy Link: https://lore.kernel.org/r/20220615101044.1972-3-shameerali.kolothum.thodi@huawei.com Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 25 +++++++++---------------- include/linux/acpi_iort.h | 6 +++--- 2 files changed, 12 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index f2f8f05662de..213f61cae176 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -811,22 +811,19 @@ static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev) * @dev: Device from iommu_get_resv_regions() * @head: Reserved region list from iommu_get_resv_regions() * - * Returns: Number of msi reserved regions on success (0 if platform - * doesn't require the reservation or no associated msi regions), - * appropriate error value otherwise. The ITS interrupt translation - * spaces (ITS_base + SZ_64K, SZ_64K) associated with the device - * are the msi reserved regions. + * The ITS interrupt translation spaces (ITS_base + SZ_64K, SZ_64K) + * associated with the device are the HW MSI reserved regions. */ -int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct acpi_iort_its_group *its; struct acpi_iort_node *iommu_node, *its_node = NULL; - int i, resv = 0; + int i; iommu_node = iort_get_msi_resv_iommu(dev); if (!iommu_node) - return 0; + return; /* * Current logic to reserve ITS regions relies on HW topologies @@ -846,7 +843,7 @@ int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) } if (!its_node) - return 0; + return; /* Move to ITS specific data */ its = (struct acpi_iort_its_group *)its_node->node_data; @@ -860,14 +857,10 @@ int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) region = iommu_alloc_resv_region(base + SZ_64K, SZ_64K, prot, IOMMU_RESV_MSI); - if (region) { + if (region) list_add_tail(®ion->list, head); - resv++; - } } } - - return (resv == its->its_count) ? resv : -ENODEV; } static inline bool iort_iommu_driver_enabled(u8 type) @@ -1034,8 +1027,8 @@ int iort_iommu_configure_id(struct device *dev, const u32 *id_in) } #else -int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) -{ return 0; } +void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +{ } int iort_iommu_configure_id(struct device *dev, const u32 *input_id) { return -ENODEV; } #endif diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index f1f0842a2cb2..a8198b83753d 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -36,7 +36,7 @@ int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id); /* IOMMU interface */ int iort_dma_get_ranges(struct device *dev, u64 *size); int iort_iommu_configure_id(struct device *dev, const u32 *id_in); -int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head); +void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head); phys_addr_t acpi_iort_dma_get_max_cpu_address(void); #else static inline void acpi_iort_init(void) { } @@ -52,8 +52,8 @@ static inline int iort_dma_get_ranges(struct device *dev, u64 *size) static inline int iort_iommu_configure_id(struct device *dev, const u32 *id_in) { return -ENODEV; } static inline -int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) -{ return 0; } +void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +{ } static inline phys_addr_t acpi_iort_dma_get_max_cpu_address(void) { return PHYS_ADDR_MAX; } -- cgit v1.2.3 From 55be25b8b5e4e2fd680cfb073b84a74a79c002fa Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Wed, 15 Jun 2022 11:10:38 +0100 Subject: ACPI/IORT: Provide a generic helper to retrieve reserve regions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently IORT provides a helper to retrieve HW MSI reserve regions. Change this to a generic helper to retrieve any IORT related reserve regions. This will be useful when we add support for RMR nodes in subsequent patches. [Lorenzo: For ACPI IORT] Reviewed-by: Lorenzo Pieralisi Reviewed-by: Christoph Hellwig Tested-by: Steven Price Tested-by: Laurentiu Tudor Tested-by: Hanjun Guo Reviewed-by: Hanjun Guo Signed-off-by: Shameer Kolothum Acked-by: Robin Murphy Link: https://lore.kernel.org/r/20220615101044.1972-4-shameerali.kolothum.thodi@huawei.com Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 22 +++++++++++++++------- drivers/iommu/dma-iommu.c | 2 +- include/linux/acpi_iort.h | 4 ++-- 3 files changed, 18 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 213f61cae176..cd5d1d7823cb 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -806,15 +806,13 @@ static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev) return NULL; } -/** - * iort_iommu_msi_get_resv_regions - Reserved region driver helper - * @dev: Device from iommu_get_resv_regions() - * @head: Reserved region list from iommu_get_resv_regions() - * +/* + * Retrieve platform specific HW MSI reserve regions. * The ITS interrupt translation spaces (ITS_base + SZ_64K, SZ_64K) * associated with the device are the HW MSI reserved regions. */ -void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +static void iort_iommu_msi_get_resv_regions(struct device *dev, + struct list_head *head) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct acpi_iort_its_group *its; @@ -863,6 +861,16 @@ void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) } } +/** + * iort_iommu_get_resv_regions - Generic helper to retrieve reserved regions. + * @dev: Device from iommu_get_resv_regions() + * @head: Reserved region list from iommu_get_resv_regions() + */ +void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head) +{ + iort_iommu_msi_get_resv_regions(dev, head); +} + static inline bool iort_iommu_driver_enabled(u8 type) { switch (type) { @@ -1027,7 +1035,7 @@ int iort_iommu_configure_id(struct device *dev, const u32 *id_in) } #else -void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head) { } int iort_iommu_configure_id(struct device *dev, const u32 *input_id) { return -ENODEV; } diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 1910f4f1612b..458fb6738223 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -387,7 +387,7 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) { if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode)) - iort_iommu_msi_get_resv_regions(dev, list); + iort_iommu_get_resv_regions(dev, list); } EXPORT_SYMBOL(iommu_dma_get_resv_regions); diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index a8198b83753d..e5d2de9caf7f 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -36,7 +36,7 @@ int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id); /* IOMMU interface */ int iort_dma_get_ranges(struct device *dev, u64 *size); int iort_iommu_configure_id(struct device *dev, const u32 *id_in); -void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head); +void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head); phys_addr_t acpi_iort_dma_get_max_cpu_address(void); #else static inline void acpi_iort_init(void) { } @@ -52,7 +52,7 @@ static inline int iort_dma_get_ranges(struct device *dev, u64 *size) static inline int iort_iommu_configure_id(struct device *dev, const u32 *id_in) { return -ENODEV; } static inline -void iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head) { } static inline phys_addr_t acpi_iort_dma_get_max_cpu_address(void) -- cgit v1.2.3 From 491cf4a6735a957cd0733365f8d17e0f4308f5a4 Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Wed, 15 Jun 2022 11:10:39 +0100 Subject: ACPI/IORT: Add support to retrieve IORT RMR reserved regions Parse through the IORT RMR nodes and populate the reserve region list corresponding to a given IOMMU and device(optional). Also, go through the ID mappings of the RMR node and retrieve all the SIDs associated with it. Reviewed-by: Lorenzo Pieralisi Tested-by: Steven Price Tested-by: Laurentiu Tudor Tested-by: Hanjun Guo Reviewed-by: Hanjun Guo Signed-off-by: Shameer Kolothum Acked-by: Robin Murphy Link: https://lore.kernel.org/r/20220615101044.1972-5-shameerali.kolothum.thodi@huawei.com Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 291 ++++++++++++++++++++++++++++++++++++++++++++++ include/linux/iommu.h | 8 ++ 2 files changed, 299 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index cd5d1d7823cb..b6273af316c6 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -788,6 +788,294 @@ void acpi_configure_pmsi_domain(struct device *dev) } #ifdef CONFIG_IOMMU_API +static void iort_rmr_free(struct device *dev, + struct iommu_resv_region *region) +{ + struct iommu_iort_rmr_data *rmr_data; + + rmr_data = container_of(region, struct iommu_iort_rmr_data, rr); + kfree(rmr_data->sids); + kfree(rmr_data); +} + +static struct iommu_iort_rmr_data *iort_rmr_alloc( + struct acpi_iort_rmr_desc *rmr_desc, + int prot, enum iommu_resv_type type, + u32 *sids, u32 num_sids) +{ + struct iommu_iort_rmr_data *rmr_data; + struct iommu_resv_region *region; + u32 *sids_copy; + u64 addr = rmr_desc->base_address, size = rmr_desc->length; + + rmr_data = kmalloc(sizeof(*rmr_data), GFP_KERNEL); + if (!rmr_data) + return NULL; + + /* Create a copy of SIDs array to associate with this rmr_data */ + sids_copy = kmemdup(sids, num_sids * sizeof(*sids), GFP_KERNEL); + if (!sids_copy) { + kfree(rmr_data); + return NULL; + } + rmr_data->sids = sids_copy; + rmr_data->num_sids = num_sids; + + if (!IS_ALIGNED(addr, SZ_64K) || !IS_ALIGNED(size, SZ_64K)) { + /* PAGE align base addr and size */ + addr &= PAGE_MASK; + size = PAGE_ALIGN(size + offset_in_page(rmr_desc->base_address)); + + pr_err(FW_BUG "RMR descriptor[0x%llx - 0x%llx] not aligned to 64K, continue with [0x%llx - 0x%llx]\n", + rmr_desc->base_address, + rmr_desc->base_address + rmr_desc->length - 1, + addr, addr + size - 1); + } + + region = &rmr_data->rr; + INIT_LIST_HEAD(®ion->list); + region->start = addr; + region->length = size; + region->prot = prot; + region->type = type; + region->free = iort_rmr_free; + + return rmr_data; +} + +static void iort_rmr_desc_check_overlap(struct acpi_iort_rmr_desc *desc, + u32 count) +{ + int i, j; + + for (i = 0; i < count; i++) { + u64 end, start = desc[i].base_address, length = desc[i].length; + + if (!length) { + pr_err(FW_BUG "RMR descriptor[0x%llx] with zero length, continue anyway\n", + start); + continue; + } + + end = start + length - 1; + + /* Check for address overlap */ + for (j = i + 1; j < count; j++) { + u64 e_start = desc[j].base_address; + u64 e_end = e_start + desc[j].length - 1; + + if (start <= e_end && end >= e_start) + pr_err(FW_BUG "RMR descriptor[0x%llx - 0x%llx] overlaps, continue anyway\n", + start, end); + } + } +} + +/* + * Please note, we will keep the already allocated RMR reserve + * regions in case of a memory allocation failure. + */ +static void iort_get_rmrs(struct acpi_iort_node *node, + struct acpi_iort_node *smmu, + u32 *sids, u32 num_sids, + struct list_head *head) +{ + struct acpi_iort_rmr *rmr = (struct acpi_iort_rmr *)node->node_data; + struct acpi_iort_rmr_desc *rmr_desc; + int i; + + rmr_desc = ACPI_ADD_PTR(struct acpi_iort_rmr_desc, node, + rmr->rmr_offset); + + iort_rmr_desc_check_overlap(rmr_desc, rmr->rmr_count); + + for (i = 0; i < rmr->rmr_count; i++, rmr_desc++) { + struct iommu_iort_rmr_data *rmr_data; + enum iommu_resv_type type; + int prot = IOMMU_READ | IOMMU_WRITE; + + if (rmr->flags & ACPI_IORT_RMR_REMAP_PERMITTED) + type = IOMMU_RESV_DIRECT_RELAXABLE; + else + type = IOMMU_RESV_DIRECT; + + if (rmr->flags & ACPI_IORT_RMR_ACCESS_PRIVILEGE) + prot |= IOMMU_PRIV; + + /* Attributes 0x00 - 0x03 represents device memory */ + if (ACPI_IORT_RMR_ACCESS_ATTRIBUTES(rmr->flags) <= + ACPI_IORT_RMR_ATTR_DEVICE_GRE) + prot |= IOMMU_MMIO; + else if (ACPI_IORT_RMR_ACCESS_ATTRIBUTES(rmr->flags) == + ACPI_IORT_RMR_ATTR_NORMAL_IWB_OWB) + prot |= IOMMU_CACHE; + + rmr_data = iort_rmr_alloc(rmr_desc, prot, type, + sids, num_sids); + if (!rmr_data) + return; + + list_add_tail(&rmr_data->rr.list, head); + } +} + +static u32 *iort_rmr_alloc_sids(u32 *sids, u32 count, u32 id_start, + u32 new_count) +{ + u32 *new_sids; + u32 total_count = count + new_count; + int i; + + new_sids = krealloc_array(sids, count + new_count, + sizeof(*new_sids), GFP_KERNEL); + if (!new_sids) + return NULL; + + for (i = count; i < total_count; i++) + new_sids[i] = id_start++; + + return new_sids; +} + +static bool iort_rmr_has_dev(struct device *dev, u32 id_start, + u32 id_count) +{ + int i; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + + /* + * Make sure the kernel has preserved the boot firmware PCIe + * configuration. This is required to ensure that the RMR PCIe + * StreamIDs are still valid (Refer: ARM DEN 0049E.d Section 3.1.1.5). + */ + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus); + + if (!host->preserve_config) + return false; + } + + for (i = 0; i < fwspec->num_ids; i++) { + if (fwspec->ids[i] >= id_start && + fwspec->ids[i] <= id_start + id_count) + return true; + } + + return false; +} + +static void iort_node_get_rmr_info(struct acpi_iort_node *node, + struct acpi_iort_node *iommu, + struct device *dev, struct list_head *head) +{ + struct acpi_iort_node *smmu = NULL; + struct acpi_iort_rmr *rmr; + struct acpi_iort_id_mapping *map; + u32 *sids = NULL; + u32 num_sids = 0; + int i; + + if (!node->mapping_offset || !node->mapping_count) { + pr_err(FW_BUG "Invalid ID mapping, skipping RMR node %p\n", + node); + return; + } + + rmr = (struct acpi_iort_rmr *)node->node_data; + if (!rmr->rmr_offset || !rmr->rmr_count) + return; + + map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node, + node->mapping_offset); + + /* + * Go through the ID mappings and see if we have a match for SMMU + * and dev(if !NULL). If found, get the sids for the Node. + * Please note, id_count is equal to the number of IDs in the + * range minus one. + */ + for (i = 0; i < node->mapping_count; i++, map++) { + struct acpi_iort_node *parent; + + if (!map->id_count) + continue; + + parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table, + map->output_reference); + if (parent != iommu) + continue; + + /* If dev is valid, check RMR node corresponds to the dev SID */ + if (dev && !iort_rmr_has_dev(dev, map->output_base, + map->id_count)) + continue; + + /* Retrieve SIDs associated with the Node. */ + sids = iort_rmr_alloc_sids(sids, num_sids, map->output_base, + map->id_count + 1); + if (!sids) + return; + + num_sids += map->id_count + 1; + } + + if (!sids) + return; + + iort_get_rmrs(node, smmu, sids, num_sids, head); + kfree(sids); +} + +static void iort_find_rmrs(struct acpi_iort_node *iommu, struct device *dev, + struct list_head *head) +{ + struct acpi_table_iort *iort; + struct acpi_iort_node *iort_node, *iort_end; + int i; + + /* Only supports ARM DEN 0049E.d onwards */ + if (iort_table->revision < 5) + return; + + iort = (struct acpi_table_iort *)iort_table; + + iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort, + iort->node_offset); + iort_end = ACPI_ADD_PTR(struct acpi_iort_node, iort, + iort_table->length); + + for (i = 0; i < iort->node_count; i++) { + if (WARN_TAINT(iort_node >= iort_end, TAINT_FIRMWARE_WORKAROUND, + "IORT node pointer overflows, bad table!\n")) + return; + + if (iort_node->type == ACPI_IORT_NODE_RMR) + iort_node_get_rmr_info(iort_node, iommu, dev, head); + + iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node, + iort_node->length); + } +} + +/* + * Populate the RMR list associated with a given IOMMU and dev(if provided). + * If dev is NULL, the function populates all the RMRs associated with the + * given IOMMU. + */ +static void iort_iommu_rmr_get_resv_regions(struct fwnode_handle *iommu_fwnode, + struct device *dev, + struct list_head *head) +{ + struct acpi_iort_node *iommu; + + iommu = iort_get_iort_node(iommu_fwnode); + if (!iommu) + return; + + iort_find_rmrs(iommu, dev, head); +} + static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev) { struct acpi_iort_node *iommu; @@ -868,7 +1156,10 @@ static void iort_iommu_msi_get_resv_regions(struct device *dev, */ void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + iort_iommu_msi_get_resv_regions(dev, head); + iort_iommu_rmr_get_resv_regions(fwspec->iommu_fwnode, dev, head); } static inline bool iort_iommu_driver_enabled(u8 type) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index b22ffa6bc4a9..e6abd998dbe7 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -146,6 +146,14 @@ struct iommu_resv_region { void (*free)(struct device *dev, struct iommu_resv_region *region); }; +struct iommu_iort_rmr_data { + struct iommu_resv_region rr; + + /* Stream IDs associated with IORT RMR entry */ + const u32 *sids; + u32 num_sids; +}; + /** * enum iommu_dev_features - Per device IOMMU features * @IOMMU_DEV_FEAT_SVA: Shared Virtual Addresses -- cgit v1.2.3 From e302eea8f49717253ac64fd45b7cc719e87fa010 Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Wed, 15 Jun 2022 11:10:40 +0100 Subject: ACPI/IORT: Add a helper to retrieve RMR info directly This will provide a way for SMMU drivers to retrieve StreamIDs associated with IORT RMR nodes and use that to set bypass settings for those IDs. Tested-by: Steven Price Tested-by: Laurentiu Tudor Tested-by: Hanjun Guo Reviewed-by: Hanjun Guo Signed-off-by: Shameer Kolothum Acked-by: Robin Murphy Link: https://lore.kernel.org/r/20220615101044.1972-6-shameerali.kolothum.thodi@huawei.com Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 28 ++++++++++++++++++++++++++++ include/linux/acpi_iort.h | 8 ++++++++ 2 files changed, 36 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index b6273af316c6..cd1349d3544e 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1394,6 +1394,34 @@ int iort_dma_get_ranges(struct device *dev, u64 *size) return nc_dma_get_range(dev, size); } +/** + * iort_get_rmr_sids - Retrieve IORT RMR node reserved regions with + * associated StreamIDs information. + * @iommu_fwnode: fwnode associated with IOMMU + * @head: Resereved region list + */ +void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode, + struct list_head *head) +{ + iort_iommu_rmr_get_resv_regions(iommu_fwnode, NULL, head); +} +EXPORT_SYMBOL_GPL(iort_get_rmr_sids); + +/** + * iort_put_rmr_sids - Free memory allocated for RMR reserved regions. + * @iommu_fwnode: fwnode associated with IOMMU + * @head: Resereved region list + */ +void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode, + struct list_head *head) +{ + struct iommu_resv_region *entry, *next; + + list_for_each_entry_safe(entry, next, head, list) + entry->free(NULL, entry); +} +EXPORT_SYMBOL_GPL(iort_put_rmr_sids); + static void __init acpi_iort_register_irq(int hwirq, const char *name, int trigger, struct resource *res) diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index e5d2de9caf7f..b43be0987b19 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -33,6 +33,10 @@ struct irq_domain *iort_get_device_domain(struct device *dev, u32 id, enum irq_domain_bus_token bus_token); void acpi_configure_pmsi_domain(struct device *dev); int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id); +void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode, + struct list_head *head); +void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode, + struct list_head *head); /* IOMMU interface */ int iort_dma_get_ranges(struct device *dev, u64 *size); int iort_iommu_configure_id(struct device *dev, const u32 *id_in); @@ -46,6 +50,10 @@ static inline struct irq_domain *iort_get_device_domain( struct device *dev, u32 id, enum irq_domain_bus_token bus_token) { return NULL; } static inline void acpi_configure_pmsi_domain(struct device *dev) { } +static inline +void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *head) { } +static inline +void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *head) { } /* IOMMU interface */ static inline int iort_dma_get_ranges(struct device *dev, u64 *size) { return -ENODEV; } -- cgit v1.2.3 From 88527790c079fb1ea41cbcfa4450ee37906a2fb0 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 5 Jul 2022 16:59:24 -0700 Subject: tls: rx: add sockopt for enabling optimistic decrypt with TLS 1.3 Since optimisitic decrypt may add extra load in case of retries require socket owner to explicitly opt-in. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- Documentation/networking/tls.rst | 18 ++++++++++ include/linux/sockptr.h | 8 +++++ include/net/tls.h | 3 ++ include/uapi/linux/snmp.h | 1 + include/uapi/linux/tls.h | 2 ++ net/tls/tls_main.c | 75 ++++++++++++++++++++++++++++++++++++++++ net/tls/tls_proc.c | 1 + net/tls/tls_sw.c | 21 +++++++---- 8 files changed, 122 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/tls.rst b/Documentation/networking/tls.rst index be8e10c14b05..7a6643836e42 100644 --- a/Documentation/networking/tls.rst +++ b/Documentation/networking/tls.rst @@ -239,6 +239,19 @@ for the original TCP transmission and TCP retransmissions. To the receiver this will look like TLS records had been tampered with and will result in record authentication failures. +TLS_RX_EXPECT_NO_PAD +~~~~~~~~~~~~~~~~~~~~ + +TLS 1.3 only. Expect the sender to not pad records. This allows the data +to be decrypted directly into user space buffers with TLS 1.3. + +This optimization is safe to enable only if the remote end is trusted, +otherwise it is an attack vector to doubling the TLS processing cost. + +If the record decrypted turns out to had been padded or is not a data +record it will be decrypted again into a kernel buffer without zero copy. +Such events are counted in the ``TlsDecryptRetry`` statistic. + Statistics ========== @@ -264,3 +277,8 @@ TLS implementation exposes the following per-namespace statistics - ``TlsDeviceRxResync`` - number of RX resyncs sent to NICs handling cryptography + +- ``TlsDecryptRetry`` - + number of RX records which had to be re-decrypted due to + ``TLS_RX_EXPECT_NO_PAD`` mis-prediction. Note that this counter will + also increment for non-data records. diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h index ea193414298b..d45902fb4cad 100644 --- a/include/linux/sockptr.h +++ b/include/linux/sockptr.h @@ -102,4 +102,12 @@ static inline long strncpy_from_sockptr(char *dst, sockptr_t src, size_t count) return strncpy_from_user(dst, src.user, count); } +static inline int check_zeroed_sockptr(sockptr_t src, size_t offset, + size_t size) +{ + if (!sockptr_is_kernel(src)) + return check_zeroed_user(src.user + offset, size); + return memchr_inv(src.kernel + offset, 0, size) == NULL; +} + #endif /* _LINUX_SOCKPTR_H */ diff --git a/include/net/tls.h b/include/net/tls.h index 8017f1703447..4fc16ca5f469 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -149,6 +149,7 @@ struct tls_sw_context_rx { struct sk_buff *recv_pkt; u8 async_capable:1; + u8 zc_capable:1; atomic_t decrypt_pending; /* protect crypto_wait with decrypt_pending*/ spinlock_t decrypt_compl_lock; @@ -239,6 +240,7 @@ struct tls_context { u8 tx_conf:3; u8 rx_conf:3; u8 zerocopy_sendfile:1; + u8 rx_no_pad:1; int (*push_pending_record)(struct sock *sk, int flags); void (*sk_write_space)(struct sock *sk); @@ -358,6 +360,7 @@ int tls_sk_attach(struct sock *sk, int optname, char __user *optval, void tls_err_abort(struct sock *sk, int err); int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx); +void tls_update_rx_zc_capable(struct tls_context *tls_ctx); void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx); void tls_sw_strparser_done(struct tls_context *tls_ctx); int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h index 904909d020e2..1c9152add663 100644 --- a/include/uapi/linux/snmp.h +++ b/include/uapi/linux/snmp.h @@ -344,6 +344,7 @@ enum LINUX_MIB_TLSRXDEVICE, /* TlsRxDevice */ LINUX_MIB_TLSDECRYPTERROR, /* TlsDecryptError */ LINUX_MIB_TLSRXDEVICERESYNC, /* TlsRxDeviceResync */ + LINUX_MIN_TLSDECRYPTRETRY, /* TlsDecryptRetry */ __LINUX_MIB_TLSMAX }; diff --git a/include/uapi/linux/tls.h b/include/uapi/linux/tls.h index bb8f80812b0b..f1157d8f4acd 100644 --- a/include/uapi/linux/tls.h +++ b/include/uapi/linux/tls.h @@ -40,6 +40,7 @@ #define TLS_TX 1 /* Set transmit parameters */ #define TLS_RX 2 /* Set receive parameters */ #define TLS_TX_ZEROCOPY_RO 3 /* TX zerocopy (only sendfile now) */ +#define TLS_RX_EXPECT_NO_PAD 4 /* Attempt opportunistic zero-copy */ /* Supported versions */ #define TLS_VERSION_MINOR(ver) ((ver) & 0xFF) @@ -162,6 +163,7 @@ enum { TLS_INFO_TXCONF, TLS_INFO_RXCONF, TLS_INFO_ZC_RO_TX, + TLS_INFO_RX_NO_PAD, __TLS_INFO_MAX, }; #define TLS_INFO_MAX (__TLS_INFO_MAX - 1) diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c index 2ffede463e4a..1b3efc96db0b 100644 --- a/net/tls/tls_main.c +++ b/net/tls/tls_main.c @@ -533,6 +533,37 @@ static int do_tls_getsockopt_tx_zc(struct sock *sk, char __user *optval, return 0; } +static int do_tls_getsockopt_no_pad(struct sock *sk, char __user *optval, + int __user *optlen) +{ + struct tls_context *ctx = tls_get_ctx(sk); + unsigned int value; + int err, len; + + if (ctx->prot_info.version != TLS_1_3_VERSION) + return -EINVAL; + + if (get_user(len, optlen)) + return -EFAULT; + if (len < sizeof(value)) + return -EINVAL; + + lock_sock(sk); + err = -EINVAL; + if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) + value = ctx->rx_no_pad; + release_sock(sk); + if (err) + return err; + + if (put_user(sizeof(value), optlen)) + return -EFAULT; + if (copy_to_user(optval, &value, sizeof(value))) + return -EFAULT; + + return 0; +} + static int do_tls_getsockopt(struct sock *sk, int optname, char __user *optval, int __user *optlen) { @@ -547,6 +578,9 @@ static int do_tls_getsockopt(struct sock *sk, int optname, case TLS_TX_ZEROCOPY_RO: rc = do_tls_getsockopt_tx_zc(sk, optval, optlen); break; + case TLS_RX_EXPECT_NO_PAD: + rc = do_tls_getsockopt_no_pad(sk, optval, optlen); + break; default: rc = -ENOPROTOOPT; break; @@ -718,6 +752,38 @@ static int do_tls_setsockopt_tx_zc(struct sock *sk, sockptr_t optval, return 0; } +static int do_tls_setsockopt_no_pad(struct sock *sk, sockptr_t optval, + unsigned int optlen) +{ + struct tls_context *ctx = tls_get_ctx(sk); + u32 val; + int rc; + + if (ctx->prot_info.version != TLS_1_3_VERSION || + sockptr_is_null(optval) || optlen < sizeof(val)) + return -EINVAL; + + rc = copy_from_sockptr(&val, optval, sizeof(val)); + if (rc) + return -EFAULT; + if (val > 1) + return -EINVAL; + rc = check_zeroed_sockptr(optval, sizeof(val), optlen - sizeof(val)); + if (rc < 1) + return rc == 0 ? -EINVAL : rc; + + lock_sock(sk); + rc = -EINVAL; + if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) { + ctx->rx_no_pad = val; + tls_update_rx_zc_capable(ctx); + rc = 0; + } + release_sock(sk); + + return rc; +} + static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval, unsigned int optlen) { @@ -736,6 +802,9 @@ static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval, rc = do_tls_setsockopt_tx_zc(sk, optval, optlen); release_sock(sk); break; + case TLS_RX_EXPECT_NO_PAD: + rc = do_tls_setsockopt_no_pad(sk, optval, optlen); + break; default: rc = -ENOPROTOOPT; break; @@ -976,6 +1045,11 @@ static int tls_get_info(const struct sock *sk, struct sk_buff *skb) if (err) goto nla_failure; } + if (ctx->rx_no_pad) { + err = nla_put_flag(skb, TLS_INFO_RX_NO_PAD); + if (err) + goto nla_failure; + } rcu_read_unlock(); nla_nest_end(skb, start); @@ -997,6 +1071,7 @@ static size_t tls_get_info_size(const struct sock *sk) nla_total_size(sizeof(u16)) + /* TLS_INFO_RXCONF */ nla_total_size(sizeof(u16)) + /* TLS_INFO_TXCONF */ nla_total_size(0) + /* TLS_INFO_ZC_RO_TX */ + nla_total_size(0) + /* TLS_INFO_RX_NO_PAD */ 0; return size; diff --git a/net/tls/tls_proc.c b/net/tls/tls_proc.c index feeceb0e4cb4..0c200000cc45 100644 --- a/net/tls/tls_proc.c +++ b/net/tls/tls_proc.c @@ -18,6 +18,7 @@ static const struct snmp_mib tls_mib_list[] = { SNMP_MIB_ITEM("TlsRxDevice", LINUX_MIB_TLSRXDEVICE), SNMP_MIB_ITEM("TlsDecryptError", LINUX_MIB_TLSDECRYPTERROR), SNMP_MIB_ITEM("TlsRxDeviceResync", LINUX_MIB_TLSRXDEVICERESYNC), + SNMP_MIB_ITEM("TlsDecryptRetry", LINUX_MIN_TLSDECRYPTRETRY), SNMP_MIB_SENTINEL }; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 2bac57684429..7592b6519953 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -1601,6 +1601,7 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, if (unlikely(darg->zc && prot->version == TLS_1_3_VERSION && darg->tail != TLS_RECORD_TYPE_DATA)) { darg->zc = false; + TLS_INC_STATS(sock_net(sk), LINUX_MIN_TLSDECRYPTRETRY); return decrypt_skb_update(sk, skb, dest, darg); } @@ -1787,7 +1788,7 @@ int tls_sw_recvmsg(struct sock *sk, timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); zc_capable = !bpf_strp_enabled && !is_kvec && !is_peek && - prot->version != TLS_1_3_VERSION; + ctx->zc_capable; decrypted = 0; while (len && (decrypted + copied < target || ctx->recv_pkt)) { struct tls_decrypt_arg darg = {}; @@ -2269,6 +2270,14 @@ void tls_sw_strparser_arm(struct sock *sk, struct tls_context *tls_ctx) strp_check_rcv(&rx_ctx->strp); } +void tls_update_rx_zc_capable(struct tls_context *tls_ctx) +{ + struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx); + + rx_ctx->zc_capable = tls_ctx->rx_no_pad || + tls_ctx->prot_info.version != TLS_1_3_VERSION; +} + int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) { struct tls_context *tls_ctx = tls_get_ctx(sk); @@ -2504,12 +2513,10 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) if (sw_ctx_rx) { tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv); - if (crypto_info->version == TLS_1_3_VERSION) - sw_ctx_rx->async_capable = 0; - else - sw_ctx_rx->async_capable = - !!(tfm->__crt_alg->cra_flags & - CRYPTO_ALG_ASYNC); + tls_update_rx_zc_capable(ctx); + sw_ctx_rx->async_capable = + crypto_info->version != TLS_1_3_VERSION && + !!(tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC); /* Set up strparser */ memset(&cb, 0, sizeof(cb)); -- cgit v1.2.3 From 2fd26970cf66bd52dc42843c46968040caa8c9a1 Mon Sep 17 00:00:00 2001 From: Imran Khan Date: Wed, 6 Jul 2022 06:10:26 +1000 Subject: Revert "kernfs: Change kernfs_notify_list to llist." This reverts commit b8f35fa1188b84035c59d4842826c4e93a1b1c9f. This is causing regression due to same kernfs_node getting added multiple times in kernfs_notify_list so revert it until safe way of using llist in this context is found. Reported-by: Nathan Chancellor Reported-by: Michael Walle Reported-by: Marek Szyprowski Signed-off-by: Imran Khan Cc: Tejun Heo Link: https://lore.kernel.org/r/20220705201026.2487665-1-imran.f.khan@oracle.com Signed-off-by: Greg Kroah-Hartman --- fs/kernfs/file.c | 47 +++++++++++++++++++++++++++-------------------- include/linux/kernfs.h | 2 +- 2 files changed, 28 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index bb933221b4ba..baff4b1d40c7 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -25,16 +25,18 @@ struct kernfs_open_node { struct list_head files; /* goes through kernfs_open_file.list */ }; -/** - * attribute_to_node - get kernfs_node object corresponding to a kernfs attribute - * @ptr: &struct kernfs_elem_attr - * @type: struct kernfs_node - * @member: name of member (i.e attr) +/* + * kernfs_notify() may be called from any context and bounces notifications + * through a work item. To minimize space overhead in kernfs_node, the + * pending queue is implemented as a singly linked list of kernfs_nodes. + * The list is terminated with the self pointer so that whether a + * kernfs_node is on the list or not can be determined by testing the next + * pointer for NULL. */ -#define attribute_to_node(ptr, type, member) \ - container_of(ptr, type, member) +#define KERNFS_NOTIFY_EOL ((void *)&kernfs_notify_list) -static LLIST_HEAD(kernfs_notify_list); +static DEFINE_SPINLOCK(kernfs_notify_lock); +static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL; static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn) { @@ -909,16 +911,18 @@ static void kernfs_notify_workfn(struct work_struct *work) struct kernfs_node *kn; struct kernfs_super_info *info; struct kernfs_root *root; - struct llist_node *free; - struct kernfs_elem_attr *attr; repeat: /* pop one off the notify_list */ - free = llist_del_first(&kernfs_notify_list); - if (free == NULL) + spin_lock_irq(&kernfs_notify_lock); + kn = kernfs_notify_list; + if (kn == KERNFS_NOTIFY_EOL) { + spin_unlock_irq(&kernfs_notify_lock); return; + } + kernfs_notify_list = kn->attr.notify_next; + kn->attr.notify_next = NULL; + spin_unlock_irq(&kernfs_notify_lock); - attr = llist_entry(free, struct kernfs_elem_attr, notify_next); - kn = attribute_to_node(attr, struct kernfs_node, attr); root = kernfs_root(kn); /* kick fsnotify */ down_write(&root->kernfs_rwsem); @@ -974,14 +978,12 @@ repeat: void kernfs_notify(struct kernfs_node *kn) { static DECLARE_WORK(kernfs_notify_work, kernfs_notify_workfn); + unsigned long flags; struct kernfs_open_node *on; if (WARN_ON(kernfs_type(kn) != KERNFS_FILE)) return; - /* Because we are using llist for kernfs_notify_list */ - WARN_ON_ONCE(in_nmi()); - /* kick poll immediately */ rcu_read_lock(); on = rcu_dereference(kn->attr.open); @@ -992,9 +994,14 @@ void kernfs_notify(struct kernfs_node *kn) rcu_read_unlock(); /* schedule work to kick fsnotify */ - kernfs_get(kn); - llist_add(&kn->attr.notify_next, &kernfs_notify_list); - schedule_work(&kernfs_notify_work); + spin_lock_irqsave(&kernfs_notify_lock, flags); + if (!kn->attr.notify_next) { + kernfs_get(kn); + kn->attr.notify_next = kernfs_notify_list; + kernfs_notify_list = kn; + schedule_work(&kernfs_notify_work); + } + spin_unlock_irqrestore(&kernfs_notify_lock, flags); } EXPORT_SYMBOL_GPL(kernfs_notify); diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 13e703f615f7..367044d7708c 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -173,7 +173,7 @@ struct kernfs_elem_attr { const struct kernfs_ops *ops; struct kernfs_open_node __rcu *open; loff_t size; - struct llist_node notify_next; /* for kernfs_notify() */ + struct kernfs_node *notify_next; /* for kernfs_notify() */ }; /* -- cgit v1.2.3 From 99e48cd6855e9535488e3c90d65edd46c6e6fc1b Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jul 2022 20:03:50 +0800 Subject: blk-mq: Add a flag for reserved requests Add a flag for reserved requests so that drivers may know this for any special handling. Signed-off-by: John Garry Reviewed-by: Christoph Hellwig Reviewed-by: Bart Van Assche Reviewed-by: Hannes Reinecke Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/1657109034-206040-3-git-send-email-john.garry@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 ++++++ include/linux/blk-mq.h | 6 ++++++ 2 files changed, 12 insertions(+) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index 15c7c5c4ad22..a00e43cc67e5 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -475,6 +475,9 @@ retry: if (!(data->rq_flags & RQF_ELV)) blk_mq_tag_busy(data->hctx); + if (data->flags & BLK_MQ_REQ_RESERVED) + data->rq_flags |= RQF_RESV; + /* * Try batched alloc if we want more than 1 tag. */ @@ -589,6 +592,9 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, else data.rq_flags |= RQF_ELV; + if (flags & BLK_MQ_REQ_RESERVED) + data.rq_flags |= RQF_RESV; + ret = -EWOULDBLOCK; tag = blk_mq_get_tag(&data); if (tag == BLK_MQ_NO_TAG) diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 43aad0da3305..7c62b7fabec7 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -57,6 +57,7 @@ typedef __u32 __bitwise req_flags_t; #define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21)) /* queue has elevator attached */ #define RQF_ELV ((__force req_flags_t)(1 << 22)) +#define RQF_RESV ((__force req_flags_t)(1 << 23)) /* flags that prevent us from merging requests: */ #define RQF_NOMERGE_FLAGS \ @@ -825,6 +826,11 @@ static inline bool blk_mq_need_time_stamp(struct request *rq) return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV)); } +static inline bool blk_mq_is_reserved_rq(struct request *rq) +{ + return rq->rq_flags & RQF_RESV; +} + /* * Batched completions only work when there is no I/O error and no special * ->end_io handler. -- cgit v1.2.3 From 9bdb4833dd399cbff82cc20893f52bdec66a9eca Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jul 2022 20:03:51 +0800 Subject: blk-mq: Drop blk_mq_ops.timeout 'reserved' arg With new API blk_mq_is_reserved_rq() we can tell if a request is from the reserved pool, so stop passing 'reserved' arg. There is actually only a single user of that arg for all the callback implementations, which can use blk_mq_is_reserved_rq() instead. This will also allow us to stop passing the same 'reserved' around the blk-mq iter functions next. Signed-off-by: John Garry Reviewed-by: Christoph Hellwig Reviewed-by: Bart Van Assche Reviewed-by: Hannes Reinecke Acked-by: Ulf Hansson # For MMC Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/1657109034-206040-4-git-send-email-john.garry@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 +++--- block/bsg-lib.c | 2 +- drivers/block/mtip32xx/mtip32xx.c | 5 ++--- drivers/block/nbd.c | 3 +-- drivers/block/null_blk/main.c | 2 +- drivers/mmc/core/queue.c | 3 +-- drivers/nvme/host/apple.c | 3 +-- drivers/nvme/host/fc.c | 3 +-- drivers/nvme/host/pci.c | 2 +- drivers/nvme/host/rdma.c | 3 +-- drivers/nvme/host/tcp.c | 3 +-- drivers/s390/block/dasd.c | 2 +- drivers/s390/block/dasd_int.h | 2 +- drivers/scsi/scsi_error.c | 3 +-- drivers/scsi/scsi_priv.h | 3 +-- include/linux/blk-mq.h | 2 +- 16 files changed, 19 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index a00e43cc67e5..cedbec36e907 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1427,13 +1427,13 @@ bool blk_mq_queue_inflight(struct request_queue *q) } EXPORT_SYMBOL_GPL(blk_mq_queue_inflight); -static void blk_mq_rq_timed_out(struct request *req, bool reserved) +static void blk_mq_rq_timed_out(struct request *req) { req->rq_flags |= RQF_TIMED_OUT; if (req->q->mq_ops->timeout) { enum blk_eh_timer_return ret; - ret = req->q->mq_ops->timeout(req, reserved); + ret = req->q->mq_ops->timeout(req); if (ret == BLK_EH_DONE) return; WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); @@ -1482,7 +1482,7 @@ static bool blk_mq_check_expired(struct request *rq, void *priv, bool reserved) * from blk_mq_check_expired(). */ if (blk_mq_req_expired(rq, next)) - blk_mq_rq_timed_out(rq, reserved); + blk_mq_rq_timed_out(rq); return true; } diff --git a/block/bsg-lib.c b/block/bsg-lib.c index fd4cd5e68282..d6f5dcdce748 100644 --- a/block/bsg-lib.c +++ b/block/bsg-lib.c @@ -331,7 +331,7 @@ void bsg_remove_queue(struct request_queue *q) } EXPORT_SYMBOL_GPL(bsg_remove_queue); -static enum blk_eh_timer_return bsg_timeout(struct request *rq, bool reserved) +static enum blk_eh_timer_return bsg_timeout(struct request *rq) { struct bsg_set *bset = container_of(rq->q->tag_set, struct bsg_set, tag_set); diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index e116c6cf56f5..5073cb407500 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3357,12 +3357,11 @@ static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq, return 0; } -static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req, - bool reserved) +static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req) { struct driver_data *dd = req->q->queuedata; - if (reserved) { + if (blk_mq_is_reserved_rq(req)) { struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req); cmd->status = BLK_STS_TIMEOUT; diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 5c4c9c45c6ac..028f23c965df 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -393,8 +393,7 @@ static u32 req_to_nbd_cmd_type(struct request *req) } } -static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, - bool reserved) +static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); struct nbd_device *nbd = cmd->nbd; diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index d695ea29efa6..4e03a020ee3c 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1578,7 +1578,7 @@ static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) return nr; } -static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res) +static enum blk_eh_timer_return null_timeout_rq(struct request *rq) { struct blk_mq_hw_ctx *hctx = rq->mq_hctx; struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq); diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index f824cfdab75a..fefaa901b50f 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -116,8 +116,7 @@ static enum blk_eh_timer_return mmc_cqe_timed_out(struct request *req) } } -static enum blk_eh_timer_return mmc_mq_timed_out(struct request *req, - bool reserved) +static enum blk_eh_timer_return mmc_mq_timed_out(struct request *req) { struct request_queue *q = req->q; struct mmc_queue *mq = q->queuedata; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 2d23b7d41f7e..5c352d5d8ee6 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -862,8 +862,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) } } -static enum blk_eh_timer_return apple_nvme_timeout(struct request *req, - bool reserved) +static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) { struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); struct apple_nvme_queue *q = iod->q; diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index a96aa831684c..07fd6db5869c 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2565,8 +2565,7 @@ nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg) nvme_reset_ctrl(&ctrl->ctrl); } -static enum blk_eh_timer_return -nvme_fc_timeout(struct request *rq, bool reserved) +static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq) { struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); struct nvme_fc_ctrl *ctrl = op->ctrl; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 247a74aba336..4232192e10dd 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1344,7 +1344,7 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off\" and report a bug\n"); } -static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) +static enum blk_eh_timer_return nvme_timeout(struct request *req) { struct nvme_iod *iod = blk_mq_rq_to_pdu(req); struct nvme_queue *nvmeq = iod->nvmeq; diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 0fb7c8e7ab0b..a6eaf38b9646 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -2013,8 +2013,7 @@ static void nvme_rdma_complete_timed_out(struct request *rq) nvmf_complete_timed_out_request(rq); } -static enum blk_eh_timer_return -nvme_rdma_timeout(struct request *rq, bool reserved) +static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq) { struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); struct nvme_rdma_queue *queue = req->queue; diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index b81942fa5f95..ff502172accd 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -2321,8 +2321,7 @@ static void nvme_tcp_complete_timed_out(struct request *rq) nvmf_complete_timed_out_request(rq); } -static enum blk_eh_timer_return -nvme_tcp_timeout(struct request *rq, bool reserved) +static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) { struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index e8489331f12b..4df8bf6505fc 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -3145,7 +3145,7 @@ out: * BLK_EH_DONE if the request is handled or terminated * by the driver. */ -enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved) +enum blk_eh_timer_return dasd_times_out(struct request *req) { struct dasd_block *block = req->q->queuedata; struct dasd_device *device; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 83b918b84b4a..333a399f754e 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -795,7 +795,7 @@ void dasd_free_device(struct dasd_device *); struct dasd_block *dasd_alloc_block(void); void dasd_free_block(struct dasd_block *); -enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved); +enum blk_eh_timer_return dasd_times_out(struct request *req); void dasd_enable_device(struct dasd_device *); void dasd_set_target_state(struct dasd_device *, int); diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index a8b71b73a5a5..266ce414589c 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -318,7 +318,6 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd) /** * scsi_timeout - Timeout function for normal scsi commands. * @req: request that is timing out. - * @reserved: whether the request is a reserved request. * * Notes: * We do not need to lock this. There is the potential for a race @@ -326,7 +325,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd) * normal completion function determines that the timer has already * fired, then it mustn't do anything. */ -enum blk_eh_timer_return scsi_timeout(struct request *req, bool reserved) +enum blk_eh_timer_return scsi_timeout(struct request *req) { struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req); enum blk_eh_timer_return rtn = BLK_EH_DONE; diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h index 695d0c83ffe0..6eeaa0a7f86d 100644 --- a/drivers/scsi/scsi_priv.h +++ b/drivers/scsi/scsi_priv.h @@ -72,8 +72,7 @@ extern void scsi_exit_devinfo(void); /* scsi_error.c */ extern void scmd_eh_abort_handler(struct work_struct *work); -extern enum blk_eh_timer_return scsi_timeout(struct request *req, - bool reserved); +extern enum blk_eh_timer_return scsi_timeout(struct request *req); extern int scsi_error_handler(void *host); extern enum scsi_disposition scsi_decide_disposition(struct scsi_cmnd *cmd); extern void scsi_eh_wakeup(struct Scsi_Host *shost); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 7c62b7fabec7..c84c56d296fe 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -575,7 +575,7 @@ struct blk_mq_ops { /** * @timeout: Called on request timeout. */ - enum blk_eh_timer_return (*timeout)(struct request *, bool); + enum blk_eh_timer_return (*timeout)(struct request *); /** * @poll: Called to poll for completion of a specific tag. -- cgit v1.2.3 From 2dd6532e9591f201e7571b30915db807603ab924 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jul 2022 20:03:53 +0800 Subject: blk-mq: Drop 'reserved' arg of busy_tag_iter_fn We no longer use the 'reserved' arg in busy_tag_iter_fn for any iter function so it may be dropped. Signed-off-by: John Garry Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Reviewed-by: Martin K. Petersen Reviewed-by: Sagi Grimberg #nvme Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/1657109034-206040-6-git-send-email-john.garry@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq-debugfs.c | 2 +- block/blk-mq-tag.c | 7 +++---- block/blk-mq.c | 10 ++++------ drivers/block/mtip32xx/mtip32xx.c | 4 ++-- drivers/block/nbd.c | 2 +- drivers/infiniband/ulp/srp/ib_srp.c | 3 +-- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/fc.c | 3 +-- drivers/nvme/host/nvme.h | 2 +- drivers/scsi/aacraid/comminit.c | 2 +- drivers/scsi/aacraid/linit.c | 2 +- drivers/scsi/fnic/fnic_scsi.c | 12 ++++-------- drivers/scsi/hosts.c | 14 ++++++-------- drivers/scsi/mpi3mr/mpi3mr_os.c | 16 ++++------------ include/linux/blk-mq.h | 2 +- include/scsi/scsi_host.h | 2 +- 16 files changed, 33 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index b80fae7ab1d9..b11add9a95e2 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -375,7 +375,7 @@ struct show_busy_params { * e.g. due to a concurrent blk_mq_finish_request() call. Returns true to * keep iterating requests. */ -static bool hctx_show_busy_rq(struct request *rq, void *data, bool reserved) +static bool hctx_show_busy_rq(struct request *rq, void *data) { const struct show_busy_params *params = data; diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 3cfffef1feb3..4e9b8ec55bda 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -283,7 +283,7 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) return true; if (rq->q == q && (!hctx || rq->mq_hctx == hctx)) - ret = iter_data->fn(rq, iter_data->data, reserved); + ret = iter_data->fn(rq, iter_data->data); blk_mq_put_rq_ref(rq); return ret; } @@ -354,7 +354,7 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) if (!(iter_data->flags & BT_TAG_ITER_STARTED) || blk_mq_request_started(rq)) - ret = iter_data->fn(rq, iter_data->data, reserved); + ret = iter_data->fn(rq, iter_data->data); if (!iter_static_rqs) blk_mq_put_rq_ref(rq); return ret; @@ -444,8 +444,7 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, } EXPORT_SYMBOL(blk_mq_tagset_busy_iter); -static bool blk_mq_tagset_count_completed_rqs(struct request *rq, - void *data, bool reserved) +static bool blk_mq_tagset_count_completed_rqs(struct request *rq, void *data) { unsigned *count = data; diff --git a/block/blk-mq.c b/block/blk-mq.c index cedbec36e907..63385742b8a8 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -129,8 +129,7 @@ struct mq_inflight { unsigned int inflight[2]; }; -static bool blk_mq_check_inflight(struct request *rq, void *priv, - bool reserved) +static bool blk_mq_check_inflight(struct request *rq, void *priv) { struct mq_inflight *mi = priv; @@ -1400,8 +1399,7 @@ void blk_mq_delay_kick_requeue_list(struct request_queue *q, } EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); -static bool blk_mq_rq_inflight(struct request *rq, void *priv, - bool reserved) +static bool blk_mq_rq_inflight(struct request *rq, void *priv) { /* * If we find a request that isn't idle we know the queue is busy @@ -1470,7 +1468,7 @@ void blk_mq_put_rq_ref(struct request *rq) __blk_mq_free_request(rq); } -static bool blk_mq_check_expired(struct request *rq, void *priv, bool reserved) +static bool blk_mq_check_expired(struct request *rq, void *priv) { unsigned long *next = priv; @@ -3289,7 +3287,7 @@ struct rq_iter_data { bool has_rq; }; -static bool blk_mq_has_request(struct request *rq, void *data, bool reserved) +static bool blk_mq_has_request(struct request *rq, void *data) { struct rq_iter_data *iter_data = data; diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 5073cb407500..562725d222a7 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -2441,7 +2441,7 @@ static void mtip_softirq_done_fn(struct request *rq) blk_mq_end_request(rq, cmd->status); } -static bool mtip_abort_cmd(struct request *req, void *data, bool reserved) +static bool mtip_abort_cmd(struct request *req, void *data) { struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req); struct driver_data *dd = data; @@ -2454,7 +2454,7 @@ static bool mtip_abort_cmd(struct request *req, void *data, bool reserved) return true; } -static bool mtip_queue_cmd(struct request *req, void *data, bool reserved) +static bool mtip_queue_cmd(struct request *req, void *data) { struct driver_data *dd = data; diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 028f23c965df..f5d098a148cb 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -879,7 +879,7 @@ static void recv_work(struct work_struct *work) kfree(args); } -static bool nbd_clear_req(struct request *req, void *data, bool reserved) +static bool nbd_clear_req(struct request *req, void *data) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 6058abf42ba7..7720ea270ed8 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -1282,8 +1282,7 @@ struct srp_terminate_context { int scsi_result; }; -static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr, - bool reserved) +static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr) { struct srp_terminate_context *context = context_ptr; struct srp_target_port *target = context->srp_target; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index b5b24998a5ab..9031d10c97dc 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -418,7 +418,7 @@ blk_status_t nvme_host_path_error(struct request *req) } EXPORT_SYMBOL_GPL(nvme_host_path_error); -bool nvme_cancel_request(struct request *req, void *data, bool reserved) +bool nvme_cancel_request(struct request *req, void *data) { dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device, "Cancelling I/O %d", req->tag); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 07fd6db5869c..9987797620b6 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2456,8 +2456,7 @@ nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl) * status. The done path will return the io request back to the block * layer with an error status. */ -static bool -nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved) +static bool nvme_fc_terminate_exchange(struct request *req, void *data) { struct nvme_ctrl *nctrl = data; struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 0da94b233fed..e4daa57f8bd5 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -697,7 +697,7 @@ static __always_inline void nvme_complete_batch(struct io_comp_batch *iob, } blk_status_t nvme_host_path_error(struct request *req); -bool nvme_cancel_request(struct request *req, void *data, bool reserved); +bool nvme_cancel_request(struct request *req, void *data); void nvme_cancel_tagset(struct nvme_ctrl *ctrl); void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl); bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, diff --git a/drivers/scsi/aacraid/comminit.c b/drivers/scsi/aacraid/comminit.c index 940a6deab38f..bd99c5492b7d 100644 --- a/drivers/scsi/aacraid/comminit.c +++ b/drivers/scsi/aacraid/comminit.c @@ -272,7 +272,7 @@ static void aac_queue_init(struct aac_dev * dev, struct aac_queue * q, u32 *mem, q->entries = qsize; } -static bool wait_for_io_iter(struct scsi_cmnd *cmd, void *data, bool rsvd) +static bool wait_for_io_iter(struct scsi_cmnd *cmd, void *data) { int *active = data; diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c index 9c27bc37e5de..5ba5c18b77b4 100644 --- a/drivers/scsi/aacraid/linit.c +++ b/drivers/scsi/aacraid/linit.c @@ -633,7 +633,7 @@ struct fib_count_data { int krlcnt; }; -static bool fib_count_iter(struct scsi_cmnd *scmnd, void *data, bool reserved) +static bool fib_count_iter(struct scsi_cmnd *scmnd, void *data) { struct fib_count_data *fib_count = data; diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c index e7b7f6d73429..77a4d9f8aa83 100644 --- a/drivers/scsi/fnic/fnic_scsi.c +++ b/drivers/scsi/fnic/fnic_scsi.c @@ -1350,8 +1350,7 @@ int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do) return wq_work_done; } -static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data, - bool reserved) +static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data) { const int tag = scsi_cmd_to_rq(sc)->tag; struct fnic *fnic = data; @@ -1548,8 +1547,7 @@ struct fnic_rport_abort_io_iter_data { int term_cnt; }; -static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data, - bool reserved) +static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) { struct fnic_rport_abort_io_iter_data *iter_data = data; struct fnic *fnic = iter_data->fnic; @@ -2003,8 +2001,7 @@ struct fnic_pending_aborts_iter_data { int ret; }; -static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, - void *data, bool reserved) +static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) { struct fnic_pending_aborts_iter_data *iter_data = data; struct fnic *fnic = iter_data->fnic; @@ -2668,8 +2665,7 @@ call_fc_exch_mgr_reset: } -static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data, - bool reserved) +static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data) { struct fnic_pending_aborts_iter_data *iter_data = data; struct fnic *fnic = iter_data->fnic; diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index 8352f90d997d..315c7ac730e9 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -566,8 +566,7 @@ struct Scsi_Host *scsi_host_get(struct Scsi_Host *shost) } EXPORT_SYMBOL(scsi_host_get); -static bool scsi_host_check_in_flight(struct request *rq, void *data, - bool reserved) +static bool scsi_host_check_in_flight(struct request *rq, void *data) { int *count = data; struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); @@ -662,7 +661,7 @@ void scsi_flush_work(struct Scsi_Host *shost) } EXPORT_SYMBOL_GPL(scsi_flush_work); -static bool complete_all_cmds_iter(struct request *rq, void *data, bool rsvd) +static bool complete_all_cmds_iter(struct request *rq, void *data) { struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq); enum scsi_host_status status = *(enum scsi_host_status *)data; @@ -693,17 +692,16 @@ void scsi_host_complete_all_commands(struct Scsi_Host *shost, EXPORT_SYMBOL_GPL(scsi_host_complete_all_commands); struct scsi_host_busy_iter_data { - bool (*fn)(struct scsi_cmnd *, void *, bool); + bool (*fn)(struct scsi_cmnd *, void *); void *priv; }; -static bool __scsi_host_busy_iter_fn(struct request *req, void *priv, - bool reserved) +static bool __scsi_host_busy_iter_fn(struct request *req, void *priv) { struct scsi_host_busy_iter_data *iter_data = priv; struct scsi_cmnd *sc = blk_mq_rq_to_pdu(req); - return iter_data->fn(sc, iter_data->priv, reserved); + return iter_data->fn(sc, iter_data->priv); } /** @@ -716,7 +714,7 @@ static bool __scsi_host_busy_iter_fn(struct request *req, void *priv, * ithas to be provided by the caller **/ void scsi_host_busy_iter(struct Scsi_Host *shost, - bool (*fn)(struct scsi_cmnd *, void *, bool), + bool (*fn)(struct scsi_cmnd *, void *), void *priv) { struct scsi_host_busy_iter_data iter_data = { diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c index d8c195b7ca57..59a18769a4fe 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_os.c +++ b/drivers/scsi/mpi3mr/mpi3mr_os.c @@ -381,14 +381,12 @@ void mpi3mr_invalidate_devhandles(struct mpi3mr_ioc *mrioc) * mpi3mr_print_scmd - print individual SCSI command * @rq: Block request * @data: Adapter instance reference - * @reserved: N/A. Currently not used * * Print the SCSI command details if it is in LLD scope. * * Return: true always. */ -static bool mpi3mr_print_scmd(struct request *rq, - void *data, bool reserved) +static bool mpi3mr_print_scmd(struct request *rq, void *data) { struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data; struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq); @@ -412,7 +410,6 @@ out: * mpi3mr_flush_scmd - Flush individual SCSI command * @rq: Block request * @data: Adapter instance reference - * @reserved: N/A. Currently not used * * Return the SCSI command to the upper layers if it is in LLD * scope. @@ -420,8 +417,7 @@ out: * Return: true always. */ -static bool mpi3mr_flush_scmd(struct request *rq, - void *data, bool reserved) +static bool mpi3mr_flush_scmd(struct request *rq, void *data) { struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data; struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq); @@ -451,7 +447,6 @@ out: * mpi3mr_count_dev_pending - Count commands pending for a lun * @rq: Block request * @data: SCSI device reference - * @reserved: Unused * * This is an iterator function called for each SCSI command in * a host and if the command is pending in the LLD for the @@ -461,8 +456,7 @@ out: * Return: true always. */ -static bool mpi3mr_count_dev_pending(struct request *rq, - void *data, bool reserved) +static bool mpi3mr_count_dev_pending(struct request *rq, void *data) { struct scsi_device *sdev = (struct scsi_device *)data; struct mpi3mr_sdev_priv_data *sdev_priv_data = sdev->hostdata; @@ -485,7 +479,6 @@ out: * mpi3mr_count_tgt_pending - Count commands pending for target * @rq: Block request * @data: SCSI target reference - * @reserved: Unused * * This is an iterator function called for each SCSI command in * a host and if the command is pending in the LLD for the @@ -495,8 +488,7 @@ out: * Return: true always. */ -static bool mpi3mr_count_tgt_pending(struct request *rq, - void *data, bool reserved) +static bool mpi3mr_count_tgt_pending(struct request *rq, void *data) { struct scsi_target *starget = (struct scsi_target *)data; struct mpi3mr_stgt_priv_data *stgt_priv_data = starget->hostdata; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index c84c56d296fe..810a24884f7e 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -520,7 +520,7 @@ struct blk_mq_queue_data { bool last; }; -typedef bool (busy_tag_iter_fn)(struct request *, void *, bool); +typedef bool (busy_tag_iter_fn)(struct request *, void *); /** * struct blk_mq_ops - Callback functions that implements block driver diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 667d889b92b5..65082ecdd557 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -786,7 +786,7 @@ extern int scsi_host_block(struct Scsi_Host *shost); extern int scsi_host_unblock(struct Scsi_Host *shost, int new_state); void scsi_host_busy_iter(struct Scsi_Host *, - bool (*fn)(struct scsi_cmnd *, void *, bool), void *priv); + bool (*fn)(struct scsi_cmnd *, void *), void *priv); struct class_container; -- cgit v1.2.3 From f1a8bbd1100d9cd117bc8b7fc0903982bbaf474f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Jul 2022 09:03:35 +0200 Subject: block: remove a superflous ifdef in blkdev.h It doesn't hurt to always have the blk_zone_cond_str prototype, and the two inlines can also be defined unconditionally. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20220706070350.1703384-2-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b9a94c53c6cd..270cd0c55292 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -899,8 +899,6 @@ static inline struct request_queue *bdev_get_queue(struct block_device *bdev) return bdev->bd_queue; /* this is never NULL */ } -#ifdef CONFIG_BLK_DEV_ZONED - /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */ const char *blk_zone_cond_str(enum blk_zone_cond zone_cond); @@ -915,7 +913,6 @@ static inline unsigned int bio_zone_is_seq(struct bio *bio) return blk_queue_zone_is_seq(bdev_get_queue(bio->bi_bdev), bio->bi_iter.bi_sector); } -#endif /* CONFIG_BLK_DEV_ZONED */ /* * Return how much of the chunk is left to be used for I/O at a given offset. -- cgit v1.2.3 From 6b2bd274744e6454ba7bbbe6a09b44866f2f414a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Jul 2022 09:03:40 +0200 Subject: block: pass a gendisk to blk_queue_set_zoned Prepare for storing the zone related field in struct gendisk instead of struct request_queue. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20220706070350.1703384-7-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-settings.c | 9 +++++---- block/partitions/core.c | 2 +- drivers/block/null_blk/zoned.c | 2 +- drivers/nvme/host/zns.c | 2 +- drivers/scsi/sd.c | 6 +++--- drivers/scsi/sd_zbc.c | 2 +- include/linux/blkdev.h | 2 +- 7 files changed, 13 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/block/blk-settings.c b/block/blk-settings.c index 6ccceb421ed2..35b7bba306a8 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -893,18 +893,19 @@ static bool disk_has_partitions(struct gendisk *disk) } /** - * blk_queue_set_zoned - configure a disk queue zoned model. + * disk_set_zoned - configure the zoned model for a disk * @disk: the gendisk of the queue to configure * @model: the zoned model to set * - * Set the zoned model of the request queue of @disk according to @model. + * Set the zoned model of @disk to @model. + * * When @model is BLK_ZONED_HM (host managed), this should be called only * if zoned block device support is enabled (CONFIG_BLK_DEV_ZONED option). * If @model specifies BLK_ZONED_HA (host aware), the effective model used * depends on CONFIG_BLK_DEV_ZONED settings and on the existence of partitions * on the disk. */ -void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) +void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model) { struct request_queue *q = disk->queue; @@ -948,7 +949,7 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) blk_queue_clear_zone_settings(q); } } -EXPORT_SYMBOL_GPL(blk_queue_set_zoned); +EXPORT_SYMBOL_GPL(disk_set_zoned); int bdev_alignment_offset(struct block_device *bdev) { diff --git a/block/partitions/core.c b/block/partitions/core.c index 7dc487f5b03c..1a45b1dd6491 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -330,7 +330,7 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, case BLK_ZONED_HA: pr_info("%s: disabling host aware zoned block device support due to partitions\n", disk->disk_name); - blk_queue_set_zoned(disk, BLK_ZONED_NONE); + disk_set_zoned(disk, BLK_ZONED_NONE); break; case BLK_ZONED_NONE: break; diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 2fdd7b20c224..b47bbd114058 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -159,7 +159,7 @@ int null_register_zoned_dev(struct nullb *nullb) struct nullb_device *dev = nullb->dev; struct request_queue *q = nullb->q; - blk_queue_set_zoned(nullb->disk, BLK_ZONED_HM); + disk_set_zoned(nullb->disk, BLK_ZONED_HM); blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE); diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index 9f81beb4df4e..0ed15c2fd56d 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -109,7 +109,7 @@ int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf) goto free_data; } - blk_queue_set_zoned(ns->disk, BLK_ZONED_HM); + disk_set_zoned(ns->disk, BLK_ZONED_HM); blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); blk_queue_max_open_zones(q, le32_to_cpu(id->mor) + 1); blk_queue_max_active_zones(q, le32_to_cpu(id->mar) + 1); diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index cb587e488601..eb02d939dd44 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -2934,15 +2934,15 @@ static void sd_read_block_characteristics(struct scsi_disk *sdkp) if (sdkp->device->type == TYPE_ZBC) { /* Host-managed */ - blk_queue_set_zoned(sdkp->disk, BLK_ZONED_HM); + disk_set_zoned(sdkp->disk, BLK_ZONED_HM); } else { sdkp->zoned = zoned; if (sdkp->zoned == 1) { /* Host-aware */ - blk_queue_set_zoned(sdkp->disk, BLK_ZONED_HA); + disk_set_zoned(sdkp->disk, BLK_ZONED_HA); } else { /* Regular disk or drive managed disk */ - blk_queue_set_zoned(sdkp->disk, BLK_ZONED_NONE); + disk_set_zoned(sdkp->disk, BLK_ZONED_NONE); } } diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 6acc4f406eb8..0f5823b67468 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -929,7 +929,7 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]) /* * This can happen for a host aware disk with partitions. * The block device zone model was already cleared by - * blk_queue_set_zoned(). Only free the scsi disk zone + * disk_set_zoned(). Only free the scsi disk zone * information and exit early. */ sd_zbc_free_zone_info(sdkp); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 270cd0c55292..416faa013782 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -291,7 +291,7 @@ struct queue_limits { typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, void *data); -void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model); +void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model); #ifdef CONFIG_BLK_DEV_ZONED -- cgit v1.2.3 From 1dc0172027b0aa09823b430e395b1116d2745f36 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Jul 2022 09:03:43 +0200 Subject: block: remove queue_max_open_zones and queue_max_active_zones Always use the bdev based helpers instead. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20220706070350.1703384-10-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-sysfs.c | 4 ++-- include/linux/blkdev.h | 37 ++++++++++--------------------------- 2 files changed, 12 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 7590810cf13f..5ce72345ac66 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -330,12 +330,12 @@ static ssize_t queue_nr_zones_show(struct request_queue *q, char *page) static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page) { - return queue_var_show(queue_max_open_zones(q), page); + return queue_var_show(bdev_max_open_zones(q->disk->part0), page); } static ssize_t queue_max_active_zones_show(struct request_queue *q, char *page) { - return queue_var_show(queue_max_active_zones(q), page); + return queue_var_show(bdev_max_active_zones(q->disk->part0), page); } static ssize_t queue_nomerges_show(struct request_queue *q, char *page) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 416faa013782..7d4105d23b0a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -702,21 +702,22 @@ static inline void blk_queue_max_open_zones(struct request_queue *q, q->max_open_zones = max_open_zones; } -static inline unsigned int queue_max_open_zones(const struct request_queue *q) -{ - return q->max_open_zones; -} - static inline void blk_queue_max_active_zones(struct request_queue *q, unsigned int max_active_zones) { q->max_active_zones = max_active_zones; } -static inline unsigned int queue_max_active_zones(const struct request_queue *q) +static inline unsigned int bdev_max_open_zones(struct block_device *bdev) +{ + return bdev->bd_disk->queue->max_open_zones; +} + +static inline unsigned int bdev_max_active_zones(struct block_device *bdev) { - return q->max_active_zones; + return bdev->bd_disk->queue->max_active_zones; } + #else /* CONFIG_BLK_DEV_ZONED */ static inline unsigned int blk_queue_nr_zones(struct request_queue *q) { @@ -732,11 +733,11 @@ static inline unsigned int blk_queue_zone_no(struct request_queue *q, { return 0; } -static inline unsigned int queue_max_open_zones(const struct request_queue *q) +static inline unsigned int bdev_max_open_zones(struct block_device *bdev) { return 0; } -static inline unsigned int queue_max_active_zones(const struct request_queue *q) +static inline unsigned int bdev_max_active_zones(struct block_device *bdev) { return 0; } @@ -1314,24 +1315,6 @@ static inline sector_t bdev_zone_sectors(struct block_device *bdev) return 0; } -static inline unsigned int bdev_max_open_zones(struct block_device *bdev) -{ - struct request_queue *q = bdev_get_queue(bdev); - - if (q) - return queue_max_open_zones(q); - return 0; -} - -static inline unsigned int bdev_max_active_zones(struct block_device *bdev) -{ - struct request_queue *q = bdev_get_queue(bdev); - - if (q) - return queue_max_active_zones(q); - return 0; -} - static inline int queue_dma_alignment(const struct request_queue *q) { return q ? q->dma_alignment : 511; -- cgit v1.2.3 From 982977df48179c8c690868f398051074e68eef0f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Jul 2022 09:03:44 +0200 Subject: block: pass a gendisk to blk_queue_max_open_zones and blk_queue_max_active_zones Switch to a gendisk based API in preparation for moving all zone related fields from the request_queue to the gendisk. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20220706070350.1703384-11-hch@lst.de Signed-off-by: Jens Axboe --- drivers/block/null_blk/zoned.c | 4 ++-- drivers/nvme/host/zns.c | 4 ++-- drivers/scsi/sd_zbc.c | 6 +++--- include/linux/blkdev.h | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index b47bbd114058..576ab3ed082a 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -174,8 +174,8 @@ int null_register_zoned_dev(struct nullb *nullb) } blk_queue_max_zone_append_sectors(q, dev->zone_size_sects); - blk_queue_max_open_zones(q, dev->zone_max_open); - blk_queue_max_active_zones(q, dev->zone_max_active); + disk_set_max_open_zones(nullb->disk, dev->zone_max_open); + disk_set_max_active_zones(nullb->disk, dev->zone_max_active); return 0; } diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index 0ed15c2fd56d..12316ab51bda 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -111,8 +111,8 @@ int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf) disk_set_zoned(ns->disk, BLK_ZONED_HM); blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); - blk_queue_max_open_zones(q, le32_to_cpu(id->mor) + 1); - blk_queue_max_active_zones(q, le32_to_cpu(id->mar) + 1); + disk_set_max_open_zones(ns->disk, le32_to_cpu(id->mor) + 1); + disk_set_max_active_zones(ns->disk, le32_to_cpu(id->mar) + 1); free_data: kfree(id); return status; diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 0f5823b67468..b4106f899734 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -950,10 +950,10 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]) blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE); if (sdkp->zones_max_open == U32_MAX) - blk_queue_max_open_zones(q, 0); + disk_set_max_open_zones(disk, 0); else - blk_queue_max_open_zones(q, sdkp->zones_max_open); - blk_queue_max_active_zones(q, 0); + disk_set_max_open_zones(disk, sdkp->zones_max_open); + disk_set_max_active_zones(disk, 0); nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks); /* diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 7d4105d23b0a..c05e1cc05c26 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -696,16 +696,16 @@ static inline bool blk_queue_zone_is_seq(struct request_queue *q, return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap); } -static inline void blk_queue_max_open_zones(struct request_queue *q, +static inline void disk_set_max_open_zones(struct gendisk *disk, unsigned int max_open_zones) { - q->max_open_zones = max_open_zones; + disk->queue->max_open_zones = max_open_zones; } -static inline void blk_queue_max_active_zones(struct request_queue *q, +static inline void disk_set_max_active_zones(struct gendisk *disk, unsigned int max_active_zones) { - q->max_active_zones = max_active_zones; + disk->queue->max_active_zones = max_active_zones; } static inline unsigned int bdev_max_open_zones(struct block_device *bdev) -- cgit v1.2.3 From b623e347323f6464b20fb0d899a0a73522ed8f6c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Jul 2022 09:03:45 +0200 Subject: block: replace blkdev_nr_zones with bdev_nr_zones Pass a block_device instead of a request_queue as that is what most callers have at hand. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Johannes Thumshirn Reviewed-by: Damien Le Moal Acked-by: Damien Le Moal Link: https://lore.kernel.org/r/20220706070350.1703384-12-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-zoned.c | 15 ++++++++------- block/ioctl.c | 2 +- drivers/block/null_blk/zoned.c | 2 +- drivers/md/dm-zone.c | 2 +- drivers/md/dm-zoned-target.c | 5 ++--- drivers/nvme/target/zns.c | 6 +++--- fs/zonefs/super.c | 17 ++++++++--------- include/linux/blkdev.h | 4 ++-- 8 files changed, 26 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 0d431394cf90..2dec25d8aa3b 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -108,21 +108,22 @@ void __blk_req_zone_write_unlock(struct request *rq) EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock); /** - * blkdev_nr_zones - Get number of zones - * @disk: Target gendisk + * bdev_nr_zones - Get number of zones + * @bdev: Target device * * Return the total number of zones of a zoned block device. For a block * device without zone capabilities, the number of zones is always 0. */ -unsigned int blkdev_nr_zones(struct gendisk *disk) +unsigned int bdev_nr_zones(struct block_device *bdev) { - sector_t zone_sectors = blk_queue_zone_sectors(disk->queue); + sector_t zone_sectors = bdev_zone_sectors(bdev); - if (!blk_queue_is_zoned(disk->queue)) + if (!bdev_is_zoned(bdev)) return 0; - return (get_capacity(disk) + zone_sectors - 1) >> ilog2(zone_sectors); + return (bdev_nr_sectors(bdev) + zone_sectors - 1) >> + ilog2(zone_sectors); } -EXPORT_SYMBOL_GPL(blkdev_nr_zones); +EXPORT_SYMBOL_GPL(bdev_nr_zones); /** * blkdev_report_zones - Get zones information diff --git a/block/ioctl.c b/block/ioctl.c index 46949f1b0dba..60121e89052b 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -495,7 +495,7 @@ static int blkdev_common_ioctl(struct block_device *bdev, fmode_t mode, case BLKGETZONESZ: return put_uint(argp, bdev_zone_sectors(bdev)); case BLKGETNRZONES: - return put_uint(argp, blkdev_nr_zones(bdev->bd_disk)); + return put_uint(argp, bdev_nr_zones(bdev)); case BLKROGET: return put_int(argp, bdev_read_only(bdev) != 0); case BLKSSZGET: /* get block device logical block size */ diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 576ab3ed082a..e62c52e96425 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -170,7 +170,7 @@ int null_register_zoned_dev(struct nullb *nullb) return ret; } else { blk_queue_chunk_sectors(q, dev->zone_size_sects); - q->nr_zones = blkdev_nr_zones(nullb->disk); + q->nr_zones = bdev_nr_zones(nullb->disk->part0); } blk_queue_max_zone_append_sectors(q, dev->zone_size_sects); diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index ae616b87c91a..6d105abe1241 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -301,7 +301,7 @@ int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q) * correct value to be exposed in sysfs queue/nr_zones. */ WARN_ON_ONCE(queue_is_mq(q)); - q->nr_zones = blkdev_nr_zones(md->disk); + q->nr_zones = bdev_nr_zones(md->disk->part0); /* Check if zone append is natively supported */ if (dm_table_supports_zone_append(t)) { diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c index 0ec5d8b9b1a4..6ba6ef44b00e 100644 --- a/drivers/md/dm-zoned-target.c +++ b/drivers/md/dm-zoned-target.c @@ -793,8 +793,7 @@ static int dmz_fixup_devices(struct dm_target *ti) } zone_nr_sectors = blk_queue_zone_sectors(q); zoned_dev->zone_nr_sectors = zone_nr_sectors; - zoned_dev->nr_zones = - blkdev_nr_zones(zoned_dev->bdev->bd_disk); + zoned_dev->nr_zones = bdev_nr_zones(zoned_dev->bdev); } } else { reg_dev = NULL; @@ -805,7 +804,7 @@ static int dmz_fixup_devices(struct dm_target *ti) } q = bdev_get_queue(zoned_dev->bdev); zoned_dev->zone_nr_sectors = blk_queue_zone_sectors(q); - zoned_dev->nr_zones = blkdev_nr_zones(zoned_dev->bdev->bd_disk); + zoned_dev->nr_zones = bdev_nr_zones(zoned_dev->bdev); } if (reg_dev) { diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index 82b61acf7a72..c4c99b832daf 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -60,7 +60,7 @@ bool nvmet_bdev_zns_enable(struct nvmet_ns *ns) if (ns->bdev->bd_disk->queue->conv_zones_bitmap) return false; - ret = blkdev_report_zones(ns->bdev, 0, blkdev_nr_zones(bd_disk), + ret = blkdev_report_zones(ns->bdev, 0, bdev_nr_zones(ns->bdev), validate_conv_zones_cb, NULL); if (ret < 0) return false; @@ -241,7 +241,7 @@ static unsigned long nvmet_req_nr_zones_from_slba(struct nvmet_req *req) { unsigned int sect = nvmet_lba_to_sect(req->ns, req->cmd->zmr.slba); - return blkdev_nr_zones(req->ns->bdev->bd_disk) - + return bdev_nr_zones(req->ns->bdev) - (sect >> ilog2(bdev_zone_sectors(req->ns->bdev))); } @@ -386,7 +386,7 @@ static int zmgmt_send_scan_cb(struct blk_zone *z, unsigned i, void *d) static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req) { struct block_device *bdev = req->ns->bdev; - unsigned int nr_zones = blkdev_nr_zones(bdev->bd_disk); + unsigned int nr_zones = bdev_nr_zones(bdev); struct request_queue *q = bdev_get_queue(bdev); struct bio *bio = NULL; sector_t sector = 0; diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 053299758deb..9c0eef1ff32a 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1394,7 +1394,7 @@ static void zonefs_init_dir_inode(struct inode *parent, struct inode *inode, { struct super_block *sb = parent->i_sb; - inode->i_ino = blkdev_nr_zones(sb->s_bdev->bd_disk) + type + 1; + inode->i_ino = bdev_nr_zones(sb->s_bdev) + type + 1; inode_init_owner(&init_user_ns, inode, parent, S_IFDIR | 0555); inode->i_op = &zonefs_dir_inode_operations; inode->i_fop = &simple_dir_operations; @@ -1540,7 +1540,7 @@ static int zonefs_create_zgroup(struct zonefs_zone_data *zd, /* * The first zone contains the super block: skip it. */ - end = zd->zones + blkdev_nr_zones(sb->s_bdev->bd_disk); + end = zd->zones + bdev_nr_zones(sb->s_bdev); for (zone = &zd->zones[1]; zone < end; zone = next) { next = zone + 1; @@ -1635,8 +1635,8 @@ static int zonefs_get_zone_info(struct zonefs_zone_data *zd) struct block_device *bdev = zd->sb->s_bdev; int ret; - zd->zones = kvcalloc(blkdev_nr_zones(bdev->bd_disk), - sizeof(struct blk_zone), GFP_KERNEL); + zd->zones = kvcalloc(bdev_nr_zones(bdev), sizeof(struct blk_zone), + GFP_KERNEL); if (!zd->zones) return -ENOMEM; @@ -1648,9 +1648,9 @@ static int zonefs_get_zone_info(struct zonefs_zone_data *zd) return ret; } - if (ret != blkdev_nr_zones(bdev->bd_disk)) { + if (ret != bdev_nr_zones(bdev)) { zonefs_err(zd->sb, "Invalid zone report (%d/%u zones)\n", - ret, blkdev_nr_zones(bdev->bd_disk)); + ret, bdev_nr_zones(bdev)); return -EIO; } @@ -1816,8 +1816,7 @@ static int zonefs_fill_super(struct super_block *sb, void *data, int silent) if (ret) goto cleanup; - zonefs_info(sb, "Mounting %u zones", - blkdev_nr_zones(sb->s_bdev->bd_disk)); + zonefs_info(sb, "Mounting %u zones", bdev_nr_zones(sb->s_bdev)); if (!sbi->s_max_wro_seq_files && !sbi->s_max_active_seq_files && @@ -1833,7 +1832,7 @@ static int zonefs_fill_super(struct super_block *sb, void *data, int silent) if (!inode) goto cleanup; - inode->i_ino = blkdev_nr_zones(sb->s_bdev->bd_disk); + inode->i_ino = bdev_nr_zones(sb->s_bdev); inode->i_mode = S_IFDIR | 0555; inode->i_ctime = inode->i_mtime = inode->i_atime = current_time(inode); inode->i_op = &zonefs_dir_inode_operations; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c05e1cc05c26..fa2757ef4a84 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -298,7 +298,7 @@ void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model); #define BLK_ALL_ZONES ((unsigned int)-1) int blkdev_report_zones(struct block_device *bdev, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data); -unsigned int blkdev_nr_zones(struct gendisk *disk); +unsigned int bdev_nr_zones(struct block_device *bdev); extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op, sector_t sectors, sector_t nr_sectors, gfp_t gfp_mask); @@ -312,7 +312,7 @@ extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, #else /* CONFIG_BLK_DEV_ZONED */ -static inline unsigned int blkdev_nr_zones(struct gendisk *disk) +static inline unsigned int bdev_nr_zones(struct block_device *bdev) { return 0; } -- cgit v1.2.3 From de71973c2951cb2ce4b46560f021f03b15906408 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Jul 2022 09:03:49 +0200 Subject: block: remove blk_queue_zone_sectors Always use bdev_zone_sectors instead. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20220706070350.1703384-16-hch@lst.de Signed-off-by: Jens Axboe --- drivers/md/dm-table.c | 4 +--- drivers/md/dm-zone.c | 10 ++++++---- include/linux/blkdev.h | 11 +++-------- 3 files changed, 10 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index b36b528e56cf..df904b7e95ce 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -1620,13 +1620,11 @@ static bool dm_table_supports_zoned_model(struct dm_table *t, static int device_not_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { - struct request_queue *q = bdev_get_queue(dev->bdev); unsigned int *zone_sectors = data; if (!bdev_is_zoned(dev->bdev)) return 0; - - return blk_queue_zone_sectors(q) != *zone_sectors; + return bdev_zone_sectors(dev->bdev) != *zone_sectors; } /* diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index 6d105abe1241..842c31019b51 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -334,7 +334,7 @@ static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx, static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno, unsigned int *wp_ofst) { - sector_t sector = zno * blk_queue_zone_sectors(md->queue); + sector_t sector = zno * bdev_zone_sectors(md->disk->part0); unsigned int noio_flag; struct dm_table *t; int srcu_idx, ret; @@ -373,7 +373,7 @@ struct orig_bio_details { static bool dm_zone_map_bio_begin(struct mapped_device *md, unsigned int zno, struct bio *clone) { - sector_t zsectors = blk_queue_zone_sectors(md->queue); + sector_t zsectors = bdev_zone_sectors(md->disk->part0); unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]); /* @@ -443,7 +443,7 @@ static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int z return BLK_STS_OK; case REQ_OP_ZONE_FINISH: WRITE_ONCE(md->zwp_offset[zno], - blk_queue_zone_sectors(md->queue)); + bdev_zone_sectors(md->disk->part0)); return BLK_STS_OK; case REQ_OP_WRITE_ZEROES: case REQ_OP_WRITE: @@ -593,6 +593,7 @@ void dm_zone_endio(struct dm_io *io, struct bio *clone) { struct mapped_device *md = io->md; struct request_queue *q = md->queue; + struct gendisk *disk = md->disk; struct bio *orig_bio = io->orig_bio; unsigned int zwp_offset; unsigned int zno; @@ -608,7 +609,8 @@ void dm_zone_endio(struct dm_io *io, struct bio *clone) */ if (clone->bi_status == BLK_STS_OK && bio_op(clone) == REQ_OP_ZONE_APPEND) { - sector_t mask = (sector_t)blk_queue_zone_sectors(q) - 1; + sector_t mask = + (sector_t)bdev_zone_sectors(disk->part0) - 1; orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index fa2757ef4a84..21b97f7115dc 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -667,11 +667,6 @@ static inline bool blk_queue_is_zoned(struct request_queue *q) } } -static inline sector_t blk_queue_zone_sectors(struct request_queue *q) -{ - return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0; -} - #ifdef CONFIG_BLK_DEV_ZONED static inline unsigned int blk_queue_nr_zones(struct request_queue *q) { @@ -1310,9 +1305,9 @@ static inline sector_t bdev_zone_sectors(struct block_device *bdev) { struct request_queue *q = bdev_get_queue(bdev); - if (q) - return blk_queue_zone_sectors(q); - return 0; + if (!blk_queue_is_zoned(q)) + return 0; + return q->limits.chunk_sectors; } static inline int queue_dma_alignment(const struct request_queue *q) -- cgit v1.2.3 From d86e716aa40643e3eb8c69fab3a198146bf76dd6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Jul 2022 09:03:50 +0200 Subject: block: move zone related fields to struct gendisk Move the zone related fields that are currently stored in struct request_queue to struct gendisk as these are part of the highlevel block layer API and are only used for non-passthrough I/O that requires the gendisk. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20220706070350.1703384-17-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-mq-debugfs-zoned.c | 6 +-- block/blk-sysfs.c | 2 +- block/blk-zoned.c | 45 ++++++++++----------- drivers/block/null_blk/zoned.c | 2 +- drivers/md/dm-zone.c | 74 ++++++++++++++++------------------ drivers/nvme/host/multipath.c | 2 +- drivers/nvme/target/zns.c | 4 +- drivers/scsi/sd_zbc.c | 2 +- include/linux/blk-mq.h | 8 ++-- include/linux/blkdev.h | 91 +++++++++++++++++++----------------------- 10 files changed, 111 insertions(+), 125 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq-debugfs-zoned.c b/block/blk-mq-debugfs-zoned.c index 038cb627c868..a77b099c34b7 100644 --- a/block/blk-mq-debugfs-zoned.c +++ b/block/blk-mq-debugfs-zoned.c @@ -11,11 +11,11 @@ int queue_zone_wlock_show(void *data, struct seq_file *m) struct request_queue *q = data; unsigned int i; - if (!q->seq_zones_wlock) + if (!q->disk->seq_zones_wlock) return 0; - for (i = 0; i < q->nr_zones; i++) - if (test_bit(i, q->seq_zones_wlock)) + for (i = 0; i < q->disk->nr_zones; i++) + if (test_bit(i, q->disk->seq_zones_wlock)) seq_printf(m, "%u\n", i); return 0; diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 5ce72345ac66..c0303026752d 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -325,7 +325,7 @@ static ssize_t queue_zoned_show(struct request_queue *q, char *page) static ssize_t queue_nr_zones_show(struct request_queue *q, char *page) { - return queue_var_show(blk_queue_nr_zones(q), page); + return queue_var_show(disk_nr_zones(q->disk), page); } static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page) diff --git a/block/blk-zoned.c b/block/blk-zoned.c index c2d8a38f449a..7c017458d5ce 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -57,10 +57,10 @@ EXPORT_SYMBOL_GPL(blk_zone_cond_str); */ bool blk_req_needs_zone_write_lock(struct request *rq) { - if (!rq->q->seq_zones_wlock) + if (blk_rq_is_passthrough(rq)) return false; - if (blk_rq_is_passthrough(rq)) + if (!rq->q->disk->seq_zones_wlock) return false; switch (req_op(rq)) { @@ -77,7 +77,7 @@ bool blk_req_zone_write_trylock(struct request *rq) { unsigned int zno = blk_rq_zone_no(rq); - if (test_and_set_bit(zno, rq->q->seq_zones_wlock)) + if (test_and_set_bit(zno, rq->q->disk->seq_zones_wlock)) return false; WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED); @@ -90,7 +90,7 @@ EXPORT_SYMBOL_GPL(blk_req_zone_write_trylock); void __blk_req_zone_write_lock(struct request *rq) { if (WARN_ON_ONCE(test_and_set_bit(blk_rq_zone_no(rq), - rq->q->seq_zones_wlock))) + rq->q->disk->seq_zones_wlock))) return; WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED); @@ -101,9 +101,9 @@ EXPORT_SYMBOL_GPL(__blk_req_zone_write_lock); void __blk_req_zone_write_unlock(struct request *rq) { rq->rq_flags &= ~RQF_ZONE_WRITE_LOCKED; - if (rq->q->seq_zones_wlock) + if (rq->q->disk->seq_zones_wlock) WARN_ON_ONCE(!test_and_clear_bit(blk_rq_zone_no(rq), - rq->q->seq_zones_wlock)); + rq->q->disk->seq_zones_wlock)); } EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock); @@ -189,7 +189,7 @@ static int blk_zone_need_reset_cb(struct blk_zone *zone, unsigned int idx, static int blkdev_zone_reset_all_emulated(struct block_device *bdev, gfp_t gfp_mask) { - struct request_queue *q = bdev_get_queue(bdev); + struct gendisk *disk = bdev->bd_disk; sector_t capacity = bdev_nr_sectors(bdev); sector_t zone_sectors = bdev_zone_sectors(bdev); unsigned long *need_reset; @@ -197,19 +197,18 @@ static int blkdev_zone_reset_all_emulated(struct block_device *bdev, sector_t sector = 0; int ret; - need_reset = blk_alloc_zone_bitmap(q->node, q->nr_zones); + need_reset = blk_alloc_zone_bitmap(disk->queue->node, disk->nr_zones); if (!need_reset) return -ENOMEM; - ret = bdev->bd_disk->fops->report_zones(bdev->bd_disk, 0, - q->nr_zones, blk_zone_need_reset_cb, - need_reset); + ret = disk->fops->report_zones(disk, 0, disk->nr_zones, + blk_zone_need_reset_cb, need_reset); if (ret < 0) goto out_free_need_reset; ret = 0; while (sector < capacity) { - if (!test_bit(blk_queue_zone_no(q, sector), need_reset)) { + if (!test_bit(disk_zone_no(disk, sector), need_reset)) { sector += zone_sectors; continue; } @@ -452,12 +451,10 @@ fail: void disk_free_zone_bitmaps(struct gendisk *disk) { - struct request_queue *q = disk->queue; - - kfree(q->conv_zones_bitmap); - q->conv_zones_bitmap = NULL; - kfree(q->seq_zones_wlock); - q->seq_zones_wlock = NULL; + kfree(disk->conv_zones_bitmap); + disk->conv_zones_bitmap = NULL; + kfree(disk->seq_zones_wlock); + disk->seq_zones_wlock = NULL; } struct blk_revalidate_zone_args { @@ -607,9 +604,9 @@ int blk_revalidate_disk_zones(struct gendisk *disk, blk_mq_freeze_queue(q); if (ret > 0) { blk_queue_chunk_sectors(q, args.zone_sectors); - q->nr_zones = args.nr_zones; - swap(q->seq_zones_wlock, args.seq_zones_wlock); - swap(q->conv_zones_bitmap, args.conv_zones_bitmap); + disk->nr_zones = args.nr_zones; + swap(disk->seq_zones_wlock, args.seq_zones_wlock); + swap(disk->conv_zones_bitmap, args.conv_zones_bitmap); if (update_driver_data) update_driver_data(disk); ret = 0; @@ -634,9 +631,9 @@ void disk_clear_zone_settings(struct gendisk *disk) disk_free_zone_bitmaps(disk); blk_queue_flag_clear(QUEUE_FLAG_ZONE_RESETALL, q); q->required_elevator_features &= ~ELEVATOR_F_ZBD_SEQ_WRITE; - q->nr_zones = 0; - q->max_open_zones = 0; - q->max_active_zones = 0; + disk->nr_zones = 0; + disk->max_open_zones = 0; + disk->max_active_zones = 0; q->limits.chunk_sectors = 0; q->limits.zone_write_granularity = 0; q->limits.max_zone_append_sectors = 0; diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index e62c52e96425..64b06caab984 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -170,7 +170,7 @@ int null_register_zoned_dev(struct nullb *nullb) return ret; } else { blk_queue_chunk_sectors(q, dev->zone_size_sects); - q->nr_zones = bdev_nr_zones(nullb->disk->part0); + nullb->disk->nr_zones = bdev_nr_zones(nullb->disk->part0); } blk_queue_max_zone_append_sectors(q, dev->zone_size_sects); diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index 842c31019b51..2b89cde30c9e 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -139,13 +139,11 @@ bool dm_is_zone_write(struct mapped_device *md, struct bio *bio) void dm_cleanup_zoned_dev(struct mapped_device *md) { - struct request_queue *q = md->queue; - - if (q) { - kfree(q->conv_zones_bitmap); - q->conv_zones_bitmap = NULL; - kfree(q->seq_zones_wlock); - q->seq_zones_wlock = NULL; + if (md->disk) { + kfree(md->disk->conv_zones_bitmap); + md->disk->conv_zones_bitmap = NULL; + kfree(md->disk->seq_zones_wlock); + md->disk->seq_zones_wlock = NULL; } kvfree(md->zwp_offset); @@ -179,31 +177,31 @@ static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx, void *data) { struct mapped_device *md = data; - struct request_queue *q = md->queue; + struct gendisk *disk = md->disk; switch (zone->type) { case BLK_ZONE_TYPE_CONVENTIONAL: - if (!q->conv_zones_bitmap) { - q->conv_zones_bitmap = - kcalloc(BITS_TO_LONGS(q->nr_zones), + if (!disk->conv_zones_bitmap) { + disk->conv_zones_bitmap = + kcalloc(BITS_TO_LONGS(disk->nr_zones), sizeof(unsigned long), GFP_NOIO); - if (!q->conv_zones_bitmap) + if (!disk->conv_zones_bitmap) return -ENOMEM; } - set_bit(idx, q->conv_zones_bitmap); + set_bit(idx, disk->conv_zones_bitmap); break; case BLK_ZONE_TYPE_SEQWRITE_REQ: case BLK_ZONE_TYPE_SEQWRITE_PREF: - if (!q->seq_zones_wlock) { - q->seq_zones_wlock = - kcalloc(BITS_TO_LONGS(q->nr_zones), + if (!disk->seq_zones_wlock) { + disk->seq_zones_wlock = + kcalloc(BITS_TO_LONGS(disk->nr_zones), sizeof(unsigned long), GFP_NOIO); - if (!q->seq_zones_wlock) + if (!disk->seq_zones_wlock) return -ENOMEM; } if (!md->zwp_offset) { md->zwp_offset = - kvcalloc(q->nr_zones, sizeof(unsigned int), + kvcalloc(disk->nr_zones, sizeof(unsigned int), GFP_KERNEL); if (!md->zwp_offset) return -ENOMEM; @@ -228,7 +226,7 @@ static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx, */ static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t) { - struct request_queue *q = md->queue; + struct gendisk *disk = md->disk; unsigned int noio_flag; int ret; @@ -236,7 +234,7 @@ static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t) * Check if something changed. If yes, cleanup the current resources * and reallocate everything. */ - if (!q->nr_zones || q->nr_zones != md->nr_zones) + if (!disk->nr_zones || disk->nr_zones != md->nr_zones) dm_cleanup_zoned_dev(md); if (md->nr_zones) return 0; @@ -246,17 +244,17 @@ static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t) * operations in this context are done as if GFP_NOIO was specified. */ noio_flag = memalloc_noio_save(); - ret = dm_blk_do_report_zones(md, t, 0, q->nr_zones, + ret = dm_blk_do_report_zones(md, t, 0, disk->nr_zones, dm_zone_revalidate_cb, md); memalloc_noio_restore(noio_flag); if (ret < 0) goto err; - if (ret != q->nr_zones) { + if (ret != disk->nr_zones) { ret = -EIO; goto err; } - md->nr_zones = q->nr_zones; + md->nr_zones = disk->nr_zones; return 0; @@ -301,7 +299,7 @@ int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q) * correct value to be exposed in sysfs queue/nr_zones. */ WARN_ON_ONCE(queue_is_mq(q)); - q->nr_zones = bdev_nr_zones(md->disk->part0); + md->disk->nr_zones = bdev_nr_zones(md->disk->part0); /* Check if zone append is natively supported */ if (dm_table_supports_zone_append(t)) { @@ -466,26 +464,26 @@ static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int z } } -static inline void dm_zone_lock(struct request_queue *q, - unsigned int zno, struct bio *clone) +static inline void dm_zone_lock(struct gendisk *disk, unsigned int zno, + struct bio *clone) { if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))) return; - wait_on_bit_lock_io(q->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE); + wait_on_bit_lock_io(disk->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE); bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED); } -static inline void dm_zone_unlock(struct request_queue *q, - unsigned int zno, struct bio *clone) +static inline void dm_zone_unlock(struct gendisk *disk, unsigned int zno, + struct bio *clone) { if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)) return; - WARN_ON_ONCE(!test_bit(zno, q->seq_zones_wlock)); - clear_bit_unlock(zno, q->seq_zones_wlock); + WARN_ON_ONCE(!test_bit(zno, disk->seq_zones_wlock)); + clear_bit_unlock(zno, disk->seq_zones_wlock); smp_mb__after_atomic(); - wake_up_bit(q->seq_zones_wlock, zno); + wake_up_bit(disk->seq_zones_wlock, zno); bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED); } @@ -520,7 +518,6 @@ int dm_zone_map_bio(struct dm_target_io *tio) struct dm_io *io = tio->io; struct dm_target *ti = tio->ti; struct mapped_device *md = io->md; - struct request_queue *q = md->queue; struct bio *clone = &tio->clone; struct orig_bio_details orig_bio_details; unsigned int zno; @@ -536,7 +533,7 @@ int dm_zone_map_bio(struct dm_target_io *tio) /* Lock the target zone */ zno = bio_zone_no(clone); - dm_zone_lock(q, zno, clone); + dm_zone_lock(md->disk, zno, clone); orig_bio_details.nr_sectors = bio_sectors(clone); orig_bio_details.op = bio_op(clone); @@ -546,7 +543,7 @@ int dm_zone_map_bio(struct dm_target_io *tio) * both valid, and if the bio is a zone append, remap it to a write. */ if (!dm_zone_map_bio_begin(md, zno, clone)) { - dm_zone_unlock(q, zno, clone); + dm_zone_unlock(md->disk, zno, clone); return DM_MAPIO_KILL; } @@ -570,12 +567,12 @@ int dm_zone_map_bio(struct dm_target_io *tio) sts = dm_zone_map_bio_end(md, zno, &orig_bio_details, *tio->len_ptr); if (sts != BLK_STS_OK) - dm_zone_unlock(q, zno, clone); + dm_zone_unlock(md->disk, zno, clone); break; case DM_MAPIO_REQUEUE: case DM_MAPIO_KILL: default: - dm_zone_unlock(q, zno, clone); + dm_zone_unlock(md->disk, zno, clone); sts = BLK_STS_IOERR; break; } @@ -592,7 +589,6 @@ int dm_zone_map_bio(struct dm_target_io *tio) void dm_zone_endio(struct dm_io *io, struct bio *clone) { struct mapped_device *md = io->md; - struct request_queue *q = md->queue; struct gendisk *disk = md->disk; struct bio *orig_bio = io->orig_bio; unsigned int zwp_offset; @@ -651,5 +647,5 @@ void dm_zone_endio(struct dm_io *io, struct bio *clone) zwp_offset - bio_sectors(orig_bio); } - dm_zone_unlock(q, zno, clone); + dm_zone_unlock(disk, zno, clone); } diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index ccf9a6da8f6e..f26640ccb955 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -830,7 +830,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) ns->head->disk->queue); #ifdef CONFIG_BLK_DEV_ZONED if (blk_queue_is_zoned(ns->queue) && ns->head->disk) - ns->head->disk->queue->nr_zones = ns->queue->nr_zones; + ns->head->disk->nr_zones = ns->disk->nr_zones; #endif } diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index 9d8717126ab3..c0ee21fcab81 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -57,7 +57,7 @@ bool nvmet_bdev_zns_enable(struct nvmet_ns *ns) * zones, reject the device. Otherwise, use report zones to detect if * the device has conventional zones. */ - if (ns->bdev->bd_disk->queue->conv_zones_bitmap) + if (ns->bdev->bd_disk->conv_zones_bitmap) return false; ret = blkdev_report_zones(ns->bdev, 0, bdev_nr_zones(ns->bdev), @@ -414,7 +414,7 @@ static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req) } while (sector < bdev_nr_sectors(bdev)) { - if (test_bit(blk_queue_zone_no(q, sector), d.zbitmap)) { + if (test_bit(disk_zone_no(bdev->bd_disk, sector), d.zbitmap)) { bio = blk_next_bio(bio, bdev, 0, zsa_req_op(req->cmd->zms.zsa) | REQ_SYNC, GFP_KERNEL); diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index b4106f899734..b8c97456506a 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -855,7 +855,7 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) if (sdkp->zone_info.zone_blocks == zone_blocks && sdkp->zone_info.nr_zones == nr_zones && - disk->queue->nr_zones == nr_zones) + disk->nr_zones == nr_zones) goto unlock; flags = memalloc_noio_save(); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 810a24884f7e..d74f6a6b7e69 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -1129,12 +1129,12 @@ void blk_dump_rq_flags(struct request *, char *); #ifdef CONFIG_BLK_DEV_ZONED static inline unsigned int blk_rq_zone_no(struct request *rq) { - return blk_queue_zone_no(rq->q, blk_rq_pos(rq)); + return disk_zone_no(rq->q->disk, blk_rq_pos(rq)); } static inline unsigned int blk_rq_zone_is_seq(struct request *rq) { - return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq)); + return disk_zone_is_seq(rq->q->disk, blk_rq_pos(rq)); } bool blk_req_needs_zone_write_lock(struct request *rq); @@ -1156,8 +1156,8 @@ static inline void blk_req_zone_write_unlock(struct request *rq) static inline bool blk_req_zone_is_write_locked(struct request *rq) { - return rq->q->seq_zones_wlock && - test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock); + return rq->q->disk->seq_zones_wlock && + test_bit(blk_rq_zone_no(rq), rq->q->disk->seq_zones_wlock); } static inline bool blk_req_can_dispatch_to_zone(struct request *rq) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 21b97f7115dc..22c477fadc0f 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -164,6 +164,29 @@ struct gendisk { #ifdef CONFIG_BLK_DEV_INTEGRITY struct kobject integrity_kobj; #endif /* CONFIG_BLK_DEV_INTEGRITY */ + +#ifdef CONFIG_BLK_DEV_ZONED + /* + * Zoned block device information for request dispatch control. + * nr_zones is the total number of zones of the device. This is always + * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones + * bits which indicates if a zone is conventional (bit set) or + * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones + * bits which indicates if a zone is write locked, that is, if a write + * request targeting the zone was dispatched. + * + * Reads of this information must be protected with blk_queue_enter() / + * blk_queue_exit(). Modifying this information is only allowed while + * no requests are being processed. See also blk_mq_freeze_queue() and + * blk_mq_unfreeze_queue(). + */ + unsigned int nr_zones; + unsigned int max_open_zones; + unsigned int max_active_zones; + unsigned long *conv_zones_bitmap; + unsigned long *seq_zones_wlock; +#endif /* CONFIG_BLK_DEV_ZONED */ + #if IS_ENABLED(CONFIG_CDROM) struct cdrom_device_info *cdi; #endif @@ -467,31 +490,6 @@ struct request_queue { unsigned int required_elevator_features; -#ifdef CONFIG_BLK_DEV_ZONED - /* - * Zoned block device information for request dispatch control. - * nr_zones is the total number of zones of the device. This is always - * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones - * bits which indicates if a zone is conventional (bit set) or - * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones - * bits which indicates if a zone is write locked, that is, if a write - * request targeting the zone was dispatched. All three fields are - * initialized by the low level device driver (e.g. scsi/sd.c). - * Stacking drivers (device mappers) may or may not initialize - * these fields. - * - * Reads of this information must be protected with blk_queue_enter() / - * blk_queue_exit(). Modifying this information is only allowed while - * no requests are being processed. See also blk_mq_freeze_queue() and - * blk_mq_unfreeze_queue(). - */ - unsigned int nr_zones; - unsigned long *conv_zones_bitmap; - unsigned long *seq_zones_wlock; - unsigned int max_open_zones; - unsigned int max_active_zones; -#endif /* CONFIG_BLK_DEV_ZONED */ - int node; #ifdef CONFIG_BLK_DEV_IO_TRACE struct blk_trace __rcu *blk_trace; @@ -668,63 +666,59 @@ static inline bool blk_queue_is_zoned(struct request_queue *q) } #ifdef CONFIG_BLK_DEV_ZONED -static inline unsigned int blk_queue_nr_zones(struct request_queue *q) +static inline unsigned int disk_nr_zones(struct gendisk *disk) { - return blk_queue_is_zoned(q) ? q->nr_zones : 0; + return blk_queue_is_zoned(disk->queue) ? disk->nr_zones : 0; } -static inline unsigned int blk_queue_zone_no(struct request_queue *q, - sector_t sector) +static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector) { - if (!blk_queue_is_zoned(q)) + if (!blk_queue_is_zoned(disk->queue)) return 0; - return sector >> ilog2(q->limits.chunk_sectors); + return sector >> ilog2(disk->queue->limits.chunk_sectors); } -static inline bool blk_queue_zone_is_seq(struct request_queue *q, - sector_t sector) +static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector) { - if (!blk_queue_is_zoned(q)) + if (!blk_queue_is_zoned(disk->queue)) return false; - if (!q->conv_zones_bitmap) + if (!disk->conv_zones_bitmap) return true; - return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap); + return !test_bit(disk_zone_no(disk, sector), disk->conv_zones_bitmap); } static inline void disk_set_max_open_zones(struct gendisk *disk, unsigned int max_open_zones) { - disk->queue->max_open_zones = max_open_zones; + disk->max_open_zones = max_open_zones; } static inline void disk_set_max_active_zones(struct gendisk *disk, unsigned int max_active_zones) { - disk->queue->max_active_zones = max_active_zones; + disk->max_active_zones = max_active_zones; } static inline unsigned int bdev_max_open_zones(struct block_device *bdev) { - return bdev->bd_disk->queue->max_open_zones; + return bdev->bd_disk->max_open_zones; } static inline unsigned int bdev_max_active_zones(struct block_device *bdev) { - return bdev->bd_disk->queue->max_active_zones; + return bdev->bd_disk->max_active_zones; } #else /* CONFIG_BLK_DEV_ZONED */ -static inline unsigned int blk_queue_nr_zones(struct request_queue *q) +static inline unsigned int disk_nr_zones(struct gendisk *disk) { return 0; } -static inline bool blk_queue_zone_is_seq(struct request_queue *q, - sector_t sector) +static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector) { return false; } -static inline unsigned int blk_queue_zone_no(struct request_queue *q, - sector_t sector) +static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector) { return 0; } @@ -732,6 +726,7 @@ static inline unsigned int bdev_max_open_zones(struct block_device *bdev) { return 0; } + static inline unsigned int bdev_max_active_zones(struct block_device *bdev) { return 0; @@ -900,14 +895,12 @@ const char *blk_zone_cond_str(enum blk_zone_cond zone_cond); static inline unsigned int bio_zone_no(struct bio *bio) { - return blk_queue_zone_no(bdev_get_queue(bio->bi_bdev), - bio->bi_iter.bi_sector); + return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector); } static inline unsigned int bio_zone_is_seq(struct bio *bio) { - return blk_queue_zone_is_seq(bdev_get_queue(bio->bi_bdev), - bio->bi_iter.bi_sector); + return disk_zone_is_seq(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector); } /* -- cgit v1.2.3 From 2d693ed436a67db06d1473c22d4caee899a4e9d2 Mon Sep 17 00:00:00 2001 From: James Clark Date: Wed, 11 May 2022 15:45:58 +0100 Subject: coresight: Add config flag to enable branch broadcast When enabled, all taken branch addresses are output, even if the branch was because of a direct branch instruction. This enables reconstruction of the program flow without having access to the memory image of the code being executed. Use bit 8 for the config option which would be the correct bit for programming ETMv3. Although branch broadcast can't be enabled on ETMv3 because it's not in the define ETM3X_SUPPORTED_OPTIONS, using the correct bit might help prevent future collisions or allow it to be enabled if needed. Signed-off-by: James Clark Reviewed-by: Mike Leach Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20220511144601.2257870-2-james.clark@arm.com --- drivers/hwtracing/coresight/coresight-etm-perf.c | 2 ++ drivers/hwtracing/coresight/coresight-etm4x-core.c | 14 ++++++++++++++ include/linux/coresight-pmu.h | 2 ++ 3 files changed, 18 insertions(+) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index c039b6ae206f..43bbd5dc3d3b 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -52,6 +52,7 @@ static DEFINE_PER_CPU(struct coresight_device *, csdev_src); * The PMU formats were orignally for ETMv3.5/PTM's ETMCR 'config'; * now take them as general formats and apply on all ETMs. */ +PMU_FORMAT_ATTR(branch_broadcast, "config:"__stringify(ETM_OPT_BRANCH_BROADCAST)); PMU_FORMAT_ATTR(cycacc, "config:" __stringify(ETM_OPT_CYCACC)); /* contextid1 enables tracing CONTEXTIDR_EL1 for ETMv4 */ PMU_FORMAT_ATTR(contextid1, "config:" __stringify(ETM_OPT_CTXTID)); @@ -97,6 +98,7 @@ static struct attribute *etm_config_formats_attr[] = { &format_attr_sinkid.attr, &format_attr_preset.attr, &format_attr_configid.attr, + &format_attr_branch_broadcast.attr, NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 87299e99dabb..cf249ecad5a5 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -696,6 +696,20 @@ static int etm4_parse_event_config(struct coresight_device *csdev, ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); } + /* branch broadcast - enable if selected and supported */ + if (attr->config & BIT(ETM_OPT_BRANCH_BROADCAST)) { + if (!drvdata->trcbb) { + /* + * Missing BB support could cause silent decode errors + * so fail to open if it's not supported. + */ + ret = -EINVAL; + goto out; + } else { + config->cfg |= BIT(ETM4_CFG_BIT_BB); + } + } + out: return ret; } diff --git a/include/linux/coresight-pmu.h b/include/linux/coresight-pmu.h index 4ac5c081af93..6c2fd6cc5a98 100644 --- a/include/linux/coresight-pmu.h +++ b/include/linux/coresight-pmu.h @@ -18,6 +18,7 @@ * ETMv3.5/PTM doesn't define ETMCR config bits with prefix "ETM3_" and * directly use below macros as config bits. */ +#define ETM_OPT_BRANCH_BROADCAST 8 #define ETM_OPT_CYCACC 12 #define ETM_OPT_CTXTID 14 #define ETM_OPT_CTXTID2 15 @@ -25,6 +26,7 @@ #define ETM_OPT_RETSTK 29 /* ETMv4 CONFIGR programming bits for the ETM OPTs */ +#define ETM4_CFG_BIT_BB 3 #define ETM4_CFG_BIT_CYCACC 4 #define ETM4_CFG_BIT_CTXTID 6 #define ETM4_CFG_BIT_VMID 7 -- cgit v1.2.3 From bfdd231374181254742c5e2faef0bef2d30c0ee4 Mon Sep 17 00:00:00 2001 From: Yunfei Wang Date: Thu, 30 Jun 2022 17:29:25 +0800 Subject: iommu/io-pgtable-arm-v7s: Add a quirk to allow pgtable PA up to 35bit Single memory zone feature will remove ZONE_DMA32 and ZONE_DMA and cause pgtable PA size larger than 32bit. Since Mediatek IOMMU hardware support at most 35bit PA in pgtable, so add a quirk to allow the PA of pgtables support up to bit35. Signed-off-by: Ning Li Signed-off-by: Yunfei Wang Reviewed-by: Robin Murphy Acked-by: Will Deacon Link: https://lore.kernel.org/r/20220630092927.24925-2-yf.wang@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm-v7s.c | 75 ++++++++++++++++++++++++++++---------- include/linux/io-pgtable.h | 15 +++++--- 2 files changed, 66 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index be066c1503d3..ba3115fd0f86 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -182,14 +182,8 @@ static bool arm_v7s_is_mtk_enabled(struct io_pgtable_cfg *cfg) (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT); } -static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl, - struct io_pgtable_cfg *cfg) +static arm_v7s_iopte to_mtk_iopte(phys_addr_t paddr, arm_v7s_iopte pte) { - arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl); - - if (!arm_v7s_is_mtk_enabled(cfg)) - return pte; - if (paddr & BIT_ULL(32)) pte |= ARM_V7S_ATTR_MTK_PA_BIT32; if (paddr & BIT_ULL(33)) @@ -199,6 +193,17 @@ static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl, return pte; } +static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl, + struct io_pgtable_cfg *cfg) +{ + arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl); + + if (arm_v7s_is_mtk_enabled(cfg)) + return to_mtk_iopte(paddr, pte); + + return pte; +} + static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl, struct io_pgtable_cfg *cfg) { @@ -240,10 +245,17 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, dma_addr_t dma; size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg); void *table = NULL; + gfp_t gfp_l1; + + /* + * ARM_MTK_TTBR_EXT extend the translation table base support larger + * memory address. + */ + gfp_l1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? + GFP_KERNEL : ARM_V7S_TABLE_GFP_DMA; if (lvl == 1) - table = (void *)__get_free_pages( - __GFP_ZERO | ARM_V7S_TABLE_GFP_DMA, get_order(size)); + table = (void *)__get_free_pages(gfp_l1 | __GFP_ZERO, get_order(size)); else if (lvl == 2) table = kmem_cache_zalloc(data->l2_tables, gfp); @@ -251,7 +263,8 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, return NULL; phys = virt_to_phys(table); - if (phys != (arm_v7s_iopte)phys) { + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? + phys >= (1ULL << cfg->oas) : phys != (arm_v7s_iopte)phys) { /* Doesn't fit in PTE */ dev_err(dev, "Page table does not fit in PTE: %pa", &phys); goto out_free; @@ -457,9 +470,14 @@ static arm_v7s_iopte arm_v7s_install_table(arm_v7s_iopte *table, arm_v7s_iopte curr, struct io_pgtable_cfg *cfg) { + phys_addr_t phys = virt_to_phys(table); arm_v7s_iopte old, new; - new = virt_to_phys(table) | ARM_V7S_PTE_TYPE_TABLE; + new = phys | ARM_V7S_PTE_TYPE_TABLE; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) + new = to_mtk_iopte(phys, new); + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) new |= ARM_V7S_ATTR_NS_TABLE; @@ -779,6 +797,8 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) { struct arm_v7s_io_pgtable *data; + slab_flags_t slab_flag; + phys_addr_t paddr; if (cfg->ias > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS)) return NULL; @@ -788,7 +808,8 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | - IO_PGTABLE_QUIRK_ARM_MTK_EXT)) + IO_PGTABLE_QUIRK_ARM_MTK_EXT | + IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT)) return NULL; /* If ARM_MTK_4GB is enabled, the NO_PERMS is also expected. */ @@ -796,15 +817,27 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS)) return NULL; + if ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) && + !arm_v7s_is_mtk_enabled(cfg)) + return NULL; + data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return NULL; spin_lock_init(&data->split_lock); + + /* + * ARM_MTK_TTBR_EXT extend the translation table base support larger + * memory address. + */ + slab_flag = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? + 0 : ARM_V7S_TABLE_SLAB_FLAGS; + data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2", ARM_V7S_TABLE_SIZE(2, cfg), ARM_V7S_TABLE_SIZE(2, cfg), - ARM_V7S_TABLE_SLAB_FLAGS, NULL); + slab_flag, NULL); if (!data->l2_tables) goto out_free_data; @@ -850,12 +883,16 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, wmb(); /* TTBR */ - cfg->arm_v7s_cfg.ttbr = virt_to_phys(data->pgd) | ARM_V7S_TTBR_S | - (cfg->coherent_walk ? (ARM_V7S_TTBR_NOS | - ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | - ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) : - (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) | - ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC))); + paddr = virt_to_phys(data->pgd); + if (arm_v7s_is_mtk_enabled(cfg)) + cfg->arm_v7s_cfg.ttbr = paddr | upper_32_bits(paddr); + else + cfg->arm_v7s_cfg.ttbr = paddr | ARM_V7S_TTBR_S | + (cfg->coherent_walk ? (ARM_V7S_TTBR_NOS | + ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) : + (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC))); return &data->iop; out_free_data: diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 86af6f0a00a2..ca98aeadcc80 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -74,17 +74,22 @@ struct io_pgtable_cfg { * to support up to 35 bits PA where the bit32, bit33 and bit34 are * encoded in the bit9, bit4 and bit5 of the PTE respectively. * + * IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT: (ARM v7s format) MediaTek IOMMUs + * extend the translation table base support up to 35 bits PA, the + * encoding format is same with IO_PGTABLE_QUIRK_ARM_MTK_EXT. + * * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table * for use in the upper half of a split address space. * * IO_PGTABLE_QUIRK_ARM_OUTER_WBWA: Override the outer-cacheability * attributes set in the TCR for a non-coherent page-table walker. */ - #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) - #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) - #define IO_PGTABLE_QUIRK_ARM_MTK_EXT BIT(3) - #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(5) - #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA BIT(6) + #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) + #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) + #define IO_PGTABLE_QUIRK_ARM_MTK_EXT BIT(3) + #define IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT BIT(4) + #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(5) + #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA BIT(6) unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; -- cgit v1.2.3 From 2aec377a29250b942f14d3c16d49783da3e9df11 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 5 Jul 2022 14:00:36 -0400 Subject: dm table: remove dm_table_get_num_targets() wrapper More efficient and readable to just access table->num_targets directly. Suggested-by: Linus Torvalds Signed-off-by: Mike Snitzer --- drivers/md/dm-ima.c | 2 +- drivers/md/dm-ioctl.c | 6 +++--- drivers/md/dm-table.c | 37 ++++++++++++++++--------------------- drivers/md/dm-zone.c | 2 +- drivers/md/dm.c | 4 ++-- include/linux/device-mapper.h | 1 - 6 files changed, 23 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/drivers/md/dm-ima.c b/drivers/md/dm-ima.c index 1842d3a958ef..3c15c069947f 100644 --- a/drivers/md/dm-ima.c +++ b/drivers/md/dm-ima.c @@ -208,7 +208,7 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl if (!target_data_buf) goto error; - num_targets = dm_table_get_num_targets(table); + num_targets = table->num_targets; if (dm_ima_alloc_and_copy_device_data(table->md, &device_data_buf, num_targets, noio)) goto error; diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 87310fceb0d8..98976aaa9db9 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -832,7 +832,7 @@ static void __dev_status(struct mapped_device *md, struct dm_ioctl *param) if (!(param->flags & DM_QUERY_INACTIVE_TABLE_FLAG)) { if (get_disk_ro(disk)) param->flags |= DM_READONLY_FLAG; - param->target_count = dm_table_get_num_targets(table); + param->target_count = table->num_targets; } param->flags |= DM_ACTIVE_PRESENT_FLAG; @@ -845,7 +845,7 @@ static void __dev_status(struct mapped_device *md, struct dm_ioctl *param) if (table) { if (!(dm_table_get_mode(table) & FMODE_WRITE)) param->flags |= DM_READONLY_FLAG; - param->target_count = dm_table_get_num_targets(table); + param->target_count = table->num_targets; } dm_put_live_table(md, srcu_idx); } @@ -1248,7 +1248,7 @@ static void retrieve_status(struct dm_table *table, type = STATUSTYPE_INFO; /* Get all the target info */ - num_targets = dm_table_get_num_targets(table); + num_targets = table->num_targets; for (i = 0; i < num_targets; i++) { struct dm_target *ti = dm_table_get_target(table, i); size_t l; diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 3f29b1113294..b4af34041a6f 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -593,7 +593,7 @@ static int validate_hardware_logical_block_alignment(struct dm_table *table, /* * Check each entry in the table in turn. */ - for (i = 0; i < dm_table_get_num_targets(table); i++) { + for (i = 0; i < table->num_targets; i++) { ti = dm_table_get_target(table, i); blk_set_stacking_limits(&ti_limits); @@ -832,7 +832,7 @@ static bool dm_table_supports_dax(struct dm_table *t, unsigned i; /* Ensure that all targets support DAX. */ - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->type->direct_access) @@ -987,7 +987,7 @@ struct dm_target *dm_table_get_wildcard_target(struct dm_table *t) struct dm_target *ti; unsigned i; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (dm_target_is_wildcard(ti->type)) return ti; @@ -1127,7 +1127,7 @@ static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t) struct gendisk *prev_disk = NULL, *template_disk = NULL; unsigned i; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { struct dm_target *ti = dm_table_get_target(t, i); if (!dm_target_passes_integrity(ti->type)) goto no_integrity; @@ -1248,7 +1248,7 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile, t = dm_get_live_table(md, &srcu_idx); if (!t) return 0; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->type->iterate_devices) continue; @@ -1318,7 +1318,7 @@ static int dm_table_construct_crypto_profile(struct dm_table *t) memset(profile->modes_supported, 0xFF, sizeof(profile->modes_supported)); - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!dm_target_passes_crypto(ti->type)) { @@ -1540,7 +1540,7 @@ static bool dm_table_any_dev_attr(struct dm_table *t, struct dm_target *ti; unsigned int i; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (ti->type->iterate_devices && @@ -1566,7 +1566,7 @@ static bool dm_table_supports_poll(struct dm_table *t) struct dm_target *ti; unsigned i = 0; - while (i < dm_table_get_num_targets(t)) { + while (i < t->num_targets) { ti = dm_table_get_target(t, i++); if (!ti->type->iterate_devices || @@ -1588,7 +1588,7 @@ bool dm_table_has_no_data_devices(struct dm_table *table) struct dm_target *ti; unsigned i, num_devices; - for (i = 0; i < dm_table_get_num_targets(table); i++) { + for (i = 0; i < table->num_targets; i++) { ti = dm_table_get_target(table, i); if (!ti->type->iterate_devices) @@ -1625,7 +1625,7 @@ static bool dm_table_supports_zoned_model(struct dm_table *t, struct dm_target *ti; unsigned i; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (dm_target_supports_zoned_hm(ti->type)) { @@ -1699,7 +1699,7 @@ int dm_calculate_queue_limits(struct dm_table *table, blk_set_stacking_limits(limits); - for (i = 0; i < dm_table_get_num_targets(table); i++) { + for (i = 0; i < table->num_targets; i++) { blk_set_stacking_limits(&ti_limits); ti = dm_table_get_target(table, i); @@ -1819,7 +1819,7 @@ static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush) * so we need to use iterate_devices here, which targets * supporting flushes must provide. */ - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->num_flush_bios) @@ -1877,7 +1877,7 @@ static bool dm_table_supports_write_zeroes(struct dm_table *t) struct dm_target *ti; unsigned i = 0; - while (i < dm_table_get_num_targets(t)) { + while (i < t->num_targets) { ti = dm_table_get_target(t, i++); if (!ti->num_write_zeroes_bios) @@ -1904,7 +1904,7 @@ static bool dm_table_supports_nowait(struct dm_table *t) struct dm_target *ti; unsigned i = 0; - while (i < dm_table_get_num_targets(t)) { + while (i < t->num_targets) { ti = dm_table_get_target(t, i++); if (!dm_target_supports_nowait(ti->type)) @@ -1929,7 +1929,7 @@ static bool dm_table_supports_discards(struct dm_table *t) struct dm_target *ti; unsigned i; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->num_discard_bios) @@ -1961,7 +1961,7 @@ static bool dm_table_supports_secure_erase(struct dm_table *t) struct dm_target *ti; unsigned int i; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->num_secure_erase_bios) @@ -2092,11 +2092,6 @@ int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, return 0; } -unsigned int dm_table_get_num_targets(struct dm_table *t) -{ - return t->num_targets; -} - struct list_head *dm_table_get_devices(struct dm_table *t) { return &t->devices; diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index 3e7b1fe1580b..16dc9ec4bb11 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -278,7 +278,7 @@ static bool dm_table_supports_zone_append(struct dm_table *t) struct dm_target *ti; unsigned int i; - for (i = 0; i < dm_table_get_num_targets(t); i++) { + for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (ti->emulate_zone_append) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index fa6839141118..44dae3869f29 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -422,7 +422,7 @@ retry: return r; /* We only support devices that have a single target */ - if (dm_table_get_num_targets(map) != 1) + if (map->num_targets != 1) return r; tgt = dm_table_get_target(map, 0); @@ -3092,7 +3092,7 @@ static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, goto out; /* We only support devices that have a single target */ - if (dm_table_get_num_targets(table) != 1) + if (table->num_targets != 1) goto out; ti = dm_table_get_target(table, 0); diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 47a01c7cffdf..920085dd7f3b 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -561,7 +561,6 @@ void dm_sync_table(struct mapped_device *md); * Queries */ sector_t dm_table_get_size(struct dm_table *t); -unsigned int dm_table_get_num_targets(struct dm_table *t); fmode_t dm_table_get_mode(struct dm_table *t); struct mapped_device *dm_table_get_md(struct dm_table *t); const char *dm_table_device_name(struct dm_table *t); -- cgit v1.2.3 From 752f596371289cb3b45e99453b5c09d96ed36614 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:48 +0100 Subject: docs: filesystems: update netfs-api.rst reference Changeset efc930fa1d84 ("docs: filesystems: caching/netfs-api.txt: convert it to ReST") renamed: Documentation/filesystems/caching/netfs-api.txt to: Documentation/filesystems/caching/netfs-api.rst. Update its cross-reference accordingly. Fixes: efc930fa1d84 ("docs: filesystems: caching/netfs-api.txt: convert it to ReST") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/5f867f01d42c3e65e111167739ed1a41a26623f9.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- include/linux/fscache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 72585c9729a2..2f82ea31d4c1 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -378,7 +378,7 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, * * Request that the size of an object be changed. * - * See Documentation/filesystems/caching/netfs-api.txt for a complete + * See Documentation/filesystems/caching/netfs-api.rst for a complete * description. */ static inline -- cgit v1.2.3 From c02b872a7ca7842e4cdbbf621f77607d0a655f83 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:56 +0100 Subject: Documentation: update watch_queue.rst references Changeset f5461124d59b ("Documentation: move watch_queue to core-api") renamed: Documentation/watch_queue.rst to: Documentation/core-api/watch_queue.rst. Update the cross-references accordingly. Fixes: f5461124d59b ("Documentation: move watch_queue to core-api") Reviewed-by: Randy Dunlap Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/1c220de9c58f35e815a3df9458ac2bea323c8bfb.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/security/keys/core.rst | 2 +- include/linux/watch_queue.h | 2 +- init/Kconfig | 2 +- kernel/watch_queue.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/Documentation/security/keys/core.rst b/Documentation/security/keys/core.rst index b3ed5c581034..811b905b56bf 100644 --- a/Documentation/security/keys/core.rst +++ b/Documentation/security/keys/core.rst @@ -1046,7 +1046,7 @@ The keyctl syscall functions are: "filter" is either NULL to remove a watch or a filter specification to indicate what events are required from the key. - See Documentation/watch_queue.rst for more information. + See Documentation/core-api/watch_queue.rst for more information. Note that only one watch may be emplaced for any particular { key, queue_fd } combination. diff --git a/include/linux/watch_queue.h b/include/linux/watch_queue.h index 3b9a40ae8bdb..fc6bba20273b 100644 --- a/include/linux/watch_queue.h +++ b/include/linux/watch_queue.h @@ -4,7 +4,7 @@ * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) * - * See Documentation/watch_queue.rst + * See Documentation/core-api/watch_queue.rst */ #ifndef _LINUX_WATCH_QUEUE_H diff --git a/init/Kconfig b/init/Kconfig index c7900e8975f1..6f1a5355dc61 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -414,7 +414,7 @@ config WATCH_QUEUE with watches for key/keyring change notifications and device notifications. - See Documentation/watch_queue.rst + See Documentation/core-api/watch_queue.rst config CROSS_MEMORY_ATTACH bool "Enable process_vm_readv/writev syscalls" diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c index 230038d4f908..869fea4fe26b 100644 --- a/kernel/watch_queue.c +++ b/kernel/watch_queue.c @@ -4,7 +4,7 @@ * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) * - * See Documentation/watch_queue.rst + * See Documentation/core-api/watch_queue.rst */ #define pr_fmt(fmt) "watchq: " fmt -- cgit v1.2.3 From d6a21f2d73258f2a4cd2e7806f5755ee73fddced Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:11:01 +0100 Subject: objtool: update objtool.txt references Changeset a8e35fece49b ("objtool: Update documentation") renamed: tools/objtool/Documentation/stack-validation.txt to: tools/objtool/Documentation/objtool.txt. Update the cross-references accordingly. Fixes: a8e35fece49b ("objtool: Update documentation") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/ec285ece6348a5be191aebe45f78d06b3319056b.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/x86/orc-unwinder.rst | 2 +- include/linux/objtool.h | 2 +- lib/Kconfig.debug | 2 +- tools/include/linux/objtool.h | 2 +- tools/objtool/check.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/Documentation/x86/orc-unwinder.rst b/Documentation/x86/orc-unwinder.rst index 9a66a88be765..cdb257015bd9 100644 --- a/Documentation/x86/orc-unwinder.rst +++ b/Documentation/x86/orc-unwinder.rst @@ -140,7 +140,7 @@ Unwinder implementation details Objtool generates the ORC data by integrating with the compile-time stack metadata validation feature, which is described in detail in -tools/objtool/Documentation/stack-validation.txt. After analyzing all +tools/objtool/Documentation/objtool.txt. After analyzing all the code paths of a .o file, it creates an array of orc_entry structs, and a parallel array of instruction addresses associated with those structs, and writes them to the .orc_unwind and .orc_unwind_ip sections diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 6491fa8fba6d..6f89471005ee 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -62,7 +62,7 @@ struct unwind_hint { * It should only be used in special cases where you're 100% sure it won't * affect the reliability of frame pointers and kernel stack traces. * - * For more information, see tools/objtool/Documentation/stack-validation.txt. + * For more information, see tools/objtool/Documentation/objtool.txt. */ #define STACK_FRAME_NON_STANDARD(func) \ static void __used __section(".discard.func_stack_frame_non_standard") \ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2e24db4bff19..79a71eb96111 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -498,7 +498,7 @@ config STACK_VALIDATION runtime stack traces are more reliable. For more information, see - tools/objtool/Documentation/stack-validation.txt. + tools/objtool/Documentation/objtool.txt. config NOINSTR_VALIDATION bool diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h index 6491fa8fba6d..6f89471005ee 100644 --- a/tools/include/linux/objtool.h +++ b/tools/include/linux/objtool.h @@ -62,7 +62,7 @@ struct unwind_hint { * It should only be used in special cases where you're 100% sure it won't * affect the reliability of frame pointers and kernel stack traces. * - * For more information, see tools/objtool/Documentation/stack-validation.txt. + * For more information, see tools/objtool/Documentation/objtool.txt. */ #define STACK_FRAME_NON_STANDARD(func) \ static void __used __section(".discard.func_stack_frame_non_standard") \ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 864bb9dd3584..970844ceecdc 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -3190,7 +3190,7 @@ static struct instruction *next_insn_to_validate(struct objtool_file *file, * Follow the branch starting at the given instruction, and recursively follow * any other branches (jumps). Meanwhile, track the frame pointer state at * each instruction and validate all the rules described in - * tools/objtool/Documentation/stack-validation.txt. + * tools/objtool/Documentation/objtool.txt. */ static int validate_branch(struct objtool_file *file, struct symbol *func, struct instruction *insn, struct insn_state state) -- cgit v1.2.3 From 3566ee1d776c1393393564b2514f9cd52a49c16e Mon Sep 17 00:00:00 2001 From: Michael Kawano Date: Thu, 7 Jul 2022 15:57:27 +0200 Subject: vfio/ccw: Remove UUID from s390 debug log As vfio-ccw devices are created/destroyed, the uuid of the associated mdevs that are recorded in $S390DBF/vfio_ccw_msg/sprintf get lost. This is because a pointer to the UUID is stored instead of the UUID itself, and that memory may have been repurposed if/when the logs are examined. The result is usually garbage UUID data in the logs, though there is an outside chance of an oops happening here. Simply remove the UUID from the traces, as the subchannel number will provide useful configuration information for problem determination, and is stored directly into the log instead of a pointer. As we were the only consumer of mdev_uuid(), remove that too. Cc: Kirti Wankhede Signed-off-by: Michael Kawano Fixes: 60e05d1cf0875 ("vfio-ccw: add some logging") Fixes: b7701dfbf9832 ("vfio-ccw: Register a chp_event callback for vfio-ccw") [farman: reworded commit message, added Fixes: tags] Signed-off-by: Eric Farman Reviewed-by: Jason Gunthorpe Reviewed-by: Matthew Rosato Reviewed-by: Kirti Wankhede Link: https://lore.kernel.org/r/20220707135737.720765-2-farman@linux.ibm.com Signed-off-by: Alex Williamson --- drivers/s390/cio/vfio_ccw_drv.c | 5 ++--- drivers/s390/cio/vfio_ccw_fsm.c | 26 ++++++++++++-------------- drivers/s390/cio/vfio_ccw_ops.c | 8 ++++---- include/linux/mdev.h | 5 ----- 4 files changed, 18 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index ee182cfb467d..35055eb94115 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -358,8 +357,8 @@ static int vfio_ccw_chp_event(struct subchannel *sch, return 0; trace_vfio_ccw_chp_event(private->sch->schid, mask, event); - VFIO_CCW_MSG_EVENT(2, "%pUl (%x.%x.%04x): mask=0x%x event=%d\n", - mdev_uuid(private->mdev), sch->schid.cssid, + VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: mask=0x%x event=%d\n", + sch->schid.cssid, sch->schid.ssid, sch->schid.sch_no, mask, event); diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c index 8483a266051c..bbcc5b486749 100644 --- a/drivers/s390/cio/vfio_ccw_fsm.c +++ b/drivers/s390/cio/vfio_ccw_fsm.c @@ -10,7 +10,6 @@ */ #include -#include #include "ioasm.h" #include "vfio_ccw_private.h" @@ -242,7 +241,6 @@ static void fsm_io_request(struct vfio_ccw_private *private, union orb *orb; union scsw *scsw = &private->scsw; struct ccw_io_region *io_region = private->io_region; - struct mdev_device *mdev = private->mdev; char *errstr = "request"; struct subchannel_id schid = get_schid(private); @@ -256,8 +254,8 @@ static void fsm_io_request(struct vfio_ccw_private *private, if (orb->tm.b) { io_region->ret_code = -EOPNOTSUPP; VFIO_CCW_MSG_EVENT(2, - "%pUl (%x.%x.%04x): transport mode\n", - mdev_uuid(mdev), schid.cssid, + "sch %x.%x.%04x: transport mode\n", + schid.cssid, schid.ssid, schid.sch_no); errstr = "transport mode"; goto err_out; @@ -265,8 +263,8 @@ static void fsm_io_request(struct vfio_ccw_private *private, io_region->ret_code = cp_init(&private->cp, orb); if (io_region->ret_code) { VFIO_CCW_MSG_EVENT(2, - "%pUl (%x.%x.%04x): cp_init=%d\n", - mdev_uuid(mdev), schid.cssid, + "sch %x.%x.%04x: cp_init=%d\n", + schid.cssid, schid.ssid, schid.sch_no, io_region->ret_code); errstr = "cp init"; @@ -276,8 +274,8 @@ static void fsm_io_request(struct vfio_ccw_private *private, io_region->ret_code = cp_prefetch(&private->cp); if (io_region->ret_code) { VFIO_CCW_MSG_EVENT(2, - "%pUl (%x.%x.%04x): cp_prefetch=%d\n", - mdev_uuid(mdev), schid.cssid, + "sch %x.%x.%04x: cp_prefetch=%d\n", + schid.cssid, schid.ssid, schid.sch_no, io_region->ret_code); errstr = "cp prefetch"; @@ -289,8 +287,8 @@ static void fsm_io_request(struct vfio_ccw_private *private, io_region->ret_code = fsm_io_helper(private); if (io_region->ret_code) { VFIO_CCW_MSG_EVENT(2, - "%pUl (%x.%x.%04x): fsm_io_helper=%d\n", - mdev_uuid(mdev), schid.cssid, + "sch %x.%x.%04x: fsm_io_helper=%d\n", + schid.cssid, schid.ssid, schid.sch_no, io_region->ret_code); errstr = "cp fsm_io_helper"; @@ -300,16 +298,16 @@ static void fsm_io_request(struct vfio_ccw_private *private, return; } else if (scsw->cmd.fctl & SCSW_FCTL_HALT_FUNC) { VFIO_CCW_MSG_EVENT(2, - "%pUl (%x.%x.%04x): halt on io_region\n", - mdev_uuid(mdev), schid.cssid, + "sch %x.%x.%04x: halt on io_region\n", + schid.cssid, schid.ssid, schid.sch_no); /* halt is handled via the async cmd region */ io_region->ret_code = -EOPNOTSUPP; goto err_out; } else if (scsw->cmd.fctl & SCSW_FCTL_CLEAR_FUNC) { VFIO_CCW_MSG_EVENT(2, - "%pUl (%x.%x.%04x): clear on io_region\n", - mdev_uuid(mdev), schid.cssid, + "sch %x.%x.%04x: clear on io_region\n", + schid.cssid, schid.ssid, schid.sch_no); /* clear is handled via the async cmd region */ io_region->ret_code = -EOPNOTSUPP; diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index b49e2e9db2dc..0e05bff78b8e 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -131,8 +131,8 @@ static int vfio_ccw_mdev_probe(struct mdev_device *mdev) private->mdev = mdev; private->state = VFIO_CCW_STATE_IDLE; - VFIO_CCW_MSG_EVENT(2, "mdev %pUl, sch %x.%x.%04x: create\n", - mdev_uuid(mdev), private->sch->schid.cssid, + VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", + private->sch->schid.cssid, private->sch->schid.ssid, private->sch->schid.sch_no); @@ -154,8 +154,8 @@ static void vfio_ccw_mdev_remove(struct mdev_device *mdev) { struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); - VFIO_CCW_MSG_EVENT(2, "mdev %pUl, sch %x.%x.%04x: remove\n", - mdev_uuid(mdev), private->sch->schid.cssid, + VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", + private->sch->schid.cssid, private->sch->schid.ssid, private->sch->schid.sch_no); diff --git a/include/linux/mdev.h b/include/linux/mdev.h index bb539794f54a..47ad3b104d9e 100644 --- a/include/linux/mdev.h +++ b/include/linux/mdev.h @@ -65,11 +65,6 @@ struct mdev_driver { struct device_driver driver; }; -static inline const guid_t *mdev_uuid(struct mdev_device *mdev) -{ - return &mdev->uuid; -} - extern struct bus_type mdev_bus_type; int mdev_register_device(struct device *dev, struct mdev_driver *mdev_driver); -- cgit v1.2.3 From 87686cc845c3be7dea777f1dbf2de0767007cda8 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 4 Jul 2022 16:10:39 +0530 Subject: OPP: Make dev_pm_opp_set_regulators() accept NULL terminated list Make dev_pm_opp_set_regulators() accept a NULL terminated list of names instead of making the callers keep the two parameters in sync, which creates an opportunity for bugs to get in. Suggested-by: Greg Kroah-Hartman Reviewed-by: Steven Price # panfrost Reviewed-by: Chanwoo Choi Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq-dt.c | 9 ++++----- drivers/cpufreq/ti-cpufreq.c | 7 +++---- drivers/devfreq/exynos-bus.c | 4 ++-- drivers/gpu/drm/lima/lima_devfreq.c | 3 ++- drivers/gpu/drm/panfrost/panfrost_devfreq.c | 3 +-- drivers/gpu/drm/panfrost/panfrost_drv.c | 15 ++++++++++----- drivers/opp/core.c | 18 ++++++++++++------ drivers/soc/tegra/pmc.c | 4 ++-- include/linux/pm_opp.h | 9 ++++----- 9 files changed, 40 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 8fcaba541539..be0c19b3ffa5 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -193,7 +193,7 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) struct private_data *priv; struct device *cpu_dev; bool fallback = false; - const char *reg_name; + const char *reg_name[] = { NULL, NULL }; int ret; /* Check if this CPU is already covered by some other policy */ @@ -218,10 +218,9 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) * OPP layer will be taking care of regulators now, but it needs to know * the name of the regulator first. */ - reg_name = find_supply_name(cpu_dev); - if (reg_name) { - priv->opp_table = dev_pm_opp_set_regulators(cpu_dev, ®_name, - 1); + reg_name[0] = find_supply_name(cpu_dev); + if (reg_name[0]) { + priv->opp_table = dev_pm_opp_set_regulators(cpu_dev, reg_name); if (IS_ERR(priv->opp_table)) { ret = PTR_ERR(priv->opp_table); if (ret != -EPROBE_DEFER) diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c index 8f9fdd864391..560d67a6bef1 100644 --- a/drivers/cpufreq/ti-cpufreq.c +++ b/drivers/cpufreq/ti-cpufreq.c @@ -173,7 +173,7 @@ static struct ti_cpufreq_soc_data omap34xx_soc_data = { * seems to always read as 0). */ -static const char * const omap3_reg_names[] = {"cpu0", "vbb"}; +static const char * const omap3_reg_names[] = {"cpu0", "vbb", NULL}; static struct ti_cpufreq_soc_data omap36xx_soc_data = { .reg_names = omap3_reg_names, @@ -326,7 +326,7 @@ static int ti_cpufreq_probe(struct platform_device *pdev) const struct of_device_id *match; struct opp_table *ti_opp_table; struct ti_cpufreq_data *opp_data; - const char * const default_reg_names[] = {"vdd", "vbb"}; + const char * const default_reg_names[] = {"vdd", "vbb", NULL}; int ret; match = dev_get_platdata(&pdev->dev); @@ -387,8 +387,7 @@ static int ti_cpufreq_probe(struct platform_device *pdev) if (opp_data->soc_data->reg_names) reg_names = opp_data->soc_data->reg_names; ti_opp_table = dev_pm_opp_set_regulators(opp_data->cpu_dev, - reg_names, - ARRAY_SIZE(default_reg_names)); + reg_names); if (IS_ERR(ti_opp_table)) { dev_pm_opp_put_supported_hw(opp_data->opp_table); ret = PTR_ERR(ti_opp_table); diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index e689101abc93..541baff93ee8 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -180,10 +180,10 @@ static int exynos_bus_parent_parse_of(struct device_node *np, { struct device *dev = bus->dev; struct opp_table *opp_table; - const char *vdd = "vdd"; + const char *supplies[] = { "vdd", NULL }; int i, ret, count, size; - opp_table = dev_pm_opp_set_regulators(dev, &vdd, 1); + opp_table = dev_pm_opp_set_regulators(dev, supplies); if (IS_ERR(opp_table)) { ret = PTR_ERR(opp_table); dev_err(dev, "failed to set regulators %d\n", ret); diff --git a/drivers/gpu/drm/lima/lima_devfreq.c b/drivers/gpu/drm/lima/lima_devfreq.c index 8989e215dfc9..dc83c5421125 100644 --- a/drivers/gpu/drm/lima/lima_devfreq.c +++ b/drivers/gpu/drm/lima/lima_devfreq.c @@ -111,6 +111,7 @@ int lima_devfreq_init(struct lima_device *ldev) struct dev_pm_opp *opp; unsigned long cur_freq; int ret; + const char *regulator_names[] = { "mali", NULL }; if (!device_property_present(dev, "operating-points-v2")) /* Optional, continue without devfreq */ @@ -122,7 +123,7 @@ int lima_devfreq_init(struct lima_device *ldev) if (ret) return ret; - ret = devm_pm_opp_set_regulators(dev, (const char *[]){ "mali" }, 1); + ret = devm_pm_opp_set_regulators(dev, regulator_names); if (ret) { /* Continue if the optional regulator is missing */ if (ret != -ENODEV) diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c index 194af7f607a6..5110cd9b2425 100644 --- a/drivers/gpu/drm/panfrost/panfrost_devfreq.c +++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c @@ -101,8 +101,7 @@ int panfrost_devfreq_init(struct panfrost_device *pfdev) return 0; } - ret = devm_pm_opp_set_regulators(dev, pfdev->comp->supply_names, - pfdev->comp->num_supplies); + ret = devm_pm_opp_set_regulators(dev, pfdev->comp->supply_names); if (ret) { /* Continue if the optional regulator is missing */ if (ret != -ENODEV) { diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index 7fcbc2a5b6cd..8a4bef65d38c 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -625,24 +625,29 @@ static int panfrost_remove(struct platform_device *pdev) return 0; } -static const char * const default_supplies[] = { "mali" }; +/* + * The OPP core wants the supply names to be NULL terminated, but we need the + * correct num_supplies value for regulator core. Hence, we NULL terminate here + * and then initialize num_supplies with ARRAY_SIZE - 1. + */ +static const char * const default_supplies[] = { "mali", NULL }; static const struct panfrost_compatible default_data = { - .num_supplies = ARRAY_SIZE(default_supplies), + .num_supplies = ARRAY_SIZE(default_supplies) - 1, .supply_names = default_supplies, .num_pm_domains = 1, /* optional */ .pm_domain_names = NULL, }; static const struct panfrost_compatible amlogic_data = { - .num_supplies = ARRAY_SIZE(default_supplies), + .num_supplies = ARRAY_SIZE(default_supplies) - 1, .supply_names = default_supplies, .vendor_quirk = panfrost_gpu_amlogic_quirk, }; -static const char * const mediatek_mt8183_supplies[] = { "mali", "sram" }; +static const char * const mediatek_mt8183_supplies[] = { "mali", "sram", NULL }; static const char * const mediatek_mt8183_pm_domains[] = { "core0", "core1", "core2" }; static const struct panfrost_compatible mediatek_mt8183_data = { - .num_supplies = ARRAY_SIZE(mediatek_mt8183_supplies), + .num_supplies = ARRAY_SIZE(mediatek_mt8183_supplies) - 1, .supply_names = mediatek_mt8183_supplies, .num_pm_domains = ARRAY_SIZE(mediatek_mt8183_pm_domains), .pm_domain_names = mediatek_mt8183_pm_domains, diff --git a/drivers/opp/core.c b/drivers/opp/core.c index e166bfe5fc90..4e4593957ec5 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2105,13 +2105,20 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); * This must be called before any OPPs are initialized for the device. */ struct opp_table *dev_pm_opp_set_regulators(struct device *dev, - const char * const names[], - unsigned int count) + const char * const names[]) { struct dev_pm_opp_supply *supplies; + const char * const *temp = names; struct opp_table *opp_table; struct regulator *reg; - int ret, i; + int count = 0, ret, i; + + /* Count number of regulators */ + while (*temp++) + count++; + + if (!count) + return ERR_PTR(-EINVAL); opp_table = _add_opp_table(dev, false); if (IS_ERR(opp_table)) @@ -2236,12 +2243,11 @@ static void devm_pm_opp_regulators_release(void *data) * Return: 0 on success and errorno otherwise. */ int devm_pm_opp_set_regulators(struct device *dev, - const char * const names[], - unsigned int count) + const char * const names[]) { struct opp_table *opp_table; - opp_table = dev_pm_opp_set_regulators(dev, names, count); + opp_table = dev_pm_opp_set_regulators(dev, names); if (IS_ERR(opp_table)) return PTR_ERR(opp_table); diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index 5611d14d3ba2..6a4b8f7e7948 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -1384,7 +1384,7 @@ tegra_pmc_core_pd_opp_to_performance_state(struct generic_pm_domain *genpd, static int tegra_pmc_core_pd_add(struct tegra_pmc *pmc, struct device_node *np) { struct generic_pm_domain *genpd; - const char *rname = "core"; + const char *rname[] = { "core", NULL}; int err; genpd = devm_kzalloc(pmc->dev, sizeof(*genpd), GFP_KERNEL); @@ -1395,7 +1395,7 @@ static int tegra_pmc_core_pd_add(struct tegra_pmc *pmc, struct device_node *np) genpd->set_performance_state = tegra_pmc_core_pd_set_performance_state; genpd->opp_to_performance_state = tegra_pmc_core_pd_opp_to_performance_state; - err = devm_pm_opp_set_regulators(pmc->dev, &rname, 1); + err = devm_pm_opp_set_regulators(pmc->dev, rname); if (err) return dev_err_probe(pmc->dev, err, "failed to set core OPP regulator\n"); diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 6708b4ec244d..4c490865d574 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -159,9 +159,9 @@ void dev_pm_opp_put_supported_hw(struct opp_table *opp_table); int devm_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, unsigned int count); struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct opp_table *opp_table); -struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count); +struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[]); void dev_pm_opp_put_regulators(struct opp_table *opp_table); -int devm_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count); +int devm_pm_opp_set_regulators(struct device *dev, const char * const names[]); struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name); void dev_pm_opp_put_clkname(struct opp_table *opp_table); int devm_pm_opp_set_clkname(struct device *dev, const char *name); @@ -379,7 +379,7 @@ static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, con static inline void dev_pm_opp_put_prop_name(struct opp_table *opp_table) {} -static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count) +static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[]) { return ERR_PTR(-EOPNOTSUPP); } @@ -387,8 +387,7 @@ static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, co static inline void dev_pm_opp_put_regulators(struct opp_table *opp_table) {} static inline int devm_pm_opp_set_regulators(struct device *dev, - const char * const names[], - unsigned int count) + const char * const names[]) { return -EOPNOTSUPP; } -- cgit v1.2.3 From 11b9b663585c4f8b00846089ebbca4d1e3283e86 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 May 2022 15:23:16 +0530 Subject: OPP: Add dev_pm_opp_set_config() and friends The OPP core already have few configuration specific APIs and it is getting complex or messy for both the OPP core and its users. Lets introduce a new set of API which will be used for all kind of different configurations, and shall eventually be used by all the existing ones. The new API, returns a unique token instead of a pointer to the OPP table, which allows the OPP core to drop the resources selectively later on. Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 229 ++++++++++++++++++++++++++++++++++++++++++++++++- drivers/opp/opp.h | 21 +++++ include/linux/pm_opp.h | 42 +++++++++ 3 files changed, 291 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 4e4593957ec5..7ab20c3b91ed 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -13,11 +13,12 @@ #include #include #include -#include #include #include #include #include +#include +#include #include "opp.h" @@ -36,6 +37,9 @@ DEFINE_MUTEX(opp_table_lock); /* Flag indicating that opp_tables list is being updated at the moment */ static bool opp_tables_busy; +/* OPP ID allocator */ +static DEFINE_XARRAY_ALLOC1(opp_configs); + static bool _find_opp_dev(const struct device *dev, struct opp_table *opp_table) { struct opp_device *opp_dev; @@ -2624,6 +2628,229 @@ int devm_pm_opp_attach_genpd(struct device *dev, const char * const *names, } EXPORT_SYMBOL_GPL(devm_pm_opp_attach_genpd); +static void _opp_clear_config(struct opp_config_data *data) +{ + if (data->flags & OPP_CONFIG_GENPD) + dev_pm_opp_detach_genpd(data->opp_table); + if (data->flags & OPP_CONFIG_REGULATOR) + dev_pm_opp_put_regulators(data->opp_table); + if (data->flags & OPP_CONFIG_SUPPORTED_HW) + dev_pm_opp_put_supported_hw(data->opp_table); + if (data->flags & OPP_CONFIG_REGULATOR_HELPER) + dev_pm_opp_unregister_set_opp_helper(data->opp_table); + if (data->flags & OPP_CONFIG_PROP_NAME) + dev_pm_opp_put_prop_name(data->opp_table); + if (data->flags & OPP_CONFIG_CLK) + dev_pm_opp_put_clkname(data->opp_table); + + dev_pm_opp_put_opp_table(data->opp_table); + kfree(data); +} + +/** + * dev_pm_opp_set_config() - Set OPP configuration for the device. + * @dev: Device for which configuration is being set. + * @config: OPP configuration. + * + * This allows all device OPP configurations to be performed at once. + * + * This must be called before any OPPs are initialized for the device. This may + * be called multiple times for the same OPP table, for example once for each + * CPU that share the same table. This must be balanced by the same number of + * calls to dev_pm_opp_clear_config() in order to free the OPP table properly. + * + * This returns a token to the caller, which must be passed to + * dev_pm_opp_clear_config() to free the resources later. The value of the + * returned token will be >= 1 for success and negative for errors. The minimum + * value of 1 is chosen here to make it easy for callers to manage the resource. + */ +int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) +{ + struct opp_table *opp_table, *err; + struct opp_config_data *data; + unsigned int id; + int ret; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + opp_table = _add_opp_table(dev, false); + if (IS_ERR(opp_table)) { + kfree(data); + return PTR_ERR(opp_table); + } + + data->opp_table = opp_table; + data->flags = 0; + + /* This should be called before OPPs are initialized */ + if (WARN_ON(!list_empty(&opp_table->opp_list))) { + ret = -EBUSY; + goto err; + } + + /* Configure clocks */ + if (config->clk_names) { + const char * const *temp = config->clk_names; + int count = 0; + + /* Count number of clks */ + while (*temp++) + count++; + + /* + * This is a special case where we have a single clock, whose + * connection id name is NULL, i.e. first two entries are NULL + * in the array. + */ + if (!count && !config->clk_names[1]) + count = 1; + + /* We support only one clock name for now */ + if (count != 1) { + ret = -EINVAL; + goto err; + } + + err = dev_pm_opp_set_clkname(dev, config->clk_names[0]); + if (IS_ERR(err)) { + ret = PTR_ERR(err); + goto err; + } + + data->flags |= OPP_CONFIG_CLK; + } + + /* Configure property names */ + if (config->prop_name) { + err = dev_pm_opp_set_prop_name(dev, config->prop_name); + if (IS_ERR(err)) { + ret = PTR_ERR(err); + goto err; + } + + data->flags |= OPP_CONFIG_PROP_NAME; + } + + /* Configure opp helper */ + if (config->set_opp) { + err = dev_pm_opp_register_set_opp_helper(dev, config->set_opp); + if (IS_ERR(err)) { + ret = PTR_ERR(err); + goto err; + } + + data->flags |= OPP_CONFIG_REGULATOR_HELPER; + } + + /* Configure supported hardware */ + if (config->supported_hw) { + err = dev_pm_opp_set_supported_hw(dev, config->supported_hw, + config->supported_hw_count); + if (IS_ERR(err)) { + ret = PTR_ERR(err); + goto err; + } + + data->flags |= OPP_CONFIG_SUPPORTED_HW; + } + + /* Configure supplies */ + if (config->regulator_names) { + err = dev_pm_opp_set_regulators(dev, config->regulator_names); + if (IS_ERR(err)) { + ret = PTR_ERR(err); + goto err; + } + + data->flags |= OPP_CONFIG_REGULATOR; + } + + /* Attach genpds */ + if (config->genpd_names) { + err = dev_pm_opp_attach_genpd(dev, config->genpd_names, + config->virt_devs); + if (IS_ERR(err)) { + ret = PTR_ERR(err); + goto err; + } + + data->flags |= OPP_CONFIG_GENPD; + } + + ret = xa_alloc(&opp_configs, &id, data, XA_LIMIT(1, INT_MAX), + GFP_KERNEL); + if (ret) + goto err; + + return id; + +err: + _opp_clear_config(data); + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_config); + +/** + * dev_pm_opp_clear_config() - Releases resources blocked for OPP configuration. + * @opp_table: OPP table returned from dev_pm_opp_set_config(). + * + * This allows all device OPP configurations to be cleared at once. This must be + * called once for each call made to dev_pm_opp_set_config(), in order to free + * the OPPs properly. + * + * Currently the first call itself ends up freeing all the OPP configurations, + * while the later ones only drop the OPP table reference. This works well for + * now as we would never want to use an half initialized OPP table and want to + * remove the configurations together. + */ +void dev_pm_opp_clear_config(int token) +{ + struct opp_config_data *data; + + /* + * This lets the callers call this unconditionally and keep their code + * simple. + */ + if (unlikely(token <= 0)) + return; + + data = xa_erase(&opp_configs, token); + if (WARN_ON(!data)) + return; + + _opp_clear_config(data); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_clear_config); + +static void devm_pm_opp_config_release(void *token) +{ + dev_pm_opp_clear_config((unsigned long)token); +} + +/** + * devm_pm_opp_set_config() - Set OPP configuration for the device. + * @dev: Device for which configuration is being set. + * @config: OPP configuration. + * + * This allows all device OPP configurations to be performed at once. + * This is a resource-managed variant of dev_pm_opp_set_config(). + * + * Return: 0 on success and errorno otherwise. + */ +int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) +{ + int token = dev_pm_opp_set_config(dev, config); + + if (token < 0) + return token; + + return devm_add_action_or_reset(dev, devm_pm_opp_config_release, + (void *) ((unsigned long) token)); +} +EXPORT_SYMBOL_GPL(devm_pm_opp_set_config); + /** * dev_pm_opp_xlate_required_opp() - Find required OPP for @src_table OPP. * @src_table: OPP table which has @dst_table as one of its required OPP table. diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h index 9e1cfcb0ea98..d652f0cc84f1 100644 --- a/drivers/opp/opp.h +++ b/drivers/opp/opp.h @@ -28,6 +28,27 @@ extern struct mutex opp_table_lock; extern struct list_head opp_tables, lazy_opp_tables; +/* OPP Config flags */ +#define OPP_CONFIG_CLK BIT(0) +#define OPP_CONFIG_REGULATOR BIT(1) +#define OPP_CONFIG_REGULATOR_HELPER BIT(2) +#define OPP_CONFIG_PROP_NAME BIT(3) +#define OPP_CONFIG_SUPPORTED_HW BIT(4) +#define OPP_CONFIG_GENPD BIT(5) + +/** + * struct opp_config_data - data for set config operations + * @opp_table: OPP table + * @flags: OPP config flags + * + * This structure stores the OPP config information for each OPP table + * configuration by the callers. + */ +struct opp_config_data { + struct opp_table *opp_table; + unsigned int flags; +}; + /* * Internal data structure organization with the OPP layer library is as * follows: diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 4c490865d574..a08f9481efb3 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -90,6 +90,32 @@ struct dev_pm_set_opp_data { struct device *dev; }; +/** + * struct dev_pm_opp_config - Device OPP configuration values + * @clk_names: Clk names, NULL terminated array, max 1 clock for now. + * @prop_name: Name to postfix to properties. + * @set_opp: Custom set OPP helper. + * @supported_hw: Array of hierarchy of versions to match. + * @supported_hw_count: Number of elements in the array. + * @regulator_names: Array of pointers to the names of the regulator, NULL terminated. + * @genpd_names: Null terminated array of pointers containing names of genpd to + * attach. + * @virt_devs: Pointer to return the array of virtual devices. + * + * This structure contains platform specific OPP configurations for the device. + */ +struct dev_pm_opp_config { + /* NULL terminated */ + const char * const *clk_names; + const char *prop_name; + int (*set_opp)(struct dev_pm_set_opp_data *data); + const unsigned int *supported_hw; + unsigned int supported_hw_count; + const char * const *regulator_names; + const char * const *genpd_names; + struct device ***virt_devs; +}; + #if defined(CONFIG_PM_OPP) struct opp_table *dev_pm_opp_get_opp_table(struct device *dev); @@ -154,6 +180,10 @@ int dev_pm_opp_disable(struct device *dev, unsigned long freq); int dev_pm_opp_register_notifier(struct device *dev, struct notifier_block *nb); int dev_pm_opp_unregister_notifier(struct device *dev, struct notifier_block *nb); +int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); +int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); +void dev_pm_opp_clear_config(int token); + struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, unsigned int count); void dev_pm_opp_put_supported_hw(struct opp_table *opp_table); int devm_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, unsigned int count); @@ -418,6 +448,18 @@ static inline int devm_pm_opp_attach_genpd(struct device *dev, return -EOPNOTSUPP; } +static inline int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) +{ + return -EOPNOTSUPP; +} + +static inline int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) +{ + return -EOPNOTSUPP; +} + +static inline void dev_pm_opp_clear_config(int token) {} + static inline struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, struct opp_table *dst_table, struct dev_pm_opp *src_opp) { -- cgit v1.2.3 From b0ec09428621daee5101130c307634a390b0213b Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 4 Jul 2022 16:36:26 +0530 Subject: OPP: Migrate set-regulators API to use set-config helpers Now that we have a central API to handle all OPP table configurations, migrate the set-regulators family of helpers to use the new infrastructure. The return type and parameter to the APIs change a bit due to this, update the current users as well in the same commit in order to avoid breaking builds. Reviewed-by: Chanwoo Choi Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq-dt.c | 12 +++--- drivers/devfreq/exynos-bus.c | 19 ++++----- drivers/opp/core.c | 91 +++++++++----------------------------------- include/linux/pm_opp.h | 44 +++++++++++++-------- 4 files changed, 60 insertions(+), 106 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index be0c19b3ffa5..d69d13a26414 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -29,9 +29,9 @@ struct private_data { cpumask_var_t cpus; struct device *cpu_dev; - struct opp_table *opp_table; struct cpufreq_frequency_table *freq_table; bool have_static_opps; + int opp_token; }; static LIST_HEAD(priv_list); @@ -220,9 +220,9 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) */ reg_name[0] = find_supply_name(cpu_dev); if (reg_name[0]) { - priv->opp_table = dev_pm_opp_set_regulators(cpu_dev, reg_name); - if (IS_ERR(priv->opp_table)) { - ret = PTR_ERR(priv->opp_table); + priv->opp_token = dev_pm_opp_set_regulators(cpu_dev, reg_name); + if (priv->opp_token < 0) { + ret = priv->opp_token; if (ret != -EPROBE_DEFER) dev_err(cpu_dev, "failed to set regulators: %d\n", ret); @@ -294,7 +294,7 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) out: if (priv->have_static_opps) dev_pm_opp_of_cpumask_remove_table(priv->cpus); - dev_pm_opp_put_regulators(priv->opp_table); + dev_pm_opp_put_regulators(priv->opp_token); free_cpumask: free_cpumask_var(priv->cpus); return ret; @@ -308,7 +308,7 @@ static void dt_cpufreq_release(void) dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &priv->freq_table); if (priv->have_static_opps) dev_pm_opp_of_cpumask_remove_table(priv->cpus); - dev_pm_opp_put_regulators(priv->opp_table); + dev_pm_opp_put_regulators(priv->opp_token); free_cpumask_var(priv->cpus); list_del(&priv->node); } diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 541baff93ee8..d1235242367f 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -33,7 +33,7 @@ struct exynos_bus { unsigned long curr_freq; - struct opp_table *opp_table; + int opp_token; struct clk *clk; unsigned int ratio; }; @@ -161,8 +161,7 @@ static void exynos_bus_exit(struct device *dev) dev_pm_opp_of_remove_table(dev); clk_disable_unprepare(bus->clk); - dev_pm_opp_put_regulators(bus->opp_table); - bus->opp_table = NULL; + dev_pm_opp_put_regulators(bus->opp_token); } static void exynos_bus_passive_exit(struct device *dev) @@ -179,18 +178,16 @@ static int exynos_bus_parent_parse_of(struct device_node *np, struct exynos_bus *bus) { struct device *dev = bus->dev; - struct opp_table *opp_table; const char *supplies[] = { "vdd", NULL }; int i, ret, count, size; - opp_table = dev_pm_opp_set_regulators(dev, supplies); - if (IS_ERR(opp_table)) { - ret = PTR_ERR(opp_table); + ret = dev_pm_opp_set_regulators(dev, supplies); + if (ret < 0) { dev_err(dev, "failed to set regulators %d\n", ret); return ret; } - bus->opp_table = opp_table; + bus->opp_token = ret; /* * Get the devfreq-event devices to get the current utilization of @@ -236,8 +233,7 @@ static int exynos_bus_parent_parse_of(struct device_node *np, return 0; err_regulator: - dev_pm_opp_put_regulators(bus->opp_table); - bus->opp_table = NULL; + dev_pm_opp_put_regulators(bus->opp_token); return ret; } @@ -459,8 +455,7 @@ err: dev_pm_opp_of_remove_table(dev); clk_disable_unprepare(bus->clk); err_reg: - dev_pm_opp_put_regulators(bus->opp_table); - bus->opp_table = NULL; + dev_pm_opp_put_regulators(bus->opp_token); return ret; } diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 7ab20c3b91ed..6ff9b5b69d07 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -991,8 +991,8 @@ static int _set_opp_custom(const struct opp_table *opp_table, int size; /* - * We support this only if dev_pm_opp_set_regulators() was called - * earlier. + * We support this only if dev_pm_opp_set_config() was called + * earlier to set regulators. */ if (opp_table->sod_supplies) { size = sizeof(*old_opp->supplies) * opp_table->regulator_count; @@ -2097,7 +2097,7 @@ void dev_pm_opp_put_prop_name(struct opp_table *opp_table) EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); /** - * dev_pm_opp_set_regulators() - Set regulator names for the device + * _opp_set_regulators() - Set regulator names for the device * @dev: Device for which regulator name is being set. * @names: Array of pointers to the names of the regulator. * @count: Number of regulators. @@ -2108,12 +2108,11 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); * * This must be called before any OPPs are initialized for the device. */ -struct opp_table *dev_pm_opp_set_regulators(struct device *dev, - const char * const names[]) +static int _opp_set_regulators(struct opp_table *opp_table, struct device *dev, + const char * const names[]) { struct dev_pm_opp_supply *supplies; const char * const *temp = names; - struct opp_table *opp_table; struct regulator *reg; int count = 0, ret, i; @@ -2122,29 +2121,17 @@ struct opp_table *dev_pm_opp_set_regulators(struct device *dev, count++; if (!count) - return ERR_PTR(-EINVAL); - - opp_table = _add_opp_table(dev, false); - if (IS_ERR(opp_table)) - return opp_table; - - /* This should be called before OPPs are initialized */ - if (WARN_ON(!list_empty(&opp_table->opp_list))) { - ret = -EBUSY; - goto err; - } + return -EINVAL; /* Another CPU that shares the OPP table has set the regulators ? */ if (opp_table->regulators) - return opp_table; + return 0; opp_table->regulators = kmalloc_array(count, sizeof(*opp_table->regulators), GFP_KERNEL); - if (!opp_table->regulators) { - ret = -ENOMEM; - goto err; - } + if (!opp_table->regulators) + return -ENOMEM; for (i = 0; i < count; i++) { reg = regulator_get_optional(dev, names[i]); @@ -2174,7 +2161,7 @@ struct opp_table *dev_pm_opp_set_regulators(struct device *dev, } mutex_unlock(&opp_table->lock); - return opp_table; + return 0; free_regulators: while (i != 0) @@ -2183,26 +2170,20 @@ free_regulators: kfree(opp_table->regulators); opp_table->regulators = NULL; opp_table->regulator_count = -1; -err: - dev_pm_opp_put_opp_table(opp_table); - return ERR_PTR(ret); + return ret; } -EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulators); /** - * dev_pm_opp_put_regulators() - Releases resources blocked for regulator - * @opp_table: OPP table returned from dev_pm_opp_set_regulators(). + * _opp_put_regulators() - Releases resources blocked for regulator + * @opp_table: OPP table returned from _opp_set_regulators(). */ -void dev_pm_opp_put_regulators(struct opp_table *opp_table) +static void _opp_put_regulators(struct opp_table *opp_table) { int i; - if (unlikely(!opp_table)) - return; - if (!opp_table->regulators) - goto put_opp_table; + return; if (opp_table->enabled) { for (i = opp_table->regulator_count - 1; i >= 0; i--) @@ -2225,40 +2206,7 @@ void dev_pm_opp_put_regulators(struct opp_table *opp_table) kfree(opp_table->regulators); opp_table->regulators = NULL; opp_table->regulator_count = -1; - -put_opp_table: - dev_pm_opp_put_opp_table(opp_table); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulators); - -static void devm_pm_opp_regulators_release(void *data) -{ - dev_pm_opp_put_regulators(data); -} - -/** - * devm_pm_opp_set_regulators() - Set regulator names for the device - * @dev: Device for which regulator name is being set. - * @names: Array of pointers to the names of the regulator. - * @count: Number of regulators. - * - * This is a resource-managed variant of dev_pm_opp_set_regulators(). - * - * Return: 0 on success and errorno otherwise. - */ -int devm_pm_opp_set_regulators(struct device *dev, - const char * const names[]) -{ - struct opp_table *opp_table; - - opp_table = dev_pm_opp_set_regulators(dev, names); - if (IS_ERR(opp_table)) - return PTR_ERR(opp_table); - - return devm_add_action_or_reset(dev, devm_pm_opp_regulators_release, - opp_table); } -EXPORT_SYMBOL_GPL(devm_pm_opp_set_regulators); /** * dev_pm_opp_set_clkname() - Set clk name for the device @@ -2633,7 +2581,7 @@ static void _opp_clear_config(struct opp_config_data *data) if (data->flags & OPP_CONFIG_GENPD) dev_pm_opp_detach_genpd(data->opp_table); if (data->flags & OPP_CONFIG_REGULATOR) - dev_pm_opp_put_regulators(data->opp_table); + _opp_put_regulators(data->opp_table); if (data->flags & OPP_CONFIG_SUPPORTED_HW) dev_pm_opp_put_supported_hw(data->opp_table); if (data->flags & OPP_CONFIG_REGULATOR_HELPER) @@ -2758,11 +2706,10 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) /* Configure supplies */ if (config->regulator_names) { - err = dev_pm_opp_set_regulators(dev, config->regulator_names); - if (IS_ERR(err)) { - ret = PTR_ERR(err); + ret = _opp_set_regulators(opp_table, dev, + config->regulator_names); + if (ret) goto err; - } data->flags |= OPP_CONFIG_REGULATOR; } diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index a08f9481efb3..f014bd172c99 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -189,9 +189,6 @@ void dev_pm_opp_put_supported_hw(struct opp_table *opp_table); int devm_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, unsigned int count); struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct opp_table *opp_table); -struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[]); -void dev_pm_opp_put_regulators(struct opp_table *opp_table); -int devm_pm_opp_set_regulators(struct device *dev, const char * const names[]); struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name); void dev_pm_opp_put_clkname(struct opp_table *opp_table); int devm_pm_opp_set_clkname(struct device *dev, const char *name); @@ -409,19 +406,6 @@ static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, con static inline void dev_pm_opp_put_prop_name(struct opp_table *opp_table) {} -static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[]) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void dev_pm_opp_put_regulators(struct opp_table *opp_table) {} - -static inline int devm_pm_opp_set_regulators(struct device *dev, - const char * const names[]) -{ - return -EOPNOTSUPP; -} - static inline struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name) { return ERR_PTR(-EOPNOTSUPP); @@ -606,4 +590,32 @@ static inline int dev_pm_opp_of_find_icc_paths(struct device *dev, struct opp_ta } #endif +/* OPP Configuration helpers */ + +/* Regulators helpers */ +static inline int dev_pm_opp_set_regulators(struct device *dev, + const char * const names[]) +{ + struct dev_pm_opp_config config = { + .regulator_names = names, + }; + + return dev_pm_opp_set_config(dev, &config); +} + +static inline void dev_pm_opp_put_regulators(int token) +{ + dev_pm_opp_clear_config(token); +} + +static inline int devm_pm_opp_set_regulators(struct device *dev, + const char * const names[]) +{ + struct dev_pm_opp_config config = { + .regulator_names = names, + }; + + return devm_pm_opp_set_config(dev, &config); +} + #endif /* __LINUX_OPP_H__ */ -- cgit v1.2.3 From 89f03984fa2abface1ffb1fe050b7c175651ffc7 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 26 May 2022 09:36:27 +0530 Subject: OPP: Migrate set-supported-hw API to use set-config helpers Now that we have a central API to handle all OPP table configurations, migrate the set-supported-hw family of helpers to use the new infrastructure. The return type and parameter to the APIs change a bit due to this, update the current users as well in the same commit in order to avoid breaking builds. Signed-off-by: Viresh Kumar --- drivers/cpufreq/imx-cpufreq-dt.c | 12 ++--- drivers/cpufreq/tegra20-cpufreq.c | 12 +++-- drivers/memory/tegra/tegra124-emc.c | 11 +++-- drivers/opp/core.c | 87 +++++++++---------------------------- include/linux/pm_opp.h | 49 +++++++++++++-------- 5 files changed, 66 insertions(+), 105 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/imx-cpufreq-dt.c b/drivers/cpufreq/imx-cpufreq-dt.c index 3fe9125156b4..76e553af2071 100644 --- a/drivers/cpufreq/imx-cpufreq-dt.c +++ b/drivers/cpufreq/imx-cpufreq-dt.c @@ -31,8 +31,8 @@ /* cpufreq-dt device registered by imx-cpufreq-dt */ static struct platform_device *cpufreq_dt_pdev; -static struct opp_table *cpufreq_opp_table; static struct device *cpu_dev; +static int cpufreq_opp_token; enum IMX7ULP_CPUFREQ_CLKS { ARM, @@ -153,9 +153,9 @@ static int imx_cpufreq_dt_probe(struct platform_device *pdev) dev_info(&pdev->dev, "cpu speed grade %d mkt segment %d supported-hw %#x %#x\n", speed_grade, mkt_segment, supported_hw[0], supported_hw[1]); - cpufreq_opp_table = dev_pm_opp_set_supported_hw(cpu_dev, supported_hw, 2); - if (IS_ERR(cpufreq_opp_table)) { - ret = PTR_ERR(cpufreq_opp_table); + cpufreq_opp_token = dev_pm_opp_set_supported_hw(cpu_dev, supported_hw, 2); + if (cpufreq_opp_token < 0) { + ret = cpufreq_opp_token; dev_err(&pdev->dev, "Failed to set supported opp: %d\n", ret); return ret; } @@ -163,7 +163,7 @@ static int imx_cpufreq_dt_probe(struct platform_device *pdev) cpufreq_dt_pdev = platform_device_register_data( &pdev->dev, "cpufreq-dt", -1, NULL, 0); if (IS_ERR(cpufreq_dt_pdev)) { - dev_pm_opp_put_supported_hw(cpufreq_opp_table); + dev_pm_opp_put_supported_hw(cpufreq_opp_token); ret = PTR_ERR(cpufreq_dt_pdev); dev_err(&pdev->dev, "Failed to register cpufreq-dt: %d\n", ret); return ret; @@ -176,7 +176,7 @@ static int imx_cpufreq_dt_remove(struct platform_device *pdev) { platform_device_unregister(cpufreq_dt_pdev); if (!of_machine_is_compatible("fsl,imx7ulp")) - dev_pm_opp_put_supported_hw(cpufreq_opp_table); + dev_pm_opp_put_supported_hw(cpufreq_opp_token); else clk_bulk_put(ARRAY_SIZE(imx7ulp_clks), imx7ulp_clks); diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c index e8db3d75be25..ab7ac7df9e62 100644 --- a/drivers/cpufreq/tegra20-cpufreq.c +++ b/drivers/cpufreq/tegra20-cpufreq.c @@ -32,9 +32,9 @@ static bool cpu0_node_has_opp_v2_prop(void) return ret; } -static void tegra20_cpufreq_put_supported_hw(void *opp_table) +static void tegra20_cpufreq_put_supported_hw(void *opp_token) { - dev_pm_opp_put_supported_hw(opp_table); + dev_pm_opp_put_supported_hw((unsigned long) opp_token); } static void tegra20_cpufreq_dt_unregister(void *cpufreq_dt) @@ -45,7 +45,6 @@ static void tegra20_cpufreq_dt_unregister(void *cpufreq_dt) static int tegra20_cpufreq_probe(struct platform_device *pdev) { struct platform_device *cpufreq_dt; - struct opp_table *opp_table; struct device *cpu_dev; u32 versions[2]; int err; @@ -71,16 +70,15 @@ static int tegra20_cpufreq_probe(struct platform_device *pdev) if (WARN_ON(!cpu_dev)) return -ENODEV; - opp_table = dev_pm_opp_set_supported_hw(cpu_dev, versions, 2); - err = PTR_ERR_OR_ZERO(opp_table); - if (err) { + err = dev_pm_opp_set_supported_hw(cpu_dev, versions, 2); + if (err < 0) { dev_err(&pdev->dev, "failed to set supported hw: %d\n", err); return err; } err = devm_add_action_or_reset(&pdev->dev, tegra20_cpufreq_put_supported_hw, - opp_table); + (void *)((unsigned long) err)); if (err) return err; diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c index 908f8d5392b2..85bc936c02f9 100644 --- a/drivers/memory/tegra/tegra124-emc.c +++ b/drivers/memory/tegra/tegra124-emc.c @@ -1395,15 +1395,14 @@ err_msg: static int tegra_emc_opp_table_init(struct tegra_emc *emc) { u32 hw_version = BIT(tegra_sku_info.soc_speedo_id); - struct opp_table *hw_opp_table; - int err; + int opp_token, err; - hw_opp_table = dev_pm_opp_set_supported_hw(emc->dev, &hw_version, 1); - err = PTR_ERR_OR_ZERO(hw_opp_table); - if (err) { + err = dev_pm_opp_set_supported_hw(emc->dev, &hw_version, 1); + if (err < 0) { dev_err(emc->dev, "failed to set OPP supported HW: %d\n", err); return err; } + opp_token = err; err = dev_pm_opp_of_add_table(emc->dev); if (err) { @@ -1430,7 +1429,7 @@ static int tegra_emc_opp_table_init(struct tegra_emc *emc) remove_table: dev_pm_opp_of_remove_table(emc->dev); put_hw_table: - dev_pm_opp_put_supported_hw(hw_opp_table); + dev_pm_opp_put_supported_hw(opp_token); return err; } diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 6ff9b5b69d07..8dbdfff38973 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1952,7 +1952,7 @@ free_opp: } /** - * dev_pm_opp_set_supported_hw() - Set supported platforms + * _opp_set_supported_hw() - Set supported platforms * @dev: Device for which supported-hw has to be set. * @versions: Array of hierarchy of versions to match. * @count: Number of elements in the array. @@ -1962,84 +1962,39 @@ free_opp: * OPPs, which are available for those versions, based on its 'opp-supported-hw' * property. */ -struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev, - const u32 *versions, unsigned int count) +static int _opp_set_supported_hw(struct opp_table *opp_table, + const u32 *versions, unsigned int count) { - struct opp_table *opp_table; - - opp_table = _add_opp_table(dev, false); - if (IS_ERR(opp_table)) - return opp_table; - - /* Make sure there are no concurrent readers while updating opp_table */ - WARN_ON(!list_empty(&opp_table->opp_list)); - /* Another CPU that shares the OPP table has set the property ? */ if (opp_table->supported_hw) - return opp_table; + return 0; opp_table->supported_hw = kmemdup(versions, count * sizeof(*versions), GFP_KERNEL); - if (!opp_table->supported_hw) { - dev_pm_opp_put_opp_table(opp_table); - return ERR_PTR(-ENOMEM); - } + if (!opp_table->supported_hw) + return -ENOMEM; opp_table->supported_hw_count = count; - return opp_table; + return 0; } -EXPORT_SYMBOL_GPL(dev_pm_opp_set_supported_hw); /** - * dev_pm_opp_put_supported_hw() - Releases resources blocked for supported hw - * @opp_table: OPP table returned by dev_pm_opp_set_supported_hw(). + * _opp_put_supported_hw() - Releases resources blocked for supported hw + * @opp_table: OPP table returned by _opp_set_supported_hw(). * * This is required only for the V2 bindings, and is called for a matching - * dev_pm_opp_set_supported_hw(). Until this is called, the opp_table structure + * _opp_set_supported_hw(). Until this is called, the opp_table structure * will not be freed. */ -void dev_pm_opp_put_supported_hw(struct opp_table *opp_table) +static void _opp_put_supported_hw(struct opp_table *opp_table) { - if (unlikely(!opp_table)) - return; - - kfree(opp_table->supported_hw); - opp_table->supported_hw = NULL; - opp_table->supported_hw_count = 0; - - dev_pm_opp_put_opp_table(opp_table); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_put_supported_hw); - -static void devm_pm_opp_supported_hw_release(void *data) -{ - dev_pm_opp_put_supported_hw(data); -} - -/** - * devm_pm_opp_set_supported_hw() - Set supported platforms - * @dev: Device for which supported-hw has to be set. - * @versions: Array of hierarchy of versions to match. - * @count: Number of elements in the array. - * - * This is a resource-managed variant of dev_pm_opp_set_supported_hw(). - * - * Return: 0 on success and errorno otherwise. - */ -int devm_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, - unsigned int count) -{ - struct opp_table *opp_table; - - opp_table = dev_pm_opp_set_supported_hw(dev, versions, count); - if (IS_ERR(opp_table)) - return PTR_ERR(opp_table); - - return devm_add_action_or_reset(dev, devm_pm_opp_supported_hw_release, - opp_table); + if (opp_table->supported_hw) { + kfree(opp_table->supported_hw); + opp_table->supported_hw = NULL; + opp_table->supported_hw_count = 0; + } } -EXPORT_SYMBOL_GPL(devm_pm_opp_set_supported_hw); /** * dev_pm_opp_set_prop_name() - Set prop-extn name @@ -2583,7 +2538,7 @@ static void _opp_clear_config(struct opp_config_data *data) if (data->flags & OPP_CONFIG_REGULATOR) _opp_put_regulators(data->opp_table); if (data->flags & OPP_CONFIG_SUPPORTED_HW) - dev_pm_opp_put_supported_hw(data->opp_table); + _opp_put_supported_hw(data->opp_table); if (data->flags & OPP_CONFIG_REGULATOR_HELPER) dev_pm_opp_unregister_set_opp_helper(data->opp_table); if (data->flags & OPP_CONFIG_PROP_NAME) @@ -2694,12 +2649,10 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) /* Configure supported hardware */ if (config->supported_hw) { - err = dev_pm_opp_set_supported_hw(dev, config->supported_hw, - config->supported_hw_count); - if (IS_ERR(err)) { - ret = PTR_ERR(err); + ret = _opp_set_supported_hw(opp_table, config->supported_hw, + config->supported_hw_count); + if (ret) goto err; - } data->flags |= OPP_CONFIG_SUPPORTED_HW; } diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index f014bd172c99..94d0101c254c 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -184,9 +184,6 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); void dev_pm_opp_clear_config(int token); -struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, unsigned int count); -void dev_pm_opp_put_supported_hw(struct opp_table *opp_table); -int devm_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, unsigned int count); struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct opp_table *opp_table); struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name); @@ -369,22 +366,6 @@ static inline int dev_pm_opp_unregister_notifier(struct device *dev, struct noti return -EOPNOTSUPP; } -static inline struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev, - const u32 *versions, - unsigned int count) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void dev_pm_opp_put_supported_hw(struct opp_table *opp_table) {} - -static inline int devm_pm_opp_set_supported_hw(struct device *dev, - const u32 *versions, - unsigned int count) -{ - return -EOPNOTSUPP; -} - static inline struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)) { @@ -618,4 +599,34 @@ static inline int devm_pm_opp_set_regulators(struct device *dev, return devm_pm_opp_set_config(dev, &config); } +/* Supported-hw helpers */ +static inline int dev_pm_opp_set_supported_hw(struct device *dev, + const u32 *versions, + unsigned int count) +{ + struct dev_pm_opp_config config = { + .supported_hw = versions, + .supported_hw_count = count, + }; + + return dev_pm_opp_set_config(dev, &config); +} + +static inline void dev_pm_opp_put_supported_hw(int token) +{ + dev_pm_opp_clear_config(token); +} + +static inline int devm_pm_opp_set_supported_hw(struct device *dev, + const u32 *versions, + unsigned int count) +{ + struct dev_pm_opp_config config = { + .supported_hw = versions, + .supported_hw_count = count, + }; + + return devm_pm_opp_set_config(dev, &config); +} + #endif /* __LINUX_OPP_H__ */ -- cgit v1.2.3 From 2368f57685768f9f9cd666eaa4194a359d89afb8 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 26 May 2022 09:36:27 +0530 Subject: OPP: Migrate set-clk-name API to use set-config helpers Now that we have a central API to handle all OPP table configurations, migrate the set-clk-name family of helpers to use the new infrastructure. Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 142 +++++++++++++++---------------------------------- include/linux/pm_opp.h | 41 ++++++++------ 2 files changed, 69 insertions(+), 114 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 8dbdfff38973..0a82ca7ae453 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2164,104 +2164,71 @@ static void _opp_put_regulators(struct opp_table *opp_table) } /** - * dev_pm_opp_set_clkname() - Set clk name for the device - * @dev: Device for which clk name is being set. - * @name: Clk name. - * - * In order to support OPP switching, OPP layer needs to get pointer to the - * clock for the device. Simple cases work fine without using this routine (i.e. - * by passing connection-id as NULL), but for a device with multiple clocks - * available, the OPP core needs to know the exact name of the clk to use. + * _opp_set_clknames() - Set clk names for the device + * @dev: Device for which clk names is being set. + * @names: Clk names. + * + * In order to support OPP switching, OPP layer needs to get pointers to the + * clocks for the device. Simple cases work fine without using this routine + * (i.e. by passing connection-id as NULL), but for a device with multiple + * clocks available, the OPP core needs to know the exact names of the clks to + * use. * * This must be called before any OPPs are initialized for the device. */ -struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name) +static int _opp_set_clknames(struct opp_table *opp_table, struct device *dev, + const char * const names[]) { - struct opp_table *opp_table; - int ret; + const char * const *temp = names; + int count = 0; - opp_table = _add_opp_table(dev, false); - if (IS_ERR(opp_table)) - return opp_table; + /* Count number of clks */ + while (*temp++) + count++; - /* This should be called before OPPs are initialized */ - if (WARN_ON(!list_empty(&opp_table->opp_list))) { - ret = -EBUSY; - goto err; - } + /* + * This is a special case where we have a single clock, whose connection + * id name is NULL, i.e. first two entries are NULL in the array. + */ + if (!count && !names[1]) + count = 1; + + /* We support only one clock name for now */ + if (count != 1) + return -EINVAL; /* Another CPU that shares the OPP table has set the clkname ? */ if (opp_table->clk_configured) - return opp_table; + return 0; /* clk shouldn't be initialized at this point */ - if (WARN_ON(opp_table->clk)) { - ret = -EBUSY; - goto err; - } + if (WARN_ON(opp_table->clk)) + return -EBUSY; /* Find clk for the device */ - opp_table->clk = clk_get(dev, name); + opp_table->clk = clk_get(dev, names[0]); if (IS_ERR(opp_table->clk)) { - ret = dev_err_probe(dev, PTR_ERR(opp_table->clk), + return dev_err_probe(dev, PTR_ERR(opp_table->clk), "%s: Couldn't find clock\n", __func__); - goto err; } opp_table->clk_configured = true; - return opp_table; - -err: - dev_pm_opp_put_opp_table(opp_table); - - return ERR_PTR(ret); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_set_clkname); - -/** - * dev_pm_opp_put_clkname() - Releases resources blocked for clk. - * @opp_table: OPP table returned from dev_pm_opp_set_clkname(). - */ -void dev_pm_opp_put_clkname(struct opp_table *opp_table) -{ - if (unlikely(!opp_table)) - return; - - clk_put(opp_table->clk); - opp_table->clk = ERR_PTR(-EINVAL); - opp_table->clk_configured = false; - - dev_pm_opp_put_opp_table(opp_table); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_put_clkname); - -static void devm_pm_opp_clkname_release(void *data) -{ - dev_pm_opp_put_clkname(data); + return 0; } /** - * devm_pm_opp_set_clkname() - Set clk name for the device - * @dev: Device for which clk name is being set. - * @name: Clk name. - * - * This is a resource-managed variant of dev_pm_opp_set_clkname(). - * - * Return: 0 on success and errorno otherwise. + * _opp_put_clknames() - Releases resources blocked for clks. + * @opp_table: OPP table returned from _opp_set_clknames(). */ -int devm_pm_opp_set_clkname(struct device *dev, const char *name) +static void _opp_put_clknames(struct opp_table *opp_table) { - struct opp_table *opp_table; - - opp_table = dev_pm_opp_set_clkname(dev, name); - if (IS_ERR(opp_table)) - return PTR_ERR(opp_table); - - return devm_add_action_or_reset(dev, devm_pm_opp_clkname_release, - opp_table); + if (opp_table->clk_configured) { + clk_put(opp_table->clk); + opp_table->clk = ERR_PTR(-EINVAL); + opp_table->clk_configured = false; + } } -EXPORT_SYMBOL_GPL(devm_pm_opp_set_clkname); /** * dev_pm_opp_register_set_opp_helper() - Register custom set OPP helper @@ -2544,7 +2511,7 @@ static void _opp_clear_config(struct opp_config_data *data) if (data->flags & OPP_CONFIG_PROP_NAME) dev_pm_opp_put_prop_name(data->opp_table); if (data->flags & OPP_CONFIG_CLK) - dev_pm_opp_put_clkname(data->opp_table); + _opp_put_clknames(data->opp_table); dev_pm_opp_put_opp_table(data->opp_table); kfree(data); @@ -2595,32 +2562,9 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) /* Configure clocks */ if (config->clk_names) { - const char * const *temp = config->clk_names; - int count = 0; - - /* Count number of clks */ - while (*temp++) - count++; - - /* - * This is a special case where we have a single clock, whose - * connection id name is NULL, i.e. first two entries are NULL - * in the array. - */ - if (!count && !config->clk_names[1]) - count = 1; - - /* We support only one clock name for now */ - if (count != 1) { - ret = -EINVAL; - goto err; - } - - err = dev_pm_opp_set_clkname(dev, config->clk_names[0]); - if (IS_ERR(err)) { - ret = PTR_ERR(err); + ret = _opp_set_clknames(opp_table, dev, config->clk_names); + if (ret) goto err; - } data->flags |= OPP_CONFIG_CLK; } diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 94d0101c254c..ed1906bbe8bb 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -186,9 +186,6 @@ void dev_pm_opp_clear_config(int token); struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct opp_table *opp_table); -struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name); -void dev_pm_opp_put_clkname(struct opp_table *opp_table); -int devm_pm_opp_set_clkname(struct device *dev, const char *name); struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table); int devm_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); @@ -387,18 +384,6 @@ static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, con static inline void dev_pm_opp_put_prop_name(struct opp_table *opp_table) {} -static inline struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void dev_pm_opp_put_clkname(struct opp_table *opp_table) {} - -static inline int devm_pm_opp_set_clkname(struct device *dev, const char *name) -{ - return -EOPNOTSUPP; -} - static inline struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char * const *names, struct device ***virt_devs) { return ERR_PTR(-EOPNOTSUPP); @@ -629,4 +614,30 @@ static inline int devm_pm_opp_set_supported_hw(struct device *dev, return devm_pm_opp_set_config(dev, &config); } +/* clkname helpers */ +static inline int dev_pm_opp_set_clkname(struct device *dev, const char *name) +{ + const char *names[] = { name, NULL }; + struct dev_pm_opp_config config = { + .clk_names = names, + }; + + return dev_pm_opp_set_config(dev, &config); +} + +static inline void dev_pm_opp_put_clkname(int token) +{ + dev_pm_opp_clear_config(token); +} + +static inline int devm_pm_opp_set_clkname(struct device *dev, const char *name) +{ + const char *names[] = { name, NULL }; + struct dev_pm_opp_config config = { + .clk_names = names, + }; + + return devm_pm_opp_set_config(dev, &config); +} + #endif /* __LINUX_OPP_H__ */ -- cgit v1.2.3 From 3c543b42a6df35feb3db6689e512fa167739451e Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 26 May 2022 09:36:27 +0530 Subject: OPP: Migrate set-opp-helper API to use set-config helpers Now that we have a central API to handle all OPP table configurations, migrate the set-opp-helper family of helpers to use the new infrastructure. The return type and parameter to the APIs change a bit due to this, update the current users as well in the same commit in order to avoid breaking builds. Remove devm_pm_opp_register_set_opp_helper() as it has no users currently. Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 89 ++++++++++----------------------------------- drivers/opp/ti-opp-supply.c | 6 +-- include/linux/pm_opp.h | 33 ++++++++--------- 3 files changed, 39 insertions(+), 89 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 0a82ca7ae453..9da7dcf62cab 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2231,7 +2231,7 @@ static void _opp_put_clknames(struct opp_table *opp_table) } /** - * dev_pm_opp_register_set_opp_helper() - Register custom set OPP helper + * _opp_register_set_opp_helper() - Register custom set OPP helper * @dev: Device for which the helper is getting registered. * @set_opp: Custom set OPP helper. * @@ -2240,32 +2240,18 @@ static void _opp_put_clknames(struct opp_table *opp_table) * * This must be called before any OPPs are initialized for the device. */ -struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, - int (*set_opp)(struct dev_pm_set_opp_data *data)) +static int _opp_register_set_opp_helper(struct opp_table *opp_table, + struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)) { struct dev_pm_set_opp_data *data; - struct opp_table *opp_table; - - if (!set_opp) - return ERR_PTR(-EINVAL); - - opp_table = _add_opp_table(dev, false); - if (IS_ERR(opp_table)) - return opp_table; - - /* This should be called before OPPs are initialized */ - if (WARN_ON(!list_empty(&opp_table->opp_list))) { - dev_pm_opp_put_opp_table(opp_table); - return ERR_PTR(-EBUSY); - } /* Another CPU that shares the OPP table has set the helper ? */ if (opp_table->set_opp) - return opp_table; + return 0; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) - return ERR_PTR(-ENOMEM); + return -ENOMEM; mutex_lock(&opp_table->lock); opp_table->set_opp_data = data; @@ -2278,60 +2264,26 @@ struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, opp_table->set_opp = set_opp; - return opp_table; + return 0; } -EXPORT_SYMBOL_GPL(dev_pm_opp_register_set_opp_helper); /** - * dev_pm_opp_unregister_set_opp_helper() - Releases resources blocked for - * set_opp helper - * @opp_table: OPP table returned from dev_pm_opp_register_set_opp_helper(). + * _opp_unregister_set_opp_helper() - Releases resources blocked for set_opp helper + * @opp_table: OPP table returned from _opp_register_set_opp_helper(). * * Release resources blocked for platform specific set_opp helper. */ -void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table) +static void _opp_unregister_set_opp_helper(struct opp_table *opp_table) { - if (unlikely(!opp_table)) - return; - - opp_table->set_opp = NULL; - - mutex_lock(&opp_table->lock); - kfree(opp_table->set_opp_data); - opp_table->set_opp_data = NULL; - mutex_unlock(&opp_table->lock); - - dev_pm_opp_put_opp_table(opp_table); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_unregister_set_opp_helper); - -static void devm_pm_opp_unregister_set_opp_helper(void *data) -{ - dev_pm_opp_unregister_set_opp_helper(data); -} - -/** - * devm_pm_opp_register_set_opp_helper() - Register custom set OPP helper - * @dev: Device for which the helper is getting registered. - * @set_opp: Custom set OPP helper. - * - * This is a resource-managed version of dev_pm_opp_register_set_opp_helper(). - * - * Return: 0 on success and errorno otherwise. - */ -int devm_pm_opp_register_set_opp_helper(struct device *dev, - int (*set_opp)(struct dev_pm_set_opp_data *data)) -{ - struct opp_table *opp_table; - - opp_table = dev_pm_opp_register_set_opp_helper(dev, set_opp); - if (IS_ERR(opp_table)) - return PTR_ERR(opp_table); + if (opp_table->set_opp) { + opp_table->set_opp = NULL; - return devm_add_action_or_reset(dev, devm_pm_opp_unregister_set_opp_helper, - opp_table); + mutex_lock(&opp_table->lock); + kfree(opp_table->set_opp_data); + opp_table->set_opp_data = NULL; + mutex_unlock(&opp_table->lock); + } } -EXPORT_SYMBOL_GPL(devm_pm_opp_register_set_opp_helper); static void _opp_detach_genpd(struct opp_table *opp_table) { @@ -2507,7 +2459,7 @@ static void _opp_clear_config(struct opp_config_data *data) if (data->flags & OPP_CONFIG_SUPPORTED_HW) _opp_put_supported_hw(data->opp_table); if (data->flags & OPP_CONFIG_REGULATOR_HELPER) - dev_pm_opp_unregister_set_opp_helper(data->opp_table); + _opp_unregister_set_opp_helper(data->opp_table); if (data->flags & OPP_CONFIG_PROP_NAME) dev_pm_opp_put_prop_name(data->opp_table); if (data->flags & OPP_CONFIG_CLK) @@ -2582,11 +2534,10 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) /* Configure opp helper */ if (config->set_opp) { - err = dev_pm_opp_register_set_opp_helper(dev, config->set_opp); - if (IS_ERR(err)) { - ret = PTR_ERR(err); + ret = _opp_register_set_opp_helper(opp_table, dev, + config->set_opp); + if (ret) goto err; - } data->flags |= OPP_CONFIG_REGULATOR_HELPER; } diff --git a/drivers/opp/ti-opp-supply.c b/drivers/opp/ti-opp-supply.c index bd4771f388ab..40ebc9ac82dd 100644 --- a/drivers/opp/ti-opp-supply.c +++ b/drivers/opp/ti-opp-supply.c @@ -405,9 +405,9 @@ static int ti_opp_supply_probe(struct platform_device *pdev) return ret; } - ret = PTR_ERR_OR_ZERO(dev_pm_opp_register_set_opp_helper(cpu_dev, - ti_opp_supply_set_opp)); - if (ret) + ret = dev_pm_opp_register_set_opp_helper(cpu_dev, + ti_opp_supply_set_opp); + if (ret < 0) _free_optimized_voltages(dev, &opp_data); return ret; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index ed1906bbe8bb..85a4b7353979 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -186,9 +186,6 @@ void dev_pm_opp_clear_config(int token); struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct opp_table *opp_table); -struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); -void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table); -int devm_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char * const *names, struct device ***virt_devs); void dev_pm_opp_detach_genpd(struct opp_table *opp_table); int devm_pm_opp_attach_genpd(struct device *dev, const char * const *names, struct device ***virt_devs); @@ -363,20 +360,6 @@ static inline int dev_pm_opp_unregister_notifier(struct device *dev, struct noti return -EOPNOTSUPP; } -static inline struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, - int (*set_opp)(struct dev_pm_set_opp_data *data)) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table) {} - -static inline int devm_pm_opp_register_set_opp_helper(struct device *dev, - int (*set_opp)(struct dev_pm_set_opp_data *data)) -{ - return -EOPNOTSUPP; -} - static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name) { return ERR_PTR(-EOPNOTSUPP); @@ -640,4 +623,20 @@ static inline int devm_pm_opp_set_clkname(struct device *dev, const char *name) return devm_pm_opp_set_config(dev, &config); } +/* set-opp helpers */ +static inline int dev_pm_opp_register_set_opp_helper(struct device *dev, + int (*set_opp)(struct dev_pm_set_opp_data *data)) +{ + struct dev_pm_opp_config config = { + .set_opp = set_opp, + }; + + return dev_pm_opp_set_config(dev, &config); +} + +static inline void dev_pm_opp_unregister_set_opp_helper(int token) +{ + dev_pm_opp_clear_config(token); +} + #endif /* __LINUX_OPP_H__ */ -- cgit v1.2.3 From 442e7a1786e628b38175314ec1805966b8d0c02c Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 26 May 2022 09:36:27 +0530 Subject: OPP: Migrate attach-genpd API to use set-config helpers Now that we have a central API to handle all OPP table configurations, migrate the attach-genpd family of helpers to use the new infrastructure. Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 85 +++++++++++--------------------------------------- include/linux/pm_opp.h | 48 ++++++++++++++++++---------- 2 files changed, 49 insertions(+), 84 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 9da7dcf62cab..458584994c2b 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2285,7 +2285,7 @@ static void _opp_unregister_set_opp_helper(struct opp_table *opp_table) } } -static void _opp_detach_genpd(struct opp_table *opp_table) +static void _detach_genpd(struct opp_table *opp_table) { int index; @@ -2305,7 +2305,7 @@ static void _opp_detach_genpd(struct opp_table *opp_table) } /** - * dev_pm_opp_attach_genpd - Attach genpd(s) for the device and save virtual device pointer + * _opp_attach_genpd - Attach genpd(s) for the device and save virtual device pointer * @dev: Consumer device for which the genpd is getting attached. * @names: Null terminated array of pointers containing names of genpd to attach. * @virt_devs: Pointer to return the array of virtual devices. @@ -2326,30 +2326,23 @@ static void _opp_detach_genpd(struct opp_table *opp_table) * The order of entries in the names array must match the order in which * "required-opps" are added in DT. */ -struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, - const char * const *names, struct device ***virt_devs) +static int _opp_attach_genpd(struct opp_table *opp_table, struct device *dev, + const char * const *names, struct device ***virt_devs) { - struct opp_table *opp_table; struct device *virt_dev; int index = 0, ret = -EINVAL; const char * const *name = names; - opp_table = _add_opp_table(dev, false); - if (IS_ERR(opp_table)) - return opp_table; - if (opp_table->genpd_virt_devs) - return opp_table; + return 0; /* * If the genpd's OPP table isn't already initialized, parsing of the * required-opps fail for dev. We should retry this after genpd's OPP * table is added. */ - if (!opp_table->required_opp_count) { - ret = -EPROBE_DEFER; - goto put_table; - } + if (!opp_table->required_opp_count) + return -EPROBE_DEFER; mutex_lock(&opp_table->genpd_virt_dev_lock); @@ -2382,78 +2375,38 @@ struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, *virt_devs = opp_table->genpd_virt_devs; mutex_unlock(&opp_table->genpd_virt_dev_lock); - return opp_table; + return 0; err: - _opp_detach_genpd(opp_table); + _detach_genpd(opp_table); unlock: mutex_unlock(&opp_table->genpd_virt_dev_lock); + return ret; -put_table: - dev_pm_opp_put_opp_table(opp_table); - - return ERR_PTR(ret); } -EXPORT_SYMBOL_GPL(dev_pm_opp_attach_genpd); /** - * dev_pm_opp_detach_genpd() - Detach genpd(s) from the device. - * @opp_table: OPP table returned by dev_pm_opp_attach_genpd(). + * _opp_detach_genpd() - Detach genpd(s) from the device. + * @opp_table: OPP table returned by _opp_attach_genpd(). * * This detaches the genpd(s), resets the virtual device pointers, and puts the * OPP table. */ -void dev_pm_opp_detach_genpd(struct opp_table *opp_table) +static void _opp_detach_genpd(struct opp_table *opp_table) { - if (unlikely(!opp_table)) - return; - /* * Acquire genpd_virt_dev_lock to make sure virt_dev isn't getting * used in parallel. */ mutex_lock(&opp_table->genpd_virt_dev_lock); - _opp_detach_genpd(opp_table); + _detach_genpd(opp_table); mutex_unlock(&opp_table->genpd_virt_dev_lock); - - dev_pm_opp_put_opp_table(opp_table); } -EXPORT_SYMBOL_GPL(dev_pm_opp_detach_genpd); - -static void devm_pm_opp_detach_genpd(void *data) -{ - dev_pm_opp_detach_genpd(data); -} - -/** - * devm_pm_opp_attach_genpd - Attach genpd(s) for the device and save virtual - * device pointer - * @dev: Consumer device for which the genpd is getting attached. - * @names: Null terminated array of pointers containing names of genpd to attach. - * @virt_devs: Pointer to return the array of virtual devices. - * - * This is a resource-managed version of dev_pm_opp_attach_genpd(). - * - * Return: 0 on success and errorno otherwise. - */ -int devm_pm_opp_attach_genpd(struct device *dev, const char * const *names, - struct device ***virt_devs) -{ - struct opp_table *opp_table; - - opp_table = dev_pm_opp_attach_genpd(dev, names, virt_devs); - if (IS_ERR(opp_table)) - return PTR_ERR(opp_table); - - return devm_add_action_or_reset(dev, devm_pm_opp_detach_genpd, - opp_table); -} -EXPORT_SYMBOL_GPL(devm_pm_opp_attach_genpd); static void _opp_clear_config(struct opp_config_data *data) { if (data->flags & OPP_CONFIG_GENPD) - dev_pm_opp_detach_genpd(data->opp_table); + _opp_detach_genpd(data->opp_table); if (data->flags & OPP_CONFIG_REGULATOR) _opp_put_regulators(data->opp_table); if (data->flags & OPP_CONFIG_SUPPORTED_HW) @@ -2564,12 +2517,10 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) /* Attach genpds */ if (config->genpd_names) { - err = dev_pm_opp_attach_genpd(dev, config->genpd_names, - config->virt_devs); - if (IS_ERR(err)) { - ret = PTR_ERR(err); + ret = _opp_attach_genpd(opp_table, dev, config->genpd_names, + config->virt_devs); + if (ret) goto err; - } data->flags |= OPP_CONFIG_GENPD; } diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 85a4b7353979..20e1e5060a8a 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -186,9 +186,6 @@ void dev_pm_opp_clear_config(int token); struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct opp_table *opp_table); -struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char * const *names, struct device ***virt_devs); -void dev_pm_opp_detach_genpd(struct opp_table *opp_table); -int devm_pm_opp_attach_genpd(struct device *dev, const char * const *names, struct device ***virt_devs); struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, struct opp_table *dst_table, struct dev_pm_opp *src_opp); int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); @@ -367,20 +364,6 @@ static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, con static inline void dev_pm_opp_put_prop_name(struct opp_table *opp_table) {} -static inline struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char * const *names, struct device ***virt_devs) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void dev_pm_opp_detach_genpd(struct opp_table *opp_table) {} - -static inline int devm_pm_opp_attach_genpd(struct device *dev, - const char * const *names, - struct device ***virt_devs) -{ - return -EOPNOTSUPP; -} - static inline int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) { return -EOPNOTSUPP; @@ -639,4 +622,35 @@ static inline void dev_pm_opp_unregister_set_opp_helper(int token) dev_pm_opp_clear_config(token); } +/* genpd helpers */ +static inline int dev_pm_opp_attach_genpd(struct device *dev, + const char * const *names, + struct device ***virt_devs) +{ + struct dev_pm_opp_config config = { + .genpd_names = names, + .virt_devs = virt_devs, + }; + + return dev_pm_opp_set_config(dev, &config); +} + +static inline void dev_pm_opp_detach_genpd(int token) +{ + dev_pm_opp_clear_config(token); +} + +static inline int devm_pm_opp_attach_genpd(struct device *dev, + const char * const *names, + struct device ***virt_devs) +{ + struct dev_pm_opp_config config = { + .genpd_names = names, + .virt_devs = virt_devs, + }; + + return devm_pm_opp_set_config(dev, &config); +} + + #endif /* __LINUX_OPP_H__ */ -- cgit v1.2.3 From 298098e55a6fcc176a5af52cd689f33577ead5ca Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 26 May 2022 09:36:27 +0530 Subject: OPP: Migrate set-prop-name helper API to use set-config helpers Now that we have a central API to handle all OPP table configurations, migrate the set-prop-name family of helpers to use the new infrastructure. The return type and parameter to the APIs change a bit due to this, update the current users as well in the same commit in order to avoid breaking builds. Acked-by: Samuel Holland # sun50i Signed-off-by: Viresh Kumar --- drivers/cpufreq/sun50i-cpufreq-nvmem.c | 31 +++++++++---------- drivers/opp/core.c | 55 +++++++++++----------------------- include/linux/pm_opp.h | 23 ++++++++------ 3 files changed, 46 insertions(+), 63 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/sun50i-cpufreq-nvmem.c b/drivers/cpufreq/sun50i-cpufreq-nvmem.c index 75e1bf3a08f7..a4922580ce06 100644 --- a/drivers/cpufreq/sun50i-cpufreq-nvmem.c +++ b/drivers/cpufreq/sun50i-cpufreq-nvmem.c @@ -86,20 +86,20 @@ static int sun50i_cpufreq_get_efuse(u32 *versions) static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev) { - struct opp_table **opp_tables; + int *opp_tokens; char name[MAX_NAME_LEN]; unsigned int cpu; u32 speed = 0; int ret; - opp_tables = kcalloc(num_possible_cpus(), sizeof(*opp_tables), + opp_tokens = kcalloc(num_possible_cpus(), sizeof(*opp_tokens), GFP_KERNEL); - if (!opp_tables) + if (!opp_tokens) return -ENOMEM; ret = sun50i_cpufreq_get_efuse(&speed); if (ret) { - kfree(opp_tables); + kfree(opp_tokens); return ret; } @@ -113,9 +113,9 @@ static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev) goto free_opp; } - opp_tables[cpu] = dev_pm_opp_set_prop_name(cpu_dev, name); - if (IS_ERR(opp_tables[cpu])) { - ret = PTR_ERR(opp_tables[cpu]); + opp_tokens[cpu] = dev_pm_opp_set_prop_name(cpu_dev, name); + if (opp_tokens[cpu] < 0) { + ret = opp_tokens[cpu]; pr_err("Failed to set prop name\n"); goto free_opp; } @@ -124,7 +124,7 @@ static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev) cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1, NULL, 0); if (!IS_ERR(cpufreq_dt_pdev)) { - platform_set_drvdata(pdev, opp_tables); + platform_set_drvdata(pdev, opp_tokens); return 0; } @@ -132,27 +132,24 @@ static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev) pr_err("Failed to register platform device\n"); free_opp: - for_each_possible_cpu(cpu) { - if (IS_ERR_OR_NULL(opp_tables[cpu])) - break; - dev_pm_opp_put_prop_name(opp_tables[cpu]); - } - kfree(opp_tables); + for_each_possible_cpu(cpu) + dev_pm_opp_put_prop_name(opp_tokens[cpu]); + kfree(opp_tokens); return ret; } static int sun50i_cpufreq_nvmem_remove(struct platform_device *pdev) { - struct opp_table **opp_tables = platform_get_drvdata(pdev); + int *opp_tokens = platform_get_drvdata(pdev); unsigned int cpu; platform_device_unregister(cpufreq_dt_pdev); for_each_possible_cpu(cpu) - dev_pm_opp_put_prop_name(opp_tables[cpu]); + dev_pm_opp_put_prop_name(opp_tokens[cpu]); - kfree(opp_tables); + kfree(opp_tokens); return 0; } diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 458584994c2b..1745e25c1eaf 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1997,7 +1997,7 @@ static void _opp_put_supported_hw(struct opp_table *opp_table) } /** - * dev_pm_opp_set_prop_name() - Set prop-extn name + * _opp_set_prop_name() - Set prop-extn name * @dev: Device for which the prop-name has to be set. * @name: name to postfix to properties. * @@ -2006,50 +2006,33 @@ static void _opp_put_supported_hw(struct opp_table *opp_table) * which the extension will apply are opp-microvolt and opp-microamp. OPP core * should postfix the property name with - while looking for them. */ -struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name) +static int _opp_set_prop_name(struct opp_table *opp_table, const char *name) { - struct opp_table *opp_table; - - opp_table = _add_opp_table(dev, false); - if (IS_ERR(opp_table)) - return opp_table; - - /* Make sure there are no concurrent readers while updating opp_table */ - WARN_ON(!list_empty(&opp_table->opp_list)); - /* Another CPU that shares the OPP table has set the property ? */ - if (opp_table->prop_name) - return opp_table; - - opp_table->prop_name = kstrdup(name, GFP_KERNEL); if (!opp_table->prop_name) { - dev_pm_opp_put_opp_table(opp_table); - return ERR_PTR(-ENOMEM); + opp_table->prop_name = kstrdup(name, GFP_KERNEL); + if (!opp_table->prop_name) + return -ENOMEM; } - return opp_table; + return 0; } -EXPORT_SYMBOL_GPL(dev_pm_opp_set_prop_name); /** - * dev_pm_opp_put_prop_name() - Releases resources blocked for prop-name - * @opp_table: OPP table returned by dev_pm_opp_set_prop_name(). + * _opp_put_prop_name() - Releases resources blocked for prop-name + * @opp_table: OPP table returned by _opp_set_prop_name(). * * This is required only for the V2 bindings, and is called for a matching - * dev_pm_opp_set_prop_name(). Until this is called, the opp_table structure + * _opp_set_prop_name(). Until this is called, the opp_table structure * will not be freed. */ -void dev_pm_opp_put_prop_name(struct opp_table *opp_table) +static void _opp_put_prop_name(struct opp_table *opp_table) { - if (unlikely(!opp_table)) - return; - - kfree(opp_table->prop_name); - opp_table->prop_name = NULL; - - dev_pm_opp_put_opp_table(opp_table); + if (opp_table->prop_name) { + kfree(opp_table->prop_name); + opp_table->prop_name = NULL; + } } -EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); /** * _opp_set_regulators() - Set regulator names for the device @@ -2414,7 +2397,7 @@ static void _opp_clear_config(struct opp_config_data *data) if (data->flags & OPP_CONFIG_REGULATOR_HELPER) _opp_unregister_set_opp_helper(data->opp_table); if (data->flags & OPP_CONFIG_PROP_NAME) - dev_pm_opp_put_prop_name(data->opp_table); + _opp_put_prop_name(data->opp_table); if (data->flags & OPP_CONFIG_CLK) _opp_put_clknames(data->opp_table); @@ -2441,7 +2424,7 @@ static void _opp_clear_config(struct opp_config_data *data) */ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) { - struct opp_table *opp_table, *err; + struct opp_table *opp_table; struct opp_config_data *data; unsigned int id; int ret; @@ -2476,11 +2459,9 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) /* Configure property names */ if (config->prop_name) { - err = dev_pm_opp_set_prop_name(dev, config->prop_name); - if (IS_ERR(err)) { - ret = PTR_ERR(err); + ret = _opp_set_prop_name(opp_table, config->prop_name); + if (ret) goto err; - } data->flags |= OPP_CONFIG_PROP_NAME; } diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 20e1e5060a8a..f995ca1406e8 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -184,8 +184,6 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); void dev_pm_opp_clear_config(int token); -struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name); -void dev_pm_opp_put_prop_name(struct opp_table *opp_table); struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, struct opp_table *dst_table, struct dev_pm_opp *src_opp); int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); @@ -357,13 +355,6 @@ static inline int dev_pm_opp_unregister_notifier(struct device *dev, struct noti return -EOPNOTSUPP; } -static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void dev_pm_opp_put_prop_name(struct opp_table *opp_table) {} - static inline int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) { return -EOPNOTSUPP; @@ -652,5 +643,19 @@ static inline int devm_pm_opp_attach_genpd(struct device *dev, return devm_pm_opp_set_config(dev, &config); } +/* prop-name helpers */ +static inline int dev_pm_opp_set_prop_name(struct device *dev, const char *name) +{ + struct dev_pm_opp_config config = { + .prop_name = name, + }; + + return dev_pm_opp_set_config(dev, &config); +} + +static inline void dev_pm_opp_put_prop_name(int token) +{ + dev_pm_opp_clear_config(token); +} #endif /* __LINUX_OPP_H__ */ -- cgit v1.2.3 From aee3352f6ecf8cfad1f1ee5838cfc4d37c6b8f75 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 4 Jul 2022 13:45:08 +0530 Subject: OPP: Add support for config_regulators() helper Extend the dev_pm_opp_set_config() interface to allow adding config_regulators() helpers. This helper will be called to set the voltages of the regulators from the regular path in _set_opp(), while we are trying to change the OPP. This will eventually replace the custom set_opp() helper. Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++- drivers/opp/opp.h | 2 ++ include/linux/pm_opp.h | 22 ++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 1745e25c1eaf..12bae79564f1 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1185,6 +1185,17 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, dev_err(dev, "Failed to set bw: %d\n", ret); return ret; } + + if (opp_table->config_regulators) { + ret = opp_table->config_regulators(dev, old_opp, opp, + opp_table->regulators, + opp_table->regulator_count); + if (ret) { + dev_err(dev, "Failed to set regulator voltages: %d\n", + ret); + return ret; + } + } } if (opp_table->set_opp) { @@ -1202,6 +1213,17 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, /* Scaling down? Configure required OPPs after frequency */ if (scaling_down) { + if (opp_table->config_regulators) { + ret = opp_table->config_regulators(dev, old_opp, opp, + opp_table->regulators, + opp_table->regulator_count); + if (ret) { + dev_err(dev, "Failed to set regulator voltages: %d\n", + ret); + return ret; + } + } + ret = _set_opp_bw(opp_table, opp, dev); if (ret) { dev_err(dev, "Failed to set bw: %d\n", ret); @@ -2268,6 +2290,38 @@ static void _opp_unregister_set_opp_helper(struct opp_table *opp_table) } } +/** + * _opp_set_config_regulators_helper() - Register custom set regulator helper. + * @dev: Device for which the helper is getting registered. + * @config_regulators: Custom set regulator helper. + * + * This is useful to support platforms with multiple regulators per device. + * + * This must be called before any OPPs are initialized for the device. + */ +static int _opp_set_config_regulators_helper(struct opp_table *opp_table, + struct device *dev, config_regulators_t config_regulators) +{ + /* Another CPU that shares the OPP table has set the helper ? */ + if (!opp_table->config_regulators) + opp_table->config_regulators = config_regulators; + + return 0; +} + +/** + * _opp_put_config_regulators_helper() - Releases resources blocked for + * config_regulators helper. + * @opp_table: OPP table returned from _opp_set_config_regulators_helper(). + * + * Release resources blocked for platform specific config_regulators helper. + */ +static void _opp_put_config_regulators_helper(struct opp_table *opp_table) +{ + if (opp_table->config_regulators) + opp_table->config_regulators = NULL; +} + static void _detach_genpd(struct opp_table *opp_table) { int index; @@ -2394,8 +2448,10 @@ static void _opp_clear_config(struct opp_config_data *data) _opp_put_regulators(data->opp_table); if (data->flags & OPP_CONFIG_SUPPORTED_HW) _opp_put_supported_hw(data->opp_table); - if (data->flags & OPP_CONFIG_REGULATOR_HELPER) + if (data->flags & OPP_CONFIG_REGULATOR_HELPER) { + _opp_put_config_regulators_helper(data->opp_table); _opp_unregister_set_opp_helper(data->opp_table); + } if (data->flags & OPP_CONFIG_PROP_NAME) _opp_put_prop_name(data->opp_table); if (data->flags & OPP_CONFIG_CLK) @@ -2476,6 +2532,16 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) data->flags |= OPP_CONFIG_REGULATOR_HELPER; } + /* Configure config_regulators helper */ + if (config->config_regulators) { + ret = _opp_set_config_regulators_helper(opp_table, dev, + config->config_regulators); + if (ret) + goto err; + + data->flags |= OPP_CONFIG_REGULATOR_HELPER; + } + /* Configure supported hardware */ if (config->supported_hw) { ret = _opp_set_supported_hw(opp_table, config->supported_hw, diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h index d652f0cc84f1..45fd40737159 100644 --- a/drivers/opp/opp.h +++ b/drivers/opp/opp.h @@ -172,6 +172,7 @@ enum opp_table_access { * @prop_name: A name to postfix to many DT properties, while parsing them. * @clk_configured: Clock name is configured by the platform. * @clk: Device's clock handle + * @config_regulators: Platform specific config_regulators() callback. * @regulators: Supply regulators * @regulator_count: Number of power supply regulators. Its value can be -1 * (uninitialized), 0 (no opp-microvolt property) or > 0 (has opp-microvolt @@ -224,6 +225,7 @@ struct opp_table { const char *prop_name; bool clk_configured; struct clk *clk; + config_regulators_t config_regulators; struct regulator **regulators; int regulator_count; struct icc_path **paths; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index f995ca1406e8..9f2f9a792a19 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -90,11 +90,16 @@ struct dev_pm_set_opp_data { struct device *dev; }; +typedef int (*config_regulators_t)(struct device *dev, + struct dev_pm_opp *old_opp, struct dev_pm_opp *new_opp, + struct regulator **regulators, unsigned int count); + /** * struct dev_pm_opp_config - Device OPP configuration values * @clk_names: Clk names, NULL terminated array, max 1 clock for now. * @prop_name: Name to postfix to properties. * @set_opp: Custom set OPP helper. + * @config_regulators: Custom set regulator helper. * @supported_hw: Array of hierarchy of versions to match. * @supported_hw_count: Number of elements in the array. * @regulator_names: Array of pointers to the names of the regulator, NULL terminated. @@ -109,6 +114,7 @@ struct dev_pm_opp_config { const char * const *clk_names; const char *prop_name; int (*set_opp)(struct dev_pm_set_opp_data *data); + config_regulators_t config_regulators; const unsigned int *supported_hw; unsigned int supported_hw_count; const char * const *regulator_names; @@ -613,6 +619,22 @@ static inline void dev_pm_opp_unregister_set_opp_helper(int token) dev_pm_opp_clear_config(token); } +/* config-regulators helpers */ +static inline int dev_pm_opp_set_config_regulators(struct device *dev, + config_regulators_t helper) +{ + struct dev_pm_opp_config config = { + .config_regulators = helper, + }; + + return dev_pm_opp_set_config(dev, &config); +} + +static inline void dev_pm_opp_put_config_regulators(int token) +{ + dev_pm_opp_clear_config(token); +} + /* genpd helpers */ static inline int dev_pm_opp_attach_genpd(struct device *dev, const char * const *names, -- cgit v1.2.3 From 69b1af178a3a534da2f20fa0c7356fcbbe8cce1f Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 31 May 2022 13:24:21 +0530 Subject: OPP: Add dev_pm_opp_get_supplies() We already have an API for getting voltage information for a single regulator, dev_pm_opp_get_voltage(), but there is nothing available for multiple regulator case. This patch adds a new API, dev_pm_opp_get_supplies(), to get all information related to the supplies for an OPP. Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 25 +++++++++++++++++++++++++ include/linux/pm_opp.h | 7 +++++++ 2 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 5ecf077b6b17..3a794bff2ba1 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -117,6 +117,31 @@ unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp) } EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage); +/** + * dev_pm_opp_get_supplies() - Gets the supply information corresponding to an opp + * @opp: opp for which voltage has to be returned for + * @supplies: Placeholder for copying the supply information. + * + * Return: negative error number on failure, 0 otherwise on success after + * setting @supplies. + * + * This can be used for devices with any number of power supplies. The caller + * must ensure the @supplies array must contain space for each regulator. + */ +int dev_pm_opp_get_supplies(struct dev_pm_opp *opp, + struct dev_pm_opp_supply *supplies) +{ + if (IS_ERR_OR_NULL(opp) || !supplies) { + pr_err("%s: Invalid parameters\n", __func__); + return -EINVAL; + } + + memcpy(supplies, opp->supplies, + sizeof(*supplies) * opp->opp_table->regulator_count); + return 0; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_supplies); + /** * dev_pm_opp_get_power() - Gets the power corresponding to an opp * @opp: opp for which power has to be returned for diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 9f2f9a792a19..1e2b33d79ba6 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -129,6 +129,8 @@ void dev_pm_opp_put_opp_table(struct opp_table *opp_table); unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp); +int dev_pm_opp_get_supplies(struct dev_pm_opp *opp, struct dev_pm_opp_supply *supplies); + unsigned long dev_pm_opp_get_power(struct dev_pm_opp *opp); unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp); @@ -217,6 +219,11 @@ static inline unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp) return 0; } +static inline int dev_pm_opp_get_supplies(struct dev_pm_opp *opp, struct dev_pm_opp_supply *supplies) +{ + return -EOPNOTSUPP; +} + static inline unsigned long dev_pm_opp_get_power(struct dev_pm_opp *opp) { return 0; -- cgit v1.2.3 From 1f378c6ead5c69decf36e8e61de2e50377c76ab8 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 4 Jul 2022 14:57:06 +0530 Subject: OPP: Remove custom OPP helper support The only user of the custom helper is migrated to use dev_pm_opp_set_config_regulators() interface. Remove the now unused custom OPP helper support. This cleans up _set_opp() and leaves a single code path to be used by all users. Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 132 +------------------------------------------------ drivers/opp/opp.h | 7 --- include/linux/pm_opp.h | 51 ------------------- 3 files changed, 2 insertions(+), 188 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 3a794bff2ba1..e74bdc134c5a 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -979,36 +979,6 @@ static int _set_opp_bw(const struct opp_table *opp_table, return 0; } -static int _set_opp_custom(const struct opp_table *opp_table, - struct device *dev, struct dev_pm_opp *opp, - unsigned long freq) -{ - struct dev_pm_set_opp_data *data = opp_table->set_opp_data; - struct dev_pm_opp *old_opp = opp_table->current_opp; - int size; - - /* - * We support this only if dev_pm_opp_set_config() was called - * earlier to set regulators. - */ - if (opp_table->sod_supplies) { - size = sizeof(*old_opp->supplies) * opp_table->regulator_count; - memcpy(data->old_opp.supplies, old_opp->supplies, size); - memcpy(data->new_opp.supplies, opp->supplies, size); - data->regulator_count = opp_table->regulator_count; - } else { - data->regulator_count = 0; - } - - data->regulators = opp_table->regulators; - data->clk = opp_table->clk; - data->dev = dev; - data->old_opp.rate = old_opp->rate; - data->new_opp.rate = freq; - - return opp_table->set_opp(data); -} - static int _set_required_opp(struct device *dev, struct device *pd_dev, struct dev_pm_opp *opp, int i) { @@ -1195,13 +1165,7 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, } } - if (opp_table->set_opp) { - ret = _set_opp_custom(opp_table, dev, opp, freq); - } else { - /* Only frequency scaling */ - ret = _generic_set_opp_clk_only(dev, opp_table->clk, freq); - } - + ret = _generic_set_opp_clk_only(dev, opp_table->clk, freq); if (ret) return ret; @@ -2065,7 +2029,6 @@ static void _opp_put_prop_name(struct opp_table *opp_table) static int _opp_set_regulators(struct opp_table *opp_table, struct device *dev, const char * const names[]) { - struct dev_pm_opp_supply *supplies; const char * const *temp = names; struct regulator *reg; int count = 0, ret, i; @@ -2101,20 +2064,6 @@ static int _opp_set_regulators(struct opp_table *opp_table, struct device *dev, opp_table->regulator_count = count; - supplies = kmalloc_array(count * 2, sizeof(*supplies), GFP_KERNEL); - if (!supplies) { - ret = -ENOMEM; - goto free_regulators; - } - - mutex_lock(&opp_table->lock); - opp_table->sod_supplies = supplies; - if (opp_table->set_opp_data) { - opp_table->set_opp_data->old_opp.supplies = supplies; - opp_table->set_opp_data->new_opp.supplies = supplies + count; - } - mutex_unlock(&opp_table->lock); - /* Set generic config_regulators() for single regulators here */ if (count == 1) opp_table->config_regulators = _opp_config_regulator_single; @@ -2151,16 +2100,6 @@ static void _opp_put_regulators(struct opp_table *opp_table) for (i = opp_table->regulator_count - 1; i >= 0; i--) regulator_put(opp_table->regulators[i]); - mutex_lock(&opp_table->lock); - if (opp_table->set_opp_data) { - opp_table->set_opp_data->old_opp.supplies = NULL; - opp_table->set_opp_data->new_opp.supplies = NULL; - } - - kfree(opp_table->sod_supplies); - opp_table->sod_supplies = NULL; - mutex_unlock(&opp_table->lock); - kfree(opp_table->regulators); opp_table->regulators = NULL; opp_table->regulator_count = -1; @@ -2233,61 +2172,6 @@ static void _opp_put_clknames(struct opp_table *opp_table) } } -/** - * _opp_register_set_opp_helper() - Register custom set OPP helper - * @dev: Device for which the helper is getting registered. - * @set_opp: Custom set OPP helper. - * - * This is useful to support complex platforms (like platforms with multiple - * regulators per device), instead of the generic OPP set rate helper. - * - * This must be called before any OPPs are initialized for the device. - */ -static int _opp_register_set_opp_helper(struct opp_table *opp_table, - struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)) -{ - struct dev_pm_set_opp_data *data; - - /* Another CPU that shares the OPP table has set the helper ? */ - if (opp_table->set_opp) - return 0; - - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - mutex_lock(&opp_table->lock); - opp_table->set_opp_data = data; - if (opp_table->sod_supplies) { - data->old_opp.supplies = opp_table->sod_supplies; - data->new_opp.supplies = opp_table->sod_supplies + - opp_table->regulator_count; - } - mutex_unlock(&opp_table->lock); - - opp_table->set_opp = set_opp; - - return 0; -} - -/** - * _opp_unregister_set_opp_helper() - Releases resources blocked for set_opp helper - * @opp_table: OPP table returned from _opp_register_set_opp_helper(). - * - * Release resources blocked for platform specific set_opp helper. - */ -static void _opp_unregister_set_opp_helper(struct opp_table *opp_table) -{ - if (opp_table->set_opp) { - opp_table->set_opp = NULL; - - mutex_lock(&opp_table->lock); - kfree(opp_table->set_opp_data); - opp_table->set_opp_data = NULL; - mutex_unlock(&opp_table->lock); - } -} - /** * _opp_set_config_regulators_helper() - Register custom set regulator helper. * @dev: Device for which the helper is getting registered. @@ -2446,10 +2330,8 @@ static void _opp_clear_config(struct opp_config_data *data) _opp_put_regulators(data->opp_table); if (data->flags & OPP_CONFIG_SUPPORTED_HW) _opp_put_supported_hw(data->opp_table); - if (data->flags & OPP_CONFIG_REGULATOR_HELPER) { + if (data->flags & OPP_CONFIG_REGULATOR_HELPER) _opp_put_config_regulators_helper(data->opp_table); - _opp_unregister_set_opp_helper(data->opp_table); - } if (data->flags & OPP_CONFIG_PROP_NAME) _opp_put_prop_name(data->opp_table); if (data->flags & OPP_CONFIG_CLK) @@ -2520,16 +2402,6 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) data->flags |= OPP_CONFIG_PROP_NAME; } - /* Configure opp helper */ - if (config->set_opp) { - ret = _opp_register_set_opp_helper(opp_table, dev, - config->set_opp); - if (ret) - goto err; - - data->flags |= OPP_CONFIG_REGULATOR_HELPER; - } - /* Configure config_regulators helper */ if (config->config_regulators) { ret = _opp_set_config_regulators_helper(opp_table, dev, diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h index 45fd40737159..13abe991e811 100644 --- a/drivers/opp/opp.h +++ b/drivers/opp/opp.h @@ -182,9 +182,6 @@ enum opp_table_access { * @enabled: Set to true if the device's resources are enabled/configured. * @genpd_performance_state: Device's power domain support performance state. * @is_genpd: Marks if the OPP table belongs to a genpd. - * @set_opp: Platform specific set_opp callback - * @sod_supplies: Set opp data supplies - * @set_opp_data: Data to be passed to set_opp callback * @dentry: debugfs dentry pointer of the real device directory (not links). * @dentry_name: Name of the real dentry. * @@ -234,10 +231,6 @@ struct opp_table { bool genpd_performance_state; bool is_genpd; - int (*set_opp)(struct dev_pm_set_opp_data *data); - struct dev_pm_opp_supply *sod_supplies; - struct dev_pm_set_opp_data *set_opp_data; - #ifdef CONFIG_DEBUG_FS struct dentry *dentry; char dentry_name[NAME_MAX]; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 1e2b33d79ba6..9d59aedc2be3 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -57,39 +57,6 @@ struct dev_pm_opp_icc_bw { u32 peak; }; -/** - * struct dev_pm_opp_info - OPP freq/voltage/current values - * @rate: Target clk rate in hz - * @supplies: Array of voltage/current values for all power supplies - * - * This structure stores the freq/voltage/current values for a single OPP. - */ -struct dev_pm_opp_info { - unsigned long rate; - struct dev_pm_opp_supply *supplies; -}; - -/** - * struct dev_pm_set_opp_data - Set OPP data - * @old_opp: Old OPP info - * @new_opp: New OPP info - * @regulators: Array of regulator pointers - * @regulator_count: Number of regulators - * @clk: Pointer to clk - * @dev: Pointer to the struct device - * - * This structure contains all information required for setting an OPP. - */ -struct dev_pm_set_opp_data { - struct dev_pm_opp_info old_opp; - struct dev_pm_opp_info new_opp; - - struct regulator **regulators; - unsigned int regulator_count; - struct clk *clk; - struct device *dev; -}; - typedef int (*config_regulators_t)(struct device *dev, struct dev_pm_opp *old_opp, struct dev_pm_opp *new_opp, struct regulator **regulators, unsigned int count); @@ -98,7 +65,6 @@ typedef int (*config_regulators_t)(struct device *dev, * struct dev_pm_opp_config - Device OPP configuration values * @clk_names: Clk names, NULL terminated array, max 1 clock for now. * @prop_name: Name to postfix to properties. - * @set_opp: Custom set OPP helper. * @config_regulators: Custom set regulator helper. * @supported_hw: Array of hierarchy of versions to match. * @supported_hw_count: Number of elements in the array. @@ -113,7 +79,6 @@ struct dev_pm_opp_config { /* NULL terminated */ const char * const *clk_names; const char *prop_name; - int (*set_opp)(struct dev_pm_set_opp_data *data); config_regulators_t config_regulators; const unsigned int *supported_hw; unsigned int supported_hw_count; @@ -610,22 +575,6 @@ static inline int devm_pm_opp_set_clkname(struct device *dev, const char *name) return devm_pm_opp_set_config(dev, &config); } -/* set-opp helpers */ -static inline int dev_pm_opp_register_set_opp_helper(struct device *dev, - int (*set_opp)(struct dev_pm_set_opp_data *data)) -{ - struct dev_pm_opp_config config = { - .set_opp = set_opp, - }; - - return dev_pm_opp_set_config(dev, &config); -} - -static inline void dev_pm_opp_unregister_set_opp_helper(int token) -{ - dev_pm_opp_clear_config(token); -} - /* config-regulators helpers */ static inline int dev_pm_opp_set_config_regulators(struct device *dev, config_regulators_t helper) -- cgit v1.2.3 From 9fbb62605607d8f00c32a4765cd1ae67f022b640 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 2 Jun 2022 14:05:59 +0530 Subject: OPP: Remove dev_pm_opp_find_freq_ceil_by_volt() This was added few years back, but the code that was supposed to use it never got merged. Remove the unused helper. Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 54 -------------------------------------------------- include/linux/pm_opp.h | 8 -------- 2 files changed, 62 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index e74bdc134c5a..fc0232f695c6 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -607,60 +607,6 @@ struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor); -/** - * dev_pm_opp_find_freq_ceil_by_volt() - Find OPP with highest frequency for - * target voltage. - * @dev: Device for which we do this operation. - * @u_volt: Target voltage. - * - * Search for OPP with highest (ceil) frequency and has voltage <= u_volt. - * - * Return: matching *opp, else returns ERR_PTR in case of error which should be - * handled using IS_ERR. - * - * Error return values can be: - * EINVAL: bad parameters - * - * The callers are required to call dev_pm_opp_put() for the returned OPP after - * use. - */ -struct dev_pm_opp *dev_pm_opp_find_freq_ceil_by_volt(struct device *dev, - unsigned long u_volt) -{ - struct opp_table *opp_table; - struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); - - if (!dev || !u_volt) { - dev_err(dev, "%s: Invalid argument volt=%lu\n", __func__, - u_volt); - return ERR_PTR(-EINVAL); - } - - opp_table = _find_opp_table(dev); - if (IS_ERR(opp_table)) - return ERR_CAST(opp_table); - - mutex_lock(&opp_table->lock); - - list_for_each_entry(temp_opp, &opp_table->opp_list, node) { - if (temp_opp->available) { - if (temp_opp->supplies[0].u_volt > u_volt) - break; - opp = temp_opp; - } - } - - /* Increment the reference count of OPP */ - if (!IS_ERR(opp)) - dev_pm_opp_get(opp); - - mutex_unlock(&opp_table->lock); - dev_pm_opp_put_opp_table(opp_table); - - return opp; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil_by_volt); - /** * dev_pm_opp_find_level_exact() - search for an exact level * @dev: device for which we do this operation diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 9d59aedc2be3..50cbc75bef71 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -118,8 +118,6 @@ struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, bool available); struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, unsigned long *freq); -struct dev_pm_opp *dev_pm_opp_find_freq_ceil_by_volt(struct device *dev, - unsigned long u_volt); struct dev_pm_opp *dev_pm_opp_find_level_exact(struct device *dev, unsigned int level); @@ -265,12 +263,6 @@ static inline struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, return ERR_PTR(-EOPNOTSUPP); } -static inline struct dev_pm_opp *dev_pm_opp_find_freq_ceil_by_volt(struct device *dev, - unsigned long u_volt) -{ - return ERR_PTR(-EOPNOTSUPP); -} - static inline struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, unsigned long *freq) { -- cgit v1.2.3 From 7963d4d710112bc457f99bdb56608211e561190e Mon Sep 17 00:00:00 2001 From: Xin Ji Date: Wed, 6 Jul 2022 16:34:31 +0800 Subject: usb: typec: tcpci: move tcpci.h to include/linux/usb/ USB PD controllers which consisting of a microcontroller (acting as the TCPM) and a port controller (TCPC) - may require that the driver for the PD controller accesses directly also the on-chip port controller in some cases. Move tcpci.h to include/linux/usb/ is convenience access TCPC registers. Reviewed-by: Heikki Krogerus Signed-off-by: Xin Ji Link: https://lore.kernel.org/r/20220706083433.2415524-1-xji@analogixsemi.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpci.c | 3 +- drivers/usb/typec/tcpm/tcpci.h | 209 -------------------------------- drivers/usb/typec/tcpm/tcpci_maxim.c | 3 +- drivers/usb/typec/tcpm/tcpci_mt6360.c | 3 +- drivers/usb/typec/tcpm/tcpci_rt1711h.c | 2 +- include/linux/usb/tcpci.h | 210 +++++++++++++++++++++++++++++++++ 6 files changed, 214 insertions(+), 216 deletions(-) delete mode 100644 drivers/usb/typec/tcpm/tcpci.h create mode 100644 include/linux/usb/tcpci.h (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index f33e08eb7670..812784702d53 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -13,11 +13,10 @@ #include #include #include +#include #include #include -#include "tcpci.h" - #define PD_RETRY_COUNT_DEFAULT 3 #define PD_RETRY_COUNT_3_0_OR_HIGHER 2 #define AUTO_DISCHARGE_DEFAULT_THRESHOLD_MV 3500 diff --git a/drivers/usb/typec/tcpm/tcpci.h b/drivers/usb/typec/tcpm/tcpci.h deleted file mode 100644 index b2edd45f13c6..000000000000 --- a/drivers/usb/typec/tcpm/tcpci.h +++ /dev/null @@ -1,209 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * Copyright 2015-2017 Google, Inc - * - * USB Type-C Port Controller Interface. - */ - -#ifndef __LINUX_USB_TCPCI_H -#define __LINUX_USB_TCPCI_H - -#include - -#define TCPC_VENDOR_ID 0x0 -#define TCPC_PRODUCT_ID 0x2 -#define TCPC_BCD_DEV 0x4 -#define TCPC_TC_REV 0x6 -#define TCPC_PD_REV 0x8 -#define TCPC_PD_INT_REV 0xa - -#define TCPC_ALERT 0x10 -#define TCPC_ALERT_EXTND BIT(14) -#define TCPC_ALERT_EXTENDED_STATUS BIT(13) -#define TCPC_ALERT_VBUS_DISCNCT BIT(11) -#define TCPC_ALERT_RX_BUF_OVF BIT(10) -#define TCPC_ALERT_FAULT BIT(9) -#define TCPC_ALERT_V_ALARM_LO BIT(8) -#define TCPC_ALERT_V_ALARM_HI BIT(7) -#define TCPC_ALERT_TX_SUCCESS BIT(6) -#define TCPC_ALERT_TX_DISCARDED BIT(5) -#define TCPC_ALERT_TX_FAILED BIT(4) -#define TCPC_ALERT_RX_HARD_RST BIT(3) -#define TCPC_ALERT_RX_STATUS BIT(2) -#define TCPC_ALERT_POWER_STATUS BIT(1) -#define TCPC_ALERT_CC_STATUS BIT(0) - -#define TCPC_ALERT_MASK 0x12 -#define TCPC_POWER_STATUS_MASK 0x14 -#define TCPC_FAULT_STATUS_MASK 0x15 - -#define TCPC_EXTENDED_STATUS_MASK 0x16 -#define TCPC_EXTENDED_STATUS_MASK_VSAFE0V BIT(0) - -#define TCPC_ALERT_EXTENDED_MASK 0x17 -#define TCPC_SINK_FAST_ROLE_SWAP BIT(0) - -#define TCPC_CONFIG_STD_OUTPUT 0x18 - -#define TCPC_TCPC_CTRL 0x19 -#define TCPC_TCPC_CTRL_ORIENTATION BIT(0) -#define PLUG_ORNT_CC1 0 -#define PLUG_ORNT_CC2 1 -#define TCPC_TCPC_CTRL_BIST_TM BIT(1) -#define TCPC_TCPC_CTRL_EN_LK4CONN_ALRT BIT(6) - -#define TCPC_EXTENDED_STATUS 0x20 -#define TCPC_EXTENDED_STATUS_VSAFE0V BIT(0) - -#define TCPC_ROLE_CTRL 0x1a -#define TCPC_ROLE_CTRL_DRP BIT(6) -#define TCPC_ROLE_CTRL_RP_VAL_SHIFT 4 -#define TCPC_ROLE_CTRL_RP_VAL_MASK 0x3 -#define TCPC_ROLE_CTRL_RP_VAL_DEF 0x0 -#define TCPC_ROLE_CTRL_RP_VAL_1_5 0x1 -#define TCPC_ROLE_CTRL_RP_VAL_3_0 0x2 -#define TCPC_ROLE_CTRL_CC2_SHIFT 2 -#define TCPC_ROLE_CTRL_CC2_MASK 0x3 -#define TCPC_ROLE_CTRL_CC1_SHIFT 0 -#define TCPC_ROLE_CTRL_CC1_MASK 0x3 -#define TCPC_ROLE_CTRL_CC_RA 0x0 -#define TCPC_ROLE_CTRL_CC_RP 0x1 -#define TCPC_ROLE_CTRL_CC_RD 0x2 -#define TCPC_ROLE_CTRL_CC_OPEN 0x3 - -#define TCPC_FAULT_CTRL 0x1b - -#define TCPC_POWER_CTRL 0x1c -#define TCPC_POWER_CTRL_VCONN_ENABLE BIT(0) -#define TCPC_POWER_CTRL_BLEED_DISCHARGE BIT(3) -#define TCPC_POWER_CTRL_AUTO_DISCHARGE BIT(4) -#define TCPC_DIS_VOLT_ALRM BIT(5) -#define TCPC_POWER_CTRL_VBUS_VOLT_MON BIT(6) -#define TCPC_FAST_ROLE_SWAP_EN BIT(7) - -#define TCPC_CC_STATUS 0x1d -#define TCPC_CC_STATUS_TOGGLING BIT(5) -#define TCPC_CC_STATUS_TERM BIT(4) -#define TCPC_CC_STATUS_TERM_RP 0 -#define TCPC_CC_STATUS_TERM_RD 1 -#define TCPC_CC_STATE_SRC_OPEN 0 -#define TCPC_CC_STATUS_CC2_SHIFT 2 -#define TCPC_CC_STATUS_CC2_MASK 0x3 -#define TCPC_CC_STATUS_CC1_SHIFT 0 -#define TCPC_CC_STATUS_CC1_MASK 0x3 - -#define TCPC_POWER_STATUS 0x1e -#define TCPC_POWER_STATUS_DBG_ACC_CON BIT(7) -#define TCPC_POWER_STATUS_UNINIT BIT(6) -#define TCPC_POWER_STATUS_SOURCING_VBUS BIT(4) -#define TCPC_POWER_STATUS_VBUS_DET BIT(3) -#define TCPC_POWER_STATUS_VBUS_PRES BIT(2) -#define TCPC_POWER_STATUS_VCONN_PRES BIT(1) -#define TCPC_POWER_STATUS_SINKING_VBUS BIT(0) - -#define TCPC_FAULT_STATUS 0x1f - -#define TCPC_ALERT_EXTENDED 0x21 - -#define TCPC_COMMAND 0x23 -#define TCPC_CMD_WAKE_I2C 0x11 -#define TCPC_CMD_DISABLE_VBUS_DETECT 0x22 -#define TCPC_CMD_ENABLE_VBUS_DETECT 0x33 -#define TCPC_CMD_DISABLE_SINK_VBUS 0x44 -#define TCPC_CMD_SINK_VBUS 0x55 -#define TCPC_CMD_DISABLE_SRC_VBUS 0x66 -#define TCPC_CMD_SRC_VBUS_DEFAULT 0x77 -#define TCPC_CMD_SRC_VBUS_HIGH 0x88 -#define TCPC_CMD_LOOK4CONNECTION 0x99 -#define TCPC_CMD_RXONEMORE 0xAA -#define TCPC_CMD_I2C_IDLE 0xFF - -#define TCPC_DEV_CAP_1 0x24 -#define TCPC_DEV_CAP_2 0x26 -#define TCPC_STD_INPUT_CAP 0x28 -#define TCPC_STD_OUTPUT_CAP 0x29 - -#define TCPC_MSG_HDR_INFO 0x2e -#define TCPC_MSG_HDR_INFO_DATA_ROLE BIT(3) -#define TCPC_MSG_HDR_INFO_PWR_ROLE BIT(0) -#define TCPC_MSG_HDR_INFO_REV_SHIFT 1 -#define TCPC_MSG_HDR_INFO_REV_MASK 0x3 - -#define TCPC_RX_DETECT 0x2f -#define TCPC_RX_DETECT_HARD_RESET BIT(5) -#define TCPC_RX_DETECT_SOP BIT(0) -#define TCPC_RX_DETECT_SOP1 BIT(1) -#define TCPC_RX_DETECT_SOP2 BIT(2) -#define TCPC_RX_DETECT_DBG1 BIT(3) -#define TCPC_RX_DETECT_DBG2 BIT(4) - -#define TCPC_RX_BYTE_CNT 0x30 -#define TCPC_RX_BUF_FRAME_TYPE 0x31 -#define TCPC_RX_BUF_FRAME_TYPE_SOP 0 -#define TCPC_RX_HDR 0x32 -#define TCPC_RX_DATA 0x34 /* through 0x4f */ - -#define TCPC_TRANSMIT 0x50 -#define TCPC_TRANSMIT_RETRY_SHIFT 4 -#define TCPC_TRANSMIT_RETRY_MASK 0x3 -#define TCPC_TRANSMIT_TYPE_SHIFT 0 -#define TCPC_TRANSMIT_TYPE_MASK 0x7 - -#define TCPC_TX_BYTE_CNT 0x51 -#define TCPC_TX_HDR 0x52 -#define TCPC_TX_DATA 0x54 /* through 0x6f */ - -#define TCPC_VBUS_VOLTAGE 0x70 -#define TCPC_VBUS_VOLTAGE_MASK 0x3ff -#define TCPC_VBUS_VOLTAGE_LSB_MV 25 -#define TCPC_VBUS_SINK_DISCONNECT_THRESH 0x72 -#define TCPC_VBUS_SINK_DISCONNECT_THRESH_LSB_MV 25 -#define TCPC_VBUS_SINK_DISCONNECT_THRESH_MAX 0x3ff -#define TCPC_VBUS_STOP_DISCHARGE_THRESH 0x74 -#define TCPC_VBUS_VOLTAGE_ALARM_HI_CFG 0x76 -#define TCPC_VBUS_VOLTAGE_ALARM_LO_CFG 0x78 - -/* I2C_WRITE_BYTE_COUNT + 1 when TX_BUF_BYTE_x is only accessible I2C_WRITE_BYTE_COUNT */ -#define TCPC_TRANSMIT_BUFFER_MAX_LEN 31 - -struct tcpci; - -/* - * @TX_BUF_BYTE_x_hidden: - * optional; Set when TX_BUF_BYTE_x can only be accessed through I2C_WRITE_BYTE_COUNT. - * @frs_sourcing_vbus: - * Optional; Callback to perform chip specific operations when FRS - * is sourcing vbus. - * @auto_discharge_disconnect: - * Optional; Enables TCPC to autonously discharge vbus on disconnect. - * @vbus_vsafe0v: - * optional; Set when TCPC can detect whether vbus is at VSAFE0V. - * @set_partner_usb_comm_capable: - * Optional; The USB Communications Capable bit indicates if port - * partner is capable of communication over the USB data lines - * (e.g. D+/- or SS Tx/Rx). Called to notify the status of the bit. - */ -struct tcpci_data { - struct regmap *regmap; - unsigned char TX_BUF_BYTE_x_hidden:1; - unsigned char auto_discharge_disconnect:1; - unsigned char vbus_vsafe0v:1; - - int (*init)(struct tcpci *tcpci, struct tcpci_data *data); - int (*set_vconn)(struct tcpci *tcpci, struct tcpci_data *data, - bool enable); - int (*start_drp_toggling)(struct tcpci *tcpci, struct tcpci_data *data, - enum typec_cc_status cc); - int (*set_vbus)(struct tcpci *tcpci, struct tcpci_data *data, bool source, bool sink); - void (*frs_sourcing_vbus)(struct tcpci *tcpci, struct tcpci_data *data); - void (*set_partner_usb_comm_capable)(struct tcpci *tcpci, struct tcpci_data *data, - bool capable); -}; - -struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data); -void tcpci_unregister_port(struct tcpci *tcpci); -irqreturn_t tcpci_irq(struct tcpci *tcpci); - -struct tcpm_port; -struct tcpm_port *tcpci_get_tcpm_port(struct tcpci *tcpci); -#endif /* __LINUX_USB_TCPCI_H */ diff --git a/drivers/usb/typec/tcpm/tcpci_maxim.c b/drivers/usb/typec/tcpm/tcpci_maxim.c index df2505570f07..4b6705f3d7b7 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim.c +++ b/drivers/usb/typec/tcpm/tcpci_maxim.c @@ -11,11 +11,10 @@ #include #include #include +#include #include #include -#include "tcpci.h" - #define PD_ACTIVITY_TIMEOUT_MS 10000 #define TCPC_VENDOR_ALERT 0x80 diff --git a/drivers/usb/typec/tcpm/tcpci_mt6360.c b/drivers/usb/typec/tcpm/tcpci_mt6360.c index 8a952eaf9016..1b7c31278ebb 100644 --- a/drivers/usb/typec/tcpm/tcpci_mt6360.c +++ b/drivers/usb/typec/tcpm/tcpci_mt6360.c @@ -11,10 +11,9 @@ #include #include #include +#include #include -#include "tcpci.h" - #define MT6360_REG_PHYCTRL1 0x80 #define MT6360_REG_PHYCTRL3 0x82 #define MT6360_REG_PHYCTRL7 0x86 diff --git a/drivers/usb/typec/tcpm/tcpci_rt1711h.c b/drivers/usb/typec/tcpm/tcpci_rt1711h.c index b56a0880a044..3291ca4948da 100644 --- a/drivers/usb/typec/tcpm/tcpci_rt1711h.c +++ b/drivers/usb/typec/tcpm/tcpci_rt1711h.c @@ -10,9 +10,9 @@ #include #include #include +#include #include #include -#include "tcpci.h" #define RT1711H_VID 0x29CF #define RT1711H_PID 0x1711 diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h new file mode 100644 index 000000000000..20c0bedb8ec8 --- /dev/null +++ b/include/linux/usb/tcpci.h @@ -0,0 +1,210 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright 2015-2017 Google, Inc + * + * USB Type-C Port Controller Interface. + */ + +#ifndef __LINUX_USB_TCPCI_H +#define __LINUX_USB_TCPCI_H + +#include +#include + +#define TCPC_VENDOR_ID 0x0 +#define TCPC_PRODUCT_ID 0x2 +#define TCPC_BCD_DEV 0x4 +#define TCPC_TC_REV 0x6 +#define TCPC_PD_REV 0x8 +#define TCPC_PD_INT_REV 0xa + +#define TCPC_ALERT 0x10 +#define TCPC_ALERT_EXTND BIT(14) +#define TCPC_ALERT_EXTENDED_STATUS BIT(13) +#define TCPC_ALERT_VBUS_DISCNCT BIT(11) +#define TCPC_ALERT_RX_BUF_OVF BIT(10) +#define TCPC_ALERT_FAULT BIT(9) +#define TCPC_ALERT_V_ALARM_LO BIT(8) +#define TCPC_ALERT_V_ALARM_HI BIT(7) +#define TCPC_ALERT_TX_SUCCESS BIT(6) +#define TCPC_ALERT_TX_DISCARDED BIT(5) +#define TCPC_ALERT_TX_FAILED BIT(4) +#define TCPC_ALERT_RX_HARD_RST BIT(3) +#define TCPC_ALERT_RX_STATUS BIT(2) +#define TCPC_ALERT_POWER_STATUS BIT(1) +#define TCPC_ALERT_CC_STATUS BIT(0) + +#define TCPC_ALERT_MASK 0x12 +#define TCPC_POWER_STATUS_MASK 0x14 +#define TCPC_FAULT_STATUS_MASK 0x15 + +#define TCPC_EXTENDED_STATUS_MASK 0x16 +#define TCPC_EXTENDED_STATUS_MASK_VSAFE0V BIT(0) + +#define TCPC_ALERT_EXTENDED_MASK 0x17 +#define TCPC_SINK_FAST_ROLE_SWAP BIT(0) + +#define TCPC_CONFIG_STD_OUTPUT 0x18 + +#define TCPC_TCPC_CTRL 0x19 +#define TCPC_TCPC_CTRL_ORIENTATION BIT(0) +#define PLUG_ORNT_CC1 0 +#define PLUG_ORNT_CC2 1 +#define TCPC_TCPC_CTRL_BIST_TM BIT(1) +#define TCPC_TCPC_CTRL_EN_LK4CONN_ALRT BIT(6) + +#define TCPC_EXTENDED_STATUS 0x20 +#define TCPC_EXTENDED_STATUS_VSAFE0V BIT(0) + +#define TCPC_ROLE_CTRL 0x1a +#define TCPC_ROLE_CTRL_DRP BIT(6) +#define TCPC_ROLE_CTRL_RP_VAL_SHIFT 4 +#define TCPC_ROLE_CTRL_RP_VAL_MASK 0x3 +#define TCPC_ROLE_CTRL_RP_VAL_DEF 0x0 +#define TCPC_ROLE_CTRL_RP_VAL_1_5 0x1 +#define TCPC_ROLE_CTRL_RP_VAL_3_0 0x2 +#define TCPC_ROLE_CTRL_CC2_SHIFT 2 +#define TCPC_ROLE_CTRL_CC2_MASK 0x3 +#define TCPC_ROLE_CTRL_CC1_SHIFT 0 +#define TCPC_ROLE_CTRL_CC1_MASK 0x3 +#define TCPC_ROLE_CTRL_CC_RA 0x0 +#define TCPC_ROLE_CTRL_CC_RP 0x1 +#define TCPC_ROLE_CTRL_CC_RD 0x2 +#define TCPC_ROLE_CTRL_CC_OPEN 0x3 + +#define TCPC_FAULT_CTRL 0x1b + +#define TCPC_POWER_CTRL 0x1c +#define TCPC_POWER_CTRL_VCONN_ENABLE BIT(0) +#define TCPC_POWER_CTRL_BLEED_DISCHARGE BIT(3) +#define TCPC_POWER_CTRL_AUTO_DISCHARGE BIT(4) +#define TCPC_DIS_VOLT_ALRM BIT(5) +#define TCPC_POWER_CTRL_VBUS_VOLT_MON BIT(6) +#define TCPC_FAST_ROLE_SWAP_EN BIT(7) + +#define TCPC_CC_STATUS 0x1d +#define TCPC_CC_STATUS_TOGGLING BIT(5) +#define TCPC_CC_STATUS_TERM BIT(4) +#define TCPC_CC_STATUS_TERM_RP 0 +#define TCPC_CC_STATUS_TERM_RD 1 +#define TCPC_CC_STATE_SRC_OPEN 0 +#define TCPC_CC_STATUS_CC2_SHIFT 2 +#define TCPC_CC_STATUS_CC2_MASK 0x3 +#define TCPC_CC_STATUS_CC1_SHIFT 0 +#define TCPC_CC_STATUS_CC1_MASK 0x3 + +#define TCPC_POWER_STATUS 0x1e +#define TCPC_POWER_STATUS_DBG_ACC_CON BIT(7) +#define TCPC_POWER_STATUS_UNINIT BIT(6) +#define TCPC_POWER_STATUS_SOURCING_VBUS BIT(4) +#define TCPC_POWER_STATUS_VBUS_DET BIT(3) +#define TCPC_POWER_STATUS_VBUS_PRES BIT(2) +#define TCPC_POWER_STATUS_VCONN_PRES BIT(1) +#define TCPC_POWER_STATUS_SINKING_VBUS BIT(0) + +#define TCPC_FAULT_STATUS 0x1f + +#define TCPC_ALERT_EXTENDED 0x21 + +#define TCPC_COMMAND 0x23 +#define TCPC_CMD_WAKE_I2C 0x11 +#define TCPC_CMD_DISABLE_VBUS_DETECT 0x22 +#define TCPC_CMD_ENABLE_VBUS_DETECT 0x33 +#define TCPC_CMD_DISABLE_SINK_VBUS 0x44 +#define TCPC_CMD_SINK_VBUS 0x55 +#define TCPC_CMD_DISABLE_SRC_VBUS 0x66 +#define TCPC_CMD_SRC_VBUS_DEFAULT 0x77 +#define TCPC_CMD_SRC_VBUS_HIGH 0x88 +#define TCPC_CMD_LOOK4CONNECTION 0x99 +#define TCPC_CMD_RXONEMORE 0xAA +#define TCPC_CMD_I2C_IDLE 0xFF + +#define TCPC_DEV_CAP_1 0x24 +#define TCPC_DEV_CAP_2 0x26 +#define TCPC_STD_INPUT_CAP 0x28 +#define TCPC_STD_OUTPUT_CAP 0x29 + +#define TCPC_MSG_HDR_INFO 0x2e +#define TCPC_MSG_HDR_INFO_DATA_ROLE BIT(3) +#define TCPC_MSG_HDR_INFO_PWR_ROLE BIT(0) +#define TCPC_MSG_HDR_INFO_REV_SHIFT 1 +#define TCPC_MSG_HDR_INFO_REV_MASK 0x3 + +#define TCPC_RX_DETECT 0x2f +#define TCPC_RX_DETECT_HARD_RESET BIT(5) +#define TCPC_RX_DETECT_SOP BIT(0) +#define TCPC_RX_DETECT_SOP1 BIT(1) +#define TCPC_RX_DETECT_SOP2 BIT(2) +#define TCPC_RX_DETECT_DBG1 BIT(3) +#define TCPC_RX_DETECT_DBG2 BIT(4) + +#define TCPC_RX_BYTE_CNT 0x30 +#define TCPC_RX_BUF_FRAME_TYPE 0x31 +#define TCPC_RX_BUF_FRAME_TYPE_SOP 0 +#define TCPC_RX_HDR 0x32 +#define TCPC_RX_DATA 0x34 /* through 0x4f */ + +#define TCPC_TRANSMIT 0x50 +#define TCPC_TRANSMIT_RETRY_SHIFT 4 +#define TCPC_TRANSMIT_RETRY_MASK 0x3 +#define TCPC_TRANSMIT_TYPE_SHIFT 0 +#define TCPC_TRANSMIT_TYPE_MASK 0x7 + +#define TCPC_TX_BYTE_CNT 0x51 +#define TCPC_TX_HDR 0x52 +#define TCPC_TX_DATA 0x54 /* through 0x6f */ + +#define TCPC_VBUS_VOLTAGE 0x70 +#define TCPC_VBUS_VOLTAGE_MASK 0x3ff +#define TCPC_VBUS_VOLTAGE_LSB_MV 25 +#define TCPC_VBUS_SINK_DISCONNECT_THRESH 0x72 +#define TCPC_VBUS_SINK_DISCONNECT_THRESH_LSB_MV 25 +#define TCPC_VBUS_SINK_DISCONNECT_THRESH_MAX 0x3ff +#define TCPC_VBUS_STOP_DISCHARGE_THRESH 0x74 +#define TCPC_VBUS_VOLTAGE_ALARM_HI_CFG 0x76 +#define TCPC_VBUS_VOLTAGE_ALARM_LO_CFG 0x78 + +/* I2C_WRITE_BYTE_COUNT + 1 when TX_BUF_BYTE_x is only accessible I2C_WRITE_BYTE_COUNT */ +#define TCPC_TRANSMIT_BUFFER_MAX_LEN 31 + +struct tcpci; + +/* + * @TX_BUF_BYTE_x_hidden: + * optional; Set when TX_BUF_BYTE_x can only be accessed through I2C_WRITE_BYTE_COUNT. + * @frs_sourcing_vbus: + * Optional; Callback to perform chip specific operations when FRS + * is sourcing vbus. + * @auto_discharge_disconnect: + * Optional; Enables TCPC to autonously discharge vbus on disconnect. + * @vbus_vsafe0v: + * optional; Set when TCPC can detect whether vbus is at VSAFE0V. + * @set_partner_usb_comm_capable: + * Optional; The USB Communications Capable bit indicates if port + * partner is capable of communication over the USB data lines + * (e.g. D+/- or SS Tx/Rx). Called to notify the status of the bit. + */ +struct tcpci_data { + struct regmap *regmap; + unsigned char TX_BUF_BYTE_x_hidden:1; + unsigned char auto_discharge_disconnect:1; + unsigned char vbus_vsafe0v:1; + + int (*init)(struct tcpci *tcpci, struct tcpci_data *data); + int (*set_vconn)(struct tcpci *tcpci, struct tcpci_data *data, + bool enable); + int (*start_drp_toggling)(struct tcpci *tcpci, struct tcpci_data *data, + enum typec_cc_status cc); + int (*set_vbus)(struct tcpci *tcpci, struct tcpci_data *data, bool source, bool sink); + void (*frs_sourcing_vbus)(struct tcpci *tcpci, struct tcpci_data *data); + void (*set_partner_usb_comm_capable)(struct tcpci *tcpci, struct tcpci_data *data, + bool capable); +}; + +struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data); +void tcpci_unregister_port(struct tcpci *tcpci); +irqreturn_t tcpci_irq(struct tcpci *tcpci); + +struct tcpm_port; +struct tcpm_port *tcpci_get_tcpm_port(struct tcpci *tcpci); +#endif /* __LINUX_USB_TCPCI_H */ -- cgit v1.2.3 From 620e8e8ba6210003fa45081b63d90ef4f3a0637f Mon Sep 17 00:00:00 2001 From: Matthias Kaehlcke Date: Thu, 30 Jun 2022 12:35:27 -0700 Subject: of/platform: Add stubs for of_platform_device_create/destroy() Code for platform_device_create() and of_platform_device_destroy() is only generated if CONFIG_OF_ADDRESS=y. Add stubs to avoid unresolved symbols when CONFIG_OF_ADDRESS is not set. Reviewed-by: Stephen Boyd Reviewed-by: Douglas Anderson Acked-by: Rob Herring Signed-off-by: Matthias Kaehlcke Link: https://lore.kernel.org/r/20220630123445.v24.1.I08fd2e1c775af04f663730e9fb4d00e6bbb38541@changeid Signed-off-by: Greg Kroah-Hartman --- include/linux/of_platform.h | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/of_platform.h b/include/linux/of_platform.h index 84a966623e78..d15b6cd5e1c3 100644 --- a/include/linux/of_platform.h +++ b/include/linux/of_platform.h @@ -61,16 +61,18 @@ static inline struct platform_device *of_find_device_by_node(struct device_node } #endif +extern int of_platform_bus_probe(struct device_node *root, + const struct of_device_id *matches, + struct device *parent); + +#ifdef CONFIG_OF_ADDRESS /* Platform devices and busses creation */ extern struct platform_device *of_platform_device_create(struct device_node *np, const char *bus_id, struct device *parent); extern int of_platform_device_destroy(struct device *dev, void *data); -extern int of_platform_bus_probe(struct device_node *root, - const struct of_device_id *matches, - struct device *parent); -#ifdef CONFIG_OF_ADDRESS + extern int of_platform_populate(struct device_node *root, const struct of_device_id *matches, const struct of_dev_auxdata *lookup, @@ -84,6 +86,18 @@ extern int devm_of_platform_populate(struct device *dev); extern void devm_of_platform_depopulate(struct device *dev); #else +/* Platform devices and busses creation */ +static inline struct platform_device *of_platform_device_create(struct device_node *np, + const char *bus_id, + struct device *parent) +{ + return NULL; +} +static inline int of_platform_device_destroy(struct device *dev, void *data) +{ + return -ENODEV; +} + static inline int of_platform_populate(struct device_node *root, const struct of_device_id *matches, const struct of_dev_auxdata *lookup, -- cgit v1.2.3 From 8bc063641cebf9d555e41d135db2b5035b521768 Mon Sep 17 00:00:00 2001 From: Matthias Kaehlcke Date: Thu, 30 Jun 2022 12:35:29 -0700 Subject: usb: misc: Add onboard_usb_hub driver The main issue this driver addresses is that a USB hub needs to be powered before it can be discovered. For discrete onboard hubs (an example for such a hub is the Realtek RTS5411) this is often solved by supplying the hub with an 'always-on' regulator, which is kind of a hack. Some onboard hubs may require further initialization steps, like changing the state of a GPIO or enabling a clock, which requires even more hacks. This driver creates a platform device representing the hub which performs the necessary initialization. Currently it only supports switching on a single regulator, support for multiple regulators or other actions can be added as needed. Different initialization sequences can be supported based on the compatible string. Besides performing the initialization the driver can be configured to power the hub off during system suspend. This can help to extend battery life on battery powered devices which have no requirements to keep the hub powered during suspend. The driver can also be configured to leave the hub powered when a wakeup capable USB device is connected when suspending, and power it off otherwise. Technically the driver consists of two drivers, the platform driver described above and a very thin USB driver that subclasses the generic driver. The purpose of this driver is to provide the platform driver with the USB devices corresponding to the hub(s) (a hub controller may provide multiple 'logical' hubs, e.g. one to support USB 2.0 and another for USB 3.x). Co-developed-by: Ravi Chandra Sadineni Reviewed-by: Douglas Anderson Signed-off-by: Ravi Chandra Sadineni Signed-off-by: Matthias Kaehlcke Link: https://lore.kernel.org/r/20220630123445.v24.3.I7c9a1f1d6ced41dd8310e8a03da666a32364e790@changeid Signed-off-by: Greg Kroah-Hartman --- .../ABI/testing/sysfs-bus-platform-onboard-usb-hub | 8 + MAINTAINERS | 7 + drivers/usb/core/Makefile | 4 + drivers/usb/misc/Kconfig | 16 + drivers/usb/misc/Makefile | 1 + drivers/usb/misc/onboard_usb_hub.c | 428 +++++++++++++++++++++ drivers/usb/misc/onboard_usb_hub.h | 17 + drivers/usb/misc/onboard_usb_hub_pdevs.c | 142 +++++++ include/linux/usb/onboard_hub.h | 18 + 9 files changed, 641 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-bus-platform-onboard-usb-hub create mode 100644 drivers/usb/misc/onboard_usb_hub.c create mode 100644 drivers/usb/misc/onboard_usb_hub.h create mode 100644 drivers/usb/misc/onboard_usb_hub_pdevs.c create mode 100644 include/linux/usb/onboard_hub.h (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-bus-platform-onboard-usb-hub b/Documentation/ABI/testing/sysfs-bus-platform-onboard-usb-hub new file mode 100644 index 000000000000..42deb0552065 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-platform-onboard-usb-hub @@ -0,0 +1,8 @@ +What: /sys/bus/platform/devices//always_powered_in_suspend +Date: June 2022 +KernelVersion: 5.20 +Contact: Matthias Kaehlcke + linux-usb@vger.kernel.org +Description: + (RW) Controls whether the USB hub remains always powered + during system suspend or not. \ No newline at end of file diff --git a/MAINTAINERS b/MAINTAINERS index 7533cb27adc0..f43b126eaf59 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14848,6 +14848,13 @@ S: Maintained T: git git://linuxtv.org/media_tree.git F: drivers/media/i2c/ov9734.c +ONBOARD USB HUB DRIVER +M: Matthias Kaehlcke +L: linux-usb@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/sysfs-bus-platform-onboard-usb-hub +F: drivers/usb/misc/onboard_usb_hub.c + ONENAND FLASH DRIVER M: Kyungmin Park L: linux-mtd@lists.infradead.org diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile index 18e874b0441e..7d338e9c0657 100644 --- a/drivers/usb/core/Makefile +++ b/drivers/usb/core/Makefile @@ -12,6 +12,10 @@ usbcore-$(CONFIG_OF) += of.o usbcore-$(CONFIG_USB_PCI) += hcd-pci.o usbcore-$(CONFIG_ACPI) += usb-acpi.o +ifdef CONFIG_USB_ONBOARD_HUB +usbcore-y += ../misc/onboard_usb_hub_pdevs.o +endif + obj-$(CONFIG_USB) += usbcore.o obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT) += ledtrig-usbport.o diff --git a/drivers/usb/misc/Kconfig b/drivers/usb/misc/Kconfig index 4c5ddbd75b7e..9367c12c7e6f 100644 --- a/drivers/usb/misc/Kconfig +++ b/drivers/usb/misc/Kconfig @@ -295,3 +295,19 @@ config BRCM_USB_PINMAP This option enables support for remapping some USB external signals, which are typically on dedicated pins on the chip, to any gpio. + +config USB_ONBOARD_HUB + tristate "Onboard USB hub support" + depends on OF || COMPILE_TEST + help + Say Y here if you want to support discrete onboard USB hubs that + don't require an additional control bus for initialization, but + need some non-trivial form of initialization, such as enabling a + power regulator. An example for such a hub is the Realtek + RTS5411. + + This driver can be used as a module but its state (module vs + builtin) must match the state of the USB subsystem. Enabling + this config will enable the driver and it will automatically + match the state of the USB subsystem. If this driver is a + module it will be called onboard_usb_hub. diff --git a/drivers/usb/misc/Makefile b/drivers/usb/misc/Makefile index 35bdb4b6c3b6..93581baec3a8 100644 --- a/drivers/usb/misc/Makefile +++ b/drivers/usb/misc/Makefile @@ -33,3 +33,4 @@ obj-$(CONFIG_USB_CHAOSKEY) += chaoskey.o obj-$(CONFIG_USB_SISUSBVGA) += sisusbvga/ obj-$(CONFIG_USB_LINK_LAYER_TEST) += lvstest.o obj-$(CONFIG_BRCM_USB_PINMAP) += brcmstb-usb-pinmap.o +obj-$(CONFIG_USB_ONBOARD_HUB) += onboard_usb_hub.o diff --git a/drivers/usb/misc/onboard_usb_hub.c b/drivers/usb/misc/onboard_usb_hub.c new file mode 100644 index 000000000000..6b9b949d17d3 --- /dev/null +++ b/drivers/usb/misc/onboard_usb_hub.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Driver for onboard USB hubs + * + * Copyright (c) 2022, Google LLC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "onboard_usb_hub.h" + +static struct usb_device_driver onboard_hub_usbdev_driver; + +/************************** Platform driver **************************/ + +struct usbdev_node { + struct usb_device *udev; + struct list_head list; +}; + +struct onboard_hub { + struct regulator *vdd; + struct device *dev; + bool always_powered_in_suspend; + bool is_powered_on; + bool going_away; + struct list_head udev_list; + struct work_struct attach_usb_driver_work; + struct mutex lock; +}; + +static int onboard_hub_power_on(struct onboard_hub *hub) +{ + int err; + + err = regulator_enable(hub->vdd); + if (err) { + dev_err(hub->dev, "failed to enable regulator: %d\n", err); + return err; + } + + hub->is_powered_on = true; + + return 0; +} + +static int onboard_hub_power_off(struct onboard_hub *hub) +{ + int err; + + err = regulator_disable(hub->vdd); + if (err) { + dev_err(hub->dev, "failed to disable regulator: %d\n", err); + return err; + } + + hub->is_powered_on = false; + + return 0; +} + +static int __maybe_unused onboard_hub_suspend(struct device *dev) +{ + struct onboard_hub *hub = dev_get_drvdata(dev); + struct usbdev_node *node; + bool power_off = true; + + if (hub->always_powered_in_suspend) + return 0; + + mutex_lock(&hub->lock); + + list_for_each_entry(node, &hub->udev_list, list) { + if (!device_may_wakeup(node->udev->bus->controller)) + continue; + + if (usb_wakeup_enabled_descendants(node->udev)) { + power_off = false; + break; + } + } + + mutex_unlock(&hub->lock); + + if (!power_off) + return 0; + + return onboard_hub_power_off(hub); +} + +static int __maybe_unused onboard_hub_resume(struct device *dev) +{ + struct onboard_hub *hub = dev_get_drvdata(dev); + + if (hub->is_powered_on) + return 0; + + return onboard_hub_power_on(hub); +} + +static inline void get_udev_link_name(const struct usb_device *udev, char *buf, size_t size) +{ + snprintf(buf, size, "usb_dev.%s", dev_name(&udev->dev)); +} + +static int onboard_hub_add_usbdev(struct onboard_hub *hub, struct usb_device *udev) +{ + struct usbdev_node *node; + char link_name[64]; + int err; + + mutex_lock(&hub->lock); + + if (hub->going_away) { + err = -EINVAL; + goto error; + } + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) { + err = -ENOMEM; + goto error; + } + + node->udev = udev; + + list_add(&node->list, &hub->udev_list); + + mutex_unlock(&hub->lock); + + get_udev_link_name(udev, link_name, sizeof(link_name)); + WARN_ON(sysfs_create_link(&hub->dev->kobj, &udev->dev.kobj, link_name)); + + return 0; + +error: + mutex_unlock(&hub->lock); + + return err; +} + +static void onboard_hub_remove_usbdev(struct onboard_hub *hub, const struct usb_device *udev) +{ + struct usbdev_node *node; + char link_name[64]; + + get_udev_link_name(udev, link_name, sizeof(link_name)); + sysfs_remove_link(&hub->dev->kobj, link_name); + + mutex_lock(&hub->lock); + + list_for_each_entry(node, &hub->udev_list, list) { + if (node->udev == udev) { + list_del(&node->list); + kfree(node); + break; + } + } + + mutex_unlock(&hub->lock); +} + +static ssize_t always_powered_in_suspend_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const struct onboard_hub *hub = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", hub->always_powered_in_suspend); +} + +static ssize_t always_powered_in_suspend_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct onboard_hub *hub = dev_get_drvdata(dev); + bool val; + int ret; + + ret = kstrtobool(buf, &val); + if (ret < 0) + return ret; + + hub->always_powered_in_suspend = val; + + return count; +} +static DEVICE_ATTR_RW(always_powered_in_suspend); + +static struct attribute *onboard_hub_attrs[] = { + &dev_attr_always_powered_in_suspend.attr, + NULL, +}; +ATTRIBUTE_GROUPS(onboard_hub); + +static void onboard_hub_attach_usb_driver(struct work_struct *work) +{ + int err; + + err = driver_attach(&onboard_hub_usbdev_driver.drvwrap.driver); + if (err) + pr_err("Failed to attach USB driver: %d\n", err); +} + +static int onboard_hub_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct onboard_hub *hub; + int err; + + hub = devm_kzalloc(dev, sizeof(*hub), GFP_KERNEL); + if (!hub) + return -ENOMEM; + + hub->vdd = devm_regulator_get(dev, "vdd"); + if (IS_ERR(hub->vdd)) + return PTR_ERR(hub->vdd); + + hub->dev = dev; + mutex_init(&hub->lock); + INIT_LIST_HEAD(&hub->udev_list); + + dev_set_drvdata(dev, hub); + + err = onboard_hub_power_on(hub); + if (err) + return err; + + /* + * The USB driver might have been detached from the USB devices by + * onboard_hub_remove() (e.g. through an 'unbind' by userspace), + * make sure to re-attach it if needed. + * + * This needs to be done deferred to avoid self-deadlocks on systems + * with nested onboard hubs. + */ + INIT_WORK(&hub->attach_usb_driver_work, onboard_hub_attach_usb_driver); + schedule_work(&hub->attach_usb_driver_work); + + return 0; +} + +static int onboard_hub_remove(struct platform_device *pdev) +{ + struct onboard_hub *hub = dev_get_drvdata(&pdev->dev); + struct usbdev_node *node; + struct usb_device *udev; + + hub->going_away = true; + + if (&hub->attach_usb_driver_work != current_work()) + cancel_work_sync(&hub->attach_usb_driver_work); + + mutex_lock(&hub->lock); + + /* unbind the USB devices to avoid dangling references to this device */ + while (!list_empty(&hub->udev_list)) { + node = list_first_entry(&hub->udev_list, struct usbdev_node, list); + udev = node->udev; + + /* + * Unbinding the driver will call onboard_hub_remove_usbdev(), + * which acquires hub->lock. We must release the lock first. + */ + get_device(&udev->dev); + mutex_unlock(&hub->lock); + device_release_driver(&udev->dev); + put_device(&udev->dev); + mutex_lock(&hub->lock); + } + + mutex_unlock(&hub->lock); + + return onboard_hub_power_off(hub); +} + +MODULE_DEVICE_TABLE(of, onboard_hub_match); + +static const struct dev_pm_ops __maybe_unused onboard_hub_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(onboard_hub_suspend, onboard_hub_resume) +}; + +static struct platform_driver onboard_hub_driver = { + .probe = onboard_hub_probe, + .remove = onboard_hub_remove, + + .driver = { + .name = "onboard-usb-hub", + .of_match_table = onboard_hub_match, + .pm = pm_ptr(&onboard_hub_pm_ops), + .dev_groups = onboard_hub_groups, + }, +}; + +/************************** USB driver **************************/ + +#define VENDOR_ID_REALTEK 0x0bda + +/* + * Returns the onboard_hub platform device that is associated with the USB + * device passed as parameter. + */ +static struct onboard_hub *_find_onboard_hub(struct device *dev) +{ + struct platform_device *pdev; + struct device_node *np; + struct onboard_hub *hub; + + pdev = of_find_device_by_node(dev->of_node); + if (!pdev) { + np = of_parse_phandle(dev->of_node, "peer-hub", 0); + if (!np) { + dev_err(dev, "failed to find device node for peer hub\n"); + return ERR_PTR(-EINVAL); + } + + pdev = of_find_device_by_node(np); + of_node_put(np); + + if (!pdev) + return ERR_PTR(-ENODEV); + } + + hub = dev_get_drvdata(&pdev->dev); + put_device(&pdev->dev); + + /* + * The presence of drvdata ('hub') indicates that the platform driver + * finished probing. This handles the case where (conceivably) we could + * be running at the exact same time as the platform driver's probe. If + * we detect the race we request probe deferral and we'll come back and + * try again. + */ + if (!hub) + return ERR_PTR(-EPROBE_DEFER); + + return hub; +} + +static int onboard_hub_usbdev_probe(struct usb_device *udev) +{ + struct device *dev = &udev->dev; + struct onboard_hub *hub; + int err; + + /* ignore supported hubs without device tree node */ + if (!dev->of_node) + return -ENODEV; + + hub = _find_onboard_hub(dev); + if (IS_ERR(hub)) + return PTR_ERR(hub); + + dev_set_drvdata(dev, hub); + + err = onboard_hub_add_usbdev(hub, udev); + if (err) + return err; + + return 0; +} + +static void onboard_hub_usbdev_disconnect(struct usb_device *udev) +{ + struct onboard_hub *hub = dev_get_drvdata(&udev->dev); + + onboard_hub_remove_usbdev(hub, udev); +} + +static const struct usb_device_id onboard_hub_id_table[] = { + { USB_DEVICE(VENDOR_ID_REALTEK, 0x0411) }, /* RTS5411 USB 3.1 */ + { USB_DEVICE(VENDOR_ID_REALTEK, 0x5411) }, /* RTS5411 USB 2.1 */ + { USB_DEVICE(VENDOR_ID_REALTEK, 0x0414) }, /* RTS5414 USB 3.2 */ + { USB_DEVICE(VENDOR_ID_REALTEK, 0x5414) }, /* RTS5414 USB 2.1 */ + {} +}; +MODULE_DEVICE_TABLE(usb, onboard_hub_id_table); + +static struct usb_device_driver onboard_hub_usbdev_driver = { + .name = "onboard-usb-hub", + .probe = onboard_hub_usbdev_probe, + .disconnect = onboard_hub_usbdev_disconnect, + .generic_subclass = 1, + .supports_autosuspend = 1, + .id_table = onboard_hub_id_table, +}; + +static int __init onboard_hub_init(void) +{ + int ret; + + ret = platform_driver_register(&onboard_hub_driver); + if (ret) + return ret; + + ret = usb_register_device_driver(&onboard_hub_usbdev_driver, THIS_MODULE); + if (ret) + platform_driver_unregister(&onboard_hub_driver); + + return ret; +} +module_init(onboard_hub_init); + +static void __exit onboard_hub_exit(void) +{ + usb_deregister_device_driver(&onboard_hub_usbdev_driver); + platform_driver_unregister(&onboard_hub_driver); +} +module_exit(onboard_hub_exit); + +MODULE_AUTHOR("Matthias Kaehlcke "); +MODULE_DESCRIPTION("Driver for discrete onboard USB hubs"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/usb/misc/onboard_usb_hub.h b/drivers/usb/misc/onboard_usb_hub.h new file mode 100644 index 000000000000..d3a5b6938582 --- /dev/null +++ b/drivers/usb/misc/onboard_usb_hub.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2022, Google LLC + */ + +#ifndef _USB_MISC_ONBOARD_USB_HUB_H +#define _USB_MISC_ONBOARD_USB_HUB_H + +static const struct of_device_id onboard_hub_match[] = { + { .compatible = "usbbda,411" }, + { .compatible = "usbbda,5411" }, + { .compatible = "usbbda,414" }, + { .compatible = "usbbda,5414" }, + {} +}; + +#endif /* _USB_MISC_ONBOARD_USB_HUB_H */ diff --git a/drivers/usb/misc/onboard_usb_hub_pdevs.c b/drivers/usb/misc/onboard_usb_hub_pdevs.c new file mode 100644 index 000000000000..a0a5f719129f --- /dev/null +++ b/drivers/usb/misc/onboard_usb_hub_pdevs.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * API for creating and destroying USB onboard hub platform devices + * + * Copyright (c) 2022, Google LLC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "onboard_usb_hub.h" + +struct pdev_list_entry { + struct platform_device *pdev; + struct list_head node; +}; + +static bool of_is_onboard_usb_hub(const struct device_node *np) +{ + return !!of_match_node(onboard_hub_match, np); +} + +/** + * onboard_hub_create_pdevs -- create platform devices for onboard USB hubs + * @parent_hub : parent hub to scan for connected onboard hubs + * @pdev_list : list of onboard hub platform devices owned by the parent hub + * + * Creates a platform device for each supported onboard hub that is connected to + * the given parent hub. The platform device is in charge of initializing the + * hub (enable regulators, take the hub out of reset, ...) and can optionally + * control whether the hub remains powered during system suspend or not. + * + * To keep track of the platform devices they are added to a list that is owned + * by the parent hub. + * + * Some background about the logic in this function, which can be a bit hard + * to follow: + * + * Root hubs don't have dedicated device tree nodes, but use the node of their + * HCD. The primary and secondary HCD are usually represented by a single DT + * node. That means the root hubs of the primary and secondary HCD share the + * same device tree node (the HCD node). As a result this function can be called + * twice with the same DT node for root hubs. We only want to create a single + * platform device for each physical onboard hub, hence for root hubs the loop + * is only executed for the root hub of the primary HCD. Since the function + * scans through all child nodes it still creates pdevs for onboard hubs + * connected to the root hub of the secondary HCD if needed. + * + * Further there must be only one platform device for onboard hubs with a peer + * hub (the hub is a single physical device). To achieve this two measures are + * taken: pdevs for onboard hubs with a peer are only created when the function + * is called on behalf of the parent hub that is connected to the primary HCD + * (directly or through other hubs). For onboard hubs connected to root hubs + * the function processes the nodes of both peers. A platform device is only + * created if the peer hub doesn't have one already. + */ +void onboard_hub_create_pdevs(struct usb_device *parent_hub, struct list_head *pdev_list) +{ + int i; + struct usb_hcd *hcd = bus_to_hcd(parent_hub->bus); + struct device_node *np, *npc; + struct platform_device *pdev; + struct pdev_list_entry *pdle; + + if (!parent_hub->dev.of_node) + return; + + if (!parent_hub->parent && !usb_hcd_is_primary_hcd(hcd)) + return; + + for (i = 1; i <= parent_hub->maxchild; i++) { + np = usb_of_get_device_node(parent_hub, i); + if (!np) + continue; + + if (!of_is_onboard_usb_hub(np)) + goto node_put; + + npc = of_parse_phandle(np, "peer-hub", 0); + if (npc) { + if (!usb_hcd_is_primary_hcd(hcd)) { + of_node_put(npc); + goto node_put; + } + + pdev = of_find_device_by_node(npc); + of_node_put(npc); + + if (pdev) { + put_device(&pdev->dev); + goto node_put; + } + } + + pdev = of_platform_device_create(np, NULL, &parent_hub->dev); + if (!pdev) { + dev_err(&parent_hub->dev, + "failed to create platform device for onboard hub '%pOF'\n", np); + goto node_put; + } + + pdle = kzalloc(sizeof(*pdle), GFP_KERNEL); + if (!pdle) { + of_platform_device_destroy(&pdev->dev, NULL); + goto node_put; + } + + pdle->pdev = pdev; + list_add(&pdle->node, pdev_list); + +node_put: + of_node_put(np); + } +} +EXPORT_SYMBOL_GPL(onboard_hub_create_pdevs); + +/** + * onboard_hub_destroy_pdevs -- free resources of onboard hub platform devices + * @pdev_list : list of onboard hub platform devices + * + * Destroys the platform devices in the given list and frees the memory associated + * with the list entry. + */ +void onboard_hub_destroy_pdevs(struct list_head *pdev_list) +{ + struct pdev_list_entry *pdle, *tmp; + + list_for_each_entry_safe(pdle, tmp, pdev_list, node) { + list_del(&pdle->node); + of_platform_device_destroy(&pdle->pdev->dev, NULL); + kfree(pdle); + } +} +EXPORT_SYMBOL_GPL(onboard_hub_destroy_pdevs); diff --git a/include/linux/usb/onboard_hub.h b/include/linux/usb/onboard_hub.h new file mode 100644 index 000000000000..d9373230556e --- /dev/null +++ b/include/linux/usb/onboard_hub.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __LINUX_USB_ONBOARD_HUB_H +#define __LINUX_USB_ONBOARD_HUB_H + +struct usb_device; +struct list_head; + +#if IS_ENABLED(CONFIG_USB_ONBOARD_HUB) +void onboard_hub_create_pdevs(struct usb_device *parent_hub, struct list_head *pdev_list); +void onboard_hub_destroy_pdevs(struct list_head *pdev_list); +#else +static inline void onboard_hub_create_pdevs(struct usb_device *parent_hub, + struct list_head *pdev_list) {} +static inline void onboard_hub_destroy_pdevs(struct list_head *pdev_list) {} +#endif + +#endif /* __LINUX_USB_ONBOARD_HUB_H */ -- cgit v1.2.3 From 0139da50dc53f0ce2804e83566d290c7e626fd17 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Mon, 4 Jul 2022 12:45:14 +0300 Subject: serial: Embed rs485_supported to uart_port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embed rs485_supported to uart_port to allow serial core to tweak it as needed. Reviewed-by: Lino Sanfilippo Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20220704094515.6831-2-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_bcm2835aux.c | 2 +- drivers/tty/serial/8250/8250_dwlib.c | 4 ++-- drivers/tty/serial/8250/8250_exar.c | 2 +- drivers/tty/serial/8250/8250_fintek.c | 4 ++-- drivers/tty/serial/8250/8250_lpc18xx.c | 2 +- drivers/tty/serial/8250/8250_of.c | 2 +- drivers/tty/serial/8250/8250_pci.c | 2 +- drivers/tty/serial/amba-pl011.c | 2 +- drivers/tty/serial/ar933x_uart.c | 4 ++-- drivers/tty/serial/atmel_serial.c | 2 +- drivers/tty/serial/fsl_lpuart.c | 2 +- drivers/tty/serial/imx.c | 4 ++-- drivers/tty/serial/max310x.c | 2 +- drivers/tty/serial/mcf.c | 4 ++-- drivers/tty/serial/omap-serial.c | 2 +- drivers/tty/serial/sc16is7xx.c | 2 +- drivers/tty/serial/serial_core.c | 8 ++++---- drivers/tty/serial/stm32-usart.c | 2 +- include/linux/serial_core.h | 2 +- 19 files changed, 27 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/8250/8250_bcm2835aux.c b/drivers/tty/serial/8250/8250_bcm2835aux.c index d9f1e618cfbd..047e14ccb165 100644 --- a/drivers/tty/serial/8250/8250_bcm2835aux.c +++ b/drivers/tty/serial/8250/8250_bcm2835aux.c @@ -108,7 +108,7 @@ static int bcm2835aux_serial_probe(struct platform_device *pdev) up.port.flags = UPF_SHARE_IRQ | UPF_FIXED_PORT | UPF_FIXED_TYPE | UPF_SKIP_TEST | UPF_IOREMAP; up.port.rs485_config = serial8250_em485_config; - up.port.rs485_supported = &serial8250_em485_supported; + up.port.rs485_supported = serial8250_em485_supported; up.rs485_start_tx = bcm2835aux_rs485_start_tx; up.rs485_stop_tx = bcm2835aux_rs485_stop_tx; diff --git a/drivers/tty/serial/8250/8250_dwlib.c b/drivers/tty/serial/8250/8250_dwlib.c index a8bbed74ea70..2c3b1468bd88 100644 --- a/drivers/tty/serial/8250/8250_dwlib.c +++ b/drivers/tty/serial/8250/8250_dwlib.c @@ -255,10 +255,10 @@ void dw8250_setup_port(struct uart_port *p) if (pd->hw_rs485_support) { p->rs485_config = dw8250_rs485_config; up->lsr_save_mask = LSR_SAVE_FLAGS | DW_UART_LSR_ADDR_RCVD; - p->rs485_supported = &dw8250_rs485_supported; + p->rs485_supported = dw8250_rs485_supported; } else { p->rs485_config = serial8250_em485_config; - p->rs485_supported = &serial8250_em485_supported; + p->rs485_supported = serial8250_em485_supported; up->rs485_start_tx = serial8250_em485_start_tx; up->rs485_stop_tx = serial8250_em485_stop_tx; } diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c index f5344cfe303c..314a05e009df 100644 --- a/drivers/tty/serial/8250/8250_exar.c +++ b/drivers/tty/serial/8250/8250_exar.c @@ -550,7 +550,7 @@ pci_xr17v35x_setup(struct exar8250 *priv, struct pci_dev *pcidev, port->port.uartclk = baud * 16; port->port.rs485_config = platform->rs485_config; - port->port.rs485_supported = platform->rs485_supported; + port->port.rs485_supported = *(platform->rs485_supported); /* * Setup the UART clock for the devices on expansion slot to diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c index eea693f5b577..65b6b3cbaff6 100644 --- a/drivers/tty/serial/8250/8250_fintek.c +++ b/drivers/tty/serial/8250/8250_fintek.c @@ -433,9 +433,9 @@ static void fintek_8250_set_rs485_handler(struct uart_8250_port *uart) case CHIP_ID_F81865: uart->port.rs485_config = fintek_8250_rs485_config; if (!pdata->index) - uart->port.rs485_supported = &fintek_8250_rs485_supported_port0; + uart->port.rs485_supported = fintek_8250_rs485_supported_port0; else - uart->port.rs485_supported = &fintek_8250_rs485_supported; + uart->port.rs485_supported = fintek_8250_rs485_supported; break; default: /* No RS485 Auto direction functional */ diff --git a/drivers/tty/serial/8250/8250_lpc18xx.c b/drivers/tty/serial/8250/8250_lpc18xx.c index d7cb3bb52069..d6ca0d47e9d5 100644 --- a/drivers/tty/serial/8250/8250_lpc18xx.c +++ b/drivers/tty/serial/8250/8250_lpc18xx.c @@ -161,7 +161,7 @@ static int lpc18xx_serial_probe(struct platform_device *pdev) uart.port.uartclk = clk_get_rate(data->clk_uart); uart.port.private_data = data; uart.port.rs485_config = lpc18xx_rs485_config; - uart.port.rs485_supported = &lpc18xx_rs485_supported; + uart.port.rs485_supported = lpc18xx_rs485_supported; uart.port.serial_out = lpc18xx_uart_serial_out; uart.dma = &data->dma; diff --git a/drivers/tty/serial/8250/8250_of.c b/drivers/tty/serial/8250/8250_of.c index 65cccd559db2..1b461fba15a3 100644 --- a/drivers/tty/serial/8250/8250_of.c +++ b/drivers/tty/serial/8250/8250_of.c @@ -165,7 +165,7 @@ static int of_platform_serial_setup(struct platform_device *ofdev, port->dev = &ofdev->dev; port->rs485_config = serial8250_em485_config; - port->rs485_supported = &serial8250_em485_supported; + port->rs485_supported = serial8250_em485_supported; up->rs485_start_tx = serial8250_em485_start_tx; up->rs485_stop_tx = serial8250_em485_stop_tx; diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index d31d2350a9db..8a39ae072c65 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -1607,7 +1607,7 @@ static int pci_fintek_setup(struct serial_private *priv, port->port.iotype = UPIO_PORT; port->port.iobase = iobase; port->port.rs485_config = pci_fintek_rs485_config; - port->port.rs485_supported = &pci_fintek_rs485_supported; + port->port.rs485_supported = pci_fintek_rs485_supported; data = devm_kzalloc(&pdev->dev, sizeof(u8), GFP_KERNEL); if (!data) diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index c8f52945a4aa..abeceeefdece 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2779,7 +2779,7 @@ static int pl011_probe(struct amba_device *dev, const struct amba_id *id) uap->port.irq = dev->irq[0]; uap->port.ops = &amba_pl011_pops; uap->port.rs485_config = pl011_rs485_config; - uap->port.rs485_supported = &pl011_rs485_supported; + uap->port.rs485_supported = pl011_rs485_supported; snprintf(uap->type, sizeof(uap->type), "PL011 rev%u", amba_rev(dev)); ret = pl011_setup_port(&dev->dev, uap, &dev->res, portnr); diff --git a/drivers/tty/serial/ar933x_uart.c b/drivers/tty/serial/ar933x_uart.c index b73ce13683db..f931ecbc0bc0 100644 --- a/drivers/tty/serial/ar933x_uart.c +++ b/drivers/tty/serial/ar933x_uart.c @@ -778,7 +778,7 @@ static int ar933x_uart_probe(struct platform_device *pdev) port->fifosize = AR933X_UART_FIFO_SIZE; port->ops = &ar933x_uart_ops; port->rs485_config = ar933x_config_rs485; - port->rs485_supported = &ar933x_rs485_supported; + port->rs485_supported = ar933x_rs485_supported; baud = ar933x_uart_get_baud(port->uartclk, AR933X_UART_MAX_SCALE, 1); up->min_baud = max_t(unsigned int, baud, AR933X_UART_MIN_BAUD); @@ -802,7 +802,7 @@ static int ar933x_uart_probe(struct platform_device *pdev) !up->rts_gpiod) { dev_err(&pdev->dev, "lacking rts-gpio, disabling RS485\n"); port->rs485.flags &= ~SER_RS485_ENABLED; - port->rs485_supported = &ar933x_no_rs485; + port->rs485_supported = ar933x_no_rs485; } #ifdef CONFIG_SERIAL_AR933X_CONSOLE diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c index bc6004679585..30ba9eef7b39 100644 --- a/drivers/tty/serial/atmel_serial.c +++ b/drivers/tty/serial/atmel_serial.c @@ -2498,7 +2498,7 @@ static int atmel_init_port(struct atmel_uart_port *atmel_port, port->mapbase = mpdev->resource[0].start; port->irq = platform_get_irq(mpdev, 0); port->rs485_config = atmel_config_rs485; - port->rs485_supported = &atmel_rs485_supported; + port->rs485_supported = atmel_rs485_supported; port->iso7816_config = atmel_config_iso7816; port->membase = NULL; diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c index 8fe0494d4057..fc7d235a1e27 100644 --- a/drivers/tty/serial/fsl_lpuart.c +++ b/drivers/tty/serial/fsl_lpuart.c @@ -2655,7 +2655,7 @@ static int lpuart_probe(struct platform_device *pdev) sport->port.rs485_config = lpuart32_config_rs485; else sport->port.rs485_config = lpuart_config_rs485; - sport->port.rs485_supported = &lpuart_rs485_supported; + sport->port.rs485_supported = lpuart_rs485_supported; sport->ipg_clk = devm_clk_get(&pdev->dev, "ipg"); if (IS_ERR(sport->ipg_clk)) { diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 3457006cea3f..522445a8f666 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -2285,9 +2285,9 @@ static int imx_uart_probe(struct platform_device *pdev) sport->port.rs485_config = imx_uart_rs485_config; /* RTS is required to control the RS485 transmitter */ if (sport->have_rtscts || sport->have_rtsgpio) - sport->port.rs485_supported = &imx_rs485_supported; + sport->port.rs485_supported = imx_rs485_supported; else - sport->port.rs485_supported = &imx_no_rs485; + sport->port.rs485_supported = imx_no_rs485; sport->port.flags = UPF_BOOT_AUTOCONF; timer_setup(&sport->timer, imx_uart_timeout, 0); diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c index e162bfb44080..ab10ca4a45b5 100644 --- a/drivers/tty/serial/max310x.c +++ b/drivers/tty/serial/max310x.c @@ -1370,7 +1370,7 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty s->p[i].port.membase = (void __iomem *)~0; s->p[i].port.uartclk = uartclk; s->p[i].port.rs485_config = max310x_rs485_config; - s->p[i].port.rs485_supported = &max310x_rs485_supported; + s->p[i].port.rs485_supported = max310x_rs485_supported; s->p[i].port.ops = &max310x_ops; s->p[i].regmap = regmaps[i]; diff --git a/drivers/tty/serial/mcf.c b/drivers/tty/serial/mcf.c index 73c5287b8e5e..f4aaaadd0742 100644 --- a/drivers/tty/serial/mcf.c +++ b/drivers/tty/serial/mcf.c @@ -506,7 +506,7 @@ int __init early_mcf_setup(struct mcf_platform_uart *platp) port->uartclk = MCF_BUSCLK; port->flags = UPF_BOOT_AUTOCONF; port->rs485_config = mcf_config_rs485; - port->rs485_supported = &mcf_rs485_supported; + port->rs485_supported = mcf_rs485_supported; port->ops = &mcf_uart_ops; } @@ -634,7 +634,7 @@ static int mcf_probe(struct platform_device *pdev) port->ops = &mcf_uart_ops; port->flags = UPF_BOOT_AUTOCONF; port->rs485_config = mcf_config_rs485; - port->rs485_supported = &mcf_rs485_supported; + port->rs485_supported = mcf_rs485_supported; port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_MCF_CONSOLE); uart_add_one_port(&mcf_driver, port); diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 196bae704f85..0aa666e247d5 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -1643,7 +1643,7 @@ static int serial_omap_probe(struct platform_device *pdev) up->port.flags = omap_up_info->flags; up->port.uartclk = omap_up_info->uartclk; up->port.rs485_config = serial_omap_config_rs485; - up->port.rs485_supported = &serial_omap_rs485_supported; + up->port.rs485_supported = serial_omap_rs485_supported; if (!up->port.uartclk) { up->port.uartclk = DEFAULT_CLK_SPEED; dev_warn(&pdev->dev, diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index 8cb92a3b3fb8..259e08cc347c 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -1461,7 +1461,7 @@ static int sc16is7xx_probe(struct device *dev, s->p[i].port.iotype = UPIO_PORT; s->p[i].port.uartclk = freq; s->p[i].port.rs485_config = sc16is7xx_config_rs485; - s->p[i].port.rs485_supported = &sc16is7xx_rs485_supported; + s->p[i].port.rs485_supported = sc16is7xx_rs485_supported; s->p[i].port.ops = &sc16is7xx_ops; s->p[i].old_mctrl = 0; s->p[i].port.line = sc16is7xx_alloc_line(); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 85ef7ef00b82..a9cf1044a9fa 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -1285,7 +1285,7 @@ static int uart_check_rs485_flags(struct uart_port *port, struct serial_rs485 *r * For any bit outside of the legacy ones that is not supported by * the driver, return -EINVAL. */ - if (flags & ~port->rs485_supported->flags) + if (flags & ~port->rs485_supported.flags) return -EINVAL; /* Asking for address w/o addressing mode? */ @@ -1304,7 +1304,7 @@ static int uart_check_rs485_flags(struct uart_port *port, struct serial_rs485 *r static void uart_sanitize_serial_rs485(struct uart_port *port, struct serial_rs485 *rs485) { - u32 supported_flags = port->rs485_supported->flags; + u32 supported_flags = port->rs485_supported.flags; if (!(rs485->flags & SER_RS485_ENABLED)) { memset(rs485, 0, sizeof(*rs485)); @@ -1323,7 +1323,7 @@ static void uart_sanitize_serial_rs485(struct uart_port *port, struct serial_rs4 supported_flags |= SER_RS485_RTS_ON_SEND|SER_RS485_RTS_AFTER_SEND; } - if (!port->rs485_supported->delay_rts_before_send) { + if (!port->rs485_supported.delay_rts_before_send) { if (rs485->delay_rts_before_send) { dev_warn_ratelimited(port->dev, "%s (%d): RTS delay before sending not supported\n", @@ -1337,7 +1337,7 @@ static void uart_sanitize_serial_rs485(struct uart_port *port, struct serial_rs4 port->name, port->line, rs485->delay_rts_before_send); } - if (!port->rs485_supported->delay_rts_after_send) { + if (!port->rs485_supported.delay_rts_after_send) { if (rs485->delay_rts_after_send) { dev_warn_ratelimited(port->dev, "%s (%d): RTS delay after sending not supported\n", diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c index 13992e64a7df..ff5c7e0ebc4c 100644 --- a/drivers/tty/serial/stm32-usart.c +++ b/drivers/tty/serial/stm32-usart.c @@ -1401,7 +1401,7 @@ static int stm32_usart_init_port(struct stm32_port *stm32port, port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_STM32_CONSOLE); port->irq = irq; port->rs485_config = stm32_usart_config_rs485; - port->rs485_supported = &stm32_rs485_supported; + port->rs485_supported = stm32_rs485_supported; ret = stm32_usart_init_rs485(port, pdev); if (ret) diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index b7b86ee3cb12..a6fa7c40c330 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -255,7 +255,7 @@ struct uart_port { struct attribute_group *attr_group; /* port specific attributes */ const struct attribute_group **tty_groups; /* all attributes (serial core use only) */ struct serial_rs485 rs485; - const struct serial_rs485 *rs485_supported; /* Supported mask for serial_rs485 */ + struct serial_rs485 rs485_supported; /* Supported mask for serial_rs485 */ struct gpio_desc *rs485_term_gpio; /* enable RS485 bus termination */ struct serial_iso7816 iso7816; void *private_data; /* generic platform data pointer */ -- cgit v1.2.3 From 8aa5bcb61612060429223d1fbb7a1c30a579fc1f Mon Sep 17 00:00:00 2001 From: Mikko Perttunen Date: Mon, 27 Jun 2022 17:19:48 +0300 Subject: gpu: host1x: Add context device management code Add code to register context devices from device tree, allocate them out and manage their refcounts. Signed-off-by: Mikko Perttunen Signed-off-by: Thierry Reding --- drivers/gpu/host1x/Makefile | 3 + drivers/gpu/host1x/context.c | 160 +++++++++++++++++++++++++++++++++++++++++++ drivers/gpu/host1x/context.h | 38 ++++++++++ drivers/gpu/host1x/dev.c | 12 +++- drivers/gpu/host1x/dev.h | 2 + include/linux/host1x.h | 34 +++++++++ 6 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/host1x/context.c create mode 100644 drivers/gpu/host1x/context.h (limited to 'include/linux') diff --git a/drivers/gpu/host1x/Makefile b/drivers/gpu/host1x/Makefile index c891a3e33844..28f7aaedce0f 100644 --- a/drivers/gpu/host1x/Makefile +++ b/drivers/gpu/host1x/Makefile @@ -17,5 +17,8 @@ host1x-y = \ hw/host1x06.o \ hw/host1x07.o +host1x-$(CONFIG_IOMMU_API) += \ + context.o + obj-$(CONFIG_TEGRA_HOST1X) += host1x.o obj-$(CONFIG_TEGRA_HOST1X_CONTEXT_BUS) += context_bus.o diff --git a/drivers/gpu/host1x/context.c b/drivers/gpu/host1x/context.c new file mode 100644 index 000000000000..b08cf11f9a66 --- /dev/null +++ b/drivers/gpu/host1x/context.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, NVIDIA Corporation. + */ + +#include +#include +#include +#include +#include +#include + +#include "context.h" +#include "dev.h" + +int host1x_memory_context_list_init(struct host1x *host1x) +{ + struct host1x_memory_context_list *cdl = &host1x->context_list; + struct device_node *node = host1x->dev->of_node; + struct host1x_memory_context *ctx; + unsigned int i; + int err; + + cdl->devs = NULL; + cdl->len = 0; + mutex_init(&cdl->lock); + + err = of_property_count_u32_elems(node, "iommu-map"); + if (err < 0) + return 0; + + cdl->devs = kcalloc(err, sizeof(*cdl->devs), GFP_KERNEL); + if (!cdl->devs) + return -ENOMEM; + cdl->len = err / 4; + + for (i = 0; i < cdl->len; i++) { + struct iommu_fwspec *fwspec; + + ctx = &cdl->devs[i]; + + ctx->host = host1x; + + device_initialize(&ctx->dev); + + /* + * Due to an issue with T194 NVENC, only 38 bits can be used. + * Anyway, 256GiB of IOVA ought to be enough for anyone. + */ + ctx->dma_mask = DMA_BIT_MASK(38); + ctx->dev.dma_mask = &ctx->dma_mask; + ctx->dev.coherent_dma_mask = ctx->dma_mask; + dev_set_name(&ctx->dev, "host1x-ctx.%d", i); + ctx->dev.bus = &host1x_context_device_bus_type; + ctx->dev.parent = host1x->dev; + + dma_set_max_seg_size(&ctx->dev, UINT_MAX); + + err = device_add(&ctx->dev); + if (err) { + dev_err(host1x->dev, "could not add context device %d: %d\n", i, err); + goto del_devices; + } + + err = of_dma_configure_id(&ctx->dev, node, true, &i); + if (err) { + dev_err(host1x->dev, "IOMMU configuration failed for context device %d: %d\n", + i, err); + device_del(&ctx->dev); + goto del_devices; + } + + fwspec = dev_iommu_fwspec_get(&ctx->dev); + if (!fwspec || !device_iommu_mapped(&ctx->dev)) { + dev_err(host1x->dev, "Context device %d has no IOMMU!\n", i); + device_del(&ctx->dev); + goto del_devices; + } + + ctx->stream_id = fwspec->ids[0] & 0xffff; + } + + return 0; + +del_devices: + while (i--) + device_del(&cdl->devs[i].dev); + + kfree(cdl->devs); + cdl->len = 0; + + return err; +} + +void host1x_memory_context_list_free(struct host1x_memory_context_list *cdl) +{ + unsigned int i; + + for (i = 0; i < cdl->len; i++) + device_del(&cdl->devs[i].dev); + + kfree(cdl->devs); + cdl->len = 0; +} + +struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x, + struct pid *pid) +{ + struct host1x_memory_context_list *cdl = &host1x->context_list; + struct host1x_memory_context *free = NULL; + int i; + + if (!cdl->len) + return ERR_PTR(-EOPNOTSUPP); + + mutex_lock(&cdl->lock); + + for (i = 0; i < cdl->len; i++) { + struct host1x_memory_context *cd = &cdl->devs[i]; + + if (cd->owner == pid) { + refcount_inc(&cd->ref); + mutex_unlock(&cdl->lock); + return cd; + } else if (!cd->owner && !free) { + free = cd; + } + } + + if (!free) { + mutex_unlock(&cdl->lock); + return ERR_PTR(-EBUSY); + } + + refcount_set(&free->ref, 1); + free->owner = get_pid(pid); + + mutex_unlock(&cdl->lock); + + return free; +} +EXPORT_SYMBOL_GPL(host1x_memory_context_alloc); + +void host1x_memory_context_get(struct host1x_memory_context *cd) +{ + refcount_inc(&cd->ref); +} +EXPORT_SYMBOL_GPL(host1x_memory_context_get); + +void host1x_memory_context_put(struct host1x_memory_context *cd) +{ + struct host1x_memory_context_list *cdl = &cd->host->context_list; + + if (refcount_dec_and_mutex_lock(&cd->ref, &cdl->lock)) { + put_pid(cd->owner); + cd->owner = NULL; + mutex_unlock(&cdl->lock); + } +} +EXPORT_SYMBOL_GPL(host1x_memory_context_put); diff --git a/drivers/gpu/host1x/context.h b/drivers/gpu/host1x/context.h new file mode 100644 index 000000000000..3e03bc1d3bac --- /dev/null +++ b/drivers/gpu/host1x/context.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Host1x context devices + * + * Copyright (c) 2020, NVIDIA Corporation. + */ + +#ifndef __HOST1X_CONTEXT_H +#define __HOST1X_CONTEXT_H + +#include +#include + +struct host1x; + +extern struct bus_type host1x_context_device_bus_type; + +struct host1x_memory_context_list { + struct mutex lock; + struct host1x_memory_context *devs; + unsigned int len; +}; + +#ifdef CONFIG_IOMMU_API +int host1x_memory_context_list_init(struct host1x *host1x); +void host1x_memory_context_list_free(struct host1x_memory_context_list *cdl); +#else +static inline int host1x_memory_context_list_init(struct host1x *host1x) +{ + return 0; +} + +static inline void host1x_memory_context_list_free(struct host1x_memory_context_list *cdl) +{ +} +#endif + +#endif diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c index 80c685ab3e30..89cc79a48eab 100644 --- a/drivers/gpu/host1x/dev.c +++ b/drivers/gpu/host1x/dev.c @@ -28,6 +28,7 @@ #include "bus.h" #include "channel.h" +#include "context.h" #include "debug.h" #include "dev.h" #include "intr.h" @@ -503,10 +504,16 @@ static int host1x_probe(struct platform_device *pdev) goto iommu_exit; } + err = host1x_memory_context_list_init(host); + if (err) { + dev_err(&pdev->dev, "failed to initialize context list\n"); + goto free_channels; + } + err = host1x_syncpt_init(host); if (err) { dev_err(&pdev->dev, "failed to initialize syncpts\n"); - goto free_channels; + goto free_contexts; } err = host1x_intr_init(host, syncpt_irq); @@ -550,6 +557,8 @@ pm_disable: host1x_intr_deinit(host); deinit_syncpt: host1x_syncpt_deinit(host); +free_contexts: + host1x_memory_context_list_free(&host->context_list); free_channels: host1x_channel_list_free(&host->channel_list); iommu_exit: @@ -571,6 +580,7 @@ static int host1x_remove(struct platform_device *pdev) host1x_intr_deinit(host); host1x_syncpt_deinit(host); + host1x_memory_context_list_free(&host->context_list); host1x_channel_list_free(&host->channel_list); host1x_iommu_exit(host); host1x_bo_cache_destroy(&host->cache); diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h index ca4b082f0cd4..7552a4554534 100644 --- a/drivers/gpu/host1x/dev.h +++ b/drivers/gpu/host1x/dev.h @@ -14,6 +14,7 @@ #include "cdma.h" #include "channel.h" +#include "context.h" #include "intr.h" #include "job.h" #include "syncpt.h" @@ -141,6 +142,7 @@ struct host1x { struct mutex syncpt_mutex; struct host1x_channel_list channel_list; + struct host1x_memory_context_list context_list; struct dentry *debugfs; diff --git a/include/linux/host1x.h b/include/linux/host1x.h index c0bf4e581fe9..32a82da13fed 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -446,4 +446,38 @@ int tegra_mipi_disable(struct tegra_mipi_device *device); int tegra_mipi_start_calibration(struct tegra_mipi_device *device); int tegra_mipi_finish_calibration(struct tegra_mipi_device *device); +/* host1x memory contexts */ + +struct host1x_memory_context { + struct host1x *host; + + refcount_t ref; + struct pid *owner; + + struct device dev; + u64 dma_mask; + u32 stream_id; +}; + +#ifdef CONFIG_IOMMU_API +struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x, + struct pid *pid); +void host1x_memory_context_get(struct host1x_memory_context *cd); +void host1x_memory_context_put(struct host1x_memory_context *cd); +#else +static inline struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x, + struct pid *pid) +{ + return NULL; +} + +static inline void host1x_memory_context_get(struct host1x_memory_context *cd) +{ +} + +static inline void host1x_memory_context_put(struct host1x_memory_context *cd) +{ +} +#endif + #endif -- cgit v1.2.3 From 2486254781eab6f6119fabea78f11386c54460d2 Mon Sep 17 00:00:00 2001 From: Mikko Perttunen Date: Mon, 27 Jun 2022 17:19:49 +0300 Subject: gpu: host1x: Program context stream ID on submission Add code to do stream ID switching at the beginning of a job. The stream ID is switched to the stream ID specified by the context passed in the job structure. Before switching the stream ID, an OP_DONE wait is done on the channel's engine to ensure that there is no residual ongoing work that might do DMA using the new stream ID. Signed-off-by: Mikko Perttunen Signed-off-by: Thierry Reding --- drivers/gpu/host1x/hw/channel_hw.c | 52 ++++++++++++++++++++++++++++--- drivers/gpu/host1x/hw/host1x06_hardware.h | 10 ++++++ drivers/gpu/host1x/hw/host1x07_hardware.h | 10 ++++++ include/linux/host1x.h | 8 +++++ 4 files changed, 76 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c index 6b40e9af1e88..f84caf06621a 100644 --- a/drivers/gpu/host1x/hw/channel_hw.c +++ b/drivers/gpu/host1x/hw/channel_hw.c @@ -180,6 +180,45 @@ static void host1x_enable_gather_filter(struct host1x_channel *ch) #endif } +static void host1x_channel_program_engine_streamid(struct host1x_job *job) +{ +#if HOST1X_HW >= 6 + u32 fence; + + if (!job->memory_context) + return; + + fence = host1x_syncpt_incr_max(job->syncpt, 1); + + /* First, increment a syncpoint on OP_DONE condition.. */ + + host1x_cdma_push(&job->channel->cdma, + host1x_opcode_nonincr(HOST1X_UCLASS_INCR_SYNCPT, 1), + HOST1X_UCLASS_INCR_SYNCPT_INDX_F(job->syncpt->id) | + HOST1X_UCLASS_INCR_SYNCPT_COND_F(1)); + + /* Wait for syncpoint to increment */ + + host1x_cdma_push(&job->channel->cdma, + host1x_opcode_setclass(HOST1X_CLASS_HOST1X, + host1x_uclass_wait_syncpt_r(), 1), + host1x_class_host_wait_syncpt(job->syncpt->id, fence)); + + /* + * Now that we know the engine is idle, return to class and + * change stream ID. + */ + + host1x_cdma_push(&job->channel->cdma, + host1x_opcode_setclass(job->class, 0, 0), + HOST1X_OPCODE_NOP); + + host1x_cdma_push(&job->channel->cdma, + host1x_opcode_setpayload(job->memory_context->stream_id), + host1x_opcode_setstreamid(job->engine_streamid_offset / 4)); +#endif +} + static int channel_submit(struct host1x_job *job) { struct host1x_channel *ch = job->channel; @@ -236,18 +275,23 @@ static int channel_submit(struct host1x_job *job) if (sp->base) synchronize_syncpt_base(job); - syncval = host1x_syncpt_incr_max(sp, user_syncpt_incrs); - host1x_hw_syncpt_assign_to_channel(host, sp, ch); - job->syncpt_end = syncval; - /* add a setclass for modules that require it */ if (job->class) host1x_cdma_push(&ch->cdma, host1x_opcode_setclass(job->class, 0, 0), HOST1X_OPCODE_NOP); + /* + * Ensure engine DMA is idle and set new stream ID. May increment + * syncpt max. + */ + host1x_channel_program_engine_streamid(job); + + syncval = host1x_syncpt_incr_max(sp, user_syncpt_incrs); + job->syncpt_end = syncval; + submit_gathers(job, syncval - user_syncpt_incrs); /* end CDMA submit & stash pinned hMems into sync queue */ diff --git a/drivers/gpu/host1x/hw/host1x06_hardware.h b/drivers/gpu/host1x/hw/host1x06_hardware.h index 01a142a09800..5d515745eee7 100644 --- a/drivers/gpu/host1x/hw/host1x06_hardware.h +++ b/drivers/gpu/host1x/hw/host1x06_hardware.h @@ -127,6 +127,16 @@ static inline u32 host1x_opcode_gather_incr(unsigned offset, unsigned count) return (6 << 28) | (offset << 16) | BIT(15) | BIT(14) | count; } +static inline u32 host1x_opcode_setstreamid(unsigned streamid) +{ + return (7 << 28) | streamid; +} + +static inline u32 host1x_opcode_setpayload(unsigned payload) +{ + return (9 << 28) | payload; +} + static inline u32 host1x_opcode_gather_wide(unsigned count) { return (12 << 28) | count; diff --git a/drivers/gpu/host1x/hw/host1x07_hardware.h b/drivers/gpu/host1x/hw/host1x07_hardware.h index e6582172ebfd..82c0cc9bb0b5 100644 --- a/drivers/gpu/host1x/hw/host1x07_hardware.h +++ b/drivers/gpu/host1x/hw/host1x07_hardware.h @@ -127,6 +127,16 @@ static inline u32 host1x_opcode_gather_incr(unsigned offset, unsigned count) return (6 << 28) | (offset << 16) | BIT(15) | BIT(14) | count; } +static inline u32 host1x_opcode_setstreamid(unsigned streamid) +{ + return (7 << 28) | streamid; +} + +static inline u32 host1x_opcode_setpayload(unsigned payload) +{ + return (9 << 28) | payload; +} + static inline u32 host1x_opcode_gather_wide(unsigned count) { return (12 << 28) | count; diff --git a/include/linux/host1x.h b/include/linux/host1x.h index 32a82da13fed..cb2100d9b0ff 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -327,6 +327,14 @@ struct host1x_job { /* Whether host1x-side firewall should be ran for this job or not */ bool enable_firewall; + + /* Options for configuring engine data stream ID */ + /* Context device to use for job */ + struct host1x_memory_context *memory_context; + /* Stream ID to use if context isolation is disabled (!memory_context) */ + u32 engine_fallback_streamid; + /* Engine offset to program stream ID to */ + u32 engine_streamid_offset; }; struct host1x_job *host1x_job_alloc(struct host1x_channel *ch, -- cgit v1.2.3 From b6c1c5745ccc68ac5d57c7ffb51ea25a86d0e97b Mon Sep 17 00:00:00 2001 From: Matthias Kaehlcke Date: Mon, 27 Jun 2022 08:35:24 -0700 Subject: dm: Add verity helpers for LoadPin LoadPin limits loading of kernel modules, firmware and certain other files to a 'pinned' file system (typically a read-only rootfs). To provide more flexibility LoadPin is being extended to also allow loading these files from trusted dm-verity devices. For that purpose LoadPin can be provided with a list of verity root digests that it should consider as trusted. Add a bunch of helpers to allow LoadPin to check whether a DM device is a trusted verity device. The new functions broadly fall in two categories: those that need access to verity internals (like the root digest), and the 'glue' between LoadPin and verity. The new file dm-verity-loadpin.c contains the glue functions. Signed-off-by: Matthias Kaehlcke Acked-by: Mike Snitzer Link: https://lore.kernel.org/lkml/20220627083512.v7.1.I3e928575a23481121e73286874c4c2bdb403355d@changeid Signed-off-by: Kees Cook --- drivers/md/Makefile | 6 ++++ drivers/md/dm-verity-loadpin.c | 74 +++++++++++++++++++++++++++++++++++++++ drivers/md/dm-verity-target.c | 33 ++++++++++++++++- drivers/md/dm-verity.h | 4 +++ include/linux/dm-verity-loadpin.h | 27 ++++++++++++++ 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 drivers/md/dm-verity-loadpin.c create mode 100644 include/linux/dm-verity-loadpin.h (limited to 'include/linux') diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 0454b0885b01..71771901c823 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -108,6 +108,12 @@ ifeq ($(CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG),y) dm-verity-objs += dm-verity-verify-sig.o endif +ifeq ($(CONFIG_DM_VERITY),y) +ifeq ($(CONFIG_SECURITY_LOADPIN),y) +dm-verity-objs += dm-verity-loadpin.o +endif +endif + ifeq ($(CONFIG_DM_AUDIT),y) dm-mod-objs += dm-audit.o endif diff --git a/drivers/md/dm-verity-loadpin.c b/drivers/md/dm-verity-loadpin.c new file mode 100644 index 000000000000..10c18bc1652c --- /dev/null +++ b/drivers/md/dm-verity-loadpin.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +#include "dm.h" +#include "dm-verity.h" + +#define DM_MSG_PREFIX "verity-loadpin" + +LIST_HEAD(dm_verity_loadpin_trusted_root_digests); + +static bool is_trusted_verity_target(struct dm_target *ti) +{ + u8 *root_digest; + unsigned int digest_size; + struct dm_verity_loadpin_trusted_root_digest *trd; + bool trusted = false; + + if (!dm_is_verity_target(ti)) + return false; + + if (dm_verity_get_root_digest(ti, &root_digest, &digest_size)) + return false; + + list_for_each_entry(trd, &dm_verity_loadpin_trusted_root_digests, node) { + if ((trd->len == digest_size) && + !memcmp(trd->data, root_digest, digest_size)) { + trusted = true; + break; + } + } + + kfree(root_digest); + + return trusted; +} + +/* + * Determines whether the file system of a superblock is located on + * a verity device that is trusted by LoadPin. + */ +bool dm_verity_loadpin_is_bdev_trusted(struct block_device *bdev) +{ + struct mapped_device *md; + struct dm_table *table; + struct dm_target *ti; + int srcu_idx; + bool trusted = false; + + if (list_empty(&dm_verity_loadpin_trusted_root_digests)) + return false; + + md = dm_get_md(bdev->bd_dev); + if (!md) + return false; + + table = dm_get_live_table(md, &srcu_idx); + + if (dm_table_get_num_targets(table) != 1) + goto out; + + ti = dm_table_get_target(table, 0); + + if (is_trusted_verity_target(ti)) + trusted = true; + +out: + dm_put_live_table(md, srcu_idx); + dm_put(md); + + return trusted; +} diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index d6dbd47492a8..e5a01e2f96e9 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -19,6 +19,7 @@ #include #include #include +#include #define DM_MSG_PREFIX "verity" @@ -1310,10 +1311,40 @@ bad: return r; } +/* + * Check whether a DM target is a verity target. + */ +bool dm_is_verity_target(struct dm_target *ti) +{ + return ti->type->module == THIS_MODULE; +} + +/* + * Get the root digest of a verity target. + * + * Returns a copy of the root digest, the caller is responsible for + * freeing the memory of the digest. + */ +int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size) +{ + struct dm_verity *v = ti->private; + + if (!dm_is_verity_target(ti)) + return -EINVAL; + + *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL); + if (*root_digest == NULL) + return -ENOMEM; + + *digest_size = v->digest_size; + + return 0; +} + static struct target_type verity_target = { .name = "verity", .features = DM_TARGET_IMMUTABLE, - .version = {1, 8, 0}, + .version = {1, 8, 1}, .module = THIS_MODULE, .ctr = verity_ctr, .dtr = verity_dtr, diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index 4e769d13473a..c832cc3e3d24 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -129,4 +129,8 @@ extern int verity_hash(struct dm_verity *v, struct ahash_request *req, extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, sector_t block, u8 *digest, bool *is_zero); +extern bool dm_is_verity_target(struct dm_target *ti); +extern int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, + unsigned int *digest_size); + #endif /* DM_VERITY_H */ diff --git a/include/linux/dm-verity-loadpin.h b/include/linux/dm-verity-loadpin.h new file mode 100644 index 000000000000..fb695ecaa5d5 --- /dev/null +++ b/include/linux/dm-verity-loadpin.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __LINUX_DM_VERITY_LOADPIN_H +#define __LINUX_DM_VERITY_LOADPIN_H + +#include + +struct block_device; + +extern struct list_head dm_verity_loadpin_trusted_root_digests; + +struct dm_verity_loadpin_trusted_root_digest { + struct list_head node; + unsigned int len; + u8 data[]; +}; + +#if IS_ENABLED(CONFIG_SECURITY_LOADPIN) && IS_BUILTIN(CONFIG_DM_VERITY) +bool dm_verity_loadpin_is_bdev_trusted(struct block_device *bdev); +#else +static inline bool dm_verity_loadpin_is_bdev_trusted(struct block_device *bdev) +{ + return false; +} +#endif + +#endif /* __LINUX_DM_VERITY_LOADPIN_H */ -- cgit v1.2.3 From 231af4709018a8e4f20e511da4b6506346d662d3 Mon Sep 17 00:00:00 2001 From: Matthias Kaehlcke Date: Mon, 27 Jun 2022 08:35:26 -0700 Subject: dm: verity-loadpin: Use CONFIG_SECURITY_LOADPIN_VERITY for conditional compilation The verity glue for LoadPin is only needed when CONFIG_SECURITY_LOADPIN_VERITY is set, use this option for conditional compilation instead of the combo of CONFIG_DM_VERITY and CONFIG_SECURITY_LOADPIN. Signed-off-by: Matthias Kaehlcke Acked-by: Mike Snitzer Link: https://lore.kernel.org/lkml/20220627083512.v7.3.I5aca2dcc3b06de4bf53696cd21329dce8272b8aa@changeid Signed-off-by: Kees Cook --- drivers/md/Makefile | 7 +------ include/linux/dm-verity-loadpin.h | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 71771901c823..a96441752ec7 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -83,6 +83,7 @@ obj-$(CONFIG_DM_LOG_WRITES) += dm-log-writes.o obj-$(CONFIG_DM_INTEGRITY) += dm-integrity.o obj-$(CONFIG_DM_ZONED) += dm-zoned.o obj-$(CONFIG_DM_WRITECACHE) += dm-writecache.o +obj-$(CONFIG_SECURITY_LOADPIN_VERITY) += dm-verity-loadpin.o ifeq ($(CONFIG_DM_INIT),y) dm-mod-objs += dm-init.o @@ -108,12 +109,6 @@ ifeq ($(CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG),y) dm-verity-objs += dm-verity-verify-sig.o endif -ifeq ($(CONFIG_DM_VERITY),y) -ifeq ($(CONFIG_SECURITY_LOADPIN),y) -dm-verity-objs += dm-verity-loadpin.o -endif -endif - ifeq ($(CONFIG_DM_AUDIT),y) dm-mod-objs += dm-audit.o endif diff --git a/include/linux/dm-verity-loadpin.h b/include/linux/dm-verity-loadpin.h index fb695ecaa5d5..552b817ab102 100644 --- a/include/linux/dm-verity-loadpin.h +++ b/include/linux/dm-verity-loadpin.h @@ -15,7 +15,7 @@ struct dm_verity_loadpin_trusted_root_digest { u8 data[]; }; -#if IS_ENABLED(CONFIG_SECURITY_LOADPIN) && IS_BUILTIN(CONFIG_DM_VERITY) +#if IS_ENABLED(CONFIG_SECURITY_LOADPIN_VERITY) bool dm_verity_loadpin_is_bdev_trusted(struct block_device *bdev); #else static inline bool dm_verity_loadpin_is_bdev_trusted(struct block_device *bdev) -- cgit v1.2.3 From 1dd685c414a7b9fdb3d23aca3aedae84f0b998ae Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 6 Jul 2022 14:51:00 -0400 Subject: XArray: Add calls to might_alloc() Catch bogus GFP flags deterministically, instead of occasionally when we actually have to allocate memory. Reported-by: Nikolay Borisov Signed-off-by: Matthew Wilcox (Oracle) --- include/linux/xarray.h | 15 +++++++++++++++ tools/include/linux/sched/mm.h | 2 ++ 2 files changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index c29e11b2c073..44dd6d6e01bc 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -586,6 +587,7 @@ static inline void *xa_store_bh(struct xarray *xa, unsigned long index, { void *curr; + might_alloc(gfp); xa_lock_bh(xa); curr = __xa_store(xa, index, entry, gfp); xa_unlock_bh(xa); @@ -612,6 +614,7 @@ static inline void *xa_store_irq(struct xarray *xa, unsigned long index, { void *curr; + might_alloc(gfp); xa_lock_irq(xa); curr = __xa_store(xa, index, entry, gfp); xa_unlock_irq(xa); @@ -687,6 +690,7 @@ static inline void *xa_cmpxchg(struct xarray *xa, unsigned long index, { void *curr; + might_alloc(gfp); xa_lock(xa); curr = __xa_cmpxchg(xa, index, old, entry, gfp); xa_unlock(xa); @@ -714,6 +718,7 @@ static inline void *xa_cmpxchg_bh(struct xarray *xa, unsigned long index, { void *curr; + might_alloc(gfp); xa_lock_bh(xa); curr = __xa_cmpxchg(xa, index, old, entry, gfp); xa_unlock_bh(xa); @@ -741,6 +746,7 @@ static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index, { void *curr; + might_alloc(gfp); xa_lock_irq(xa); curr = __xa_cmpxchg(xa, index, old, entry, gfp); xa_unlock_irq(xa); @@ -770,6 +776,7 @@ static inline int __must_check xa_insert(struct xarray *xa, { int err; + might_alloc(gfp); xa_lock(xa); err = __xa_insert(xa, index, entry, gfp); xa_unlock(xa); @@ -799,6 +806,7 @@ static inline int __must_check xa_insert_bh(struct xarray *xa, { int err; + might_alloc(gfp); xa_lock_bh(xa); err = __xa_insert(xa, index, entry, gfp); xa_unlock_bh(xa); @@ -828,6 +836,7 @@ static inline int __must_check xa_insert_irq(struct xarray *xa, { int err; + might_alloc(gfp); xa_lock_irq(xa); err = __xa_insert(xa, index, entry, gfp); xa_unlock_irq(xa); @@ -857,6 +866,7 @@ static inline __must_check int xa_alloc(struct xarray *xa, u32 *id, { int err; + might_alloc(gfp); xa_lock(xa); err = __xa_alloc(xa, id, entry, limit, gfp); xa_unlock(xa); @@ -886,6 +896,7 @@ static inline int __must_check xa_alloc_bh(struct xarray *xa, u32 *id, { int err; + might_alloc(gfp); xa_lock_bh(xa); err = __xa_alloc(xa, id, entry, limit, gfp); xa_unlock_bh(xa); @@ -915,6 +926,7 @@ static inline int __must_check xa_alloc_irq(struct xarray *xa, u32 *id, { int err; + might_alloc(gfp); xa_lock_irq(xa); err = __xa_alloc(xa, id, entry, limit, gfp); xa_unlock_irq(xa); @@ -948,6 +960,7 @@ static inline int xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, { int err; + might_alloc(gfp); xa_lock(xa); err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); xa_unlock(xa); @@ -981,6 +994,7 @@ static inline int xa_alloc_cyclic_bh(struct xarray *xa, u32 *id, void *entry, { int err; + might_alloc(gfp); xa_lock_bh(xa); err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); xa_unlock_bh(xa); @@ -1014,6 +1028,7 @@ static inline int xa_alloc_cyclic_irq(struct xarray *xa, u32 *id, void *entry, { int err; + might_alloc(gfp); xa_lock_irq(xa); err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); xa_unlock_irq(xa); diff --git a/tools/include/linux/sched/mm.h b/tools/include/linux/sched/mm.h index c8d9f19c1f35..967294b8edcf 100644 --- a/tools/include/linux/sched/mm.h +++ b/tools/include/linux/sched/mm.h @@ -1,4 +1,6 @@ #ifndef _TOOLS_PERF_LINUX_SCHED_MM_H #define _TOOLS_PERF_LINUX_SCHED_MM_H +#define might_alloc(gfp) do { } while (0) + #endif /* _TOOLS_PERF_LINUX_SCHED_MM_H */ -- cgit v1.2.3 From c435c54639aa5513ab877c8b014dd83d4ce6b40e Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Mon, 6 Jun 2022 16:33:14 -0400 Subject: vfio/pci: introduce CONFIG_VFIO_PCI_ZDEV_KVM The current contents of vfio-pci-zdev are today only useful in a KVM environment; let's tie everything currently under vfio-pci-zdev to this Kconfig statement and require KVM in this case, reducing complexity (e.g. symbol lookups). Signed-off-by: Matthew Rosato Acked-by: Alex Williamson Reviewed-by: Pierre Morel Link: https://lore.kernel.org/r/20220606203325.110625-11-mjrosato@linux.ibm.com Signed-off-by: Christian Borntraeger --- drivers/vfio/pci/Kconfig | 11 +++++++++++ drivers/vfio/pci/Makefile | 2 +- include/linux/vfio_pci_core.h | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 4da1914425e1..f9d0c908e738 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -44,6 +44,17 @@ config VFIO_PCI_IGD To enable Intel IGD assignment through vfio-pci, say Y. endif +config VFIO_PCI_ZDEV_KVM + bool "VFIO PCI extensions for s390x KVM passthrough" + depends on S390 && KVM + default y + help + Support s390x-specific extensions to enable support for enhancements + to KVM passthrough capabilities, such as interpretive execution of + zPCI instructions. + + To enable s390x KVM vfio-pci extensions, say Y. + source "drivers/vfio/pci/mlx5/Kconfig" source "drivers/vfio/pci/hisilicon/Kconfig" diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index 7052ebd893e0..24c524224da5 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o -vfio-pci-core-$(CONFIG_S390) += vfio_pci_zdev.o +vfio-pci-core-$(CONFIG_VFIO_PCI_ZDEV_KVM) += vfio_pci_zdev.o obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o vfio-pci-y := vfio_pci.o diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 23c176d4b073..63af2897939c 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -206,7 +206,7 @@ static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) } #endif -#ifdef CONFIG_S390 +#ifdef CONFIG_VFIO_PCI_ZDEV_KVM extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, struct vfio_info_cap *caps); #else -- cgit v1.2.3 From 3c5a1b6f0a18520a0edd0600fef6f1a8553b8fdc Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Mon, 6 Jun 2022 16:33:19 -0400 Subject: KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding These routines will be wired into a kvm ioctl in order to respond to requests to enable / disable a device for Adapter Event Notifications / Adapter Interuption Forwarding. Reviewed-by: Christian Borntraeger Acked-by: Niklas Schnelle Signed-off-by: Matthew Rosato Link: https://lore.kernel.org/r/20220606203325.110625-16-mjrosato@linux.ibm.com Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 247 +++++++++++++++++++++++++++++++++++++++++++++ arch/s390/kvm/pci.h | 1 + arch/s390/pci/pci_insn.c | 1 + include/linux/sched/user.h | 3 +- 4 files changed, 251 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index d566551d3018..b232c8cbaa81 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "pci.h" struct zpci_aift *aift; @@ -152,6 +153,252 @@ unlock: return rc; } +/* Modify PCI: Register floating adapter interruption forwarding */ +static int kvm_zpci_set_airq(struct zpci_dev *zdev) +{ + u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); + struct zpci_fib fib = {}; + u8 status; + + fib.fmt0.isc = zdev->kzdev->fib.fmt0.isc; + fib.fmt0.sum = 1; /* enable summary notifications */ + fib.fmt0.noi = airq_iv_end(zdev->aibv); + fib.fmt0.aibv = virt_to_phys(zdev->aibv->vector); + fib.fmt0.aibvo = 0; + fib.fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8); + fib.fmt0.aisbo = zdev->aisb & 63; + fib.gd = zdev->gisa; + + return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; +} + +/* Modify PCI: Unregister floating adapter interruption forwarding */ +static int kvm_zpci_clear_airq(struct zpci_dev *zdev) +{ + u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT); + struct zpci_fib fib = {}; + u8 cc, status; + + fib.gd = zdev->gisa; + + cc = zpci_mod_fc(req, &fib, &status); + if (cc == 3 || (cc == 1 && status == 24)) + /* Function already gone or IRQs already deregistered. */ + cc = 0; + + return cc ? -EIO : 0; +} + +static inline void unaccount_mem(unsigned long nr_pages) +{ + struct user_struct *user = get_uid(current_user()); + + if (user) + atomic_long_sub(nr_pages, &user->locked_vm); + if (current->mm) + atomic64_sub(nr_pages, ¤t->mm->pinned_vm); +} + +static inline int account_mem(unsigned long nr_pages) +{ + struct user_struct *user = get_uid(current_user()); + unsigned long page_limit, cur_pages, new_pages; + + page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + + do { + cur_pages = atomic_long_read(&user->locked_vm); + new_pages = cur_pages + nr_pages; + if (new_pages > page_limit) + return -ENOMEM; + } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages, + new_pages) != cur_pages); + + atomic64_add(nr_pages, ¤t->mm->pinned_vm); + + return 0; +} + +static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, + bool assist) +{ + struct page *pages[1], *aibv_page, *aisb_page = NULL; + unsigned int msi_vecs, idx; + struct zpci_gaite *gaite; + unsigned long hva, bit; + struct kvm *kvm; + phys_addr_t gaddr; + int rc = 0, gisc, npages, pcount = 0; + + /* + * Interrupt forwarding is only applicable if the device is already + * enabled for interpretation + */ + if (zdev->gisa == 0) + return -EINVAL; + + kvm = zdev->kzdev->kvm; + msi_vecs = min_t(unsigned int, fib->fmt0.noi, zdev->max_msi); + + /* Get the associated forwarding ISC - if invalid, return the error */ + gisc = kvm_s390_gisc_register(kvm, fib->fmt0.isc); + if (gisc < 0) + return gisc; + + /* Replace AIBV address */ + idx = srcu_read_lock(&kvm->srcu); + hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aibv)); + npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, pages); + srcu_read_unlock(&kvm->srcu, idx); + if (npages < 1) { + rc = -EIO; + goto out; + } + aibv_page = pages[0]; + pcount++; + gaddr = page_to_phys(aibv_page) + (fib->fmt0.aibv & ~PAGE_MASK); + fib->fmt0.aibv = gaddr; + + /* Pin the guest AISB if one was specified */ + if (fib->fmt0.sum == 1) { + idx = srcu_read_lock(&kvm->srcu); + hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aisb)); + npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, + pages); + srcu_read_unlock(&kvm->srcu, idx); + if (npages < 1) { + rc = -EIO; + goto unpin1; + } + aisb_page = pages[0]; + pcount++; + } + + /* Account for pinned pages, roll back on failure */ + if (account_mem(pcount)) + goto unpin2; + + /* AISB must be allocated before we can fill in GAITE */ + mutex_lock(&aift->aift_lock); + bit = airq_iv_alloc_bit(aift->sbv); + if (bit == -1UL) + goto unlock; + zdev->aisb = bit; /* store the summary bit number */ + zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | + AIRQ_IV_BITLOCK | + AIRQ_IV_GUESTVEC, + phys_to_virt(fib->fmt0.aibv)); + + spin_lock_irq(&aift->gait_lock); + gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb * + sizeof(struct zpci_gaite)); + + /* If assist not requested, host will get all alerts */ + if (assist) + gaite->gisa = (u32)virt_to_phys(&kvm->arch.sie_page2->gisa); + else + gaite->gisa = 0; + + gaite->gisc = fib->fmt0.isc; + gaite->count++; + gaite->aisbo = fib->fmt0.aisbo; + gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb & + ~PAGE_MASK)); + aift->kzdev[zdev->aisb] = zdev->kzdev; + spin_unlock_irq(&aift->gait_lock); + + /* Update guest FIB for re-issue */ + fib->fmt0.aisbo = zdev->aisb & 63; + fib->fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8); + fib->fmt0.isc = gisc; + + /* Save some guest fib values in the host for later use */ + zdev->kzdev->fib.fmt0.isc = fib->fmt0.isc; + zdev->kzdev->fib.fmt0.aibv = fib->fmt0.aibv; + mutex_unlock(&aift->aift_lock); + + /* Issue the clp to setup the irq now */ + rc = kvm_zpci_set_airq(zdev); + return rc; + +unlock: + mutex_unlock(&aift->aift_lock); +unpin2: + if (fib->fmt0.sum == 1) + unpin_user_page(aisb_page); +unpin1: + unpin_user_page(aibv_page); +out: + return rc; +} + +static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) +{ + struct kvm_zdev *kzdev = zdev->kzdev; + struct zpci_gaite *gaite; + struct page *vpage = NULL, *spage = NULL; + int rc, pcount = 0; + u8 isc; + + if (zdev->gisa == 0) + return -EINVAL; + + mutex_lock(&aift->aift_lock); + + /* + * If the clear fails due to an error, leave now unless we know this + * device is about to go away (force) -- In that case clear the GAITE + * regardless. + */ + rc = kvm_zpci_clear_airq(zdev); + if (rc && !force) + goto out; + + if (zdev->kzdev->fib.fmt0.aibv == 0) + goto out; + spin_lock_irq(&aift->gait_lock); + gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb * + sizeof(struct zpci_gaite)); + isc = gaite->gisc; + gaite->count--; + if (gaite->count == 0) { + /* Release guest AIBV and AISB */ + vpage = phys_to_page(kzdev->fib.fmt0.aibv); + if (gaite->aisb != 0) + spage = phys_to_page(gaite->aisb); + /* Clear the GAIT entry */ + gaite->aisb = 0; + gaite->gisc = 0; + gaite->aisbo = 0; + gaite->gisa = 0; + aift->kzdev[zdev->aisb] = 0; + /* Clear zdev info */ + airq_iv_free_bit(aift->sbv, zdev->aisb); + airq_iv_release(zdev->aibv); + zdev->aisb = 0; + zdev->aibv = NULL; + } + spin_unlock_irq(&aift->gait_lock); + kvm_s390_gisc_unregister(kzdev->kvm, isc); + kzdev->fib.fmt0.isc = 0; + kzdev->fib.fmt0.aibv = 0; + + if (vpage) { + unpin_user_page(vpage); + pcount++; + } + if (spage) { + unpin_user_page(spage); + pcount++; + } + if (pcount > 0) + unaccount_mem(pcount); +out: + mutex_unlock(&aift->aift_lock); + + return rc; +} + static int kvm_s390_pci_dev_open(struct zpci_dev *zdev) { struct kvm_zdev *kzdev; diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/pci.h index 9f7828d97605..9d091033fc02 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/pci.h @@ -20,6 +20,7 @@ struct kvm_zdev { struct zpci_dev *zdev; struct kvm *kvm; + struct zpci_fib fib; }; struct zpci_gaite { diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index 5798178aec9d..56480be48244 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -92,6 +92,7 @@ u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status) return cc; } +EXPORT_SYMBOL_GPL(zpci_mod_fc); /* Refresh PCI Translations */ static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h index 00ed419dd464..f054d0360a75 100644 --- a/include/linux/sched/user.h +++ b/include/linux/sched/user.h @@ -24,7 +24,8 @@ struct user_struct { kuid_t uid; #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \ - defined(CONFIG_NET) || defined(CONFIG_IO_URING) + defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \ + defined(CONFIG_VFIO_PCI_ZDEV_KVM) atomic_long_t locked_vm; #endif #ifdef CONFIG_WATCH_QUEUE -- cgit v1.2.3 From 8061d1c31f1a018281bc9877ecce44bfc779e21d Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Mon, 6 Jun 2022 16:33:21 -0400 Subject: vfio-pci/zdev: add open/close device hooks During vfio-pci open_device, pass the KVM associated with the vfio group (if one exists). This is needed in order to pass a special indicator (GISA) to firmware to allow zPCI interpretation facilities to be used for only the specific KVM associated with the vfio-pci device. During vfio-pci close_device, unregister the notifier. Signed-off-by: Matthew Rosato Acked-by: Alex Williamson Reviewed-by: Pierre Morel Link: https://lore.kernel.org/r/20220606203325.110625-18-mjrosato@linux.ibm.com Signed-off-by: Christian Borntraeger --- drivers/vfio/pci/vfio_pci_core.c | 10 +++++++++- drivers/vfio/pci/vfio_pci_zdev.c | 24 ++++++++++++++++++++++++ include/linux/vfio_pci_core.h | 10 ++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index a0d69ddaf90d..b1e5cfbadf38 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -316,10 +316,14 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) pci_write_config_word(pdev, PCI_COMMAND, cmd); } - ret = vfio_config_init(vdev); + ret = vfio_pci_zdev_open_device(vdev); if (ret) goto out_free_state; + ret = vfio_config_init(vdev); + if (ret) + goto out_free_zdev; + msix_pos = pdev->msix_cap; if (msix_pos) { u16 flags; @@ -340,6 +344,8 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) return 0; +out_free_zdev: + vfio_pci_zdev_close_device(vdev); out_free_state: kfree(vdev->pci_saved_state); vdev->pci_saved_state = NULL; @@ -418,6 +424,8 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) vdev->needs_reset = true; + vfio_pci_zdev_close_device(vdev); + /* * If we have saved state, restore it. If we can reset the device, * even better. Resetting with current state seems better than diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c index ea4c0d2b0663..686f2e75e392 100644 --- a/drivers/vfio/pci/vfio_pci_zdev.c +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -136,3 +137,26 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, return ret; } + +int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev) +{ + struct zpci_dev *zdev = to_zpci(vdev->pdev); + + if (!zdev) + return -ENODEV; + + if (!vdev->vdev.kvm) + return 0; + + return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm); +} + +void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) +{ + struct zpci_dev *zdev = to_zpci(vdev->pdev); + + if (!zdev || !vdev->vdev.kvm) + return; + + kvm_s390_pci_unregister_kvm(zdev); +} diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 63af2897939c..d5d9e17f0156 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -209,12 +209,22 @@ static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) #ifdef CONFIG_VFIO_PCI_ZDEV_KVM extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, struct vfio_info_cap *caps); +int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev); +void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev); #else static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, struct vfio_info_cap *caps) { return -ENODEV; } + +static inline int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev) +{ + return 0; +} + +static inline void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) +{} #endif /* Will be exported for vfio pci drivers usage */ -- cgit v1.2.3 From 6976890e8998afd8abbbd9fe27ed71387b24f57f Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Wed, 22 Jun 2022 11:00:45 +0200 Subject: netfilter: nf_conntrack: add missing __rcu annotations Access to the hook pointers use correct helpers but the pointers lack the needed __rcu annotation. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter/nf_conntrack_sip.h | 2 +- include/net/netfilter/nf_conntrack_timeout.h | 2 +- net/netfilter/nf_conntrack_pptp.c | 2 +- net/netfilter/nf_conntrack_sip.c | 2 +- net/netfilter/nf_conntrack_timeout.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter/nf_conntrack_sip.h b/include/linux/netfilter/nf_conntrack_sip.h index c620521c42bc..dbc614dfe0d5 100644 --- a/include/linux/netfilter/nf_conntrack_sip.h +++ b/include/linux/netfilter/nf_conntrack_sip.h @@ -164,7 +164,7 @@ struct nf_nat_sip_hooks { unsigned int medialen, union nf_inet_addr *rtp_addr); }; -extern const struct nf_nat_sip_hooks *nf_nat_sip_hooks; +extern const struct nf_nat_sip_hooks __rcu *nf_nat_sip_hooks; int ct_sip_parse_request(const struct nf_conn *ct, const char *dptr, unsigned int datalen, unsigned int *matchoff, diff --git a/include/net/netfilter/nf_conntrack_timeout.h b/include/net/netfilter/nf_conntrack_timeout.h index fea258983d23..9fdaba911de6 100644 --- a/include/net/netfilter/nf_conntrack_timeout.h +++ b/include/net/netfilter/nf_conntrack_timeout.h @@ -105,7 +105,7 @@ struct nf_ct_timeout_hooks { void (*timeout_put)(struct nf_ct_timeout *timeout); }; -extern const struct nf_ct_timeout_hooks *nf_ct_timeout_hook; +extern const struct nf_ct_timeout_hooks __rcu *nf_ct_timeout_hook; #endif #endif /* _NF_CONNTRACK_TIMEOUT_H */ diff --git a/net/netfilter/nf_conntrack_pptp.c b/net/netfilter/nf_conntrack_pptp.c index f3fa367b455f..4c679638df06 100644 --- a/net/netfilter/nf_conntrack_pptp.c +++ b/net/netfilter/nf_conntrack_pptp.c @@ -45,7 +45,7 @@ MODULE_ALIAS_NFCT_HELPER("pptp"); static DEFINE_SPINLOCK(nf_pptp_lock); -const struct nf_nat_pptp_hook *nf_nat_pptp_hook; +const struct nf_nat_pptp_hook __rcu *nf_nat_pptp_hook; EXPORT_SYMBOL_GPL(nf_nat_pptp_hook); #if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG) diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index b83dc9bf0a5d..a88b43624b27 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -60,7 +60,7 @@ module_param(sip_external_media, int, 0600); MODULE_PARM_DESC(sip_external_media, "Expect Media streams between external " "endpoints (default 0)"); -const struct nf_nat_sip_hooks *nf_nat_sip_hooks; +const struct nf_nat_sip_hooks __rcu *nf_nat_sip_hooks; EXPORT_SYMBOL_GPL(nf_nat_sip_hooks); static int string_len(const struct nf_conn *ct, const char *dptr, diff --git a/net/netfilter/nf_conntrack_timeout.c b/net/netfilter/nf_conntrack_timeout.c index 0f828d05ea60..821365ed5b2c 100644 --- a/net/netfilter/nf_conntrack_timeout.c +++ b/net/netfilter/nf_conntrack_timeout.c @@ -22,7 +22,7 @@ #include #include -const struct nf_ct_timeout_hooks *nf_ct_timeout_hook __read_mostly; +const struct nf_ct_timeout_hooks __rcu *nf_ct_timeout_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_ct_timeout_hook); static int untimeout(struct nf_conn *ct, void *timeout) -- cgit v1.2.3 From d3f2d0a292c24fc624afb2b4f47f838e83775721 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Wed, 22 Jun 2022 11:00:47 +0200 Subject: netfilter: h323: merge nat hook pointers into one sparse complains about incorrect rcu usage. Code uses the correct rcu access primitives, but the function pointers lack rcu annotations. Collapse all of them into a single structure, then annotate the pointer. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter/nf_conntrack_h323.h | 109 ++++++------ net/ipv4/netfilter/nf_nat_h323.c | 42 ++--- net/netfilter/nf_conntrack_h323_main.c | 260 +++++++++++----------------- 3 files changed, 169 insertions(+), 242 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter/nf_conntrack_h323.h b/include/linux/netfilter/nf_conntrack_h323.h index 4561ec0fcea4..9e937f64a1ad 100644 --- a/include/linux/netfilter/nf_conntrack_h323.h +++ b/include/linux/netfilter/nf_conntrack_h323.h @@ -38,60 +38,63 @@ void nf_conntrack_h245_expect(struct nf_conn *new, struct nf_conntrack_expect *this); void nf_conntrack_q931_expect(struct nf_conn *new, struct nf_conntrack_expect *this); -extern int (*set_h245_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, - union nf_inet_addr *addr, - __be16 port); -extern int (*set_h225_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, - union nf_inet_addr *addr, - __be16 port); -extern int (*set_sig_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count); -extern int (*set_ras_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count); -extern int (*nat_rtp_rtcp_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - int dataoff, - H245_TransportAddress *taddr, - __be16 port, __be16 rtp_port, - struct nf_conntrack_expect *rtp_exp, - struct nf_conntrack_expect *rtcp_exp); -extern int (*nat_t120_hook) (struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, + +struct nfct_h323_nat_hooks { + int (*set_h245_addr)(struct sk_buff *skb, unsigned int protoff, unsigned char **data, int dataoff, - H245_TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp); -extern int (*nat_h245_hook) (struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, + H245_TransportAddress *taddr, + union nf_inet_addr *addr, __be16 port); + int (*set_h225_addr)(struct sk_buff *skb, unsigned int protoff, unsigned char **data, int dataoff, - TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp); -extern int (*nat_callforwarding_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, - __be16 port, - struct nf_conntrack_expect *exp); -extern int (*nat_q931_hook) (struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, TransportAddress *taddr, - int idx, __be16 port, - struct nf_conntrack_expect *exp); + TransportAddress *taddr, + union nf_inet_addr *addr, __be16 port); + int (*set_sig_addr)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, unsigned char **data, + TransportAddress *taddr, int count); + int (*set_ras_addr)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, unsigned char **data, + TransportAddress *taddr, int count); + int (*nat_rtp_rtcp)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + H245_TransportAddress *taddr, + __be16 port, __be16 rtp_port, + struct nf_conntrack_expect *rtp_exp, + struct nf_conntrack_expect *rtcp_exp); + int (*nat_t120)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + H245_TransportAddress *taddr, __be16 port, + struct nf_conntrack_expect *exp); + int (*nat_h245)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + TransportAddress *taddr, __be16 port, + struct nf_conntrack_expect *exp); + int (*nat_callforwarding)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + TransportAddress *taddr, __be16 port, + struct nf_conntrack_expect *exp); + int (*nat_q931)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, TransportAddress *taddr, int idx, + __be16 port, struct nf_conntrack_expect *exp); +}; +extern const struct nfct_h323_nat_hooks __rcu *nfct_h323_nat_hook; #endif diff --git a/net/ipv4/netfilter/nf_nat_h323.c b/net/ipv4/netfilter/nf_nat_h323.c index 76a411ae9fe6..a334f0dcc2d0 100644 --- a/net/ipv4/netfilter/nf_nat_h323.c +++ b/net/ipv4/netfilter/nf_nat_h323.c @@ -579,28 +579,22 @@ static struct nf_ct_helper_expectfn callforwarding_nat = { .expectfn = ip_nat_callforwarding_expect, }; +static const struct nfct_h323_nat_hooks nathooks = { + .set_h245_addr = set_h245_addr, + .set_h225_addr = set_h225_addr, + .set_sig_addr = set_sig_addr, + .set_ras_addr = set_ras_addr, + .nat_rtp_rtcp = nat_rtp_rtcp, + .nat_t120 = nat_t120, + .nat_h245 = nat_h245, + .nat_callforwarding = nat_callforwarding, + .nat_q931 = nat_q931, +}; + /****************************************************************************/ static int __init nf_nat_h323_init(void) { - BUG_ON(set_h245_addr_hook != NULL); - BUG_ON(set_h225_addr_hook != NULL); - BUG_ON(set_sig_addr_hook != NULL); - BUG_ON(set_ras_addr_hook != NULL); - BUG_ON(nat_rtp_rtcp_hook != NULL); - BUG_ON(nat_t120_hook != NULL); - BUG_ON(nat_h245_hook != NULL); - BUG_ON(nat_callforwarding_hook != NULL); - BUG_ON(nat_q931_hook != NULL); - - RCU_INIT_POINTER(set_h245_addr_hook, set_h245_addr); - RCU_INIT_POINTER(set_h225_addr_hook, set_h225_addr); - RCU_INIT_POINTER(set_sig_addr_hook, set_sig_addr); - RCU_INIT_POINTER(set_ras_addr_hook, set_ras_addr); - RCU_INIT_POINTER(nat_rtp_rtcp_hook, nat_rtp_rtcp); - RCU_INIT_POINTER(nat_t120_hook, nat_t120); - RCU_INIT_POINTER(nat_h245_hook, nat_h245); - RCU_INIT_POINTER(nat_callforwarding_hook, nat_callforwarding); - RCU_INIT_POINTER(nat_q931_hook, nat_q931); + RCU_INIT_POINTER(nfct_h323_nat_hook, &nathooks); nf_ct_helper_expectfn_register(&q931_nat); nf_ct_helper_expectfn_register(&callforwarding_nat); return 0; @@ -609,15 +603,7 @@ static int __init nf_nat_h323_init(void) /****************************************************************************/ static void __exit nf_nat_h323_fini(void) { - RCU_INIT_POINTER(set_h245_addr_hook, NULL); - RCU_INIT_POINTER(set_h225_addr_hook, NULL); - RCU_INIT_POINTER(set_sig_addr_hook, NULL); - RCU_INIT_POINTER(set_ras_addr_hook, NULL); - RCU_INIT_POINTER(nat_rtp_rtcp_hook, NULL); - RCU_INIT_POINTER(nat_t120_hook, NULL); - RCU_INIT_POINTER(nat_h245_hook, NULL); - RCU_INIT_POINTER(nat_callforwarding_hook, NULL); - RCU_INIT_POINTER(nat_q931_hook, NULL); + RCU_INIT_POINTER(nfct_h323_nat_hook, NULL); nf_ct_helper_expectfn_unregister(&q931_nat); nf_ct_helper_expectfn_unregister(&callforwarding_nat); synchronize_rcu(); diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index 2eb31ffb3d14..bb76305bb7ff 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -49,64 +49,8 @@ MODULE_PARM_DESC(callforward_filter, "only create call forwarding expectations " "if both endpoints are on different sides " "(determined by routing information)"); -/* Hooks for NAT */ -int (*set_h245_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, - union nf_inet_addr *addr, __be16 port) - __read_mostly; -int (*set_h225_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, - union nf_inet_addr *addr, __be16 port) - __read_mostly; -int (*set_sig_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count) __read_mostly; -int (*set_ras_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count) __read_mostly; -int (*nat_rtp_rtcp_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, - __be16 port, __be16 rtp_port, - struct nf_conntrack_expect *rtp_exp, - struct nf_conntrack_expect *rtcp_exp) __read_mostly; -int (*nat_t120_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp) __read_mostly; -int (*nat_h245_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp) __read_mostly; -int (*nat_callforwarding_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp) __read_mostly; -int (*nat_q931_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, TransportAddress *taddr, int idx, - __be16 port, struct nf_conntrack_expect *exp) - __read_mostly; +const struct nfct_h323_nat_hooks __rcu *nfct_h323_nat_hook __read_mostly; +EXPORT_SYMBOL_GPL(nfct_h323_nat_hook); static DEFINE_SPINLOCK(nf_h323_lock); static char *h323_buffer; @@ -259,6 +203,7 @@ static int expect_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, int dataoff, H245_TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; @@ -266,7 +211,6 @@ static int expect_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, union nf_inet_addr addr; struct nf_conntrack_expect *rtp_exp; struct nf_conntrack_expect *rtcp_exp; - typeof(nat_rtp_rtcp_hook) nat_rtp_rtcp; /* Read RTP or RTCP address */ if (!get_h245_addr(ct, *data, taddr, &addr, &port) || @@ -296,15 +240,16 @@ static int expect_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_UDP, NULL, &rtcp_port); + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_rtp_rtcp = rcu_dereference(nat_rtp_rtcp_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* NAT needed */ - ret = nat_rtp_rtcp(skb, ct, ctinfo, protoff, data, dataoff, - taddr, port, rtp_port, rtp_exp, rtcp_exp); + ret = nathook->nat_rtp_rtcp(skb, ct, ctinfo, protoff, data, dataoff, + taddr, port, rtp_port, rtp_exp, rtcp_exp); } else { /* Conntrack only */ if (nf_ct_expect_related(rtp_exp, 0) == 0) { if (nf_ct_expect_related(rtcp_exp, 0) == 0) { @@ -333,12 +278,12 @@ static int expect_t120(struct sk_buff *skb, unsigned char **data, int dataoff, H245_TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(nat_t120_hook) nat_t120; /* Read T.120 address */ if (!get_h245_addr(ct, *data, taddr, &addr, &port) || @@ -355,15 +300,16 @@ static int expect_t120(struct sk_buff *skb, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple channels */ + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_t120 = rcu_dereference(nat_t120_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* NAT needed */ - ret = nat_t120(skb, ct, ctinfo, protoff, data, dataoff, taddr, - port, exp); + ret = nathook->nat_t120(skb, ct, ctinfo, protoff, data, + dataoff, taddr, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_h323: expect T.120 "); @@ -664,18 +610,19 @@ int get_h225_addr(struct nf_conn *ct, unsigned char *data, return 1; } +EXPORT_SYMBOL_GPL(get_h225_addr); static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo, unsigned int protoff, unsigned char **data, int dataoff, TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(nat_h245_hook) nat_h245; /* Read h245Address */ if (!get_h225_addr(ct, *data, taddr, &addr, &port) || @@ -692,15 +639,16 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, IPPROTO_TCP, NULL, &port); exp->helper = &nf_conntrack_helper_h245; + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_h245 = rcu_dereference(nat_h245_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* NAT needed */ - ret = nat_h245(skb, ct, ctinfo, protoff, data, dataoff, taddr, - port, exp); + ret = nathook->nat_h245(skb, ct, ctinfo, protoff, data, + dataoff, taddr, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_q931: expect H.245 "); @@ -785,13 +733,13 @@ static int expect_callforwarding(struct sk_buff *skb, unsigned char **data, int dataoff, TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; struct net *net = nf_ct_net(ct); - typeof(nat_callforwarding_hook) nat_callforwarding; /* Read alternativeAddress */ if (!get_h225_addr(ct, *data, taddr, &addr, &port) || port == 0) @@ -815,16 +763,17 @@ static int expect_callforwarding(struct sk_buff *skb, IPPROTO_TCP, NULL, &port); exp->helper = nf_conntrack_helper_q931; + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_callforwarding = rcu_dereference(nat_callforwarding_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* Need NAT */ - ret = nat_callforwarding(skb, ct, ctinfo, - protoff, data, dataoff, - taddr, port, exp); + ret = nathook->nat_callforwarding(skb, ct, ctinfo, + protoff, data, dataoff, + taddr, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_q931: expect Call Forwarding "); @@ -844,12 +793,12 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, int dataoff, Setup_UUIE *setup) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret; int i; __be16 port; union nf_inet_addr addr; - typeof(set_h225_addr_hook) set_h225_addr; pr_debug("nf_ct_q931: Setup\n"); @@ -860,9 +809,9 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, return -1; } - set_h225_addr = rcu_dereference(set_h225_addr_hook); + nathook = rcu_dereference(nfct_h323_nat_hook); if ((setup->options & eSetup_UUIE_destCallSignalAddress) && - (set_h225_addr) && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK && get_h225_addr(ct, *data, &setup->destCallSignalAddress, &addr, &port) && @@ -870,16 +819,16 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, pr_debug("nf_ct_q931: set destCallSignalAddress %pI6:%hu->%pI6:%hu\n", &addr, ntohs(port), &ct->tuplehash[!dir].tuple.src.u3, ntohs(ct->tuplehash[!dir].tuple.src.u.tcp.port)); - ret = set_h225_addr(skb, protoff, data, dataoff, - &setup->destCallSignalAddress, - &ct->tuplehash[!dir].tuple.src.u3, - ct->tuplehash[!dir].tuple.src.u.tcp.port); + ret = nathook->set_h225_addr(skb, protoff, data, dataoff, + &setup->destCallSignalAddress, + &ct->tuplehash[!dir].tuple.src.u3, + ct->tuplehash[!dir].tuple.src.u.tcp.port); if (ret < 0) return -1; } if ((setup->options & eSetup_UUIE_sourceCallSignalAddress) && - (set_h225_addr) && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK && get_h225_addr(ct, *data, &setup->sourceCallSignalAddress, &addr, &port) && @@ -887,10 +836,10 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, pr_debug("nf_ct_q931: set sourceCallSignalAddress %pI6:%hu->%pI6:%hu\n", &addr, ntohs(port), &ct->tuplehash[!dir].tuple.dst.u3, ntohs(ct->tuplehash[!dir].tuple.dst.u.tcp.port)); - ret = set_h225_addr(skb, protoff, data, dataoff, - &setup->sourceCallSignalAddress, - &ct->tuplehash[!dir].tuple.dst.u3, - ct->tuplehash[!dir].tuple.dst.u.tcp.port); + ret = nathook->set_h225_addr(skb, protoff, data, dataoff, + &setup->sourceCallSignalAddress, + &ct->tuplehash[!dir].tuple.dst.u3, + ct->tuplehash[!dir].tuple.dst.u.tcp.port); if (ret < 0) return -1; } @@ -1249,13 +1198,13 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, TransportAddress *taddr, int count) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; int i; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(nat_q931_hook) nat_q931; /* Look for the first related address */ for (i = 0; i < count; i++) { @@ -1279,11 +1228,11 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, exp->helper = nf_conntrack_helper_q931; exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */ - nat_q931 = rcu_dereference(nat_q931_hook); - if (nat_q931 && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* Need NAT */ - ret = nat_q931(skb, ct, ctinfo, protoff, data, - taddr, i, port, exp); + ret = nathook->nat_q931(skb, ct, ctinfo, protoff, data, + taddr, i, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); @@ -1305,15 +1254,15 @@ static int process_grq(struct sk_buff *skb, struct nf_conn *ct, unsigned int protoff, unsigned char **data, GatekeeperRequest *grq) { - typeof(set_ras_addr_hook) set_ras_addr; + const struct nfct_h323_nat_hooks *nathook; pr_debug("nf_ct_ras: GRQ\n"); - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) /* NATed */ - return set_ras_addr(skb, ct, ctinfo, protoff, data, - &grq->rasAddress, 1); + return nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + &grq->rasAddress, 1); return 0; } @@ -1367,8 +1316,8 @@ static int process_rrq(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, RegistrationRequest *rrq) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int ret; - typeof(set_ras_addr_hook) set_ras_addr; pr_debug("nf_ct_ras: RRQ\n"); @@ -1378,12 +1327,12 @@ static int process_rrq(struct sk_buff *skb, struct nf_conn *ct, if (ret < 0) return -1; - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_ras_addr(skb, ct, ctinfo, protoff, data, - rrq->rasAddress.item, - rrq->rasAddress.count); + ret = nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + rrq->rasAddress.item, + rrq->rasAddress.count); if (ret < 0) return -1; } @@ -1403,19 +1352,19 @@ static int process_rcf(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, RegistrationConfirm *rcf) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret; struct nf_conntrack_expect *exp; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: RCF\n"); - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_sig_addr(skb, ct, ctinfo, protoff, data, - rcf->callSignalAddress.item, - rcf->callSignalAddress.count); + ret = nathook->set_sig_addr(skb, ct, ctinfo, protoff, data, + rcf->callSignalAddress.item, + rcf->callSignalAddress.count); if (ret < 0) return -1; } @@ -1454,18 +1403,18 @@ static int process_urq(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, UnregistrationRequest *urq) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: URQ\n"); - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_sig_addr(skb, ct, ctinfo, protoff, data, - urq->callSignalAddress.item, - urq->callSignalAddress.count); + ret = nathook->set_sig_addr(skb, ct, ctinfo, protoff, data, + urq->callSignalAddress.item, + urq->callSignalAddress.count); if (ret < 0) return -1; } @@ -1487,39 +1436,42 @@ static int process_arq(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, AdmissionRequest *arq) { const struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); __be16 port; union nf_inet_addr addr; - typeof(set_h225_addr_hook) set_h225_addr; pr_debug("nf_ct_ras: ARQ\n"); - set_h225_addr = rcu_dereference(set_h225_addr_hook); + nathook = rcu_dereference(nfct_h323_nat_hook); + if (!nathook) + return 0; + if ((arq->options & eAdmissionRequest_destCallSignalAddress) && get_h225_addr(ct, *data, &arq->destCallSignalAddress, &addr, &port) && !memcmp(&addr, &ct->tuplehash[dir].tuple.src.u3, sizeof(addr)) && port == info->sig_port[dir] && nf_ct_l3num(ct) == NFPROTO_IPV4 && - set_h225_addr && ct->status & IPS_NAT_MASK) { + ct->status & IPS_NAT_MASK) { /* Answering ARQ */ - return set_h225_addr(skb, protoff, data, 0, - &arq->destCallSignalAddress, - &ct->tuplehash[!dir].tuple.dst.u3, - info->sig_port[!dir]); + return nathook->set_h225_addr(skb, protoff, data, 0, + &arq->destCallSignalAddress, + &ct->tuplehash[!dir].tuple.dst.u3, + info->sig_port[!dir]); } if ((arq->options & eAdmissionRequest_srcCallSignalAddress) && get_h225_addr(ct, *data, &arq->srcCallSignalAddress, &addr, &port) && !memcmp(&addr, &ct->tuplehash[dir].tuple.src.u3, sizeof(addr)) && - set_h225_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* Calling ARQ */ - return set_h225_addr(skb, protoff, data, 0, - &arq->srcCallSignalAddress, - &ct->tuplehash[!dir].tuple.dst.u3, - port); + return nathook->set_h225_addr(skb, protoff, data, 0, + &arq->srcCallSignalAddress, + &ct->tuplehash[!dir].tuple.dst.u3, + port); } return 0; @@ -1535,7 +1487,6 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: ACF\n"); @@ -1544,12 +1495,15 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, return 0; if (!memcmp(&addr, &ct->tuplehash[dir].tuple.dst.u3, sizeof(addr))) { + const struct nfct_h323_nat_hooks *nathook; + /* Answering ACF */ - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) - return set_sig_addr(skb, ct, ctinfo, protoff, data, - &acf->destCallSignalAddress, 1); + return nathook->set_sig_addr(skb, ct, ctinfo, protoff, + data, + &acf->destCallSignalAddress, 1); return 0; } @@ -1578,15 +1532,15 @@ static int process_lrq(struct sk_buff *skb, struct nf_conn *ct, unsigned int protoff, unsigned char **data, LocationRequest *lrq) { - typeof(set_ras_addr_hook) set_ras_addr; + const struct nfct_h323_nat_hooks *nathook; pr_debug("nf_ct_ras: LRQ\n"); - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) - return set_ras_addr(skb, ct, ctinfo, protoff, data, - &lrq->replyAddress, 1); + return nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + &lrq->replyAddress, 1); return 0; } @@ -1634,27 +1588,22 @@ static int process_irr(struct sk_buff *skb, struct nf_conn *ct, unsigned int protoff, unsigned char **data, InfoRequestResponse *irr) { + const struct nfct_h323_nat_hooks *nathook; int ret; - typeof(set_ras_addr_hook) set_ras_addr; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: IRR\n"); - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_ras_addr(skb, ct, ctinfo, protoff, data, - &irr->rasAddress, 1); + ret = nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + &irr->rasAddress, 1); if (ret < 0) return -1; - } - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && - ct->status & IPS_NAT_MASK) { - ret = set_sig_addr(skb, ct, ctinfo, protoff, data, - irr->callSignalAddress.item, - irr->callSignalAddress.count); + ret = nathook->set_sig_addr(skb, ct, ctinfo, protoff, data, + irr->callSignalAddress.item, + irr->callSignalAddress.count); if (ret < 0) return -1; } @@ -1837,17 +1786,6 @@ err1: module_init(nf_conntrack_h323_init); module_exit(nf_conntrack_h323_fini); -EXPORT_SYMBOL_GPL(get_h225_addr); -EXPORT_SYMBOL_GPL(set_h245_addr_hook); -EXPORT_SYMBOL_GPL(set_h225_addr_hook); -EXPORT_SYMBOL_GPL(set_sig_addr_hook); -EXPORT_SYMBOL_GPL(set_ras_addr_hook); -EXPORT_SYMBOL_GPL(nat_rtp_rtcp_hook); -EXPORT_SYMBOL_GPL(nat_t120_hook); -EXPORT_SYMBOL_GPL(nat_h245_hook); -EXPORT_SYMBOL_GPL(nat_callforwarding_hook); -EXPORT_SYMBOL_GPL(nat_q931_hook); - MODULE_AUTHOR("Jing Min Zhao "); MODULE_DESCRIPTION("H.323 connection tracking helper"); MODULE_LICENSE("GPL"); -- cgit v1.2.3 From 3d6e44623841c8b82c2157f2f749019803fb238a Mon Sep 17 00:00:00 2001 From: Jeremy Kerr Date: Sat, 9 Jul 2022 11:19:57 +0800 Subject: kunit: unify module and builtin suite definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, KUnit runs built-in tests and tests loaded from modules differently. For built-in tests, the kunit_test_suite{,s}() macro adds a list of suites in the .kunit_test_suites linker section. However, for kernel modules, a module_init() function is used to run the test suites. This causes problems if tests are included in a module which already defines module_init/exit_module functions, as they'll conflict with the kunit-provided ones. This change removes the kunit-defined module inits, and instead parses the kunit tests from their own section in the module. After module init, we call __kunit_test_suites_init() on the contents of that section, which prepares and runs the suite. This essentially unifies the module- and non-module kunit init formats. Tested-by: Maíra Canal Reviewed-by: Brendan Higgins Signed-off-by: Jeremy Kerr Signed-off-by: Daniel Latypov Signed-off-by: David Gow Signed-off-by: Shuah Khan --- include/kunit/test.h | 49 ++++++----------------------------------------- include/linux/module.h | 5 +++++ kernel/module/main.c | 6 ++++++ lib/kunit/test.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/include/kunit/test.h b/include/kunit/test.h index 7646d1bcf685..cb155d3da284 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -250,42 +250,9 @@ static inline int kunit_run_all_tests(void) } #endif /* IS_BUILTIN(CONFIG_KUNIT) */ -#ifdef MODULE -/** - * kunit_test_suites_for_module() - used to register one or more - * &struct kunit_suite with KUnit. - * - * @__suites: a statically allocated list of &struct kunit_suite. - * - * Registers @__suites with the test framework. See &struct kunit_suite for - * more information. - * - * If a test suite is built-in, module_init() gets translated into - * an initcall which we don't want as the idea is that for builtins - * the executor will manage execution. So ensure we do not define - * module_{init|exit} functions for the builtin case when registering - * suites via kunit_test_suites() below. - */ -#define kunit_test_suites_for_module(__suites) \ - static int __init kunit_test_suites_init(void) \ - { \ - return __kunit_test_suites_init(__suites); \ - } \ - module_init(kunit_test_suites_init); \ - \ - static void __exit kunit_test_suites_exit(void) \ - { \ - return __kunit_test_suites_exit(__suites); \ - } \ - module_exit(kunit_test_suites_exit) \ - MODULE_INFO(test, "Y"); -#else -#define kunit_test_suites_for_module(__suites) -#endif /* MODULE */ - #define __kunit_test_suites(unique_array, unique_suites, ...) \ + MODULE_INFO(test, "Y"); \ static struct kunit_suite *unique_array[] = { __VA_ARGS__, NULL }; \ - kunit_test_suites_for_module(unique_array); \ static struct kunit_suite **unique_suites \ __used __section(".kunit_test_suites") = unique_array @@ -295,16 +262,12 @@ static inline int kunit_run_all_tests(void) * * @__suites: a statically allocated list of &struct kunit_suite. * - * Registers @suites with the test framework. See &struct kunit_suite for - * more information. - * - * When builtin, KUnit tests are all run via executor; this is done - * by placing the array of struct kunit_suite * in the .kunit_test_suites - * ELF section. + * Registers @suites with the test framework. + * This is done by placing the array of struct kunit_suite * in the + * .kunit_test_suites ELF section. * - * An alternative is to build the tests as a module. Because modules do not - * support multiple initcall()s, we need to initialize an array of suites for a - * module. + * When builtin, KUnit tests are all run via the executor at boot, and when + * built as a module, they run on module load. * */ #define kunit_test_suites(__suites...) \ diff --git a/include/linux/module.h b/include/linux/module.h index abd9fa916b7d..2490223c975d 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -505,6 +505,11 @@ struct module { int num_static_call_sites; struct static_call_site *static_call_sites; #endif +#if IS_ENABLED(CONFIG_KUNIT) + int num_kunit_suites; + struct kunit_suite ***kunit_suites; +#endif + #ifdef CONFIG_LIVEPATCH bool klp; /* Is this a livepatch module? */ diff --git a/kernel/module/main.c b/kernel/module/main.c index 4723f1316709..324a770f789c 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2094,6 +2094,12 @@ static int find_module_sections(struct module *mod, struct load_info *info) sizeof(*mod->static_call_sites), &mod->num_static_call_sites); #endif +#ifdef CONFIG_KUNIT + mod->kunit_suites = section_objs(info, ".kunit_test_suites", + sizeof(*mod->kunit_suites), + &mod->num_kunit_suites); +#endif + mod->extable = section_objs(info, "__ex_table", sizeof(*mod->extable), &mod->num_exentries); diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 8b11552dc215..246645eb3cef 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -613,6 +614,49 @@ void __kunit_test_suites_exit(struct kunit_suite **suites) } EXPORT_SYMBOL_GPL(__kunit_test_suites_exit); +#ifdef CONFIG_MODULES +static void kunit_module_init(struct module *mod) +{ + unsigned int i; + + for (i = 0; i < mod->num_kunit_suites; i++) + __kunit_test_suites_init(mod->kunit_suites[i]); +} + +static void kunit_module_exit(struct module *mod) +{ + unsigned int i; + + for (i = 0; i < mod->num_kunit_suites; i++) + __kunit_test_suites_exit(mod->kunit_suites[i]); +} + +static int kunit_module_notify(struct notifier_block *nb, unsigned long val, + void *data) +{ + struct module *mod = data; + + switch (val) { + case MODULE_STATE_LIVE: + kunit_module_init(mod); + break; + case MODULE_STATE_GOING: + kunit_module_exit(mod); + break; + case MODULE_STATE_COMING: + case MODULE_STATE_UNFORMED: + break; + } + + return 0; +} + +static struct notifier_block kunit_mod_nb = { + .notifier_call = kunit_module_notify, + .priority = 0, +}; +#endif + struct kunit_kmalloc_array_params { size_t n; size_t size; @@ -707,13 +751,19 @@ EXPORT_SYMBOL_GPL(kunit_cleanup); static int __init kunit_init(void) { kunit_debugfs_init(); - +#ifdef CONFIG_MODULES + return register_module_notifier(&kunit_mod_nb); +#else return 0; +#endif } late_initcall(kunit_init); static void __exit kunit_exit(void) { +#ifdef CONFIG_MODULES + unregister_module_notifier(&kunit_mod_nb); +#endif kunit_debugfs_cleanup(); } module_exit(kunit_exit); -- cgit v1.2.3 From e5857d396f35e59e6fe96cf1178b0357cc3a1ea4 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Sat, 9 Jul 2022 11:19:58 +0800 Subject: kunit: flatten kunit_suite*** to kunit_suite** in .kunit_test_suites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We currently store kunit suites in the .kunit_test_suites ELF section as a `struct kunit_suite***` (modulo some `const`s). For every test file, we store a struct kunit_suite** NULL-terminated array. This adds quite a bit of complexity to the test filtering code in the executor. Instead, let's just make the .kunit_test_suites section contain a single giant array of struct kunit_suite pointers, which can then be directly manipulated. This array is not NULL-terminated, and so none of the test filtering code needs to NULL-terminate anything. Tested-by: Maíra Canal Reviewed-by: Brendan Higgins Signed-off-by: Daniel Latypov Co-developed-by: David Gow Signed-off-by: David Gow Signed-off-by: Shuah Khan --- include/kunit/test.h | 13 ++--- include/linux/module.h | 2 +- lib/kunit/executor.c | 115 +++++++++--------------------------- lib/kunit/executor_test.c | 144 +++++++++++++--------------------------------- lib/kunit/test.c | 18 ++---- 5 files changed, 82 insertions(+), 210 deletions(-) (limited to 'include/linux') diff --git a/include/kunit/test.h b/include/kunit/test.h index cb155d3da284..c958855681cc 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -237,9 +237,9 @@ size_t kunit_suite_num_test_cases(struct kunit_suite *suite); unsigned int kunit_test_case_num(struct kunit_suite *suite, struct kunit_case *test_case); -int __kunit_test_suites_init(struct kunit_suite * const * const suites); +int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_suites); -void __kunit_test_suites_exit(struct kunit_suite **suites); +void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites); #if IS_BUILTIN(CONFIG_KUNIT) int kunit_run_all_tests(void); @@ -250,11 +250,11 @@ static inline int kunit_run_all_tests(void) } #endif /* IS_BUILTIN(CONFIG_KUNIT) */ -#define __kunit_test_suites(unique_array, unique_suites, ...) \ +#define __kunit_test_suites(unique_array, ...) \ MODULE_INFO(test, "Y"); \ - static struct kunit_suite *unique_array[] = { __VA_ARGS__, NULL }; \ - static struct kunit_suite **unique_suites \ - __used __section(".kunit_test_suites") = unique_array + static struct kunit_suite *unique_array[] \ + __aligned(sizeof(struct kunit_suite *)) \ + __used __section(".kunit_test_suites") = { __VA_ARGS__ } /** * kunit_test_suites() - used to register one or more &struct kunit_suite @@ -272,7 +272,6 @@ static inline int kunit_run_all_tests(void) */ #define kunit_test_suites(__suites...) \ __kunit_test_suites(__UNIQUE_ID(array), \ - __UNIQUE_ID(suites), \ ##__suites) #define kunit_test_suite(suite) kunit_test_suites(&suite) diff --git a/include/linux/module.h b/include/linux/module.h index 2490223c975d..518296ea7f73 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -507,7 +507,7 @@ struct module { #endif #if IS_ENABLED(CONFIG_KUNIT) int num_kunit_suites; - struct kunit_suite ***kunit_suites; + struct kunit_suite **kunit_suites; #endif diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 572f64e0a41a..6c489d6c5e5d 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -9,8 +9,8 @@ * These symbols point to the .kunit_test_suites section and are defined in * include/asm-generic/vmlinux.lds.h, and consequently must be extern. */ -extern struct kunit_suite * const * const __kunit_suites_start[]; -extern struct kunit_suite * const * const __kunit_suites_end[]; +extern struct kunit_suite * const __kunit_suites_start[]; +extern struct kunit_suite * const __kunit_suites_end[]; #if IS_BUILTIN(CONFIG_KUNIT) @@ -90,62 +90,18 @@ kunit_filter_tests(const struct kunit_suite *const suite, const char *test_glob) static char *kunit_shutdown; core_param(kunit_shutdown, kunit_shutdown, charp, 0644); -static struct kunit_suite * const * -kunit_filter_subsuite(struct kunit_suite * const * const subsuite, - struct kunit_test_filter *filter) -{ - int i, n = 0; - struct kunit_suite **filtered, *filtered_suite; - - n = 0; - for (i = 0; subsuite[i]; ++i) { - if (glob_match(filter->suite_glob, subsuite[i]->name)) - ++n; - } - - if (n == 0) - return NULL; - - filtered = kmalloc_array(n + 1, sizeof(*filtered), GFP_KERNEL); - if (!filtered) - return ERR_PTR(-ENOMEM); - - n = 0; - for (i = 0; subsuite[i] != NULL; ++i) { - if (!glob_match(filter->suite_glob, subsuite[i]->name)) - continue; - filtered_suite = kunit_filter_tests(subsuite[i], filter->test_glob); - if (IS_ERR(filtered_suite)) - return ERR_CAST(filtered_suite); - else if (filtered_suite) - filtered[n++] = filtered_suite; - } - filtered[n] = NULL; - - return filtered; -} - +/* Stores an array of suites, end points one past the end */ struct suite_set { - struct kunit_suite * const * const *start; - struct kunit_suite * const * const *end; + struct kunit_suite * const *start; + struct kunit_suite * const *end; }; -static void kunit_free_subsuite(struct kunit_suite * const *subsuite) -{ - unsigned int i; - - for (i = 0; subsuite[i]; i++) - kfree(subsuite[i]); - - kfree(subsuite); -} - static void kunit_free_suite_set(struct suite_set suite_set) { - struct kunit_suite * const * const *suites; + struct kunit_suite * const *suites; for (suites = suite_set.start; suites < suite_set.end; suites++) - kunit_free_subsuite(*suites); + kfree(*suites); kfree(suite_set.start); } @@ -154,7 +110,7 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, int *err) { int i; - struct kunit_suite * const **copy, * const *filtered_subsuite; + struct kunit_suite **copy, *filtered_suite; struct suite_set filtered; struct kunit_test_filter filter; @@ -169,14 +125,19 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, kunit_parse_filter_glob(&filter, filter_glob); - for (i = 0; i < max; ++i) { - filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], &filter); - if (IS_ERR(filtered_subsuite)) { - *err = PTR_ERR(filtered_subsuite); + for (i = 0; &suite_set->start[i] != suite_set->end; i++) { + if (!glob_match(filter.suite_glob, suite_set->start[i]->name)) + continue; + + filtered_suite = kunit_filter_tests(suite_set->start[i], filter.test_glob); + if (IS_ERR(filtered_suite)) { + *err = PTR_ERR(filtered_suite); return filtered; } - if (filtered_subsuite) - *copy++ = filtered_subsuite; + if (!filtered_suite) + continue; + + *copy++ = filtered_suite; } filtered.end = copy; @@ -199,52 +160,33 @@ static void kunit_handle_shutdown(void) } -static void kunit_print_tap_header(struct suite_set *suite_set) -{ - struct kunit_suite * const * const *suites, * const *subsuite; - int num_of_suites = 0; - - for (suites = suite_set->start; suites < suite_set->end; suites++) - for (subsuite = *suites; *subsuite != NULL; subsuite++) - num_of_suites++; - - pr_info("TAP version 14\n"); - pr_info("1..%d\n", num_of_suites); -} - static void kunit_exec_run_tests(struct suite_set *suite_set) { - struct kunit_suite * const * const *suites; + size_t num_suites = suite_set->end - suite_set->start; - kunit_print_tap_header(suite_set); + pr_info("TAP version 14\n"); + pr_info("1..%zu\n", num_suites); - for (suites = suite_set->start; suites < suite_set->end; suites++) - __kunit_test_suites_init(*suites); + __kunit_test_suites_init(suite_set->start, num_suites); } static void kunit_exec_list_tests(struct suite_set *suite_set) { - unsigned int i; - struct kunit_suite * const * const *suites; + struct kunit_suite * const *suites; struct kunit_case *test_case; /* Hack: print a tap header so kunit.py can find the start of KUnit output. */ pr_info("TAP version 14\n"); for (suites = suite_set->start; suites < suite_set->end; suites++) - for (i = 0; (*suites)[i] != NULL; i++) { - kunit_suite_for_each_test_case((*suites)[i], test_case) { - pr_info("%s.%s\n", (*suites)[i]->name, test_case->name); - } + kunit_suite_for_each_test_case((*suites), test_case) { + pr_info("%s.%s\n", (*suites)->name, test_case->name); } } int kunit_run_all_tests(void) { - struct suite_set suite_set = { - .start = __kunit_suites_start, - .end = __kunit_suites_end, - }; + struct suite_set suite_set = {__kunit_suites_start, __kunit_suites_end}; int err = 0; if (filter_glob_param) { @@ -262,11 +204,10 @@ int kunit_run_all_tests(void) else pr_err("kunit executor: unknown action '%s'\n", action_param); - if (filter_glob_param) { /* a copy was made of each array */ + if (filter_glob_param) { /* a copy was made of each suite */ kunit_free_suite_set(suite_set); } - out: kunit_handle_shutdown(); return err; diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c index eac6ff480273..0cea31c27b23 100644 --- a/lib/kunit/executor_test.c +++ b/lib/kunit/executor_test.c @@ -9,8 +9,6 @@ #include static void kfree_at_end(struct kunit *test, const void *to_free); -static void free_subsuite_at_end(struct kunit *test, - struct kunit_suite *const *to_free); static struct kunit_suite *alloc_fake_suite(struct kunit *test, const char *suite_name, struct kunit_case *test_cases); @@ -41,126 +39,80 @@ static void parse_filter_test(struct kunit *test) kfree(filter.test_glob); } -static void filter_subsuite_test(struct kunit *test) +static void filter_suites_test(struct kunit *test) { - struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; - struct kunit_suite * const *filtered; - struct kunit_test_filter filter = { - .suite_glob = "suite2", - .test_glob = NULL, - }; + struct kunit_suite *subsuite[3] = {NULL, NULL}; + struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]}; + struct suite_set got; + int err = 0; subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); /* Want: suite1, suite2, NULL -> suite2, NULL */ - filtered = kunit_filter_subsuite(subsuite, &filter); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered); - free_subsuite_at_end(test, filtered); + got = kunit_filter_suites(&suite_set, "suite2", &err); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start); + KUNIT_ASSERT_EQ(test, err, 0); + kfree_at_end(test, got.start); /* Validate we just have suite2 */ - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]); - KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2"); - KUNIT_EXPECT_FALSE(test, filtered[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]); + KUNIT_EXPECT_STREQ(test, (const char *)got.start[0]->name, "suite2"); + + /* Contains one element (end is 1 past end) */ + KUNIT_ASSERT_EQ(test, got.end - got.start, 1); } -static void filter_subsuite_test_glob_test(struct kunit *test) +static void filter_suites_test_glob_test(struct kunit *test) { - struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; - struct kunit_suite * const *filtered; - struct kunit_test_filter filter = { - .suite_glob = "suite2", - .test_glob = "test2", - }; + struct kunit_suite *subsuite[3] = {NULL, NULL}; + struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]}; + struct suite_set got; + int err = 0; subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); /* Want: suite1, suite2, NULL -> suite2 (just test1), NULL */ - filtered = kunit_filter_subsuite(subsuite, &filter); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered); - free_subsuite_at_end(test, filtered); + got = kunit_filter_suites(&suite_set, "suite2.test2", &err); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start); + KUNIT_ASSERT_EQ(test, err, 0); + kfree_at_end(test, got.start); /* Validate we just have suite2 */ - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]); - KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2"); - KUNIT_EXPECT_FALSE(test, filtered[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]); + KUNIT_EXPECT_STREQ(test, (const char *)got.start[0]->name, "suite2"); + KUNIT_ASSERT_EQ(test, got.end - got.start, 1); /* Now validate we just have test2 */ - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]->test_cases); - KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->test_cases[0].name, "test2"); - KUNIT_EXPECT_FALSE(test, filtered[0]->test_cases[1].name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]->test_cases); + KUNIT_EXPECT_STREQ(test, (const char *)got.start[0]->test_cases[0].name, "test2"); + KUNIT_EXPECT_FALSE(test, got.start[0]->test_cases[1].name); } -static void filter_subsuite_to_empty_test(struct kunit *test) +static void filter_suites_to_empty_test(struct kunit *test) { - struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; - struct kunit_suite * const *filtered; - struct kunit_test_filter filter = { - .suite_glob = "not_found", - .test_glob = NULL, - }; + struct kunit_suite *subsuite[3] = {NULL, NULL}; + struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]}; + struct suite_set got; + int err = 0; subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); - filtered = kunit_filter_subsuite(subsuite, &filter); - free_subsuite_at_end(test, filtered); /* just in case */ + got = kunit_filter_suites(&suite_set, "not_found", &err); + KUNIT_ASSERT_EQ(test, err, 0); + kfree_at_end(test, got.start); /* just in case */ - KUNIT_EXPECT_FALSE_MSG(test, filtered, - "should be NULL to indicate no match"); -} - -static void kfree_subsuites_at_end(struct kunit *test, struct suite_set *suite_set) -{ - struct kunit_suite * const * const *suites; - - kfree_at_end(test, suite_set->start); - for (suites = suite_set->start; suites < suite_set->end; suites++) - free_subsuite_at_end(test, *suites); -} - -static void filter_suites_test(struct kunit *test) -{ - /* Suites per-file are stored as a NULL terminated array */ - struct kunit_suite *subsuites[2][2] = { - {NULL, NULL}, - {NULL, NULL}, - }; - /* Match the memory layout of suite_set */ - struct kunit_suite * const * const suites[2] = { - subsuites[0], subsuites[1], - }; - - const struct suite_set suite_set = { - .start = suites, - .end = suites + 2, - }; - struct suite_set filtered = {.start = NULL, .end = NULL}; - int err = 0; - - /* Emulate two files, each having one suite */ - subsuites[0][0] = alloc_fake_suite(test, "suite0", dummy_test_cases); - subsuites[1][0] = alloc_fake_suite(test, "suite1", dummy_test_cases); - - /* Filter out suite1 */ - filtered = kunit_filter_suites(&suite_set, "suite0", &err); - kfree_subsuites_at_end(test, &filtered); /* let us use ASSERTs without leaking */ - KUNIT_EXPECT_EQ(test, err, 0); - KUNIT_ASSERT_EQ(test, filtered.end - filtered.start, (ptrdiff_t)1); - - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0]); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0][0]); - KUNIT_EXPECT_STREQ(test, (const char *)filtered.start[0][0]->name, "suite0"); + KUNIT_EXPECT_PTR_EQ_MSG(test, got.start, got.end, + "should be empty to indicate no match"); } static struct kunit_case executor_test_cases[] = { KUNIT_CASE(parse_filter_test), - KUNIT_CASE(filter_subsuite_test), - KUNIT_CASE(filter_subsuite_test_glob_test), - KUNIT_CASE(filter_subsuite_to_empty_test), KUNIT_CASE(filter_suites_test), + KUNIT_CASE(filter_suites_test_glob_test), + KUNIT_CASE(filter_suites_to_empty_test), {} }; @@ -190,20 +142,6 @@ static void kfree_at_end(struct kunit *test, const void *to_free) (void *)to_free); } -static void free_subsuite_res_free(struct kunit_resource *res) -{ - kunit_free_subsuite(res->data); -} - -static void free_subsuite_at_end(struct kunit *test, - struct kunit_suite *const *to_free) -{ - if (IS_ERR_OR_NULL(to_free)) - return; - kunit_alloc_resource(test, NULL, free_subsuite_res_free, - GFP_KERNEL, (void *)to_free); -} - static struct kunit_suite *alloc_fake_suite(struct kunit *test, const char *suite_name, struct kunit_case *test_cases) diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 246645eb3cef..b73d5bb5c473 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -586,11 +586,11 @@ static void kunit_init_suite(struct kunit_suite *suite) suite->suite_init_err = 0; } -int __kunit_test_suites_init(struct kunit_suite * const * const suites) +int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_suites) { unsigned int i; - for (i = 0; suites[i] != NULL; i++) { + for (i = 0; i < num_suites; i++) { kunit_init_suite(suites[i]); kunit_run_tests(suites[i]); } @@ -603,11 +603,11 @@ static void kunit_exit_suite(struct kunit_suite *suite) kunit_debugfs_destroy_suite(suite); } -void __kunit_test_suites_exit(struct kunit_suite **suites) +void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites) { unsigned int i; - for (i = 0; suites[i] != NULL; i++) + for (i = 0; i < num_suites; i++) kunit_exit_suite(suites[i]); kunit_suite_counter = 1; @@ -617,18 +617,12 @@ EXPORT_SYMBOL_GPL(__kunit_test_suites_exit); #ifdef CONFIG_MODULES static void kunit_module_init(struct module *mod) { - unsigned int i; - - for (i = 0; i < mod->num_kunit_suites; i++) - __kunit_test_suites_init(mod->kunit_suites[i]); + __kunit_test_suites_init(mod->kunit_suites, mod->num_kunit_suites); } static void kunit_module_exit(struct module *mod) { - unsigned int i; - - for (i = 0; i < mod->num_kunit_suites; i++) - __kunit_test_suites_exit(mod->kunit_suites[i]); + __kunit_test_suites_exit(mod->kunit_suites, mod->num_kunit_suites); } static int kunit_module_notify(struct notifier_block *nb, unsigned long val, -- cgit v1.2.3 From f16214c102f0f64b2f3546e989498525bd7b7708 Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Mon, 11 Jul 2022 10:12:00 +0200 Subject: bpf: Fix 'dubious one-bit signed bitfield' warnings Our CI[1] reported these warnings when using Sparse: $ touch net/mptcp/bpf.c $ make C=1 net/mptcp/bpf.o net/mptcp/bpf.c: note: in included file: include/linux/bpf_verifier.h:348:26: error: dubious one-bit signed bitfield include/linux/bpf_verifier.h:349:29: error: dubious one-bit signed bitfield Set them as 'unsigned' to avoid warnings. [1] https://github.com/multipath-tcp/mptcp_net-next/actions/runs/2643588487 Fixes: 1ade23711971 ("bpf: Inline calls to bpf_loop when callback is known") Signed-off-by: Matthieu Baerts Signed-off-by: Andrii Nakryiko Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20220711081200.2081262-1-matthieu.baerts@tessares.net --- include/linux/bpf_verifier.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 81b19669efba..2e3bad8640dc 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -345,10 +345,10 @@ struct bpf_verifier_state_list { }; struct bpf_loop_inline_state { - int initialized:1; /* set to true upon first entry */ - int fit_for_inline:1; /* true if callback function is the same - * at each call and flags are always zero - */ + unsigned int initialized:1; /* set to true upon first entry */ + unsigned int fit_for_inline:1; /* true if callback function is the same + * at each call and flags are always zero + */ u32 callback_subprogno; /* valid when fit_for_inline is true */ }; -- cgit v1.2.3 From 7b19119f4c7dc9412b556ea12fae6b32574e2810 Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Mon, 11 Jul 2022 01:14:06 -0700 Subject: net/mlx5: Use devl_ API in mlx5e_devlink_port_register As part of the flows invoked by mlx5_devlink_eswitch_mode_set() get to mlx5_rescan_drivers_locked() which can call mlx5e_probe()/mlx5e_remove and register/unregister mlx5e driver ports accordingly. This can lead to deadlock once mlx5_devlink_eswitch_mode_set() will use devlink lock. Use devl_port_register/unregister() instead of devlink_port_register/unregister() and add devlink instance locks in the driver paths to this function to have it locked while calling devl_ API function. If remove or probe were called by module init or module cleanup flows, need to lock devlink just before calling devl_port_register(), otherwise it is called by attach/detach or register/unregister flow and we can have the flow locked. Added flag to distinguish between these cases. This will be used by the downstream patch to invoke mlx5_devlink_eswitch_mode_set() with devlink locked. Signed-off-by: Moshe Shemesh Signed-off-by: Saeed Mahameed Signed-off-by: Paolo Abeni --- drivers/net/ethernet/mellanox/mlx5/core/dev.c | 29 ++++++++++++++++++++-- .../net/ethernet/mellanox/mlx5/core/en/devlink.c | 16 ++++++++++-- .../ethernet/mellanox/mlx5/core/eswitch_offloads.c | 2 ++ include/linux/mlx5/driver.h | 4 +++ 4 files changed, 47 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index 50422b56a64d..ccf2068d2e79 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -335,13 +335,16 @@ static void del_adev(struct auxiliary_device *adev) int mlx5_attach_device(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); struct mlx5_priv *priv = &dev->priv; struct auxiliary_device *adev; struct auxiliary_driver *adrv; int ret = 0, i; + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); priv->flags &= ~MLX5_PRIV_FLAGS_DETACH; + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; for (i = 0; i < ARRAY_SIZE(mlx5_adev_devices); i++) { if (!priv->adev[i]) { bool is_supported = false; @@ -389,19 +392,24 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) break; } } + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); return ret; } void mlx5_detach_device(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); struct mlx5_priv *priv = &dev->priv; struct auxiliary_device *adev; struct auxiliary_driver *adrv; pm_message_t pm = {}; int i; + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; for (i = ARRAY_SIZE(mlx5_adev_devices) - 1; i >= 0; i--) { if (!priv->adev[i]) continue; @@ -430,18 +438,24 @@ skip_suspend: del_adev(&priv->adev[i]->adev); priv->adev[i] = NULL; } + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; priv->flags |= MLX5_PRIV_FLAGS_DETACH; mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); } int mlx5_register_device(struct mlx5_core_dev *dev) { + struct devlink *devlink; int ret; + devlink = priv_to_devlink(dev); + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; ret = mlx5_rescan_drivers_locked(dev); mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); if (ret) mlx5_unregister_device(dev); @@ -450,10 +464,15 @@ int mlx5_register_device(struct mlx5_core_dev *dev) void mlx5_unregister_device(struct mlx5_core_dev *dev) { + struct devlink *devlink; + + devlink = priv_to_devlink(dev); + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); dev->priv.flags = MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; mlx5_rescan_drivers_locked(dev); mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); } static int add_drivers(struct mlx5_core_dev *dev) @@ -526,16 +545,22 @@ del_adev: int mlx5_rescan_drivers_locked(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; + int err = 0; lockdep_assert_held(&mlx5_intf_mutex); if (priv->flags & MLX5_PRIV_FLAGS_DETACH) return 0; + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; delete_drivers(dev); if (priv->flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) - return 0; + goto out; + + err = add_drivers(dev); - return add_drivers(dev); +out: + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; + return err; } bool mlx5_same_hw_devs(struct mlx5_core_dev *dev, struct mlx5_core_dev *peer_dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c index ae52e7f38306..b69f9d10ccbd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c @@ -21,6 +21,7 @@ int mlx5e_devlink_port_register(struct mlx5e_priv *priv) struct netdev_phys_item_id ppid = {}; struct devlink_port *dl_port; unsigned int dl_port_index; + int ret; if (mlx5_core_is_pf(priv->mdev)) { attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; @@ -41,7 +42,13 @@ int mlx5e_devlink_port_register(struct mlx5e_priv *priv) memset(dl_port, 0, sizeof(*dl_port)); devlink_port_attrs_set(dl_port, &attrs); - return devlink_port_register(devlink, dl_port, dl_port_index); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_lock(devlink); + ret = devl_port_register(devlink, dl_port, dl_port_index); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_unlock(devlink); + + return ret; } void mlx5e_devlink_port_type_eth_set(struct mlx5e_priv *priv) @@ -54,8 +61,13 @@ void mlx5e_devlink_port_type_eth_set(struct mlx5e_priv *priv) void mlx5e_devlink_port_unregister(struct mlx5e_priv *priv) { struct devlink_port *dl_port = mlx5e_devlink_get_dl_port(priv); + struct devlink *devlink = priv_to_devlink(priv->mdev); - devlink_port_unregister(dl_port); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_lock(devlink); + devl_port_unregister(dl_port); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_unlock(devlink); } struct devlink_port *mlx5e_get_devlink_port(struct net_device *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 1bfbc88f513f..ccda3a0a2594 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -3400,7 +3400,9 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, err = esw_offloads_start(esw, extack); } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { err = esw_offloads_stop(esw, extack); + devl_lock(devlink); mlx5_rescan_drivers(esw->dev); + devl_unlock(devlink); } else { err = -EINVAL; } diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 76d7661e3e63..bd882884b23c 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -551,6 +551,10 @@ enum { * creation/deletion on drivers rescan. Unset during device attach. */ MLX5_PRIV_FLAGS_DETACH = 1 << 2, + /* Distinguish between mlx5e_probe/remove called by module init/cleanup + * and called by other flows which can already hold devlink lock + */ + MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW = 1 << 3, }; struct mlx5_adev { -- cgit v1.2.3 From 91f059c95c6a5dbc0907a5f871e7915a5e93c1f9 Mon Sep 17 00:00:00 2001 From: Shaik Sajida Bhanu Date: Fri, 27 May 2022 23:23:52 +0530 Subject: mmc: core: Capture eMMC and SD card errors Add changes to capture eMMC and SD card errors. This is useful for debug and testing. Signed-off-by: Liangliang Lu Signed-off-by: Sayali Lokhande Signed-off-by: Bao D. Nguyen Signed-off-by: Ram Prakash Gupta Signed-off-by: Shaik Sajida Bhanu Acked-by: Adrian Hunter Link: https://lore.kernel.org/r/1653674036-21829-2-git-send-email-quic_c_sbhanu@quicinc.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/core.c | 10 +++++++++- include/linux/mmc/host.h | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index 4b70cbfc6d5d..ef53a2578824 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -943,9 +943,11 @@ int mmc_execute_tuning(struct mmc_card *card) } /* Only print error when we don't check for card removal */ - if (!host->detect_change) + if (!host->detect_change) { pr_err("%s: tuning execution failed: %d\n", mmc_hostname(host), err); + mmc_debugfs_err_stats_inc(host, MMC_ERR_TUNING); + } return err; } @@ -2244,6 +2246,12 @@ void mmc_rescan(struct work_struct *work) if (freqs[i] <= host->f_min) break; } + + /* + * Ignore the command timeout errors observed during + * the card init as those are excepted. + */ + host->err_stats[MMC_ERR_CMD_TIMEOUT] = 0; mmc_release_host(host); out: diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index c193c50ccd78..eb8bc5b9b0b7 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -93,6 +93,25 @@ struct mmc_clk_phase_map { struct mmc_host; +enum mmc_err_stat { + MMC_ERR_CMD_TIMEOUT, + MMC_ERR_CMD_CRC, + MMC_ERR_DAT_TIMEOUT, + MMC_ERR_DAT_CRC, + MMC_ERR_AUTO_CMD, + MMC_ERR_ADMA, + MMC_ERR_TUNING, + MMC_ERR_CMDQ_RED, + MMC_ERR_CMDQ_GCE, + MMC_ERR_CMDQ_ICCE, + MMC_ERR_REQ_TIMEOUT, + MMC_ERR_CMDQ_REQ_TIMEOUT, + MMC_ERR_ICE_CFG, + MMC_ERR_CTRL_TIMEOUT, + MMC_ERR_UNEXPECTED_IRQ, + MMC_ERR_MAX, +}; + struct mmc_host_ops { /* * It is optional for the host to implement pre_req and post_req in @@ -501,6 +520,7 @@ struct mmc_host { /* Host Software Queue support */ bool hsq_enabled; + u32 err_stats[MMC_ERR_MAX]; unsigned long private[] ____cacheline_aligned; }; @@ -635,6 +655,12 @@ static inline enum dma_data_direction mmc_get_dma_dir(struct mmc_data *data) return data->flags & MMC_DATA_WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE; } +static inline void mmc_debugfs_err_stats_inc(struct mmc_host *host, + enum mmc_err_stat stat) +{ + host->err_stats[stat] += 1; +} + int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error); int mmc_send_abort_tuning(struct mmc_host *host, u32 opcode); int mmc_get_ext_csd(struct mmc_card *card, u8 **new_ext_csd); -- cgit v1.2.3 From efe8f5c9b5e118070f424205078ababc46fd130a Mon Sep 17 00:00:00 2001 From: Shaik Sajida Bhanu Date: Fri, 27 May 2022 23:23:53 +0530 Subject: mmc: sdhci: Capture eMMC and SD card errors Add changes to capture eMMC and SD card errors. This is useful for debug and testing. Signed-off-by: Liangliang Lu Signed-off-by: Sayali Lokhande Signed-off-by: Bao D. Nguyen Signed-off-by: Shaik Sajida Bhanu Acked-by: Adrian Hunter Link: https://lore.kernel.org/r/1653674036-21829-3-git-send-email-quic_c_sbhanu@quicinc.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci.c | 59 ++++++++++++++++++++++++++++++++++++------------ drivers/mmc/host/sdhci.h | 3 +++ include/linux/mmc/mmc.h | 6 +++++ 3 files changed, 53 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index 22152029e14c..7689ffec5ad1 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -224,6 +224,7 @@ void sdhci_reset(struct sdhci_host *host, u8 mask) if (timedout) { pr_err("%s: Reset 0x%x never completed.\n", mmc_hostname(host->mmc), (int)mask); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); sdhci_dumpregs(host); return; } @@ -1716,6 +1717,7 @@ static bool sdhci_send_command_retry(struct sdhci_host *host, if (!timeout--) { pr_err("%s: Controller never released inhibit bit(s).\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); sdhci_dumpregs(host); cmd->error = -EIO; return false; @@ -1965,6 +1967,7 @@ void sdhci_enable_clk(struct sdhci_host *host, u16 clk) if (timedout) { pr_err("%s: Internal clock never stabilised.\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); sdhci_dumpregs(host); return; } @@ -1987,6 +1990,7 @@ void sdhci_enable_clk(struct sdhci_host *host, u16 clk) if (timedout) { pr_err("%s: PLL clock never stabilised.\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); sdhci_dumpregs(host); return; } @@ -3161,6 +3165,7 @@ static void sdhci_timeout_timer(struct timer_list *t) if (host->cmd && !sdhci_data_line_cmd(host->cmd)) { pr_err("%s: Timeout waiting for hardware cmd interrupt.\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, REQ_TIMEOUT); sdhci_dumpregs(host); host->cmd->error = -ETIMEDOUT; @@ -3183,6 +3188,7 @@ static void sdhci_timeout_data_timer(struct timer_list *t) (host->cmd && sdhci_data_line_cmd(host->cmd))) { pr_err("%s: Timeout waiting for hardware interrupt.\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, REQ_TIMEOUT); sdhci_dumpregs(host); if (host->data) { @@ -3234,17 +3240,21 @@ static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p) return; pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n", mmc_hostname(host->mmc), (unsigned)intmask); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); return; } if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC | SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) { - if (intmask & SDHCI_INT_TIMEOUT) + if (intmask & SDHCI_INT_TIMEOUT) { host->cmd->error = -ETIMEDOUT; - else + sdhci_err_stats_inc(host, CMD_TIMEOUT); + } else { host->cmd->error = -EILSEQ; - + if (!mmc_op_tuning(host->cmd->opcode)) + sdhci_err_stats_inc(host, CMD_CRC); + } /* Treat data command CRC error the same as data CRC error */ if (host->cmd->data && (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) == @@ -3266,6 +3276,8 @@ static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p) -ETIMEDOUT : -EILSEQ; + sdhci_err_stats_inc(host, AUTO_CMD); + if (sdhci_auto_cmd23(host, mrq)) { mrq->sbc->error = err; __sdhci_finish_mrq(host, mrq); @@ -3342,6 +3354,7 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) if (intmask & SDHCI_INT_DATA_TIMEOUT) { host->data_cmd = NULL; data_cmd->error = -ETIMEDOUT; + sdhci_err_stats_inc(host, CMD_TIMEOUT); __sdhci_finish_mrq(host, data_cmd->mrq); return; } @@ -3370,23 +3383,30 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n", mmc_hostname(host->mmc), (unsigned)intmask); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); return; } - if (intmask & SDHCI_INT_DATA_TIMEOUT) + if (intmask & SDHCI_INT_DATA_TIMEOUT) { host->data->error = -ETIMEDOUT; - else if (intmask & SDHCI_INT_DATA_END_BIT) + sdhci_err_stats_inc(host, DAT_TIMEOUT); + } else if (intmask & SDHCI_INT_DATA_END_BIT) { host->data->error = -EILSEQ; - else if ((intmask & SDHCI_INT_DATA_CRC) && + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) + sdhci_err_stats_inc(host, DAT_CRC); + } else if ((intmask & SDHCI_INT_DATA_CRC) && SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)) - != MMC_BUS_TEST_R) + != MMC_BUS_TEST_R) { host->data->error = -EILSEQ; - else if (intmask & SDHCI_INT_ADMA_ERROR) { + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) + sdhci_err_stats_inc(host, DAT_CRC); + } else if (intmask & SDHCI_INT_ADMA_ERROR) { pr_err("%s: ADMA error: 0x%08x\n", mmc_hostname(host->mmc), intmask); sdhci_adma_show_error(host); + sdhci_err_stats_inc(host, ADMA); host->data->error = -EIO; if (host->ops->adma_workaround) host->ops->adma_workaround(host, intmask); @@ -3584,6 +3604,7 @@ out: if (unexpected) { pr_err("%s: Unexpected interrupt 0x%08x.\n", mmc_hostname(host->mmc), unexpected); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); } @@ -3905,20 +3926,27 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error, if (!host->cqe_on) return false; - if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC)) + if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC)) { *cmd_error = -EILSEQ; - else if (intmask & SDHCI_INT_TIMEOUT) + if (!mmc_op_tuning(host->cmd->opcode)) + sdhci_err_stats_inc(host, CMD_CRC); + } else if (intmask & SDHCI_INT_TIMEOUT) { *cmd_error = -ETIMEDOUT; - else + sdhci_err_stats_inc(host, CMD_TIMEOUT); + } else *cmd_error = 0; - if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC)) + if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC)) { *data_error = -EILSEQ; - else if (intmask & SDHCI_INT_DATA_TIMEOUT) + if (!mmc_op_tuning(host->cmd->opcode)) + sdhci_err_stats_inc(host, DAT_CRC); + } else if (intmask & SDHCI_INT_DATA_TIMEOUT) { *data_error = -ETIMEDOUT; - else if (intmask & SDHCI_INT_ADMA_ERROR) + sdhci_err_stats_inc(host, DAT_TIMEOUT); + } else if (intmask & SDHCI_INT_ADMA_ERROR) { *data_error = -EIO; - else + sdhci_err_stats_inc(host, ADMA); + } else *data_error = 0; /* Clear selected interrupts. */ @@ -3934,6 +3962,7 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error, sdhci_writel(host, intmask, SDHCI_INT_STATUS); pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n", mmc_hostname(host->mmc), intmask); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); } diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h index d7929d725730..95a08f09df30 100644 --- a/drivers/mmc/host/sdhci.h +++ b/drivers/mmc/host/sdhci.h @@ -356,6 +356,9 @@ struct sdhci_adma2_64_desc { */ #define MMC_CMD_TRANSFER_TIME (10 * NSEC_PER_MSEC) /* max 10 ms */ +#define sdhci_err_stats_inc(host, err_name) \ + mmc_debugfs_err_stats_inc((host)->mmc, MMC_ERR_##err_name) + enum sdhci_cookie { COOKIE_UNMAPPED, COOKIE_PRE_MAPPED, /* mapped by sdhci_pre_req() */ diff --git a/include/linux/mmc/mmc.h b/include/linux/mmc/mmc.h index d9a65c6a8816..9c50bc40f8ff 100644 --- a/include/linux/mmc/mmc.h +++ b/include/linux/mmc/mmc.h @@ -99,6 +99,12 @@ static inline bool mmc_op_multi(u32 opcode) opcode == MMC_READ_MULTIPLE_BLOCK; } +static inline bool mmc_op_tuning(u32 opcode) +{ + return opcode == MMC_SEND_TUNING_BLOCK || + opcode == MMC_SEND_TUNING_BLOCK_HS200; +} + /* * MMC_SWITCH argument format: * -- cgit v1.2.3 From 2083da24eb56ce622332946800a67a7449d85fe5 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Fri, 10 Jun 2022 12:02:04 +0530 Subject: OPP: Allow multiple clocks for a device This patch adds support to allow multiple clocks for a device. The design is pretty much similar to how this is done for regulators, and platforms can supply their own version of the config_clks() callback if they have multiple clocks for their device. The core manages the calls via opp_table->config_clks() eventually. We have kept both "clk" and "clks" fields in the OPP table structure and the reason is provided as a comment in _opp_set_clknames(). The same isn't done for "rates" though and we use rates[0] at most of the places now. Co-developed-by: Krzysztof Kozlowski Signed-off-by: Krzysztof Kozlowski Tested-by: Jon Hunter Tested-by: Dmitry Osipenko Tested-by: Manivannan Sadhasivam Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 213 +++++++++++++++++++++++++++++++++++-------------- drivers/opp/debugfs.c | 27 ++++++- drivers/opp/of.c | 66 +++++++++++---- drivers/opp/opp.h | 16 ++-- include/linux/pm_opp.h | 7 +- 5 files changed, 246 insertions(+), 83 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index daabc810a1f9..e3322d54942a 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -181,7 +181,7 @@ unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp) return 0; } - return opp->rate; + return opp->rates[0]; } EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq); @@ -430,7 +430,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count); /* Helpers to read keys */ static unsigned long _read_freq(struct dev_pm_opp *opp, int index) { - return opp->rate; + return opp->rates[0]; } static unsigned long _read_level(struct dev_pm_opp *opp, int index) @@ -784,22 +784,19 @@ static int _set_opp_voltage(struct device *dev, struct regulator *reg, return ret; } -static inline int _generic_set_opp_clk_only(struct device *dev, - struct opp_table *opp_table, struct dev_pm_opp *opp, void *data) +static int +_opp_config_clk_single(struct device *dev, struct opp_table *opp_table, + struct dev_pm_opp *opp, void *data, bool scaling_down) { unsigned long *target = data; unsigned long freq; int ret; - /* We may reach here for devices which don't change frequency */ - if (IS_ERR(opp_table->clk)) - return 0; - /* One of target and opp must be available */ if (target) { freq = *target; } else if (opp) { - freq = opp->rate; + freq = opp->rates[0]; } else { WARN_ON(1); return -EINVAL; @@ -1025,11 +1022,11 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, } dev_dbg(dev, "%s: switching OPP: Freq %lu -> %lu Hz, Level %u -> %u, Bw %u -> %u\n", - __func__, old_opp->rate, opp->rate, old_opp->level, opp->level, - old_opp->bandwidth ? old_opp->bandwidth[0].peak : 0, + __func__, old_opp->rates[0], opp->rates[0], old_opp->level, + opp->level, old_opp->bandwidth ? old_opp->bandwidth[0].peak : 0, opp->bandwidth ? opp->bandwidth[0].peak : 0); - scaling_down = _opp_compare_key(old_opp, opp); + scaling_down = _opp_compare_key(opp_table, old_opp, opp); if (scaling_down == -1) scaling_down = 0; @@ -1059,9 +1056,11 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, } } - ret = _generic_set_opp_clk_only(dev, opp_table, opp, clk_data); - if (ret) - return ret; + if (opp_table->config_clks) { + ret = opp_table->config_clks(dev, opp_table, opp, clk_data, scaling_down); + if (ret) + return ret; + } /* Scaling down? Configure required OPPs after frequency */ if (scaling_down) { @@ -1133,8 +1132,8 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) * equivalent to a clk_set_rate() */ if (!_get_opp_count(opp_table)) { - ret = _generic_set_opp_clk_only(dev, opp_table, NULL, - &target_freq); + ret = opp_table->config_clks(dev, opp_table, NULL, + &target_freq, false); goto put_opp_table; } @@ -1255,6 +1254,8 @@ static struct opp_table *_allocate_opp_table(struct device *dev, int index) INIT_LIST_HEAD(&opp_table->dev_list); INIT_LIST_HEAD(&opp_table->lazy); + opp_table->clk = ERR_PTR(-ENODEV); + /* Mark regulator count uninitialized */ opp_table->regulator_count = -1; @@ -1301,20 +1302,38 @@ static struct opp_table *_update_opp_table_clk(struct device *dev, int ret; /* - * Return early if we don't need to get clk or we have already tried it + * Return early if we don't need to get clk or we have already done it * earlier. */ - if (!getclk || IS_ERR(opp_table) || opp_table->clk) + if (!getclk || IS_ERR(opp_table) || !IS_ERR(opp_table->clk) || + opp_table->clks) return opp_table; /* Find clk for the device */ opp_table->clk = clk_get(dev, NULL); ret = PTR_ERR_OR_ZERO(opp_table->clk); - if (!ret) + if (!ret) { + opp_table->config_clks = _opp_config_clk_single; + opp_table->clk_count = 1; return opp_table; + } if (ret == -ENOENT) { + /* + * There are few platforms which don't want the OPP core to + * manage device's clock settings. In such cases neither the + * platform provides the clks explicitly to us, nor the DT + * contains a valid clk entry. The OPP nodes in DT may still + * contain "opp-hz" property though, which we need to parse and + * allow the platform to find an OPP based on freq later on. + * + * This is a simple solution to take care of such corner cases, + * i.e. make the clk_count 1, which lets us allocate space for + * frequency in opp->rates and also parse the entries in DT. + */ + opp_table->clk_count = 1; + dev_dbg(dev, "%s: Couldn't find clock: %d\n", __func__, ret); return opp_table; } @@ -1417,7 +1436,7 @@ static void _opp_table_kref_release(struct kref *kref) _of_clear_opp_table(opp_table); - /* Release clk */ + /* Release automatically acquired single clk */ if (!IS_ERR(opp_table->clk)) clk_put(opp_table->clk); @@ -1505,7 +1524,7 @@ void dev_pm_opp_remove(struct device *dev, unsigned long freq) mutex_lock(&opp_table->lock); list_for_each_entry(iter, &opp_table->opp_list, node) { - if (iter->rate == freq) { + if (iter->rates[0] == freq) { opp = iter; break; } @@ -1612,24 +1631,28 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_remove_all_dynamic); struct dev_pm_opp *_opp_allocate(struct opp_table *opp_table) { struct dev_pm_opp *opp; - int supply_count, supply_size, icc_size; + int supply_count, supply_size, icc_size, clk_size; /* Allocate space for at least one supply */ supply_count = opp_table->regulator_count > 0 ? opp_table->regulator_count : 1; supply_size = sizeof(*opp->supplies) * supply_count; + clk_size = sizeof(*opp->rates) * opp_table->clk_count; icc_size = sizeof(*opp->bandwidth) * opp_table->path_count; /* allocate new OPP node and supplies structures */ - opp = kzalloc(sizeof(*opp) + supply_size + icc_size, GFP_KERNEL); - + opp = kzalloc(sizeof(*opp) + supply_size + clk_size + icc_size, GFP_KERNEL); if (!opp) return NULL; - /* Put the supplies at the end of the OPP structure as an empty array */ + /* Put the supplies, bw and clock at the end of the OPP structure */ opp->supplies = (struct dev_pm_opp_supply *)(opp + 1); + + opp->rates = (unsigned long *)(opp->supplies + supply_count); + if (icc_size) - opp->bandwidth = (struct dev_pm_opp_icc_bw *)(opp->supplies + supply_count); + opp->bandwidth = (struct dev_pm_opp_icc_bw *)(opp->rates + opp_table->clk_count); + INIT_LIST_HEAD(&opp->node); return opp; @@ -1660,21 +1683,43 @@ static bool _opp_supported_by_regulators(struct dev_pm_opp *opp, return true; } +static int _opp_compare_rate(struct opp_table *opp_table, + struct dev_pm_opp *opp1, struct dev_pm_opp *opp2) +{ + int i; + + for (i = 0; i < opp_table->clk_count; i++) { + if (opp1->rates[i] != opp2->rates[i]) + return opp1->rates[i] < opp2->rates[i] ? -1 : 1; + } + + /* Same rates for both OPPs */ + return 0; +} + /* * Returns * 0: opp1 == opp2 * 1: opp1 > opp2 * -1: opp1 < opp2 */ -int _opp_compare_key(struct dev_pm_opp *opp1, struct dev_pm_opp *opp2) +int _opp_compare_key(struct opp_table *opp_table, struct dev_pm_opp *opp1, + struct dev_pm_opp *opp2) { - if (opp1->rate != opp2->rate) - return opp1->rate < opp2->rate ? -1 : 1; + int ret; + + ret = _opp_compare_rate(opp_table, opp1, opp2); + if (ret) + return ret; + if (opp1->bandwidth && opp2->bandwidth && opp1->bandwidth[0].peak != opp2->bandwidth[0].peak) return opp1->bandwidth[0].peak < opp2->bandwidth[0].peak ? -1 : 1; + if (opp1->level != opp2->level) return opp1->level < opp2->level ? -1 : 1; + + /* Duplicate OPPs */ return 0; } @@ -1694,7 +1739,7 @@ static int _opp_is_duplicate(struct device *dev, struct dev_pm_opp *new_opp, * loop. */ list_for_each_entry(opp, &opp_table->opp_list, node) { - opp_cmp = _opp_compare_key(new_opp, opp); + opp_cmp = _opp_compare_key(opp_table, new_opp, opp); if (opp_cmp > 0) { *head = &opp->node; continue; @@ -1705,8 +1750,8 @@ static int _opp_is_duplicate(struct device *dev, struct dev_pm_opp *new_opp, /* Duplicate OPPs */ dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n", - __func__, opp->rate, opp->supplies[0].u_volt, - opp->available, new_opp->rate, + __func__, opp->rates[0], opp->supplies[0].u_volt, + opp->available, new_opp->rates[0], new_opp->supplies[0].u_volt, new_opp->available); /* Should we compare voltages for all regulators here ? */ @@ -1727,7 +1772,7 @@ void _required_opps_available(struct dev_pm_opp *opp, int count) opp->available = false; pr_warn("%s: OPP not supported by required OPP %pOF (%lu)\n", - __func__, opp->required_opps[i]->np, opp->rate); + __func__, opp->required_opps[i]->np, opp->rates[0]); return; } } @@ -1768,7 +1813,7 @@ int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, if (!_opp_supported_by_regulators(new_opp, opp_table)) { new_opp->available = false; dev_warn(dev, "%s: OPP not supported by regulators (%lu)\n", - __func__, new_opp->rate); + __func__, new_opp->rates[0]); } /* required-opps not fully initialized yet */ @@ -1814,7 +1859,7 @@ int _opp_add_v1(struct opp_table *opp_table, struct device *dev, return -ENOMEM; /* populate the opp table */ - new_opp->rate = freq; + new_opp->rates[0] = freq; tol = u_volt * opp_table->voltage_tolerance_v1 / 100; new_opp->supplies[0].u_volt = u_volt; new_opp->supplies[0].u_volt_min = u_volt - tol; @@ -2017,6 +2062,17 @@ static void _opp_put_regulators(struct opp_table *opp_table) opp_table->regulator_count = -1; } +static void _put_clks(struct opp_table *opp_table, int count) +{ + int i; + + for (i = count - 1; i >= 0; i--) + clk_put(opp_table->clks[i]); + + kfree(opp_table->clks); + opp_table->clks = NULL; +} + /** * _opp_set_clknames() - Set clk names for the device * @dev: Device for which clk names is being set. @@ -2031,10 +2087,12 @@ static void _opp_put_regulators(struct opp_table *opp_table) * This must be called before any OPPs are initialized for the device. */ static int _opp_set_clknames(struct opp_table *opp_table, struct device *dev, - const char * const names[]) + const char * const names[], + config_clks_t config_clks) { const char * const *temp = names; - int count = 0; + int count = 0, ret, i; + struct clk *clk; /* Count number of clks */ while (*temp++) @@ -2047,28 +2105,60 @@ static int _opp_set_clknames(struct opp_table *opp_table, struct device *dev, if (!count && !names[1]) count = 1; - /* We support only one clock name for now */ - if (count != 1) + /* Fail early for invalid configurations */ + if (!count || (config_clks && count == 1) || (!config_clks && count > 1)) return -EINVAL; /* Another CPU that shares the OPP table has set the clkname ? */ - if (opp_table->clk_configured) + if (opp_table->clks) return 0; - /* clk shouldn't be initialized at this point */ - if (WARN_ON(opp_table->clk)) - return -EBUSY; + opp_table->clks = kmalloc_array(count, sizeof(*opp_table->clks), + GFP_KERNEL); + if (!opp_table->clks) + return -ENOMEM; - /* Find clk for the device */ - opp_table->clk = clk_get(dev, names[0]); - if (IS_ERR(opp_table->clk)) { - return dev_err_probe(dev, PTR_ERR(opp_table->clk), - "%s: Couldn't find clock\n", __func__); + /* Find clks for the device */ + for (i = 0; i < count; i++) { + clk = clk_get(dev, names[i]); + if (IS_ERR(clk)) { + ret = dev_err_probe(dev, PTR_ERR(clk), + "%s: Couldn't find clock with name: %s\n", + __func__, names[i]); + goto free_clks; + } + + opp_table->clks[i] = clk; } - opp_table->clk_configured = true; + opp_table->clk_count = count; + + /* Set generic single clk set here */ + if (count == 1) { + opp_table->config_clks = _opp_config_clk_single; + + /* + * We could have just dropped the "clk" field and used "clks" + * everywhere. Instead we kept the "clk" field around for + * following reasons: + * + * - avoiding clks[0] everywhere else. + * - not running single clk helpers for multiple clk usecase by + * mistake. + * + * Since this is single-clk case, just update the clk pointer + * too. + */ + opp_table->clk = opp_table->clks[0]; + } else { + opp_table->config_clks = config_clks; + } return 0; + +free_clks: + _put_clks(opp_table, i); + return ret; } /** @@ -2077,11 +2167,13 @@ static int _opp_set_clknames(struct opp_table *opp_table, struct device *dev, */ static void _opp_put_clknames(struct opp_table *opp_table) { - if (opp_table->clk_configured) { - clk_put(opp_table->clk); - opp_table->clk = ERR_PTR(-EINVAL); - opp_table->clk_configured = false; - } + if (!opp_table->clks) + return; + + opp_table->config_clks = NULL; + opp_table->clk = ERR_PTR(-ENODEV); + + _put_clks(opp_table, opp_table->clk_count); } /** @@ -2298,11 +2390,16 @@ int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config) /* Configure clocks */ if (config->clk_names) { - ret = _opp_set_clknames(opp_table, dev, config->clk_names); + ret = _opp_set_clknames(opp_table, dev, config->clk_names, + config->config_clks); if (ret) goto err; data->flags |= OPP_CONFIG_CLK; + } else if (config->config_clks) { + /* Don't allow config callback without clocks */ + ret = -EINVAL; + goto err; } /* Configure property names */ @@ -2614,7 +2711,7 @@ static int _opp_set_availability(struct device *dev, unsigned long freq, /* Do we have the frequency? */ list_for_each_entry(tmp_opp, &opp_table->opp_list, node) { - if (tmp_opp->rate == freq) { + if (tmp_opp->rates[0] == freq) { opp = tmp_opp; break; } @@ -2685,7 +2782,7 @@ int dev_pm_opp_adjust_voltage(struct device *dev, unsigned long freq, /* Do we have the frequency? */ list_for_each_entry(tmp_opp, &opp_table->opp_list, node) { - if (tmp_opp->rate == freq) { + if (tmp_opp->rates[0] == freq) { opp = tmp_opp; break; } diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c index 1b6e5c55c3ed..96a30a032c5f 100644 --- a/drivers/opp/debugfs.c +++ b/drivers/opp/debugfs.c @@ -74,6 +74,24 @@ static void opp_debug_create_bw(struct dev_pm_opp *opp, } } +static void opp_debug_create_clks(struct dev_pm_opp *opp, + struct opp_table *opp_table, + struct dentry *pdentry) +{ + char name[12]; + int i; + + if (opp_table->clk_count == 1) { + debugfs_create_ulong("rate_hz", S_IRUGO, pdentry, &opp->rates[0]); + return; + } + + for (i = 0; i < opp_table->clk_count; i++) { + snprintf(name, sizeof(name), "rate_hz_%d", i); + debugfs_create_ulong(name, S_IRUGO, pdentry, &opp->rates[i]); + } +} + static void opp_debug_create_supplies(struct dev_pm_opp *opp, struct opp_table *opp_table, struct dentry *pdentry) @@ -117,10 +135,11 @@ void opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table) * Get directory name for OPP. * * - Normally rate is unique to each OPP, use it to get unique opp-name. - * - For some devices rate isn't available, use index instead. + * - For some devices rate isn't available or there are multiple, use + * index instead for them. */ - if (likely(opp->rate)) - id = opp->rate; + if (likely(opp_table->clk_count == 1 && opp->rates[0])) + id = opp->rates[0]; else id = _get_opp_count(opp_table); @@ -134,7 +153,6 @@ void opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table) debugfs_create_bool("turbo", S_IRUGO, d, &opp->turbo); debugfs_create_bool("suspend", S_IRUGO, d, &opp->suspend); debugfs_create_u32("performance_state", S_IRUGO, d, &opp->pstate); - debugfs_create_ulong("rate_hz", S_IRUGO, d, &opp->rate); debugfs_create_u32("level", S_IRUGO, d, &opp->level); debugfs_create_ulong("clock_latency_ns", S_IRUGO, d, &opp->clock_latency_ns); @@ -142,6 +160,7 @@ void opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table) opp->of_name = of_node_full_name(opp->np); debugfs_create_str("of_name", S_IRUGO, d, (char **)&opp->of_name); + opp_debug_create_clks(opp, opp_table, d); opp_debug_create_supplies(opp, opp_table, d); opp_debug_create_bw(opp, opp_table, d); diff --git a/drivers/opp/of.c b/drivers/opp/of.c index bb49057cb1fc..7607e112df3f 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -767,6 +767,50 @@ void dev_pm_opp_of_remove_table(struct device *dev) } EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table); +static int _read_rate(struct dev_pm_opp *new_opp, struct opp_table *opp_table, + struct device_node *np) +{ + struct property *prop; + int i, count, ret; + u64 *rates; + + prop = of_find_property(np, "opp-hz", NULL); + if (!prop) + return -ENODEV; + + count = prop->length / sizeof(u64); + if (opp_table->clk_count != count) { + pr_err("%s: Count mismatch between opp-hz and clk_count (%d %d)\n", + __func__, count, opp_table->clk_count); + return -EINVAL; + } + + rates = kmalloc_array(count, sizeof(*rates), GFP_KERNEL); + if (!rates) + return -ENOMEM; + + ret = of_property_read_u64_array(np, "opp-hz", rates, count); + if (ret) { + pr_err("%s: Error parsing opp-hz: %d\n", __func__, ret); + } else { + /* + * Rate is defined as an unsigned long in clk API, and so + * casting explicitly to its type. Must be fixed once rate is 64 + * bit guaranteed in clk API. + */ + for (i = 0; i < count; i++) { + new_opp->rates[i] = (unsigned long)rates[i]; + + /* This will happen for frequencies > 4.29 GHz */ + WARN_ON(new_opp->rates[i] != rates[i]); + } + } + + kfree(rates); + + return ret; +} + static int _read_bw(struct dev_pm_opp *new_opp, struct opp_table *opp_table, struct device_node *np, bool peak) { @@ -812,19 +856,13 @@ static int _read_opp_key(struct dev_pm_opp *new_opp, struct opp_table *opp_table, struct device_node *np) { bool found = false; - u64 rate; int ret; - ret = of_property_read_u64(np, "opp-hz", &rate); - if (!ret) { - /* - * Rate is defined as an unsigned long in clk API, and so - * casting explicitly to its type. Must be fixed once rate is 64 - * bit guaranteed in clk API. - */ - new_opp->rate = (unsigned long)rate; + ret = _read_rate(new_opp, opp_table, np); + if (!ret) found = true; - } + else if (ret != -ENODEV) + return ret; /* * Bandwidth consists of peak and average (optional) values: @@ -893,8 +931,8 @@ static struct dev_pm_opp *_opp_add_static_v2(struct opp_table *opp_table, /* Check if the OPP supports hardware's hierarchy of versions or not */ if (!_opp_is_supported(dev, opp_table, np)) { - dev_dbg(dev, "OPP not supported by hardware: %lu\n", - new_opp->rate); + dev_dbg(dev, "OPP not supported by hardware: %s\n", + of_node_full_name(np)); goto free_opp; } @@ -930,7 +968,7 @@ static struct dev_pm_opp *_opp_add_static_v2(struct opp_table *opp_table, if (of_property_read_bool(np, "opp-suspend")) { if (opp_table->suspend_opp) { /* Pick the OPP with higher rate/bw/level as suspend OPP */ - if (_opp_compare_key(new_opp, opp_table->suspend_opp) == 1) { + if (_opp_compare_key(opp_table, new_opp, opp_table->suspend_opp) == 1) { opp_table->suspend_opp->suspend = false; new_opp->suspend = true; opp_table->suspend_opp = new_opp; @@ -945,7 +983,7 @@ static struct dev_pm_opp *_opp_add_static_v2(struct opp_table *opp_table, opp_table->clock_latency_ns_max = new_opp->clock_latency_ns; pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu level:%u\n", - __func__, new_opp->turbo, new_opp->rate, + __func__, new_opp->turbo, new_opp->rates[0], new_opp->supplies[0].u_volt, new_opp->supplies[0].u_volt_min, new_opp->supplies[0].u_volt_max, new_opp->clock_latency_ns, new_opp->level); diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h index e481bdb59499..816009eaafee 100644 --- a/drivers/opp/opp.h +++ b/drivers/opp/opp.h @@ -79,7 +79,7 @@ struct opp_config_data { * @suspend: true if suspend OPP * @removed: flag indicating that OPP's reference is dropped by OPP core. * @pstate: Device's power domain's performance state. - * @rate: Frequency in hertz + * @rates: Frequencies in hertz * @level: Performance level * @supplies: Power supplies voltage/current values * @bandwidth: Interconnect bandwidth values @@ -102,7 +102,7 @@ struct dev_pm_opp { bool suspend; bool removed; unsigned int pstate; - unsigned long rate; + unsigned long *rates; unsigned int level; struct dev_pm_opp_supply *supplies; @@ -170,8 +170,10 @@ enum opp_table_access { * @supported_hw: Array of version number to support. * @supported_hw_count: Number of elements in supported_hw array. * @prop_name: A name to postfix to many DT properties, while parsing them. - * @clk_configured: Clock name is configured by the platform. - * @clk: Device's clock handle + * @config_clks: Platform specific config_clks() callback. + * @clks: Device's clock handles, for multiple clocks. + * @clk: Device's clock handle, for single clock. + * @clk_count: Number of clocks. * @config_regulators: Platform specific config_regulators() callback. * @regulators: Supply regulators * @regulator_count: Number of power supply regulators. Its value can be -1 @@ -220,8 +222,10 @@ struct opp_table { unsigned int *supported_hw; unsigned int supported_hw_count; const char *prop_name; - bool clk_configured; + config_clks_t config_clks; + struct clk **clks; struct clk *clk; + int clk_count; config_regulators_t config_regulators; struct regulator **regulators; int regulator_count; @@ -246,7 +250,7 @@ struct opp_table *_find_opp_table(struct device *dev); struct opp_device *_add_opp_dev(const struct device *dev, struct opp_table *opp_table); struct dev_pm_opp *_opp_allocate(struct opp_table *opp_table); void _opp_free(struct dev_pm_opp *opp); -int _opp_compare_key(struct dev_pm_opp *opp1, struct dev_pm_opp *opp2); +int _opp_compare_key(struct opp_table *opp_table, struct dev_pm_opp *opp1, struct dev_pm_opp *opp2); int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, struct opp_table *opp_table); int _opp_add_v1(struct opp_table *opp_table, struct device *dev, unsigned long freq, long u_volt, bool dynamic); void _dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask, int last_cpu); diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 50cbc75bef71..104151dfe46c 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -61,9 +61,13 @@ typedef int (*config_regulators_t)(struct device *dev, struct dev_pm_opp *old_opp, struct dev_pm_opp *new_opp, struct regulator **regulators, unsigned int count); +typedef int (*config_clks_t)(struct device *dev, struct opp_table *opp_table, + struct dev_pm_opp *opp, void *data, bool scaling_down); + /** * struct dev_pm_opp_config - Device OPP configuration values - * @clk_names: Clk names, NULL terminated array, max 1 clock for now. + * @clk_names: Clk names, NULL terminated array. + * @config_clks: Custom set clk helper. * @prop_name: Name to postfix to properties. * @config_regulators: Custom set regulator helper. * @supported_hw: Array of hierarchy of versions to match. @@ -78,6 +82,7 @@ typedef int (*config_regulators_t)(struct device *dev, struct dev_pm_opp_config { /* NULL terminated */ const char * const *clk_names; + config_clks_t config_clks; const char *prop_name; config_regulators_t config_regulators; const unsigned int *supported_hw; -- cgit v1.2.3 From 8174a3a613af1a911ab19da812824f7180b261f9 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Fri, 10 Jun 2022 12:37:25 +0530 Subject: OPP: Provide a simple implementation to configure multiple clocks This provides a simple implementation to configure multiple clocks for a device. Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 34 ++++++++++++++++++++++++++++++++++ include/linux/pm_opp.h | 10 ++++++++++ 2 files changed, 44 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 4675a00b9816..0c5b12e99d39 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -843,6 +843,40 @@ _opp_config_clk_single(struct device *dev, struct opp_table *opp_table, return ret; } +/* + * Simple implementation for configuring multiple clocks. Configure clocks in + * the order in which they are present in the array while scaling up. + */ +int dev_pm_opp_config_clks_simple(struct device *dev, + struct opp_table *opp_table, struct dev_pm_opp *opp, void *data, + bool scaling_down) +{ + int ret, i; + + if (scaling_down) { + for (i = opp_table->clk_count - 1; i >= 0; i--) { + ret = clk_set_rate(opp_table->clks[i], opp->rates[i]); + if (ret) { + dev_err(dev, "%s: failed to set clock rate: %d\n", __func__, + ret); + return ret; + } + } + } else { + for (i = 0; i < opp_table->clk_count; i++) { + ret = clk_set_rate(opp_table->clks[i], opp->rates[i]); + if (ret) { + dev_err(dev, "%s: failed to set clock rate: %d\n", __func__, + ret); + return ret; + } + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_config_clks_simple); + static int _opp_config_regulator_single(struct device *dev, struct dev_pm_opp *old_opp, struct dev_pm_opp *new_opp, struct regulator **regulators, unsigned int count) diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 104151dfe46c..683e6baf9618 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -159,6 +159,9 @@ int dev_pm_opp_unregister_notifier(struct device *dev, struct notifier_block *nb int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config); void dev_pm_opp_clear_config(int token); +int dev_pm_opp_config_clks_simple(struct device *dev, + struct opp_table *opp_table, struct dev_pm_opp *opp, void *data, + bool scaling_down); struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, struct opp_table *dst_table, struct dev_pm_opp *src_opp); int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); @@ -342,6 +345,13 @@ static inline int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_c static inline void dev_pm_opp_clear_config(int token) {} +static inline int dev_pm_opp_config_clks_simple(struct device *dev, + struct opp_table *opp_table, struct dev_pm_opp *opp, void *data, + bool scaling_down) +{ + return -EOPNOTSUPP; +} + static inline struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, struct opp_table *dst_table, struct dev_pm_opp *src_opp) { -- cgit v1.2.3 From 1e5fb38442ebaabb65057c2e58355ee36c2a178f Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 5 Jul 2022 09:41:56 +0530 Subject: OPP: Remove dev{m}_pm_opp_of_add_table_noclk() Remove the now unused variants and the now unnecessary "getclk" parameter from few routines. Signed-off-by: Viresh Kumar --- drivers/opp/of.c | 48 ++++++++---------------------------------------- include/linux/pm_opp.h | 12 ------------ 2 files changed, 8 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 7607e112df3f..8367823a2001 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -1120,7 +1120,7 @@ remove_static_opp: return ret; } -static int _of_add_table_indexed(struct device *dev, int index, bool getclk) +static int _of_add_table_indexed(struct device *dev, int index) { struct opp_table *opp_table; int ret, count; @@ -1136,7 +1136,7 @@ static int _of_add_table_indexed(struct device *dev, int index, bool getclk) index = 0; } - opp_table = _add_opp_table_indexed(dev, index, getclk); + opp_table = _add_opp_table_indexed(dev, index, true); if (IS_ERR(opp_table)) return PTR_ERR(opp_table); @@ -1160,11 +1160,11 @@ static void devm_pm_opp_of_table_release(void *data) dev_pm_opp_of_remove_table(data); } -static int _devm_of_add_table_indexed(struct device *dev, int index, bool getclk) +static int _devm_of_add_table_indexed(struct device *dev, int index) { int ret; - ret = _of_add_table_indexed(dev, index, getclk); + ret = _of_add_table_indexed(dev, index); if (ret) return ret; @@ -1192,7 +1192,7 @@ static int _devm_of_add_table_indexed(struct device *dev, int index, bool getclk */ int devm_pm_opp_of_add_table(struct device *dev) { - return _devm_of_add_table_indexed(dev, 0, true); + return _devm_of_add_table_indexed(dev, 0); } EXPORT_SYMBOL_GPL(devm_pm_opp_of_add_table); @@ -1215,7 +1215,7 @@ EXPORT_SYMBOL_GPL(devm_pm_opp_of_add_table); */ int dev_pm_opp_of_add_table(struct device *dev) { - return _of_add_table_indexed(dev, 0, true); + return _of_add_table_indexed(dev, 0); } EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table); @@ -1231,7 +1231,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table); */ int dev_pm_opp_of_add_table_indexed(struct device *dev, int index) { - return _of_add_table_indexed(dev, index, true); + return _of_add_table_indexed(dev, index); } EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table_indexed); @@ -1244,42 +1244,10 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table_indexed); */ int devm_pm_opp_of_add_table_indexed(struct device *dev, int index) { - return _devm_of_add_table_indexed(dev, index, true); + return _devm_of_add_table_indexed(dev, index); } EXPORT_SYMBOL_GPL(devm_pm_opp_of_add_table_indexed); -/** - * dev_pm_opp_of_add_table_noclk() - Initialize indexed opp table from device - * tree without getting clk for device. - * @dev: device pointer used to lookup OPP table. - * @index: Index number. - * - * Register the initial OPP table with the OPP library for given device only - * using the "operating-points-v2" property. Do not try to get the clk for the - * device. - * - * Return: Refer to dev_pm_opp_of_add_table() for return values. - */ -int dev_pm_opp_of_add_table_noclk(struct device *dev, int index) -{ - return _of_add_table_indexed(dev, index, false); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table_noclk); - -/** - * devm_pm_opp_of_add_table_noclk() - Initialize indexed opp table from device - * tree without getting clk for device. - * @dev: device pointer used to lookup OPP table. - * @index: Index number. - * - * This is a resource-managed variant of dev_pm_opp_of_add_table_noclk(). - */ -int devm_pm_opp_of_add_table_noclk(struct device *dev, int index) -{ - return _devm_of_add_table_indexed(dev, index, false); -} -EXPORT_SYMBOL_GPL(devm_pm_opp_of_add_table_noclk); - /* CPU device specific helpers */ /** diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 683e6baf9618..dc1fb5890792 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -402,8 +402,6 @@ static inline int dev_pm_opp_sync_regulators(struct device *dev) int dev_pm_opp_of_add_table(struct device *dev); int dev_pm_opp_of_add_table_indexed(struct device *dev, int index); int devm_pm_opp_of_add_table_indexed(struct device *dev, int index); -int dev_pm_opp_of_add_table_noclk(struct device *dev, int index); -int devm_pm_opp_of_add_table_noclk(struct device *dev, int index); void dev_pm_opp_of_remove_table(struct device *dev); int devm_pm_opp_of_add_table(struct device *dev); int dev_pm_opp_of_cpumask_add_table(const struct cpumask *cpumask); @@ -434,16 +432,6 @@ static inline int devm_pm_opp_of_add_table_indexed(struct device *dev, int index return -EOPNOTSUPP; } -static inline int dev_pm_opp_of_add_table_noclk(struct device *dev, int index) -{ - return -EOPNOTSUPP; -} - -static inline int devm_pm_opp_of_add_table_noclk(struct device *dev, int index) -{ - return -EOPNOTSUPP; -} - static inline void dev_pm_opp_of_remove_table(struct device *dev) { } -- cgit v1.2.3 From 79f772b9e8004c542cc88aaf680189c3f6e9a0f2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 14 Jun 2022 22:56:15 +0000 Subject: KVM: x86: Query vcpu->vcpu_idx directly and drop its accessor, again Read vcpu->vcpu_idx directly instead of bouncing through the one-line wrapper, kvm_vcpu_get_idx(), and drop the wrapper. The wrapper is a remnant of the original implementation and serves no purpose; remove it (again) before it gains more users. kvm_vcpu_get_idx() was removed in the not-too-distant past by commit 4eeef2424153 ("KVM: x86: Query vcpu->vcpu_idx directly and drop its accessor"), but was unintentionally re-introduced by commit a54d806688fe ("KVM: Keep memslots in tree-based structures instead of array-based ones"), likely due to a rebase goof. The wrapper then managed to gain users in KVM's Xen code. No functional change intended. Signed-off-by: Sean Christopherson Reviewed-by: Jim Mattson Link: https://lore.kernel.org/r/20220614225615.3843835-1-seanjc@google.com --- arch/x86/kvm/xen.c | 10 +++++----- include/linux/kvm_host.h | 5 ----- 2 files changed, 5 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 610beba35907..a0c05ccbf4b1 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1049,7 +1049,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, else vcpu->arch.xen.poll_evtchn = -1; - set_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.xen.poll_mask); + set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask); if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) { vcpu->arch.mp_state = KVM_MP_STATE_HALTED; @@ -1071,7 +1071,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, *r = 0; out: /* Really, this is only needed in case of timeout */ - clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.xen.poll_mask); + clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask); if (unlikely(sched_poll.nr_ports > 1)) kfree(ports); @@ -1311,7 +1311,7 @@ static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) int poll_evtchn = vcpu->arch.xen.poll_evtchn; if ((poll_evtchn == port || poll_evtchn == -1) && - test_and_clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.xen.poll_mask)) { + test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) { kvm_make_request(KVM_REQ_UNBLOCK, vcpu); kvm_vcpu_kick(vcpu); } @@ -1344,7 +1344,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) vcpu = kvm_get_vcpu_by_id(kvm, xe->vcpu_id); if (!vcpu) return -EINVAL; - WRITE_ONCE(xe->vcpu_idx, kvm_vcpu_get_idx(vcpu)); + WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx); } if (!vcpu->arch.xen.vcpu_info_cache.active) @@ -1540,7 +1540,7 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, */ vcpu = kvm_get_vcpu_by_id(kvm, ue->u.xen_evtchn.vcpu); if (vcpu) - e->xen_evtchn.vcpu_idx = kvm_vcpu_get_idx(vcpu); + e->xen_evtchn.vcpu_idx = vcpu->vcpu_idx; else e->xen_evtchn.vcpu_idx = -1; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 3b40f8d68fbb..cb8168cc9755 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -907,11 +907,6 @@ static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) return NULL; } -static inline int kvm_vcpu_get_idx(struct kvm_vcpu *vcpu) -{ - return vcpu->vcpu_idx; -} - void kvm_destroy_vcpus(struct kvm *kvm); void vcpu_load(struct kvm_vcpu *vcpu); -- cgit v1.2.3 From 4201d9ab3e42d9e2a20320b751a931e6239c0df2 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Mon, 11 Jul 2022 09:28:27 -0700 Subject: bpf: reparent bpf maps on memcg offlining The memory consumed by a bpf map is always accounted to the memory cgroup of the process which created the map. The map can outlive the memory cgroup if it's used by processes in other cgroups or is pinned on bpffs. In this case the map pins the original cgroup in the dying state. For other types of objects (slab objects, non-slab kernel allocations, percpu objects and recently LRU pages) there is a reparenting process implemented: on cgroup offlining charged objects are getting reassigned to the parent cgroup. Because all charges and statistics are fully recursive it's a fairly cheap operation. For efficiency and consistency with other types of objects, let's do the same for bpf maps. Fortunately thanks to the objcg API, the required changes are minimal. Please, note that individual allocations (slabs, percpu and large kmallocs) already have the reparenting mechanism. This commit adds it to the saved map->memcg pointer by replacing it to map->objcg. Because dying cgroups are not visible for a user and all charges are recursive, this commit doesn't bring any behavior changes for a user. v2: added a missing const qualifier Signed-off-by: Roman Gushchin Reviewed-by: Shakeel Butt Link: https://lore.kernel.org/r/20220711162827.184743-1-roman.gushchin@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 +- kernel/bpf/syscall.c | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2b21f2a3452f..85a4db3e0536 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -221,7 +221,7 @@ struct bpf_map { u32 btf_vmlinux_value_type_id; struct btf *btf; #ifdef CONFIG_MEMCG_KMEM - struct mem_cgroup *memcg; + struct obj_cgroup *objcg; #endif char name[BPF_OBJ_NAME_LEN]; struct bpf_map_off_arr *off_arr; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ab688d85b2c6..83c7136c5788 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -419,35 +419,53 @@ void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) #ifdef CONFIG_MEMCG_KMEM static void bpf_map_save_memcg(struct bpf_map *map) { - map->memcg = get_mem_cgroup_from_mm(current->mm); + /* Currently if a map is created by a process belonging to the root + * memory cgroup, get_obj_cgroup_from_current() will return NULL. + * So we have to check map->objcg for being NULL each time it's + * being used. + */ + map->objcg = get_obj_cgroup_from_current(); } static void bpf_map_release_memcg(struct bpf_map *map) { - mem_cgroup_put(map->memcg); + if (map->objcg) + obj_cgroup_put(map->objcg); +} + +static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map) +{ + if (map->objcg) + return get_mem_cgroup_from_objcg(map->objcg); + + return root_mem_cgroup; } void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, int node) { - struct mem_cgroup *old_memcg; + struct mem_cgroup *memcg, *old_memcg; void *ptr; - old_memcg = set_active_memcg(map->memcg); + memcg = bpf_map_get_memcg(map); + old_memcg = set_active_memcg(memcg); ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node); set_active_memcg(old_memcg); + mem_cgroup_put(memcg); return ptr; } void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) { - struct mem_cgroup *old_memcg; + struct mem_cgroup *memcg, *old_memcg; void *ptr; - old_memcg = set_active_memcg(map->memcg); + memcg = bpf_map_get_memcg(map); + old_memcg = set_active_memcg(memcg); ptr = kzalloc(size, flags | __GFP_ACCOUNT); set_active_memcg(old_memcg); + mem_cgroup_put(memcg); return ptr; } @@ -455,12 +473,14 @@ void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, gfp_t flags) { - struct mem_cgroup *old_memcg; + struct mem_cgroup *memcg, *old_memcg; void __percpu *ptr; - old_memcg = set_active_memcg(map->memcg); + memcg = bpf_map_get_memcg(map); + old_memcg = set_active_memcg(memcg); ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT); set_active_memcg(old_memcg); + mem_cgroup_put(memcg); return ptr; } -- cgit v1.2.3 From 1d5f82d9dd477d5c66e0214a68c3e4f308eadd6d Mon Sep 17 00:00:00 2001 From: Song Liu Date: Tue, 5 Jul 2022 17:26:12 -0700 Subject: bpf, x86: fix freeing of not-finalized bpf_prog_pack syzbot reported a few issues with bpf_prog_pack [1], [2]. This only happens with multiple subprogs. In jit_subprogs(), we first call bpf_int_jit_compile() on each sub program. And then, we call it on each sub program again. jit_data is not freed in the first call of bpf_int_jit_compile(). Similarly we don't call bpf_jit_binary_pack_finalize() in the first call of bpf_int_jit_compile(). If bpf_int_jit_compile() failed for one sub program, we will call bpf_jit_binary_pack_finalize() for this sub program. However, we don't have a chance to call it for other sub programs. Then we will hit "goto out_free" in jit_subprogs(), and call bpf_jit_free on some subprograms that haven't got bpf_jit_binary_pack_finalize() yet. At this point, bpf_jit_binary_pack_free() is called and the whole 2MB page is freed erroneously. Fix this with a custom bpf_jit_free() for x86_64, which calls bpf_jit_binary_pack_finalize() if necessary. Also, with custom bpf_jit_free(), bpf_prog_aux->use_bpf_prog_pack is not needed any more, remove it. Fixes: 1022a5498f6f ("bpf, x86_64: Use bpf_jit_binary_pack_alloc") [1] https://syzkaller.appspot.com/bug?extid=2f649ec6d2eea1495a8f [2] https://syzkaller.appspot.com/bug?extid=87f65c75f4a72db05445 Reported-by: syzbot+2f649ec6d2eea1495a8f@syzkaller.appspotmail.com Reported-by: syzbot+87f65c75f4a72db05445@syzkaller.appspotmail.com Signed-off-by: Song Liu Link: https://lore.kernel.org/r/20220706002612.4013790-1-song@kernel.org Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 25 +++++++++++++++++++++++++ include/linux/bpf.h | 1 - include/linux/filter.h | 8 ++++++++ kernel/bpf/core.c | 29 ++++++++++++----------------- 4 files changed, 45 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d2614f1bf838..54c7f46c453f 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2486,3 +2486,28 @@ bool bpf_jit_supports_subprog_tailcalls(void) { return true; } + +void bpf_jit_free(struct bpf_prog *prog) +{ + if (prog->jited) { + struct x64_jit_data *jit_data = prog->aux->jit_data; + struct bpf_binary_header *hdr; + + /* + * If we fail the final pass of JIT (from jit_subprogs), + * the program may not be finalized yet. Call finalize here + * before freeing it. + */ + if (jit_data) { + bpf_jit_binary_pack_finalize(prog, jit_data->header, + jit_data->rw_header); + kvfree(jit_data->addrs); + kfree(jit_data); + } + hdr = bpf_jit_binary_pack_hdr(prog); + bpf_jit_binary_pack_free(hdr, NULL); + WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); + } + + bpf_prog_unlock_free(prog); +} diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 85a4db3e0536..a5bf00649995 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1044,7 +1044,6 @@ struct bpf_prog_aux { bool sleepable; bool tail_call_reachable; bool xdp_has_frags; - bool use_bpf_prog_pack; /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto; /* function name for valid attach_btf_id */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 4c1a8b247545..a5f21dc3c432 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1027,6 +1027,14 @@ u64 bpf_jit_alloc_exec_limit(void); void *bpf_jit_alloc_exec(unsigned long size); void bpf_jit_free_exec(void *addr); void bpf_jit_free(struct bpf_prog *fp); +struct bpf_binary_header * +bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); + +static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) +{ + return list_empty(&fp->aux->ksym.lnode) || + fp->aux->ksym.lnode.prev == LIST_POISON2; +} struct bpf_binary_header * bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 805c2ad5c793..cfb8a50a9f12 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -650,12 +650,6 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) return fp->jited && !bpf_prog_was_classic(fp); } -static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) -{ - return list_empty(&fp->aux->ksym.lnode) || - fp->aux->ksym.lnode.prev == LIST_POISON2; -} - void bpf_prog_kallsyms_add(struct bpf_prog *fp) { if (!bpf_prog_kallsyms_candidate(fp) || @@ -1153,7 +1147,6 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, bpf_prog_pack_free(ro_header); return PTR_ERR(ptr); } - prog->aux->use_bpf_prog_pack = true; return 0; } @@ -1177,17 +1170,23 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, bpf_jit_uncharge_modmem(size); } +struct bpf_binary_header * +bpf_jit_binary_pack_hdr(const struct bpf_prog *fp) +{ + unsigned long real_start = (unsigned long)fp->bpf_func; + unsigned long addr; + + addr = real_start & BPF_PROG_CHUNK_MASK; + return (void *)addr; +} + static inline struct bpf_binary_header * bpf_jit_binary_hdr(const struct bpf_prog *fp) { unsigned long real_start = (unsigned long)fp->bpf_func; unsigned long addr; - if (fp->aux->use_bpf_prog_pack) - addr = real_start & BPF_PROG_CHUNK_MASK; - else - addr = real_start & PAGE_MASK; - + addr = real_start & PAGE_MASK; return (void *)addr; } @@ -1200,11 +1199,7 @@ void __weak bpf_jit_free(struct bpf_prog *fp) if (fp->jited) { struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); - if (fp->aux->use_bpf_prog_pack) - bpf_jit_binary_pack_free(hdr, NULL /* rw_buffer */); - else - bpf_jit_binary_free(hdr); - + bpf_jit_binary_free(hdr); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); } -- cgit v1.2.3 From 3beb0ab5bffba625007ea5c9e5e0ee5eef05c1ea Mon Sep 17 00:00:00 2001 From: Seunghui Lee Date: Wed, 13 Jul 2022 12:36:34 +0900 Subject: mmc: core: Use mmc_card_* macro and add a new for the sd_combo type Add mmc_card_sd_combo() macro for sd combo type card and use the mmc_card_* macro to simplify code instead of comparing card->type. Signed-off-by: Seunghui Lee Link: https://lore.kernel.org/r/20220713033635.28432-2-sh043.lee@samsung.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/block.c | 4 ++-- drivers/mmc/core/bus.c | 4 ++-- drivers/mmc/core/sd.c | 2 +- drivers/mmc/core/sdio.c | 16 ++++++++-------- include/linux/mmc/card.h | 1 + 5 files changed, 14 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index f4a1281658db..9c642b3b7c20 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -2988,7 +2988,7 @@ static int mmc_blk_probe(struct mmc_card *card) * Don't enable runtime PM for SD-combo cards here. Leave that * decision to be taken during the SDIO init sequence instead. */ - if (card->type != MMC_TYPE_SD_COMBO) { + if (!mmc_card_sd_combo(card)) { pm_runtime_set_active(&card->dev); pm_runtime_enable(&card->dev); } @@ -3015,7 +3015,7 @@ static void mmc_blk_remove(struct mmc_card *card) mmc_blk_part_switch(card, md->part_type); mmc_release_host(card->host); } - if (card->type != MMC_TYPE_SD_COMBO) + if (!mmc_card_sd_combo(card)) pm_runtime_disable(&card->dev); pm_runtime_put_noidle(&card->dev); mmc_blk_remove_req(md); diff --git a/drivers/mmc/core/bus.c b/drivers/mmc/core/bus.c index 58a60afa650b..d8762fa3d5cd 100644 --- a/drivers/mmc/core/bus.c +++ b/drivers/mmc/core/bus.c @@ -85,7 +85,7 @@ mmc_bus_uevent(struct device *dev, struct kobj_uevent_env *env) return retval; } - if (card->type == MMC_TYPE_SDIO || card->type == MMC_TYPE_SD_COMBO) { + if (mmc_card_sdio(card) || mmc_card_sd_combo(card)) { retval = add_uevent_var(env, "SDIO_ID=%04X:%04X", card->cis.vendor, card->cis.device); if (retval) @@ -107,7 +107,7 @@ mmc_bus_uevent(struct device *dev, struct kobj_uevent_env *env) * SDIO (non-combo) cards are not handled by mmc_block driver and do not * have accessible CID register which used by mmc_card_name() function. */ - if (card->type == MMC_TYPE_SDIO) + if (mmc_card_sdio(card)) return 0; retval = add_uevent_var(env, "MMC_NAME=%s", mmc_card_name(card)); diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index d2023837dd72..cee4c0b59f43 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -793,7 +793,7 @@ static umode_t sd_std_is_visible(struct kobject *kobj, struct attribute *attr, attr == &dev_attr_info2.attr || attr == &dev_attr_info3.attr || attr == &dev_attr_info4.attr - ) && card->type != MMC_TYPE_SD_COMBO) + ) &&!mmc_card_sd_combo(card)) return 0; return attr->mode; diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c index 25799accf8a0..b589df1c35e0 100644 --- a/drivers/mmc/core/sdio.c +++ b/drivers/mmc/core/sdio.c @@ -335,7 +335,7 @@ static int sdio_disable_4bit_bus(struct mmc_card *card) { int err; - if (card->type == MMC_TYPE_SDIO) + if (mmc_card_sdio(card)) goto out; if (!(card->host->caps & MMC_CAP_4_BIT_DATA)) @@ -360,7 +360,7 @@ static int sdio_enable_4bit_bus(struct mmc_card *card) err = sdio_enable_wide(card); if (err <= 0) return err; - if (card->type == MMC_TYPE_SDIO) + if (mmc_card_sdio(card)) goto out; if (card->scr.bus_widths & SD_SCR_BUS_WIDTH_4) { @@ -415,7 +415,7 @@ static int sdio_enable_hs(struct mmc_card *card) int ret; ret = mmc_sdio_switch_hs(card, true); - if (ret <= 0 || card->type == MMC_TYPE_SDIO) + if (ret <= 0 || mmc_card_sdio(card)) return ret; ret = mmc_sd_switch_hs(card); @@ -441,7 +441,7 @@ static unsigned mmc_sdio_get_max_clock(struct mmc_card *card) max_dtr = card->cis.max_dtr; } - if (card->type == MMC_TYPE_SD_COMBO) + if (mmc_card_sd_combo(card)) max_dtr = min(max_dtr, mmc_sd_get_max_clock(card)); return max_dtr; @@ -689,7 +689,7 @@ try_again: mmc_sd_get_cid(host, ocr & rocr, card->raw_cid, NULL) == 0) { card->type = MMC_TYPE_SD_COMBO; - if (oldcard && (oldcard->type != MMC_TYPE_SD_COMBO || + if (oldcard && (!mmc_card_sd_combo(oldcard) || memcmp(card->raw_cid, oldcard->raw_cid, sizeof(card->raw_cid)) != 0)) { err = -ENOENT; goto mismatch; @@ -697,7 +697,7 @@ try_again: } else { card->type = MMC_TYPE_SDIO; - if (oldcard && oldcard->type != MMC_TYPE_SDIO) { + if (oldcard && !mmc_card_sdio(oldcard)) { err = -ENOENT; goto mismatch; } @@ -754,7 +754,7 @@ try_again: /* * Read CSD, before selecting the card */ - if (!oldcard && card->type == MMC_TYPE_SD_COMBO) { + if (!oldcard && mmc_card_sd_combo(card)) { err = mmc_sd_get_csd(card); if (err) goto remove; @@ -827,7 +827,7 @@ try_again: mmc_fixup_device(card, sdio_fixup_methods); - if (card->type == MMC_TYPE_SD_COMBO) { + if (mmc_card_sd_combo(card)) { err = mmc_sd_setup_card(host, card, oldcard != NULL); /* handle as SDIO-only card if memory init failed */ if (err) { diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index 37f975875102..156a7b673a28 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -348,5 +348,6 @@ bool mmc_card_is_blockaddr(struct mmc_card *card); #define mmc_card_mmc(c) ((c)->type == MMC_TYPE_MMC) #define mmc_card_sd(c) ((c)->type == MMC_TYPE_SD) #define mmc_card_sdio(c) ((c)->type == MMC_TYPE_SDIO) +#define mmc_card_sd_combo(c) ((c)->type == MMC_TYPE_SD_COMBO) #endif /* LINUX_MMC_CARD_H */ -- cgit v1.2.3 From 20347fca71a387a3751f7bb270062616ddc5317a Mon Sep 17 00:00:00 2001 From: Tianyu Lan Date: Fri, 8 Jul 2022 12:15:44 -0400 Subject: swiotlb: split up the global swiotlb lock Traditionally swiotlb was not performance critical because it was only used for slow devices. But in some setups, like TDX/SEV confidential guests, all IO has to go through swiotlb. Currently swiotlb only has a single lock. Under high IO load with multiple CPUs this can lead to significat lock contention on the swiotlb lock. This patch splits the swiotlb bounce buffer pool into individual areas which have their own lock. Each CPU tries to allocate in its own area first. Only if that fails does it search other areas. On freeing the allocation is freed into the area where the memory was originally allocated from. Area number can be set via swiotlb kernel parameter and is default to be possible cpu number. If possible cpu number is not power of 2, area number will be round up to the next power of 2. This idea from Andi Kleen patch(https://github.com/intel/tdx/commit/ 4529b5784c141782c72ec9bd9a92df2b68cb7d45). Based-on-idea-by: Andi Kleen Signed-off-by: Tianyu Lan Signed-off-by: Christoph Hellwig --- Documentation/admin-guide/kernel-parameters.txt | 4 +- include/linux/swiotlb.h | 5 + kernel/dma/swiotlb.c | 229 +++++++++++++++++++----- 3 files changed, 197 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2522b11e593f..4a6ad177d4b8 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5904,8 +5904,10 @@ it if 0 is given (See Documentation/admin-guide/cgroup-v1/memory.rst) swiotlb= [ARM,IA-64,PPC,MIPS,X86] - Format: { | force | noforce } + Format: { [,] | force | noforce } -- Number of I/O TLB slabs + -- Second integer after comma. Number of swiotlb + areas with their own lock. Must be power of 2. force -- force using of bounce buffers even if they wouldn't be automatically used by the kernel noforce -- Never use bounce buffers (for debugging) diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index bdc58a0e20b1..f65ff1930120 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -88,6 +88,8 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, * @late_alloc: %true if allocated using the page allocator * @force_bounce: %true if swiotlb bouncing is forced * @for_alloc: %true if the pool is used for memory allocation + * @nareas: The area number in the pool. + * @area_nslabs: The slot number in the area. */ struct io_tlb_mem { phys_addr_t start; @@ -101,6 +103,9 @@ struct io_tlb_mem { bool late_alloc; bool force_bounce; bool for_alloc; + unsigned int nareas; + unsigned int area_nslabs; + struct io_tlb_area *areas; struct io_tlb_slot { phys_addr_t orig_addr; size_t alloc_size; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 909b43445574..dcf1459ce723 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -70,6 +70,43 @@ struct io_tlb_mem io_tlb_default_mem; phys_addr_t swiotlb_unencrypted_base; static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT; +static unsigned long default_nareas; + +/** + * struct io_tlb_area - IO TLB memory area descriptor + * + * This is a single area with a single lock. + * + * @used: The number of used IO TLB block. + * @index: The slot index to start searching in this area for next round. + * @lock: The lock to protect the above data structures in the map and + * unmap calls. + */ +struct io_tlb_area { + unsigned long used; + unsigned int index; + spinlock_t lock; +}; + +static void swiotlb_adjust_nareas(unsigned int nareas) +{ + if (!is_power_of_2(nareas)) + nareas = roundup_pow_of_two(nareas); + + default_nareas = nareas; + + pr_info("area num %d.\n", nareas); + /* + * Round up number of slabs to the next power of 2. + * The last area is going be smaller than the rest if + * default_nslabs is not power of two. + */ + if (nareas && !is_power_of_2(default_nslabs)) { + default_nslabs = roundup_pow_of_two(default_nslabs); + pr_info("SWIOTLB bounce buffer size roundup to %luMB", + (default_nslabs << IO_TLB_SHIFT) >> 20); + } +} static int __init setup_io_tlb_npages(char *str) @@ -79,6 +116,10 @@ setup_io_tlb_npages(char *str) default_nslabs = ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE); } + if (*str == ',') + ++str; + if (isdigit(*str)) + swiotlb_adjust_nareas(simple_strtoul(str, &str, 0)); if (*str == ',') ++str; if (!strcmp(str, "force")) @@ -112,8 +153,19 @@ void __init swiotlb_adjust_size(unsigned long size) */ if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT) return; + + /* + * Round up number of slabs to the next power of 2. + * The last area is going be smaller than the rest if + * default_nslabs is not power of two. + */ size = ALIGN(size, IO_TLB_SIZE); default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE); + if (default_nareas) { + default_nslabs = roundup_pow_of_two(default_nslabs); + size = default_nslabs << IO_TLB_SHIFT; + } + pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20); } @@ -192,7 +244,8 @@ void __init swiotlb_update_mem_attributes(void) } static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, - unsigned long nslabs, unsigned int flags, bool late_alloc) + unsigned long nslabs, unsigned int flags, + bool late_alloc, unsigned int nareas) { void *vaddr = phys_to_virt(start); unsigned long bytes = nslabs << IO_TLB_SHIFT, i; @@ -202,10 +255,17 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, mem->end = mem->start + bytes; mem->index = 0; mem->late_alloc = late_alloc; + mem->nareas = nareas; + mem->area_nslabs = nslabs / mem->nareas; mem->force_bounce = swiotlb_force_bounce || (flags & SWIOTLB_FORCE); spin_lock_init(&mem->lock); + for (i = 0; i < mem->nareas; i++) { + spin_lock_init(&mem->areas[i].lock); + mem->areas[i].index = 0; + } + for (i = 0; i < mem->nslabs; i++) { mem->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(i); mem->slots[i].orig_addr = INVALID_PHYS_ADDR; @@ -232,7 +292,7 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, int (*remap)(void *tlb, unsigned long nslabs)) { struct io_tlb_mem *mem = &io_tlb_default_mem; - unsigned long nslabs = default_nslabs; + unsigned long nslabs; size_t alloc_size; size_t bytes; void *tlb; @@ -242,6 +302,14 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, if (swiotlb_force_disable) return; + /* + * default_nslabs maybe changed when adjust area number. + * So allocate bounce buffer after adjusting area number. + */ + if (!default_nareas) + swiotlb_adjust_nareas(num_possible_cpus()); + + nslabs = default_nslabs; if (nslabs < IO_TLB_MIN_SLABS) panic("%s: nslabs = %lu too small\n", __func__, nslabs); @@ -278,7 +346,13 @@ retry: panic("%s: Failed to allocate %zu bytes align=0x%lx\n", __func__, alloc_size, PAGE_SIZE); - swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, flags, false); + mem->areas = memblock_alloc(sizeof(struct io_tlb_area) * + default_nareas, SMP_CACHE_BYTES); + if (!mem->areas) + panic("%s: Failed to allocate mem->areas.\n", __func__); + + swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, flags, false, + default_nareas); if (flags & SWIOTLB_VERBOSE) swiotlb_print_info(); @@ -300,7 +374,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, struct io_tlb_mem *mem = &io_tlb_default_mem; unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE); unsigned char *vstart = NULL; - unsigned int order; + unsigned int order, area_order; bool retried = false; int rc = 0; @@ -341,19 +415,34 @@ retry: (PAGE_SIZE << order) >> 20); } + if (!default_nareas) + swiotlb_adjust_nareas(num_possible_cpus()); + + area_order = get_order(array_size(sizeof(*mem->areas), + default_nareas)); + mem->areas = (struct io_tlb_area *) + __get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order); + if (!mem->areas) + goto error_area; + mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, get_order(array_size(sizeof(*mem->slots), nslabs))); - if (!mem->slots) { - free_pages((unsigned long)vstart, order); - return -ENOMEM; - } + if (!mem->slots) + goto error_slots; set_memory_decrypted((unsigned long)vstart, (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); - swiotlb_init_io_tlb_mem(mem, virt_to_phys(vstart), nslabs, 0, true); + swiotlb_init_io_tlb_mem(mem, virt_to_phys(vstart), nslabs, 0, true, + default_nareas); swiotlb_print_info(); return 0; + +error_slots: + free_pages((unsigned long)mem->areas, area_order); +error_area: + free_pages((unsigned long)vstart, order); + return -ENOMEM; } void __init swiotlb_exit(void) @@ -361,6 +450,7 @@ void __init swiotlb_exit(void) struct io_tlb_mem *mem = &io_tlb_default_mem; unsigned long tbl_vaddr; size_t tbl_size, slots_size; + unsigned int area_order; if (swiotlb_force_bounce) return; @@ -375,9 +465,14 @@ void __init swiotlb_exit(void) set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT); if (mem->late_alloc) { + area_order = get_order(array_size(sizeof(*mem->areas), + mem->nareas)); + free_pages((unsigned long)mem->areas, area_order); free_pages(tbl_vaddr, get_order(tbl_size)); free_pages((unsigned long)mem->slots, get_order(slots_size)); } else { + memblock_free_late(__pa(mem->areas), + mem->nareas * sizeof(struct io_tlb_area)); memblock_free_late(mem->start, tbl_size); memblock_free_late(__pa(mem->slots), slots_size); } @@ -480,9 +575,9 @@ static inline unsigned long get_max_slots(unsigned long boundary_mask) return nr_slots(boundary_mask + 1); } -static unsigned int wrap_index(struct io_tlb_mem *mem, unsigned int index) +static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index) { - if (index >= mem->nslabs) + if (index >= mem->area_nslabs) return 0; return index; } @@ -491,10 +586,11 @@ static unsigned int wrap_index(struct io_tlb_mem *mem, unsigned int index) * Find a suitable number of IO TLB entries size that will fit this request and * allocate a buffer from that IO TLB pool. */ -static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, - size_t alloc_size, unsigned int alloc_align_mask) +static int swiotlb_do_find_slots(struct io_tlb_mem *mem, + struct io_tlb_area *area, int area_index, + struct device *dev, phys_addr_t orig_addr, + size_t alloc_size, unsigned int alloc_align_mask) { - struct io_tlb_mem *mem = dev->dma_io_tlb_mem; unsigned long boundary_mask = dma_get_seg_boundary(dev); dma_addr_t tbl_dma_addr = phys_to_dma_unencrypted(dev, mem->start) & boundary_mask; @@ -505,8 +601,11 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, unsigned int index, wrap, count = 0, i; unsigned int offset = swiotlb_align_offset(dev, orig_addr); unsigned long flags; + unsigned int slot_base; + unsigned int slot_index; BUG_ON(!nslots); + BUG_ON(area_index >= mem->nareas); /* * For mappings with an alignment requirement don't bother looping to @@ -518,16 +617,20 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, stride = max(stride, stride << (PAGE_SHIFT - IO_TLB_SHIFT)); stride = max(stride, (alloc_align_mask >> IO_TLB_SHIFT) + 1); - spin_lock_irqsave(&mem->lock, flags); - if (unlikely(nslots > mem->nslabs - mem->used)) + spin_lock_irqsave(&area->lock, flags); + if (unlikely(nslots > mem->area_nslabs - area->used)) goto not_found; - index = wrap = wrap_index(mem, ALIGN(mem->index, stride)); + slot_base = area_index * mem->area_nslabs; + index = wrap = wrap_area_index(mem, ALIGN(area->index, stride)); + do { + slot_index = slot_base + index; + if (orig_addr && - (slot_addr(tbl_dma_addr, index) & iotlb_align_mask) != - (orig_addr & iotlb_align_mask)) { - index = wrap_index(mem, index + 1); + (slot_addr(tbl_dma_addr, slot_index) & + iotlb_align_mask) != (orig_addr & iotlb_align_mask)) { + index = wrap_area_index(mem, index + 1); continue; } @@ -536,26 +639,26 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, * contiguous buffers, we allocate the buffers from that slot * and mark the entries as '0' indicating unavailable. */ - if (!iommu_is_span_boundary(index, nslots, + if (!iommu_is_span_boundary(slot_index, nslots, nr_slots(tbl_dma_addr), max_slots)) { - if (mem->slots[index].list >= nslots) + if (mem->slots[slot_index].list >= nslots) goto found; } - index = wrap_index(mem, index + stride); + index = wrap_area_index(mem, index + stride); } while (index != wrap); not_found: - spin_unlock_irqrestore(&mem->lock, flags); + spin_unlock_irqrestore(&area->lock, flags); return -1; found: - for (i = index; i < index + nslots; i++) { + for (i = slot_index; i < slot_index + nslots; i++) { mem->slots[i].list = 0; - mem->slots[i].alloc_size = - alloc_size - (offset + ((i - index) << IO_TLB_SHIFT)); + mem->slots[i].alloc_size = alloc_size - (offset + + ((i - slot_index) << IO_TLB_SHIFT)); } - for (i = index - 1; + for (i = slot_index - 1; io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list; i--) mem->slots[i].list = ++count; @@ -563,14 +666,43 @@ found: /* * Update the indices to avoid searching in the next round. */ - if (index + nslots < mem->nslabs) - mem->index = index + nslots; + if (index + nslots < mem->area_nslabs) + area->index = index + nslots; else - mem->index = 0; - mem->used += nslots; + area->index = 0; + area->used += nslots; + spin_unlock_irqrestore(&area->lock, flags); + return slot_index; +} - spin_unlock_irqrestore(&mem->lock, flags); - return index; +static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, + size_t alloc_size, unsigned int alloc_align_mask) +{ + struct io_tlb_mem *mem = dev->dma_io_tlb_mem; + int start = raw_smp_processor_id() & ((1U << __fls(mem->nareas)) - 1); + int i = start, index; + + do { + index = swiotlb_do_find_slots(mem, mem->areas + i, i, + dev, orig_addr, alloc_size, + alloc_align_mask); + if (index >= 0) + return index; + if (++i >= mem->nareas) + i = 0; + } while (i != start); + + return -1; +} + +static unsigned long mem_used(struct io_tlb_mem *mem) +{ + int i; + unsigned long used = 0; + + for (i = 0; i < mem->nareas; i++) + used += mem->areas[i].used; + return used; } phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, @@ -602,7 +734,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, if (!(attrs & DMA_ATTR_NO_WARN)) dev_warn_ratelimited(dev, "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n", - alloc_size, mem->nslabs, mem->used); + alloc_size, mem->nslabs, mem_used(mem)); return (phys_addr_t)DMA_MAPPING_ERROR; } @@ -632,6 +764,8 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr) unsigned int offset = swiotlb_align_offset(dev, tlb_addr); int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT; int nslots = nr_slots(mem->slots[index].alloc_size + offset); + int aindex = index / mem->area_nslabs; + struct io_tlb_area *area = &mem->areas[aindex]; int count, i; /* @@ -640,7 +774,9 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr) * While returning the entries to the free list, we merge the entries * with slots below and above the pool being returned. */ - spin_lock_irqsave(&mem->lock, flags); + BUG_ON(aindex >= mem->nareas); + + spin_lock_irqsave(&area->lock, flags); if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE)) count = mem->slots[index + nslots].list; else @@ -664,8 +800,8 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr) io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list; i--) mem->slots[i].list = ++count; - mem->used -= nslots; - spin_unlock_irqrestore(&mem->lock, flags); + area->used -= nslots; + spin_unlock_irqrestore(&area->lock, flags); } /* @@ -763,12 +899,14 @@ EXPORT_SYMBOL_GPL(is_swiotlb_active); static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, const char *dirname) { + unsigned long used = mem_used(mem); + mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs); if (!mem->nslabs) return; debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs); - debugfs_create_ulong("io_tlb_used", 0400, mem->debugfs, &mem->used); + debugfs_create_ulong("io_tlb_used", 0400, mem->debugfs, &used); } static int __init __maybe_unused swiotlb_create_default_debugfs(void) @@ -819,6 +957,9 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem, struct io_tlb_mem *mem = rmem->priv; unsigned long nslabs = rmem->size >> IO_TLB_SHIFT; + /* Set Per-device io tlb area to one */ + unsigned int nareas = 1; + /* * Since multiple devices can share the same pool, the private data, * io_tlb_mem struct, will be initialized by the first device attached @@ -835,10 +976,18 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem, return -ENOMEM; } + mem->areas = kcalloc(nareas, sizeof(*mem->areas), + GFP_KERNEL); + if (!mem->areas) { + kfree(mem); + kfree(mem->slots); + return -ENOMEM; + } + set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), rmem->size >> PAGE_SHIFT); swiotlb_init_io_tlb_mem(mem, rmem->base, nslabs, SWIOTLB_FORCE, - false); + false, nareas); mem->for_alloc = true; rmem->priv = mem; -- cgit v1.2.3 From 6d1c1a73e1126572de0a8b063fe62fe43786ed59 Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Fri, 8 Jul 2022 14:13:11 +0800 Subject: soundwire: Intel: add trigger callback When a pipeline is split into FE and BE parts, the BE pipeline may need to be triggered separately in the BE trigger op. So add the trigger callback in the link_res ops that will be invoked during BE DAI trigger. Signed-off-by: Bard Liao Reviewed-by: Rander Wang Reviewed-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Acked-by: Vinod Koul Link: https://lore.kernel.org/r/20220708061312.25878-2-yung-chuan.liao@linux.intel.com Signed-off-by: Mark Brown --- drivers/soundwire/intel.c | 9 +++++++++ include/linux/soundwire/sdw_intel.h | 1 + 2 files changed, 10 insertions(+) (limited to 'include/linux') diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 0268fa527c0c..fed6418d6375 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -1004,9 +1004,18 @@ static int intel_trigger(struct snd_pcm_substream *substream, int cmd, struct sn { struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai); struct sdw_intel *sdw = cdns_to_intel(cdns); + struct sdw_intel_link_res *res = sdw->link_res; struct sdw_cdns_dma_data *dma; int ret = 0; + /* + * The .trigger callback is used to send required IPC to audio + * firmware. The .free_stream callback will still be called + * by intel_free_stream() in the TRIGGER_SUSPEND case. + */ + if (res->ops && res->ops->trigger) + res->ops->trigger(dai, cmd, substream->stream); + dma = snd_soc_dai_get_dma_data(dai, substream); if (!dma) { dev_err(dai->dev, "failed to get dma data in %s\n", diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h index b5b489ea1aef..ec16ae49e6a4 100644 --- a/include/linux/soundwire/sdw_intel.h +++ b/include/linux/soundwire/sdw_intel.h @@ -121,6 +121,7 @@ struct sdw_intel_ops { struct sdw_intel_stream_params_data *params_data); int (*free_stream)(struct device *dev, struct sdw_intel_stream_free_data *free_data); + int (*trigger)(struct snd_soc_dai *dai, int cmd, int stream); }; /** -- cgit v1.2.3 From 0372c546eca575445331c0ad8902210b70be6d61 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Thu, 2 Jun 2022 12:41:00 +0300 Subject: net/mlx5: Introduce ifc bits for using software vhca id Introduce ifc related stuff to enable using software vhca id functionality. Signed-off-by: Yishai Hadas Reviewed-by: Mark Bloch Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 8e87eb47f9dc..254cc22f5eec 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1826,7 +1826,14 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 max_reformat_remove_size[0x8]; u8 max_reformat_remove_offset[0x8]; - u8 reserved_at_c0[0x740]; + u8 reserved_at_c0[0x160]; + + u8 reserved_at_220[0x1]; + u8 sw_vhca_id_valid[0x1]; + u8 sw_vhca_id[0xe]; + u8 reserved_at_230[0x10]; + + u8 reserved_at_240[0x5c0]; }; enum mlx5_ifc_flow_destination_type { @@ -3782,6 +3789,11 @@ struct mlx5_ifc_rmpc_bits { struct mlx5_ifc_wq_bits wq; }; +enum { + VHCA_ID_TYPE_HW = 0, + VHCA_ID_TYPE_SW = 1, +}; + struct mlx5_ifc_nic_vport_context_bits { u8 reserved_at_0[0x5]; u8 min_wqe_inline_mode[0x3]; @@ -3798,8 +3810,8 @@ struct mlx5_ifc_nic_vport_context_bits { u8 event_on_mc_address_change[0x1]; u8 event_on_uc_address_change[0x1]; - u8 reserved_at_40[0xc]; - + u8 vhca_id_type[0x1]; + u8 reserved_at_41[0xb]; u8 affiliation_criteria[0x4]; u8 affiliated_vhca_id[0x10]; @@ -7259,7 +7271,12 @@ struct mlx5_ifc_init_hca_in_bits { u8 reserved_at_20[0x10]; u8 op_mod[0x10]; - u8 reserved_at_40[0x40]; + u8 reserved_at_40[0x20]; + + u8 reserved_at_60[0x2]; + u8 sw_vhca_id[0xe]; + u8 reserved_at_70[0x10]; + u8 sw_owner_id[4][0x20]; }; -- cgit v1.2.3 From dc402ccc0d7b55922a79505df3000da7deb77a2b Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Thu, 2 Jun 2022 12:47:34 +0300 Subject: net/mlx5: Use software VHCA id when it's supported Use software VHCA id when it's supported by the firmware. A unique id is allocated upon mlx5_mdev_init() and freed upon mlx5_mdev_uninit(), as such it stays the same during the full life cycle of the device including upon health recovery if occurred. The conjunction of sw_vhca_id with sw_owner_id will be a global unique id per function which uses mlx5_core. The sw_vhca_id is set upon init_hca command and is used to specify the VHCA that the NIC vport is affiliated with. This functionality is needed upon migration of VM which is MPV based. (i.e. multi port device). Signed-off-by: Yishai Hadas Reviewed-by: Mark Bloch Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/fw.c | 4 ++ drivers/net/ethernet/mellanox/mlx5/core/main.c | 49 +++++++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/vport.c | 14 +++++-- include/linux/mlx5/driver.h | 1 + 4 files changed, 65 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index cfb8bedba512..079fa44ada71 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -289,6 +289,10 @@ int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id) sw_owner_id[i]); } + if (MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) && + dev->priv.sw_vhca_id > 0) + MLX5_SET(init_hca_in, in, sw_vhca_id, dev->priv.sw_vhca_id); + return mlx5_cmd_exec_in(dev, init_hca, in); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index a9e51c1b7738..8b621c1ddd14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -90,6 +90,8 @@ module_param_named(prof_sel, prof_sel, uint, 0444); MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); static u32 sw_owner_id[4]; +#define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1) +static DEFINE_IDA(sw_vhca_ida); enum { MLX5_ATOMIC_REQ_MODE_BE = 0x0, @@ -492,6 +494,31 @@ static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev) return err; } +static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx) +{ + void *set_hca_cap; + int err; + + if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2)) + return 0; + + err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2); + if (err) + return err; + + if (!MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) || + !(dev->priv.sw_vhca_id > 0)) + return 0; + + set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, + capability); + memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur, + MLX5_ST_SZ_BYTES(cmd_hca_cap_2)); + MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1); + + return set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2); +} + static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) { struct mlx5_profile *prof = &dev->profile; @@ -662,6 +689,13 @@ static int set_hca_cap(struct mlx5_core_dev *dev) goto out; } + memset(set_ctx, 0, set_sz); + err = handle_hca_cap_2(dev, set_ctx); + if (err) { + mlx5_core_err(dev, "handle_hca_cap_2 failed\n"); + goto out; + } + out: kfree(set_ctx); return err; @@ -1506,6 +1540,18 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) if (err) goto err_hca_caps; + /* The conjunction of sw_vhca_id with sw_owner_id will be a global + * unique id per function which uses mlx5_core. + * Those values are supplied to FW as part of the init HCA command to + * be used by both driver and FW when it's applicable. + */ + dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1, + MAX_SW_VHCA_ID, + GFP_KERNEL); + if (dev->priv.sw_vhca_id < 0) + mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n", + dev->priv.sw_vhca_id); + return 0; err_hca_caps: @@ -1530,6 +1576,9 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; + if (priv->sw_vhca_id > 0) + ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id); + mlx5_hca_caps_free(dev); mlx5_adev_cleanup(dev); mlx5_pagealloc_cleanup(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index ac020cb78072..d5c317325030 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -1086,9 +1086,17 @@ int mlx5_nic_vport_affiliate_multiport(struct mlx5_core_dev *master_mdev, goto free; MLX5_SET(modify_nic_vport_context_in, in, field_select.affiliation, 1); - MLX5_SET(modify_nic_vport_context_in, in, - nic_vport_context.affiliated_vhca_id, - MLX5_CAP_GEN(master_mdev, vhca_id)); + if (MLX5_CAP_GEN_2(master_mdev, sw_vhca_id_valid)) { + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.vhca_id_type, VHCA_ID_TYPE_SW); + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.affiliated_vhca_id, + MLX5_CAP_GEN_2(master_mdev, sw_vhca_id)); + } else { + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.affiliated_vhca_id, + MLX5_CAP_GEN(master_mdev, vhca_id)); + } MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.affiliation_criteria, MLX5_CAP_GEN(port_mdev, affiliate_nic_vport_criteria)); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index bd882884b23c..ecda6e63d5f2 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -610,6 +610,7 @@ struct mlx5_priv { spinlock_t ctx_lock; struct mlx5_adev **adev; int adev_idx; + int sw_vhca_id; struct mlx5_events *events; struct mlx5_flow_steering *steering; -- cgit v1.2.3 From 9745fb07474f4501eff62130a78a42a8b8c18b05 Mon Sep 17 00:00:00 2001 From: Jonathan Yong Date: Mon, 6 Jun 2022 19:41:27 +0300 Subject: platform/x86/intel: Add Primary to Sideband (P2SB) bridge support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SoC features such as GPIO are accessed via a reserved MMIO area, we don't know its address but can obtain it from the BAR of the P2SB device, that device is normally hidden so we have to temporarily unhide it, read address and hide it back. There are already a few users and at least one more is coming which require an access to Primary to Sideband (P2SB) bridge in order to get IO or MMIO BAR hidden by BIOS. Create a library to access P2SB for x86 devices in a unified way. Background information ====================== Note, the term "bridge" is used in the documentation and it has nothing to do with a PCI (host) bridge as per the PCI specifications. The P2SB is an interesting device by its nature and hardware design. First of all, it has several devices in the hardware behind it. These devices may or may not be represented as ACPI devices by a firmware. It also has a hardwired (to 0s) the least significant bits of the base address register which is represented by the only 64-bit BAR0. It means that OS mustn't reallocate the BAR. On top of that in some cases P2SB is represented by function 0 on PCI slot (in terms of B:D.F) and according to the PCI specification any other function can't be seen until function 0 is present and visible. In the PCI configuration space of P2SB device the full 32-bit register is allocated for the only purpose of hiding the entire P2SB device. As per [3]: 3.1.39 P2SB Control (P2SBC)—Offset E0h Hide Device (HIDE): When this bit is set, the P2SB will return 1s on any PCI Configuration Read on IOSF-P. All other transactions including PCI Configuration Writes on IOSF-P are unaffected by this. This does not affect reads performed on the IOSF-SB interface. This doesn't prevent MMIO accesses, although preventing the OS from assigning these addresses. The firmware on the affected platforms marks the region as unusable (by cutting it off from the PCI host bridge resources) as depicted in the Apollo Lake example below: PCI host bridge to bus 0000:00 pci_bus 0000:00: root bus resource [io 0x0070-0x0077] pci_bus 0000:00: root bus resource [io 0x0000-0x006f window] pci_bus 0000:00: root bus resource [io 0x0078-0x0cf7 window] pci_bus 0000:00: root bus resource [io 0x0d00-0xffff window] pci_bus 0000:00: root bus resource [mem 0x7c000001-0x7fffffff window] pci_bus 0000:00: root bus resource [mem 0x7b800001-0x7bffffff window] pci_bus 0000:00: root bus resource [mem 0x80000000-0xcfffffff window] pci_bus 0000:00: root bus resource [mem 0xe0000000-0xefffffff window] pci_bus 0000:00: root bus resource [bus 00-ff] The P2SB 16MB BAR is located at 0xd0000000-0xd0ffffff memory window. The generic solution ==================== The generic solution for all cases when we need to access to the information behind P2SB device is a library code where users ask for necessary resources by demand and hence those users take care of not being run on the systems where this access is not required. The library provides the p2sb_bar() API to retrieve the MMIO of the BAR0 of the device from P2SB device slot. P2SB unconditional unhiding awareness ===================================== Technically it's possible to unhide the P2SB device and devices on the same PCI slot and access them at any time as needed. But there are several potential issues with that: - the systems were never tested against such configuration and hence nobody knows what kind of bugs it may bring, especially when we talk about SPI NOR case which contains Intel FirmWare Image (IFWI) code (including BIOS) and already known to be problematic in the past for end users - the PCI by its nature is a hotpluggable bus and in case somebody attaches a driver to the functions of a P2SB slot device(s) the end user experience and system behaviour can be unpredictable - the kernel code would need some ugly hacks (or code looking as an ugly hack) under arch/x86/pci in order to enable these devices on only selected platforms (which may include CPU ID table followed by a potentially growing number of DMI strings The future improvements ======================= The future improvements with this code may go in order to gain some kind of cache, if it's possible at all, to prevent unhiding and hiding many times to take static information that may be saved once per boot. Links ===== [1]: https://lab.whitequark.org/notes/2017-11-08/accessing-intel-ich-pch-gpios/ [2]: https://cdrdv2.intel.com/v1/dl/getContent/332690?wapkw=332690 [3]: https://cdrdv2.intel.com/v1/dl/getContent/332691?wapkw=332691 [4]: https://medium.com/@jacksonchen_43335/bios-gpio-p2sb-70e9b829b403 Signed-off-by: Jonathan Yong Co-developed-by: Andy Shevchenko Signed-off-by: Andy Shevchenko Tested-by: Henning Schild Acked-by: Hans de Goede Acked-by: Linus Walleij Signed-off-by: Lee Jones --- drivers/platform/x86/intel/Kconfig | 12 ++++ drivers/platform/x86/intel/Makefile | 2 + drivers/platform/x86/intel/p2sb.c | 127 +++++++++++++++++++++++++++++++++ include/linux/platform_data/x86/p2sb.h | 28 ++++++++ 4 files changed, 169 insertions(+) create mode 100644 drivers/platform/x86/intel/p2sb.c create mode 100644 include/linux/platform_data/x86/p2sb.h (limited to 'include/linux') diff --git a/drivers/platform/x86/intel/Kconfig b/drivers/platform/x86/intel/Kconfig index 794968bda115..c9cfbaae436b 100644 --- a/drivers/platform/x86/intel/Kconfig +++ b/drivers/platform/x86/intel/Kconfig @@ -70,6 +70,18 @@ config INTEL_OAKTRAIL enable/disable the Camera, WiFi, BT etc. devices. If in doubt, say Y here; it will only load on supported platforms. +config P2SB + bool "Primary to Sideband (P2SB) bridge access support" + depends on PCI + help + The Primary to Sideband (P2SB) bridge is an interface to some + PCI devices connected through it. In particular, SPI NOR controller + in Intel Apollo Lake SoC is one of such devices. + + The main purpose of this library is to unhide P2SB device in case + firmware kept it hidden on some platforms in order to access devices + behind it. + config INTEL_BXTWC_PMIC_TMU tristate "Intel Broxton Whiskey Cove TMU Driver" depends on INTEL_SOC_PMIC_BXTWC diff --git a/drivers/platform/x86/intel/Makefile b/drivers/platform/x86/intel/Makefile index 717933dd0cfd..741a9404db98 100644 --- a/drivers/platform/x86/intel/Makefile +++ b/drivers/platform/x86/intel/Makefile @@ -28,6 +28,8 @@ intel_int0002_vgpio-y := int0002_vgpio.o obj-$(CONFIG_INTEL_INT0002_VGPIO) += intel_int0002_vgpio.o intel_oaktrail-y := oaktrail.o obj-$(CONFIG_INTEL_OAKTRAIL) += intel_oaktrail.o +intel_p2sb-y := p2sb.o +obj-$(CONFIG_P2SB) += intel_p2sb.o intel_sdsi-y := sdsi.o obj-$(CONFIG_INTEL_SDSI) += intel_sdsi.o intel_vsec-y := vsec.o diff --git a/drivers/platform/x86/intel/p2sb.c b/drivers/platform/x86/intel/p2sb.c new file mode 100644 index 000000000000..b598ef14dbc6 --- /dev/null +++ b/drivers/platform/x86/intel/p2sb.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Primary to Sideband (P2SB) bridge access support + * + * Copyright (c) 2017, 2021-2022 Intel Corporation. + * + * Authors: Andy Shevchenko + * Jonathan Yong + */ + +#include +#include +#include +#include + +#include +#include + +#define P2SBC 0xe0 +#define P2SBC_HIDE BIT(8) + +static const struct x86_cpu_id p2sb_cpu_ids[] = { + X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, PCI_DEVFN(13, 0)), + {} +}; + +static int p2sb_get_devfn(unsigned int *devfn) +{ + const struct x86_cpu_id *id; + + id = x86_match_cpu(p2sb_cpu_ids); + if (!id) + return -ENODEV; + + *devfn = (unsigned int)id->driver_data; + return 0; +} + +static int p2sb_read_bar0(struct pci_dev *pdev, struct resource *mem) +{ + /* Copy resource from the first BAR of the device in question */ + *mem = pdev->resource[0]; + return 0; +} + +static int p2sb_scan_and_read(struct pci_bus *bus, unsigned int devfn, struct resource *mem) +{ + struct pci_dev *pdev; + int ret; + + pdev = pci_scan_single_device(bus, devfn); + if (!pdev) + return -ENODEV; + + ret = p2sb_read_bar0(pdev, mem); + + pci_stop_and_remove_bus_device(pdev); + return ret; +} + +/** + * p2sb_bar - Get Primary to Sideband (P2SB) bridge device BAR + * @bus: PCI bus to communicate with + * @devfn: PCI slot and function to communicate with + * @mem: memory resource to be filled in + * + * The BIOS prevents the P2SB device from being enumerated by the PCI + * subsystem, so we need to unhide and hide it back to lookup the BAR. + * + * if @bus is NULL, the bus 0 in domain 0 will be used. + * If @devfn is 0, it will be replaced by devfn of the P2SB device. + * + * Caller must provide a valid pointer to @mem. + * + * Locking is handled by pci_rescan_remove_lock mutex. + * + * Return: + * 0 on success or appropriate errno value on error. + */ +int p2sb_bar(struct pci_bus *bus, unsigned int devfn, struct resource *mem) +{ + struct pci_dev *pdev_p2sb; + unsigned int devfn_p2sb; + u32 value = P2SBC_HIDE; + int ret; + + /* Get devfn for P2SB device itself */ + ret = p2sb_get_devfn(&devfn_p2sb); + if (ret) + return ret; + + /* if @bus is NULL, use bus 0 in domain 0 */ + bus = bus ?: pci_find_bus(0, 0); + + /* + * Prevent concurrent PCI bus scan from seeing the P2SB device and + * removing via sysfs while it is temporarily exposed. + */ + pci_lock_rescan_remove(); + + /* Unhide the P2SB device, if needed */ + pci_bus_read_config_dword(bus, devfn_p2sb, P2SBC, &value); + if (value & P2SBC_HIDE) + pci_bus_write_config_dword(bus, devfn_p2sb, P2SBC, 0); + + pdev_p2sb = pci_scan_single_device(bus, devfn_p2sb); + if (devfn) + ret = p2sb_scan_and_read(bus, devfn, mem); + else + ret = p2sb_read_bar0(pdev_p2sb, mem); + pci_stop_and_remove_bus_device(pdev_p2sb); + + /* Hide the P2SB device, if it was hidden */ + if (value & P2SBC_HIDE) + pci_bus_write_config_dword(bus, devfn_p2sb, P2SBC, P2SBC_HIDE); + + pci_unlock_rescan_remove(); + + if (ret) + return ret; + + if (mem->flags == 0) + return -ENODEV; + + return 0; +} +EXPORT_SYMBOL_GPL(p2sb_bar); diff --git a/include/linux/platform_data/x86/p2sb.h b/include/linux/platform_data/x86/p2sb.h new file mode 100644 index 000000000000..a1d5fddc8f13 --- /dev/null +++ b/include/linux/platform_data/x86/p2sb.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Primary to Sideband (P2SB) bridge access support + */ + +#ifndef _PLATFORM_DATA_X86_P2SB_H +#define _PLATFORM_DATA_X86_P2SB_H + +#include +#include + +struct pci_bus; +struct resource; + +#if IS_BUILTIN(CONFIG_P2SB) + +int p2sb_bar(struct pci_bus *bus, unsigned int devfn, struct resource *mem); + +#else /* CONFIG_P2SB */ + +static inline int p2sb_bar(struct pci_bus *bus, unsigned int devfn, struct resource *mem) +{ + return -ENODEV; +} + +#endif /* CONFIG_P2SB is not set */ + +#endif /* _PLATFORM_DATA_X86_P2SB_H */ -- cgit v1.2.3 From 446f0cf9e08b483d7dc6f61eee0ee846b22f6386 Mon Sep 17 00:00:00 2001 From: Henning Schild Date: Mon, 6 Jun 2022 19:41:37 +0300 Subject: platform/x86: simatic-ipc: drop custom P2SB bar code The two drivers that used to use this have been switched over to the common P2SB accessor, so this code is not needed any longer. Signed-off-by: Henning Schild Signed-off-by: Andy Shevchenko Reviewed-by: Hans de Goede Signed-off-by: Lee Jones --- drivers/platform/x86/simatic-ipc.c | 38 ---------------------- include/linux/platform_data/x86/simatic-ipc-base.h | 2 -- 2 files changed, 40 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/x86/simatic-ipc.c b/drivers/platform/x86/simatic-ipc.c index b599cda5ba3c..26c35e1660cb 100644 --- a/drivers/platform/x86/simatic-ipc.c +++ b/drivers/platform/x86/simatic-ipc.c @@ -101,44 +101,6 @@ static int register_platform_devices(u32 station_id) return 0; } -/* FIXME: this should eventually be done with generic P2SB discovery code - * the individual drivers for watchdogs and LEDs access memory that implements - * GPIO, but pinctrl will not come up because of missing ACPI entries - * - * While there is no conflict a cleaner solution would be to somehow bring up - * pinctrl even with these ACPI entries missing, and base the drivers on pinctrl. - * After which the following function could be dropped, together with the code - * poking the memory. - */ -/* - * Get membase address from PCI, used in leds and wdt module. Here we read - * the bar0. The final address calculation is done in the appropriate modules - */ -u32 simatic_ipc_get_membase0(unsigned int p2sb) -{ - struct pci_bus *bus; - u32 bar0 = 0; - /* - * The GPIO memory is in bar0 of the hidden P2SB device. - * Unhide the device to have a quick look at it, before we hide it - * again. - * Also grab the pci rescan lock so that device does not get discovered - * and remapped while it is visible. - * This code is inspired by drivers/mfd/lpc_ich.c - */ - bus = pci_find_bus(0, 0); - pci_lock_rescan_remove(); - pci_bus_write_config_byte(bus, p2sb, 0xE1, 0x0); - pci_bus_read_config_dword(bus, p2sb, PCI_BASE_ADDRESS_0, &bar0); - - bar0 &= ~0xf; - pci_bus_write_config_byte(bus, p2sb, 0xE1, 0x1); - pci_unlock_rescan_remove(); - - return bar0; -} -EXPORT_SYMBOL(simatic_ipc_get_membase0); - static int __init simatic_ipc_init_module(void) { const struct dmi_system_id *match; diff --git a/include/linux/platform_data/x86/simatic-ipc-base.h b/include/linux/platform_data/x86/simatic-ipc-base.h index 62d2bc774067..39fefd48cf4d 100644 --- a/include/linux/platform_data/x86/simatic-ipc-base.h +++ b/include/linux/platform_data/x86/simatic-ipc-base.h @@ -24,6 +24,4 @@ struct simatic_ipc_platform { u8 devmode; }; -u32 simatic_ipc_get_membase0(unsigned int p2sb); - #endif /* __PLATFORM_DATA_X86_SIMATIC_IPC_BASE_H */ -- cgit v1.2.3 From 4fa05a67b558d2cb3acd2bb299b91220d405ca5e Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 11 Jul 2022 16:48:01 +0200 Subject: dma-buf: revert "return only unsignaled fences in dma_fence_unwrap_for_each v3" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 8f61973718485f3e89bc4f408f929048b7b47c83. It turned out that this is not correct. Especially the sync_file info IOCTL needs to see even signaled fences to correctly report back their status to userspace. Instead add the filter in the merge function again where it makes sense. Signed-off-by: Christian König Tested-by: Karolina Drobnik Reviewed-by: Alex Deucher Link: https://patchwork.freedesktop.org/patch/msgid/20220712102849.1562-1-christian.koenig@amd.com --- drivers/dma-buf/dma-fence-unwrap.c | 3 ++- include/linux/dma-fence-unwrap.h | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma-buf/dma-fence-unwrap.c b/drivers/dma-buf/dma-fence-unwrap.c index 502a65ea6d44..7002bca792ff 100644 --- a/drivers/dma-buf/dma-fence-unwrap.c +++ b/drivers/dma-buf/dma-fence-unwrap.c @@ -72,7 +72,8 @@ struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, count = 0; for (i = 0; i < num_fences; ++i) { dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) - ++count; + if (!dma_fence_is_signaled(tmp)) + ++count; } if (count == 0) diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h index 390de1ee9d35..66b1e56fbb81 100644 --- a/include/linux/dma-fence-unwrap.h +++ b/include/linux/dma-fence-unwrap.h @@ -43,14 +43,10 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor); * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all * potential fences in them. If @head is just a normal fence only that one is * returned. - * - * Note that signalled fences are opportunistically filtered out, which - * means the iteration is potentially over no fence at all. */ #define dma_fence_unwrap_for_each(fence, cursor, head) \ for (fence = dma_fence_unwrap_first(head, cursor); fence; \ - fence = dma_fence_unwrap_next(cursor)) \ - if (!dma_fence_is_signaled(fence)) + fence = dma_fence_unwrap_next(cursor)) struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, struct dma_fence **fences, -- cgit v1.2.3 From ddaf8d96f93bccb3f2b1f4f156c098b272440004 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 11 Jul 2022 07:22:55 +0000 Subject: usb: typec: Add support for retimers Introduce a retimer device class and associated functions that register and use retimer "switch" devices. These operate in a manner similar to the "mode-switch" and help configure retimers that exist between the Type-C connector and host controller(s). Type C ports can be linked to retimers using firmware node device references (again, in a manner similar to "mode-switch"). There are no new sysfs files being created; there is the new retimer class directory, but there are no class-specific files being created there. Signed-off-by: Prashant Malani Acked-by: Heikki Krogerus Link: https://lore.kernel.org/r/20220711072333.2064341-2-pmalani@chromium.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/Makefile | 2 +- drivers/usb/typec/class.c | 9 +- drivers/usb/typec/class.h | 1 + drivers/usb/typec/retimer.c | 168 ++++++++++++++++++++++++++++++++++++++ drivers/usb/typec/retimer.h | 15 ++++ include/linux/usb/typec_retimer.h | 45 ++++++++++ 6 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 drivers/usb/typec/retimer.c create mode 100644 drivers/usb/typec/retimer.h create mode 100644 include/linux/usb/typec_retimer.h (limited to 'include/linux') diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index dac5c11a3234..4a83dad51a6c 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_TYPEC) += typec.o -typec-y := class.o mux.o bus.o pd.o +typec-y := class.o mux.o bus.o pd.o retimer.o typec-$(CONFIG_ACPI) += port-mapper.o obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index bbc46b14f99a..9062836bb638 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -2299,10 +2299,14 @@ static int __init typec_init(void) if (ret) goto err_unregister_bus; - ret = class_register(&typec_class); + ret = class_register(&retimer_class); if (ret) goto err_unregister_mux_class; + ret = class_register(&typec_class); + if (ret) + goto err_unregister_retimer_class; + ret = usb_power_delivery_init(); if (ret) goto err_unregister_class; @@ -2312,6 +2316,9 @@ static int __init typec_init(void) err_unregister_class: class_unregister(&typec_class); +err_unregister_retimer_class: + class_unregister(&retimer_class); + err_unregister_mux_class: class_unregister(&typec_mux_class); diff --git a/drivers/usb/typec/class.h b/drivers/usb/typec/class.h index b531f9853bc0..43fcf9e37a8c 100644 --- a/drivers/usb/typec/class.h +++ b/drivers/usb/typec/class.h @@ -76,6 +76,7 @@ extern const struct device_type typec_port_dev_type; #define is_typec_port(dev) ((dev)->type == &typec_port_dev_type) extern struct class typec_mux_class; +extern struct class retimer_class; extern struct class typec_class; #if defined(CONFIG_ACPI) diff --git a/drivers/usb/typec/retimer.c b/drivers/usb/typec/retimer.c new file mode 100644 index 000000000000..051eaa7d2899 --- /dev/null +++ b/drivers/usb/typec/retimer.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2022 Google LLC + * + * USB Type-C Retimer support. + * Author: Prashant Malani + * + */ + +#include +#include +#include +#include +#include +#include + +#include "class.h" +#include "retimer.h" + +static bool dev_name_ends_with(struct device *dev, const char *suffix) +{ + const char *name = dev_name(dev); + const int name_len = strlen(name); + const int suffix_len = strlen(suffix); + + if (suffix_len > name_len) + return false; + + return strcmp(name + (name_len - suffix_len), suffix) == 0; +} + +static int retimer_fwnode_match(struct device *dev, const void *fwnode) +{ + return dev_fwnode(dev) == fwnode && dev_name_ends_with(dev, "-retimer"); +} + +static void *typec_retimer_match(struct fwnode_handle *fwnode, const char *id, void *data) +{ + struct device *dev = class_find_device(&retimer_class, NULL, fwnode, + retimer_fwnode_match); + + return dev ? to_typec_retimer(dev) : ERR_PTR(-EPROBE_DEFER); +} + +/** + * fwnode_typec_retimer_get - Find USB Type-C retimer. + * @fwnode: The caller device node. + * + * Finds a retimer linked to the caller. This function is primarily meant for the + * Type-C drivers. Returns a reference to the retimer on success, NULL if no + * matching connection was found, or ERR_PTR(-EPROBE_DEFER) when a connection + * was found but the retimer has not been enumerated yet. + */ +struct typec_retimer *fwnode_typec_retimer_get(struct fwnode_handle *fwnode) +{ + struct typec_retimer *retimer; + + retimer = fwnode_connection_find_match(fwnode, "retimer-switch", NULL, typec_retimer_match); + if (!IS_ERR_OR_NULL(retimer)) + WARN_ON(!try_module_get(retimer->dev.parent->driver->owner)); + + return retimer; +} +EXPORT_SYMBOL_GPL(fwnode_typec_retimer_get); + +/** + * typec_retimer_put - Release handle to a retimer. + * @retimer: USB Type-C Connector Retimer. + * + * Decrements reference count for @retimer. + */ +void typec_retimer_put(struct typec_retimer *retimer) +{ + if (!IS_ERR_OR_NULL(retimer)) { + module_put(retimer->dev.parent->driver->owner); + put_device(&retimer->dev); + } +} +EXPORT_SYMBOL_GPL(typec_retimer_put); + +int typec_retimer_set(struct typec_retimer *retimer, struct typec_retimer_state *state) +{ + if (IS_ERR_OR_NULL(retimer)) + return 0; + + return retimer->set(retimer, state); +} +EXPORT_SYMBOL_GPL(typec_retimer_set); + +static void typec_retimer_release(struct device *dev) +{ + kfree(to_typec_retimer(dev)); +} + +static const struct device_type typec_retimer_dev_type = { + .name = "typec_retimer", + .release = typec_retimer_release, +}; + +/** + * typec_retimer_register - Register a retimer device. + * @parent: Parent device. + * @desc: Retimer description. + * + * Some USB Type-C connectors have their physical lines routed through retimers before they + * reach muxes or host controllers. In some cases (for example: using alternate modes) + * these retimers need to be reconfigured appropriately. This function registers retimer + * switches which route and potentially modify the signals on the Type C physical lines + * enroute to the host controllers. + */ +struct typec_retimer * +typec_retimer_register(struct device *parent, const struct typec_retimer_desc *desc) +{ + struct typec_retimer *retimer; + int ret; + + if (!desc || !desc->set) + return ERR_PTR(-EINVAL); + + retimer = kzalloc(sizeof(*retimer), GFP_KERNEL); + if (!retimer) + return ERR_PTR(-ENOMEM); + + retimer->set = desc->set; + + device_initialize(&retimer->dev); + retimer->dev.parent = parent; + retimer->dev.fwnode = desc->fwnode; + retimer->dev.class = &retimer_class; + retimer->dev.type = &typec_retimer_dev_type; + retimer->dev.driver_data = desc->drvdata; + dev_set_name(&retimer->dev, "%s-retimer", + desc->name ? desc->name : dev_name(parent)); + + ret = device_add(&retimer->dev); + if (ret) { + dev_err(parent, "failed to register retimer (%d)\n", ret); + put_device(&retimer->dev); + return ERR_PTR(ret); + } + + return retimer; +} +EXPORT_SYMBOL_GPL(typec_retimer_register); + +/** + * typec_retimer_unregister - Unregister retimer device. + * @retimer: USB Type-C Connector retimer. + * + * Unregister retimer that was registered with typec_retimer_register(). + */ +void typec_retimer_unregister(struct typec_retimer *retimer) +{ + if (!IS_ERR_OR_NULL(retimer)) + device_unregister(&retimer->dev); +} +EXPORT_SYMBOL_GPL(typec_retimer_unregister); + +void *typec_retimer_get_drvdata(struct typec_retimer *retimer) +{ + return dev_get_drvdata(&retimer->dev); +} +EXPORT_SYMBOL_GPL(typec_retimer_get_drvdata); + +struct class retimer_class = { + .name = "retimer", + .owner = THIS_MODULE, +}; diff --git a/drivers/usb/typec/retimer.h b/drivers/usb/typec/retimer.h new file mode 100644 index 000000000000..fa15951d4846 --- /dev/null +++ b/drivers/usb/typec/retimer.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __USB_TYPEC_RETIMER__ +#define __USB_TYPEC_RETIMER__ + +#include + +struct typec_retimer { + struct device dev; + typec_retimer_set_fn_t set; +}; + +#define to_typec_retimer(_dev_) container_of(_dev_, struct typec_retimer, dev) + +#endif /* __USB_TYPEC_RETIMER__ */ diff --git a/include/linux/usb/typec_retimer.h b/include/linux/usb/typec_retimer.h new file mode 100644 index 000000000000..5e036b3360e2 --- /dev/null +++ b/include/linux/usb/typec_retimer.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __USB_TYPEC_RETIMER +#define __USB_TYPEC_RETIMER + +#include +#include + +struct device; +struct typec_retimer; +struct typec_altmode; +struct fwnode_handle; + +struct typec_retimer_state { + struct typec_altmode *alt; + unsigned long mode; + void *data; +}; + +typedef int (*typec_retimer_set_fn_t)(struct typec_retimer *retimer, + struct typec_retimer_state *state); + +struct typec_retimer_desc { + struct fwnode_handle *fwnode; + typec_retimer_set_fn_t set; + const char *name; + void *drvdata; +}; + +struct typec_retimer *fwnode_typec_retimer_get(struct fwnode_handle *fwnode); +void typec_retimer_put(struct typec_retimer *retimer); +int typec_retimer_set(struct typec_retimer *retimer, struct typec_retimer_state *state); + +static inline struct typec_retimer *typec_retimer_get(struct device *dev) +{ + return fwnode_typec_retimer_get(dev_fwnode(dev)); +} + +struct typec_retimer * +typec_retimer_register(struct device *parent, const struct typec_retimer_desc *desc); +void typec_retimer_unregister(struct typec_retimer *retimer); + +void *typec_retimer_get_drvdata(struct typec_retimer *retimer); + +#endif /* __USB_TYPEC_RETIMER */ -- cgit v1.2.3 From e6281c26674e037798bf674f26a7593a324cdf39 Mon Sep 17 00:00:00 2001 From: Ang Tien Sung Date: Mon, 11 Jul 2022 17:31:35 -0500 Subject: firmware: stratix10-svc: Add support for FCS Extend Intel service layer driver to support FPGA Crypto service(FCS) features on Intel Soc platforms. Adding an additional channel and FCS platform driver ("intel_fcs") as part of the probe method. FCS driver uses the driver to send crypto operations' commands to the secure device manager(SDM) on Intel Soc platforms Stratix10 and Agilex. Signed-off-by: Ang Tien Sung Signed-off-by: Dinh Nguyen Link: https://lore.kernel.org/r/20220711223140.2307945-1-dinguyen@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/stratix10-svc.c | 30 ++++++++++++++++++---- .../linux/firmware/intel/stratix10-svc-client.h | 1 + 2 files changed, 26 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 14663f671323..1fe748c047df 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -34,12 +34,13 @@ * timeout is set to 30 seconds (30 * 1000) at Intel Stratix10 SoC. */ #define SVC_NUM_DATA_IN_FIFO 32 -#define SVC_NUM_CHANNEL 2 +#define SVC_NUM_CHANNEL 3 #define FPGA_CONFIG_DATA_CLAIM_TIMEOUT_MS 200 #define FPGA_CONFIG_STATUS_TIMEOUT_SEC 30 /* stratix10 service layer clients */ #define STRATIX10_RSU "stratix10-rsu" +#define INTEL_FCS "intel-fcs" typedef void (svc_invoke_fn)(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, @@ -53,6 +54,7 @@ struct stratix10_svc_chan; */ struct stratix10_svc { struct platform_device *stratix10_svc_rsu; + struct platform_device *intel_svc_fcs; }; /** @@ -1036,6 +1038,11 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) chans[1].name = SVC_CLIENT_RSU; spin_lock_init(&chans[1].lock); + chans[2].scl = NULL; + chans[2].ctrl = controller; + chans[2].name = SVC_CLIENT_FCS; + spin_lock_init(&chans[2].lock); + list_add_tail(&controller->node, &svc_ctrl); platform_set_drvdata(pdev, controller); @@ -1054,8 +1061,22 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) } ret = platform_device_add(svc->stratix10_svc_rsu); - if (ret) - goto err_put_device; + if (ret) { + platform_device_put(svc->stratix10_svc_rsu); + return ret; + } + + svc->intel_svc_fcs = platform_device_alloc(INTEL_FCS, 1); + if (!svc->intel_svc_fcs) { + dev_err(dev, "failed to allocate %s device\n", INTEL_FCS); + return -ENOMEM; + } + + ret = platform_device_add(svc->intel_svc_fcs); + if (ret) { + platform_device_put(svc->intel_svc_fcs); + return ret; + } dev_set_drvdata(dev, svc); @@ -1063,8 +1084,6 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) return 0; -err_put_device: - platform_device_put(svc->stratix10_svc_rsu); err_free_kfifo: kfifo_free(&controller->svc_fifo); return ret; @@ -1075,6 +1094,7 @@ static int stratix10_svc_drv_remove(struct platform_device *pdev) struct stratix10_svc *svc = dev_get_drvdata(&pdev->dev); struct stratix10_svc_controller *ctrl = platform_get_drvdata(pdev); + platform_device_unregister(svc->intel_svc_fcs); platform_device_unregister(svc->stratix10_svc_rsu); kfifo_free(&ctrl->svc_fifo); diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index 18c1841fdb1f..24121f8e0d7c 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -14,6 +14,7 @@ */ #define SVC_CLIENT_FPGA "fpga" #define SVC_CLIENT_RSU "rsu" +#define SVC_CLIENT_FCS "fcs" /* * Status of the sent command, in bit number -- cgit v1.2.3 From 79b936254aa0eb4e3bc73fecaacf049145613c0a Mon Sep 17 00:00:00 2001 From: Ang Tien Sung Date: Mon, 11 Jul 2022 17:31:36 -0500 Subject: firmware: stratix10-svc: add FCS polling command Introduce a new SMC command INTEL_SIP_SMC_FUNCID_SERVICE_COMPLETED that polls if a previous asynchronous command was completed. This SMC command is used by the new FPGA Crypto Service (FCS). A basic example is that the FCS sends an AES data encryption call to the secure device manager(SDM) and waits for the completion of the operation by continuously polling the results with the new command. Signed-off-by: Ang Tien Sung Signed-off-by: Dinh Nguyen Link: https://lore.kernel.org/r/20220711223140.2307945-2-dinguyen@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/stratix10-svc.c | 42 +++++++++++++++++++--- include/linux/firmware/intel/stratix10-smc.h | 25 +++++++++++++ .../linux/firmware/intel/stratix10-svc-client.h | 5 +++ 3 files changed, 67 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 1fe748c047df..439a4bdf8207 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -248,6 +248,7 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl, { struct arm_smccc_res res; int count_in_sec; + unsigned long a0, a1, a2; cb_data->kaddr1 = NULL; cb_data->kaddr2 = NULL; @@ -256,24 +257,45 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl, pr_debug("%s: polling config status\n", __func__); + a0 = INTEL_SIP_SMC_FPGA_CONFIG_ISDONE; + a1 = (unsigned long)p_data->paddr; + a2 = (unsigned long)p_data->size; + + if (p_data->command == COMMAND_POLL_SERVICE_STATUS) + a0 = INTEL_SIP_SMC_SERVICE_COMPLETED; + count_in_sec = FPGA_CONFIG_STATUS_TIMEOUT_SEC; while (count_in_sec) { - ctrl->invoke_fn(INTEL_SIP_SMC_FPGA_CONFIG_ISDONE, - 0, 0, 0, 0, 0, 0, 0, &res); + ctrl->invoke_fn(a0, a1, a2, 0, 0, 0, 0, 0, &res); if ((res.a0 == INTEL_SIP_SMC_STATUS_OK) || - (res.a0 == INTEL_SIP_SMC_STATUS_ERROR)) + (res.a0 == INTEL_SIP_SMC_STATUS_ERROR) || + (res.a0 == INTEL_SIP_SMC_STATUS_REJECTED)) break; /* - * configuration is still in progress, wait one second then + * request is still in progress, wait one second then * poll again */ msleep(1000); count_in_sec--; } - if (res.a0 == INTEL_SIP_SMC_STATUS_OK && count_in_sec) + if (!count_in_sec) { + pr_err("%s: poll status timeout\n", __func__); + cb_data->status = BIT(SVC_STATUS_BUSY); + } else if (res.a0 == INTEL_SIP_SMC_STATUS_OK) { cb_data->status = BIT(SVC_STATUS_COMPLETED); + cb_data->kaddr2 = (res.a2) ? + svc_pa_to_va(res.a2) : NULL; + cb_data->kaddr3 = (res.a3) ? &res.a3 : NULL; + } else { + pr_err("%s: poll status error\n", __func__); + cb_data->kaddr1 = &res.a1; + cb_data->kaddr2 = (res.a2) ? + svc_pa_to_va(res.a2) : NULL; + cb_data->kaddr3 = (res.a3) ? &res.a3 : NULL; + cb_data->status = BIT(SVC_STATUS_ERROR); + } p_data->chan->scl->receive_cb(p_data->chan->scl, cb_data); } @@ -298,6 +320,7 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, case COMMAND_RECONFIG: case COMMAND_RSU_UPDATE: case COMMAND_RSU_NOTIFY: + case COMMAND_POLL_SERVICE_STATUS: cb_data->status = BIT(SVC_STATUS_OK); break; case COMMAND_RECONFIG_DATA_SUBMIT: @@ -430,6 +453,14 @@ static int svc_normal_to_secure_thread(void *data) a1 = 0; a2 = 0; break; + /* for polling */ + case COMMAND_POLL_SERVICE_STATUS: + a0 = INTEL_SIP_SMC_SERVICE_COMPLETED; + a1 = (unsigned long)pdata->paddr; + a2 = (unsigned long)pdata->size; + + break; + default: pr_warn("it shouldn't happen\n"); break; @@ -470,6 +501,7 @@ static int svc_normal_to_secure_thread(void *data) pdata, cbdata); break; case COMMAND_RECONFIG_STATUS: + case COMMAND_POLL_SERVICE_STATUS: svc_thread_cmd_config_status(ctrl, pdata, cbdata); break; diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index aad497a9ad8b..0de104009566 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -403,6 +403,31 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) #define INTEL_SIP_SMC_RSU_MAX_RETRY \ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_RSU_MAX_RETRY) +/** + * Request INTEL_SIP_SMC_SERVICE_COMPLETED + * Sync call to check if the secure world have completed service request + * or not. + * + * Call register usage: + * a0: INTEL_SIP_SMC_SERVICE_COMPLETED + * a1: this register is optional. If used, it is the physical address for + * secure firmware to put output data + * a2: this register is optional. If used, it is the size of output data + * a3-a7: not used + * + * Return status: + * a0: INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_STATUS_ERROR, + * INTEL_SIP_SMC_REJECTED or INTEL_SIP_SMC_STATUS_BUSY + * a1: mailbox error if a0 is INTEL_SIP_SMC_STATUS_ERROR + * a2: physical address containing the process info + * for FCS certificate -- the data contains the certificate status + * for FCS cryption -- the data contains the actual data size FW processes + * a3: output data size + */ +#define INTEL_SIP_SMC_FUNCID_SERVICE_COMPLETED 30 +#define INTEL_SIP_SMC_SERVICE_COMPLETED \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_SERVICE_COMPLETED) + /** * Request INTEL_SIP_SMC_FIRMWARE_VERSION * diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index 24121f8e0d7c..5d0e814e0c41 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -106,6 +106,9 @@ struct stratix10_svc_chan; * @COMMAND_RSU_DCMF_VERSION: query firmware for the DCMF version, return status * is SVC_STATUS_OK or SVC_STATUS_ERROR * + * @COMMAND_POLL_SERVICE_STATUS: poll if the service request is complete, + * return statis is SVC_STATUS_OK, SVC_STATUS_ERROR or SVC_STATUS_BUSY + * * @COMMAND_FIRMWARE_VERSION: query running firmware version, return status * is SVC_STATUS_OK or SVC_STATUS_ERROR */ @@ -122,6 +125,8 @@ enum stratix10_svc_command_code { COMMAND_RSU_MAX_RETRY, COMMAND_RSU_DCMF_VERSION, COMMAND_FIRMWARE_VERSION, + /* for general status poll */ + COMMAND_POLL_SERVICE_STATUS = 40, }; /** -- cgit v1.2.3 From 4a4709d470e624e65a879b5c430dee5e27e9ac83 Mon Sep 17 00:00:00 2001 From: Ang Tien Sung Date: Mon, 11 Jul 2022 17:31:37 -0500 Subject: firmware: stratix10-svc: add new FCS commands Extending the fpga svc driver to support 6 new FPGA Crypto Service(FCS) commands. We are adding FCS SDOS data encryption and decryption, random number generator, image validation request, reading the data provision and certificate validation. Signed-off-by: Ang Tien Sung Signed-off-by: Dinh Nguyen Link: https://lore.kernel.org/r/20220711223140.2307945-3-dinguyen@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/stratix10-svc.c | 105 +++++++++++++++++-- include/linux/firmware/intel/stratix10-smc.h | 111 +++++++++++++++++++++ .../linux/firmware/intel/stratix10-svc-client.h | 41 +++++++- 3 files changed, 245 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 439a4bdf8207..1cf9f4f47b94 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -99,8 +99,10 @@ struct stratix10_svc_data_mem { /** * struct stratix10_svc_data - service data structure * @chan: service channel - * @paddr: playload physical address - * @size: playload size + * @paddr: physical address of to be processed payload + * @size: to be processed playload size + * @paddr_output: physical address of processed payload + * @size_output: processed payload size * @command: service command requested by client * @flag: configuration type (full or partial) * @arg: args to be passed via registers and not physically mapped buffers @@ -111,6 +113,8 @@ struct stratix10_svc_data { struct stratix10_svc_chan *chan; phys_addr_t paddr; size_t size; + phys_addr_t paddr_output; + size_t size_output; u32 command; u32 flag; u64 arg[3]; @@ -320,7 +324,10 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, case COMMAND_RECONFIG: case COMMAND_RSU_UPDATE: case COMMAND_RSU_NOTIFY: - case COMMAND_POLL_SERVICE_STATUS: + case COMMAND_FCS_REQUEST_SERVICE: + case COMMAND_FCS_SEND_CERTIFICATE: + case COMMAND_FCS_DATA_ENCRYPTION: + case COMMAND_FCS_DATA_DECRYPTION: cb_data->status = BIT(SVC_STATUS_OK); break; case COMMAND_RECONFIG_DATA_SUBMIT: @@ -340,6 +347,14 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, cb_data->kaddr1 = &res.a1; cb_data->kaddr2 = &res.a2; break; + case COMMAND_FCS_RANDOM_NUMBER_GEN: + case COMMAND_FCS_GET_PROVISION_DATA: + case COMMAND_POLL_SERVICE_STATUS: + cb_data->status = BIT(SVC_STATUS_OK); + cb_data->kaddr1 = &res.a1; + cb_data->kaddr2 = svc_pa_to_va(res.a2); + cb_data->kaddr3 = &res.a3; + break; default: pr_warn("it shouldn't happen\n"); break; @@ -366,7 +381,7 @@ static int svc_normal_to_secure_thread(void *data) struct stratix10_svc_data *pdata; struct stratix10_svc_cb_data *cbdata; struct arm_smccc_res res; - unsigned long a0, a1, a2; + unsigned long a0, a1, a2, a3, a4, a5, a6, a7; int ret_fifo = 0; pdata = kmalloc(sizeof(*pdata), GFP_KERNEL); @@ -383,6 +398,11 @@ static int svc_normal_to_secure_thread(void *data) a0 = INTEL_SIP_SMC_FPGA_CONFIG_LOOPBACK; a1 = 0; a2 = 0; + a3 = 0; + a4 = 0; + a5 = 0; + a6 = 0; + a7 = 0; pr_debug("smc_hvc_shm_thread is running\n"); @@ -453,12 +473,50 @@ static int svc_normal_to_secure_thread(void *data) a1 = 0; a2 = 0; break; + + /* for FCS */ + case COMMAND_FCS_DATA_ENCRYPTION: + a0 = INTEL_SIP_SMC_FCS_CRYPTION; + a1 = 1; + a2 = (unsigned long)pdata->paddr; + a3 = (unsigned long)pdata->size; + a4 = (unsigned long)pdata->paddr_output; + a5 = (unsigned long)pdata->size_output; + break; + case COMMAND_FCS_DATA_DECRYPTION: + a0 = INTEL_SIP_SMC_FCS_CRYPTION; + a1 = 0; + a2 = (unsigned long)pdata->paddr; + a3 = (unsigned long)pdata->size; + a4 = (unsigned long)pdata->paddr_output; + a5 = (unsigned long)pdata->size_output; + break; + case COMMAND_FCS_RANDOM_NUMBER_GEN: + a0 = INTEL_SIP_SMC_FCS_RANDOM_NUMBER; + a1 = (unsigned long)pdata->paddr; + a2 = 0; + break; + case COMMAND_FCS_REQUEST_SERVICE: + a0 = INTEL_SIP_SMC_FCS_SERVICE_REQUEST; + a1 = (unsigned long)pdata->paddr; + a2 = (unsigned long)pdata->size; + break; + case COMMAND_FCS_SEND_CERTIFICATE: + a0 = INTEL_SIP_SMC_FCS_SEND_CERTIFICATE; + a1 = (unsigned long)pdata->paddr; + a2 = (unsigned long)pdata->size; + break; + case COMMAND_FCS_GET_PROVISION_DATA: + a0 = INTEL_SIP_SMC_FCS_GET_PROVISION_DATA; + a1 = (unsigned long)pdata->paddr; + a2 = 0; + break; + /* for polling */ case COMMAND_POLL_SERVICE_STATUS: a0 = INTEL_SIP_SMC_SERVICE_COMPLETED; a1 = (unsigned long)pdata->paddr; a2 = (unsigned long)pdata->size; - break; default: @@ -466,10 +524,14 @@ static int svc_normal_to_secure_thread(void *data) break; } pr_debug("%s: before SMC call -- a0=0x%016x a1=0x%016x", - __func__, (unsigned int)a0, (unsigned int)a1); + __func__, + (unsigned int)a0, + (unsigned int)a1); pr_debug(" a2=0x%016x\n", (unsigned int)a2); - - ctrl->invoke_fn(a0, a1, a2, 0, 0, 0, 0, 0, &res); + pr_debug(" a3=0x%016x\n", (unsigned int)a3); + pr_debug(" a4=0x%016x\n", (unsigned int)a4); + pr_debug(" a5=0x%016x\n", (unsigned int)a5); + ctrl->invoke_fn(a0, a1, a2, a3, a4, a5, a6, a7, &res); pr_debug("%s: after SMC call -- res.a0=0x%016x", __func__, (unsigned int)res.a0); @@ -512,6 +574,22 @@ static int svc_normal_to_secure_thread(void *data) break; case INTEL_SIP_SMC_STATUS_REJECTED: pr_debug("%s: STATUS_REJECTED\n", __func__); + /* for FCS */ + switch (pdata->command) { + case COMMAND_FCS_REQUEST_SERVICE: + case COMMAND_FCS_SEND_CERTIFICATE: + case COMMAND_FCS_GET_PROVISION_DATA: + case COMMAND_FCS_DATA_ENCRYPTION: + case COMMAND_FCS_DATA_DECRYPTION: + case COMMAND_FCS_RANDOM_NUMBER_GEN: + cbdata->status = BIT(SVC_STATUS_INVALID_PARAM); + cbdata->kaddr1 = NULL; + cbdata->kaddr2 = NULL; + cbdata->kaddr3 = NULL; + pdata->chan->scl->receive_cb(pdata->chan->scl, + cbdata); + break; + } break; case INTEL_SIP_SMC_STATUS_ERROR: case INTEL_SIP_SMC_RSU_ERROR: @@ -886,8 +964,19 @@ int stratix10_svc_send(struct stratix10_svc_chan *chan, void *msg) list_for_each_entry(p_mem, &svc_data_mem, node) if (p_mem->vaddr == p_msg->payload) { p_data->paddr = p_mem->paddr; + p_data->size = p_msg->payload_length; break; } + if (p_msg->payload_output) { + list_for_each_entry(p_mem, &svc_data_mem, node) + if (p_mem->vaddr == p_msg->payload_output) { + p_data->paddr_output = + p_mem->paddr; + p_data->size_output = + p_msg->payload_length_output; + break; + } + } } p_data->command = p_msg->command; diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index 0de104009566..8c198984a47c 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -445,4 +445,115 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) #define INTEL_SIP_SMC_FIRMWARE_VERSION \ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FIRMWARE_VERSION) +/** + * SMC call protocol for FPGA Crypto Service (FCS) + * FUNCID starts from 90 + */ + +/** + * Request INTEL_SIP_SMC_FCS_RANDOM_NUMBER + * + * Sync call used to query the random number generated by the firmware + * + * Call register usage: + * a0 INTEL_SIP_SMC_FCS_RANDOM_NUMBER + * a1 the physical address for firmware to write generated random data + * a2-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_FCS_ERROR or + * INTEL_SIP_SMC_FCS_REJECTED + * a1 mailbox error + * a2 the physical address of generated random number + * a3 size + */ +#define INTEL_SIP_SMC_FUNCID_FCS_RANDOM_NUMBER 90 +#define INTEL_SIP_SMC_FCS_RANDOM_NUMBER \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_RANDOM_NUMBER) + +/** + * Request INTEL_SIP_SMC_FCS_CRYPTION + * Async call for data encryption and HMAC signature generation, or for + * data decryption and HMAC verification. + * + * Call INTEL_SIP_SMC_SERVICE_COMPLETED to get the output encrypted or + * decrypted data + * + * Call register usage: + * a0 INTEL_SIP_SMC_FCS_CRYPTION + * a1 cryption mode (1 for encryption and 0 for decryption) + * a2 physical address which stores to be encrypted or decrypted data + * a3 input data size + * a4 physical address which will hold the encrypted or decrypted output data + * a5 output data size + * a6-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_STATUS_ERROR or + * INTEL_SIP_SMC_STATUS_REJECTED + * a1-3 not used + */ +#define INTEL_SIP_SMC_FUNCID_FCS_CRYPTION 91 +#define INTEL_SIP_SMC_FCS_CRYPTION \ + INTEL_SIP_SMC_STD_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_CRYPTION) + +/** + * Request INTEL_SIP_SMC_FCS_SERVICE_REQUEST + * Async call for authentication service of HPS software + * + * Call register usage: + * a0 INTEL_SIP_SMC_FCS_SERVICE_REQUEST + * a1 the physical address of data block + * a2 size of data block + * a3-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_ERROR or + * INTEL_SIP_SMC_REJECTED + * a1-a3 not used + */ +#define INTEL_SIP_SMC_FUNCID_FCS_SERVICE_REQUEST 92 +#define INTEL_SIP_SMC_FCS_SERVICE_REQUEST \ + INTEL_SIP_SMC_STD_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_SERVICE_REQUEST) + +/** + * Request INTEL_SIP_SMC_FUNCID_FCS_SEND_CERTIFICATE + * Sync call to send a signed certificate + * + * Call register usage: + * a0 INTEL_SIP_SMC_FCS_SEND_CERTIFICATE + * a1 the physical address of CERTIFICATE block + * a2 size of data block + * a3-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK or INTEL_SIP_SMC_FCS_REJECTED + * a1-a3 not used + */ +#define INTEL_SIP_SMC_FUNCID_FCS_SEND_CERTIFICATE 93 +#define INTEL_SIP_SMC_FCS_SEND_CERTIFICATE \ + INTEL_SIP_SMC_STD_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_SEND_CERTIFICATE) + +/** + * Request INTEL_SIP_SMC_FCS_GET_PROVISION_DATA + * Sync call to dump all the fuses and key hashes + * + * Call register usage: + * a0 INTEL_SIP_SMC_FCS_GET_PROVISION_DATA + * a1 the physical address for firmware to write structure of fuse and + * key hashes + * a2-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_FCS_ERROR or + * INTEL_SIP_SMC_FCS_REJECTED + * a1 mailbox error + * a2 physical address for the structure of fuse and key hashes + * a3 the size of structure + * + */ +#define INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA 94 +#define INTEL_SIP_SMC_FCS_GET_PROVISION_DATA \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA) + #endif diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index 5d0e814e0c41..677720792259 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -50,8 +50,8 @@ #define SVC_STATUS_BUSY 4 #define SVC_STATUS_ERROR 5 #define SVC_STATUS_NO_SUPPORT 6 - -/* +#define SVC_STATUS_INVALID_PARAM 7 +/** * Flag bit for COMMAND_RECONFIG * * COMMAND_RECONFIG_FLAG_PARTIAL: @@ -67,6 +67,8 @@ #define SVC_RECONFIG_REQUEST_TIMEOUT_MS 300 #define SVC_RECONFIG_BUFFER_TIMEOUT_MS 720 #define SVC_RSU_REQUEST_TIMEOUT_MS 300 +#define SVC_FCS_REQUEST_TIMEOUT_MS 2000 +#define SVC_COMPLETED_TIMEOUT_MS 30000 struct stratix10_svc_chan; @@ -111,20 +113,47 @@ struct stratix10_svc_chan; * * @COMMAND_FIRMWARE_VERSION: query running firmware version, return status * is SVC_STATUS_OK or SVC_STATUS_ERROR + * + * @COMMAND_FCS_REQUEST_SERVICE: request validation of image from firmware, + * return status is SVC_STATUS_OK, SVC_STATUS_INVALID_PARAM + * + * @COMMAND_FCS_SEND_CERTIFICATE: send a certificate, return status is + * SVC_STATUS_OK, SVC_STATUS_INVALID_PARAM, SVC_STATUS_ERROR + * + * @COMMAND_FCS_GET_PROVISION_DATA: read the provisioning data, return status is + * SVC_STATUS_OK, SVC_STATUS_INVALID_PARAM, SVC_STATUS_ERROR + * + * @COMMAND_FCS_DATA_ENCRYPTION: encrypt the data, return status is + * SVC_STATUS_OK, SVC_STATUS_INVALID_PARAM, SVC_STATUS_ERROR + * + * @COMMAND_FCS_DATA_DECRYPTION: decrypt the data, return status is + * SVC_STATUS_OK, SVC_STATUS_INVALID_PARAM, SVC_STATUS_ERROR + * + * @COMMAND_FCS_RANDOM_NUMBER_GEN: generate a random number, return status + * is SVC_STATUS_OK, SVC_STATUS_ERROR */ enum stratix10_svc_command_code { + /* for FPGA */ COMMAND_NOOP = 0, COMMAND_RECONFIG, COMMAND_RECONFIG_DATA_SUBMIT, COMMAND_RECONFIG_DATA_CLAIM, COMMAND_RECONFIG_STATUS, - COMMAND_RSU_STATUS, + /* for RSU */ + COMMAND_RSU_STATUS = 10, COMMAND_RSU_UPDATE, COMMAND_RSU_NOTIFY, COMMAND_RSU_RETRY, COMMAND_RSU_MAX_RETRY, COMMAND_RSU_DCMF_VERSION, COMMAND_FIRMWARE_VERSION, + /* for FCS */ + COMMAND_FCS_REQUEST_SERVICE = 20, + COMMAND_FCS_SEND_CERTIFICATE, + COMMAND_FCS_GET_PROVISION_DATA, + COMMAND_FCS_DATA_ENCRYPTION, + COMMAND_FCS_DATA_DECRYPTION, + COMMAND_FCS_RANDOM_NUMBER_GEN, /* for general status poll */ COMMAND_POLL_SERVICE_STATUS = 40, }; @@ -132,13 +161,17 @@ enum stratix10_svc_command_code { /** * struct stratix10_svc_client_msg - message sent by client to service * @payload: starting address of data need be processed - * @payload_length: data size in bytes + * @payload_length: to be processed data size in bytes + * @payload_output: starting address of processed data + * @payload_length_output: processed data size in bytes * @command: service command * @arg: args to be passed via registers and not physically mapped buffers */ struct stratix10_svc_client_msg { void *payload; size_t payload_length; + void *payload_output; + size_t payload_length_output; enum stratix10_svc_command_code command; u64 arg[3]; }; -- cgit v1.2.3 From 1b4394c5d731593063f53df9d72467335c3b0367 Mon Sep 17 00:00:00 2001 From: Kah Jing Lee Date: Mon, 11 Jul 2022 17:31:39 -0500 Subject: firmware: stratix10-svc: extend svc to support RSU feature Extend Intel Stratix10 service layer driver to support new RSU DCMF status reporting. The status of each DCMF is reported. The currently used DCMF is used as reference, while the other three are compared against it to determine if they are corrupted. DCMF = Decision Configuration Management Firmware RSU = Remote System Update Signed-off-by: Radu Bacrau Signed-off-by: Ang Tien Sung Signed-off-by: Kah Jing Lee Signed-off-by: Dinh Nguyen Link: https://lore.kernel.org/r/20220711223140.2307945-5-dinguyen@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/stratix10-svc.c | 20 ++++++++++++-------- include/linux/firmware/intel/stratix10-smc.h | 21 +++++++++++++++++++++ include/linux/firmware/intel/stratix10-svc-client.h | 4 ++++ 3 files changed, 37 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 1cf9f4f47b94..7fea94fe2ca0 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -338,6 +338,7 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, break; case COMMAND_RSU_RETRY: case COMMAND_RSU_MAX_RETRY: + case COMMAND_RSU_DCMF_STATUS: case COMMAND_FIRMWARE_VERSION: cb_data->status = BIT(SVC_STATUS_OK); cb_data->kaddr1 = &res.a1; @@ -518,7 +519,11 @@ static int svc_normal_to_secure_thread(void *data) a1 = (unsigned long)pdata->paddr; a2 = (unsigned long)pdata->size; break; - + case COMMAND_RSU_DCMF_STATUS: + a0 = INTEL_SIP_SMC_RSU_DCMF_STATUS; + a1 = 0; + a2 = 0; + break; default: pr_warn("it shouldn't happen\n"); break; @@ -596,8 +601,9 @@ static int svc_normal_to_secure_thread(void *data) pr_err("%s: STATUS_ERROR\n", __func__); cbdata->status = BIT(SVC_STATUS_ERROR); cbdata->kaddr1 = &res.a1; - cbdata->kaddr2 = NULL; - cbdata->kaddr3 = NULL; + cbdata->kaddr2 = (res.a2) ? + svc_pa_to_va(res.a2) : NULL; + cbdata->kaddr3 = (res.a3) ? &res.a3 : NULL; pdata->chan->scl->receive_cb(pdata->chan->scl, cbdata); break; default: @@ -605,12 +611,10 @@ static int svc_normal_to_secure_thread(void *data) /* * be compatible with older version firmware which - * doesn't support RSU notify or retry + * doesn't support newer RSU commands */ - if ((pdata->command == COMMAND_RSU_RETRY) || - (pdata->command == COMMAND_RSU_MAX_RETRY) || - (pdata->command == COMMAND_RSU_NOTIFY) || - (pdata->command == COMMAND_FIRMWARE_VERSION)) { + if ((pdata->command != COMMAND_RSU_UPDATE) && + (pdata->command != COMMAND_RSU_STATUS)) { cbdata->status = BIT(SVC_STATUS_NO_SUPPORT); cbdata->kaddr1 = NULL; diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index 8c198984a47c..8fc4aa450517 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -403,6 +403,27 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) #define INTEL_SIP_SMC_RSU_MAX_RETRY \ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_RSU_MAX_RETRY) +/** + * Request INTEL_SIP_SMC_RSU_DCMF_STATUS + * + * Sync call used by service driver at EL1 to query DCMF status from FW + * + * Call register usage: + * a0 INTEL_SIP_SMC_RSU_DCMF_STATUS + * a1-7 not used + * + * Return status + * a0 INTEL_SIP_SMC_STATUS_OK + * a1 dcmf3 | dcmf2 | dcmf1 | dcmf0 + * + * Or + * + * a0 INTEL_SIP_SMC_RSU_ERROR + */ +#define INTEL_SIP_SMC_FUNCID_RSU_DCMF_STATUS 20 +#define INTEL_SIP_SMC_RSU_DCMF_STATUS \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_RSU_DCMF_STATUS) + /** * Request INTEL_SIP_SMC_SERVICE_COMPLETED * Sync call to check if the secure world have completed service request diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index 677720792259..82a20e62f15f 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -114,6 +114,9 @@ struct stratix10_svc_chan; * @COMMAND_FIRMWARE_VERSION: query running firmware version, return status * is SVC_STATUS_OK or SVC_STATUS_ERROR * + * @COMMAND_RSU_DCMF_STATUS: query firmware for the DCMF status + * return status is SVC_STATUS_OK or SVC_STATUS_ERROR + * * @COMMAND_FCS_REQUEST_SERVICE: request validation of image from firmware, * return status is SVC_STATUS_OK, SVC_STATUS_INVALID_PARAM * @@ -146,6 +149,7 @@ enum stratix10_svc_command_code { COMMAND_RSU_RETRY, COMMAND_RSU_MAX_RETRY, COMMAND_RSU_DCMF_VERSION, + COMMAND_RSU_DCMF_STATUS, COMMAND_FIRMWARE_VERSION, /* for FCS */ COMMAND_FCS_REQUEST_SERVICE = 20, -- cgit v1.2.3 From 7935e899b35c93faa26e1d272a51b3d9cb39f23f Mon Sep 17 00:00:00 2001 From: Ang Tien Sung Date: Mon, 11 Jul 2022 17:31:40 -0500 Subject: firmware: stratix10-svc: To support a command ATF Get Version We are to support a new SMC Command of hexadecimal 0x200 that returns the ATF Firmware major and minor version. Signed-off-by: Ang Tien Sung Signed-off-by: Dinh Nguyen Link: https://lore.kernel.org/r/20220711223140.2307945-6-dinguyen@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/stratix10-svc.c | 10 ++++++++++ include/linux/firmware/intel/stratix10-smc.h | 18 ++++++++++++++++++ include/linux/firmware/intel/stratix10-svc-client.h | 5 +++++ 3 files changed, 33 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 7fea94fe2ca0..b4081f4d88a3 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -343,6 +343,11 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, cb_data->status = BIT(SVC_STATUS_OK); cb_data->kaddr1 = &res.a1; break; + case COMMAND_SMC_SVC_VERSION: + cb_data->status = BIT(SVC_STATUS_OK); + cb_data->kaddr1 = &res.a1; + cb_data->kaddr2 = &res.a2; + break; case COMMAND_RSU_DCMF_VERSION: cb_data->status = BIT(SVC_STATUS_OK); cb_data->kaddr1 = &res.a1; @@ -524,6 +529,11 @@ static int svc_normal_to_secure_thread(void *data) a1 = 0; a2 = 0; break; + case COMMAND_SMC_SVC_VERSION: + a0 = INTEL_SIP_SMC_SVC_VERSION; + a1 = 0; + a2 = 0; + break; default: pr_warn("it shouldn't happen\n"); break; diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index 8fc4aa450517..a718f853d457 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -466,6 +466,24 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) #define INTEL_SIP_SMC_FIRMWARE_VERSION \ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FIRMWARE_VERSION) +/** + * Request INTEL_SIP_SMC_SVC_VERSION + * + * Sync call used to query the SIP SMC API Version + * + * Call register usage: + * a0 INTEL_SIP_SMC_SVC_VERSION + * a1-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK + * a1 Major + * a2 Minor + */ +#define INTEL_SIP_SMC_SVC_FUNCID_VERSION 512 +#define INTEL_SIP_SMC_SVC_VERSION \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_SVC_FUNCID_VERSION) + /** * SMC call protocol for FPGA Crypto Service (FCS) * FUNCID starts from 90 diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index 82a20e62f15f..a776a787ac75 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -114,6 +114,9 @@ struct stratix10_svc_chan; * @COMMAND_FIRMWARE_VERSION: query running firmware version, return status * is SVC_STATUS_OK or SVC_STATUS_ERROR * + * @COMMAND_SMC_SVC_VERSION: Non-mailbox SMC SVC API Version, + * return status is SVC_STATUS_OK + * * @COMMAND_RSU_DCMF_STATUS: query firmware for the DCMF status * return status is SVC_STATUS_OK or SVC_STATUS_ERROR * @@ -160,6 +163,8 @@ enum stratix10_svc_command_code { COMMAND_FCS_RANDOM_NUMBER_GEN, /* for general status poll */ COMMAND_POLL_SERVICE_STATUS = 40, + /* Non-mailbox SMC Call */ + COMMAND_SMC_SVC_VERSION = 200, }; /** -- cgit v1.2.3 From 900d156bac2bc474cf7c7bee4efbc6c83ec5ae58 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 13 Jul 2022 07:53:17 +0200 Subject: block: remove bdevname Replace the remaining calls of bdevname with snprintf using the %pg format specifier. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Reviewed-by: Chaitanya Kulkarni Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20220713055317.1888500-10-hch@lst.de Signed-off-by: Jens Axboe --- block/genhd.c | 23 ----------------------- drivers/md/md.c | 2 +- drivers/md/raid1.c | 2 +- drivers/md/raid10.c | 2 +- fs/ext4/mmp.c | 3 ++- fs/jbd2/journal.c | 6 ++++-- include/linux/blkdev.h | 1 - kernel/trace/blktrace.c | 4 ++-- 8 files changed, 11 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/block/genhd.c b/block/genhd.c index 9d30f159c59a..44dfcf67ed96 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -101,29 +101,6 @@ bool set_capacity_and_notify(struct gendisk *disk, sector_t size) } EXPORT_SYMBOL_GPL(set_capacity_and_notify); -/* - * Format the device name of the indicated block device into the supplied buffer - * and return a pointer to that same buffer for convenience. - * - * Note: do not use this in new code, use the %pg specifier to sprintf and - * printk insted. - */ -const char *bdevname(struct block_device *bdev, char *buf) -{ - struct gendisk *hd = bdev->bd_disk; - int partno = bdev->bd_partno; - - if (!partno) - snprintf(buf, BDEVNAME_SIZE, "%s", hd->disk_name); - else if (isdigit(hd->disk_name[strlen(hd->disk_name)-1])) - snprintf(buf, BDEVNAME_SIZE, "%sp%d", hd->disk_name, partno); - else - snprintf(buf, BDEVNAME_SIZE, "%s%d", hd->disk_name, partno); - - return buf; -} -EXPORT_SYMBOL(bdevname); - static void part_stat_read_all(struct block_device *part, struct disk_stats *stat) { diff --git a/drivers/md/md.c b/drivers/md/md.c index 076255ec9ba1..4be9d8173071 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -2438,7 +2438,7 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev) mdname(mddev), mddev->max_disks); return -EBUSY; } - bdevname(rdev->bdev,b); + snprintf(b, sizeof(b), "%pg", rdev->bdev); strreplace(b, '/', '!'); rdev->mddev = mddev; diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 258d4eb2d63c..65cd90f0b2a8 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1240,7 +1240,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, rcu_read_lock(); rdev = rcu_dereference(conf->mirrors[r1_bio->read_disk].rdev); if (rdev) - bdevname(rdev->bdev, b); + snprintf(b, sizeof(b), "%pg", rdev->bdev); else strcpy(b, "???"); rcu_read_unlock(); diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index d589f823feb1..a7dcb1bf6b0a 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1164,7 +1164,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, disk = r10_bio->devs[slot].devnum; err_rdev = rcu_dereference(conf->mirrors[disk].rdev); if (err_rdev) - bdevname(err_rdev->bdev, b); + snprintf(b, sizeof(b), "%pg", err_rdev->bdev); else { strcpy(b, "???"); /* This never gets dereferenced */ diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c index b7a850b0070b..b221f313ded6 100644 --- a/fs/ext4/mmp.c +++ b/fs/ext4/mmp.c @@ -371,7 +371,8 @@ skip: EXT4_SB(sb)->s_mmp_bh = bh; BUILD_BUG_ON(sizeof(mmp->mmp_bdevname) < BDEVNAME_SIZE); - bdevname(bh->b_bdev, mmp->mmp_bdevname); + snprintf(mmp->mmp_bdevname, sizeof(mmp->mmp_bdevname), + "%pg", bh->b_bdev); /* * Start a kernel thread to update the MMP block periodically. diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index c0cbeeaec2d1..9015f5fa2862 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -1465,7 +1465,8 @@ journal_t *jbd2_journal_init_dev(struct block_device *bdev, if (!journal) return NULL; - bdevname(journal->j_dev, journal->j_devname); + snprintf(journal->j_devname, sizeof(journal->j_devname), + "%pg", journal->j_dev); strreplace(journal->j_devname, '/', '!'); jbd2_stats_proc_init(journal); @@ -1507,7 +1508,8 @@ journal_t *jbd2_journal_init_inode(struct inode *inode) return NULL; journal->j_inode = inode; - bdevname(journal->j_dev, journal->j_devname); + snprintf(journal->j_devname, sizeof(journal->j_devname), + "%pg", journal->j_dev); p = strreplace(journal->j_devname, '/', '!'); sprintf(p, "-%lu", journal->j_inode->i_ino); jbd2_stats_proc_init(journal); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 22c477fadc0f..2775763c51b9 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1457,7 +1457,6 @@ static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) int bdev_read_only(struct block_device *bdev); int set_blocksize(struct block_device *bdev, int size); -const char *bdevname(struct block_device *bdev, char *buffer); int lookup_bdev(const char *pathname, dev_t *dev); void blkdev_show(struct seq_file *seqf, off_t offset); diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index c584effe5fe9..4752bda1b1a0 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -736,12 +736,12 @@ int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) switch (cmd) { case BLKTRACESETUP: - bdevname(bdev, b); + snprintf(b, sizeof(b), "%pg", bdev); ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); break; #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) case BLKTRACESETUP32: - bdevname(bdev, b); + snprintf(b, sizeof(b), "%pg", bdev); ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); break; #endif -- cgit v1.2.3 From ff07a02e9e8e6489db841e0c48a5c78e7e78d572 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:27 -0700 Subject: treewide: Rename enum req_opf into enum req_op The type name enum req_opf is misleading since it suggests that values of this type include both an operation type and flags. Since values of this type represent an operation only, change the type name into enum req_op. Convert the enum req_op documentation into kernel-doc format. Move a few definitions such that the enum req_op documentation occurs just above the enum req_op definition. The name "req_opf" was introduced by commit ef295ecf090d ("block: better op and flags encoding"). Cc: Christoph Hellwig Cc: Ming Lei Cc: Hannes Reinecke Cc: Damien Le Moal Cc: Johannes Thumshirn Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-2-bvanassche@acm.org Signed-off-by: Jens Axboe --- block/blk-zoned.c | 7 +++---- drivers/block/null_blk/main.c | 9 ++++----- drivers/block/null_blk/null_blk.h | 12 +++++------- drivers/block/null_blk/trace.h | 2 +- drivers/block/null_blk/zoned.c | 4 ++-- drivers/md/dm-integrity.c | 2 +- drivers/nvme/target/zns.c | 4 ++-- drivers/scsi/sd_zbc.c | 2 +- drivers/ufs/core/ufshpb.c | 5 ++--- fs/zonefs/super.c | 5 ++--- fs/zonefs/trace.h | 2 +- include/linux/blk_types.h | 16 ++++++++-------- include/linux/blkdev.h | 2 +- 13 files changed, 33 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 7c017458d5ce..a264621d4905 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -256,9 +256,8 @@ static int blkdev_zone_reset_all(struct block_device *bdev, gfp_t gfp_mask) * The operation to execute on each zone can be a zone reset, open, close * or finish request. */ -int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op, - sector_t sector, sector_t nr_sectors, - gfp_t gfp_mask) +int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, + sector_t sector, sector_t nr_sectors, gfp_t gfp_mask) { struct request_queue *q = bdev_get_queue(bdev); sector_t zone_sectors = bdev_zone_sectors(bdev); @@ -397,7 +396,7 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, void __user *argp = (void __user *)arg; struct request_queue *q; struct blk_zone_range zrange; - enum req_opf op; + enum req_op op; int ret; if (!argp) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 4e03a020ee3c..8b224ede2e33 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1310,7 +1310,7 @@ static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd, } static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd, - enum req_opf op, + enum req_op op, sector_t sector, sector_t nr_sectors) { @@ -1381,9 +1381,8 @@ static inline void nullb_complete_cmd(struct nullb_cmd *cmd) } } -blk_status_t null_process_cmd(struct nullb_cmd *cmd, - enum req_opf op, sector_t sector, - unsigned int nr_sectors) +blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op, + sector_t sector, unsigned int nr_sectors) { struct nullb_device *dev = cmd->nq->dev; blk_status_t ret; @@ -1401,7 +1400,7 @@ blk_status_t null_process_cmd(struct nullb_cmd *cmd, } static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector, - sector_t nr_sectors, enum req_opf op) + sector_t nr_sectors, enum req_op op) { struct nullb_device *dev = cmd->nq->dev; struct nullb *nullb = dev->nullb; diff --git a/drivers/block/null_blk/null_blk.h b/drivers/block/null_blk/null_blk.h index 8359b43842f2..6fbf0a1b2622 100644 --- a/drivers/block/null_blk/null_blk.h +++ b/drivers/block/null_blk/null_blk.h @@ -136,9 +136,8 @@ struct nullb { blk_status_t null_handle_discard(struct nullb_device *dev, sector_t sector, sector_t nr_sectors); -blk_status_t null_process_cmd(struct nullb_cmd *cmd, - enum req_opf op, sector_t sector, - unsigned int nr_sectors); +blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op, + sector_t sector, unsigned int nr_sectors); #ifdef CONFIG_BLK_DEV_ZONED int null_init_zoned_dev(struct nullb_device *dev, struct request_queue *q); @@ -146,9 +145,8 @@ int null_register_zoned_dev(struct nullb *nullb); void null_free_zoned_dev(struct nullb_device *dev); int null_report_zones(struct gendisk *disk, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data); -blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, - enum req_opf op, sector_t sector, - sector_t nr_sectors); +blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_op op, + sector_t sector, sector_t nr_sectors); size_t null_zone_valid_read_len(struct nullb *nullb, sector_t sector, unsigned int len); #else @@ -164,7 +162,7 @@ static inline int null_register_zoned_dev(struct nullb *nullb) } static inline void null_free_zoned_dev(struct nullb_device *dev) {} static inline blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, - enum req_opf op, sector_t sector, sector_t nr_sectors) + enum req_op op, sector_t sector, sector_t nr_sectors) { return BLK_STS_NOTSUPP; } diff --git a/drivers/block/null_blk/trace.h b/drivers/block/null_blk/trace.h index 86d6c12c603c..6b2b370e786f 100644 --- a/drivers/block/null_blk/trace.h +++ b/drivers/block/null_blk/trace.h @@ -36,7 +36,7 @@ TRACE_EVENT(nullb_zone_op, TP_ARGS(cmd, zone_no, zone_cond), TP_STRUCT__entry( __array(char, disk, DISK_NAME_LEN) - __field(enum req_opf, op) + __field(enum req_op, op) __field(unsigned int, zone_no) __field(unsigned int, zone_cond) ), diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 64b06caab984..55a69e48ef8b 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -600,7 +600,7 @@ static blk_status_t null_reset_zone(struct nullb_device *dev, return BLK_STS_OK; } -static blk_status_t null_zone_mgmt(struct nullb_cmd *cmd, enum req_opf op, +static blk_status_t null_zone_mgmt(struct nullb_cmd *cmd, enum req_op op, sector_t sector) { struct nullb_device *dev = cmd->nq->dev; @@ -653,7 +653,7 @@ static blk_status_t null_zone_mgmt(struct nullb_cmd *cmd, enum req_opf op, return ret; } -blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_opf op, +blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_op op, sector_t sector, sector_t nr_sectors) { struct nullb_device *dev; diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 3d5a0ce123c9..148978ad03a8 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -298,7 +298,7 @@ struct dm_integrity_io { struct work_struct work; struct dm_integrity_c *ic; - enum req_opf op; + enum req_op op; bool fua; struct dm_integrity_range range; diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index c0ee21fcab81..b233c0943fec 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -308,7 +308,7 @@ void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req) queue_work(zbd_wq, &req->z.zmgmt_work); } -static inline enum req_opf zsa_req_op(u8 zsa) +static inline enum req_op zsa_req_op(u8 zsa) { switch (zsa) { case NVME_ZONE_OPEN: @@ -465,7 +465,7 @@ static void nvmet_bdev_zmgmt_send_work(struct work_struct *w) { struct nvmet_req *req = container_of(w, struct nvmet_req, z.zmgmt_work); sector_t sect = nvmet_lba_to_sect(req->ns, req->cmd->zms.slba); - enum req_opf op = zsa_req_op(req->cmd->zms.zsa); + enum req_op op = zsa_req_op(req->cmd->zms.zsa); struct block_device *bdev = req->ns->bdev; sector_t zone_sectors = bdev_zone_sectors(bdev); u16 status = NVME_SC_SUCCESS; diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index b8c97456506a..bd15624c6322 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -529,7 +529,7 @@ static unsigned int sd_zbc_zone_wp_update(struct scsi_cmnd *cmd, struct request *rq = scsi_cmd_to_rq(cmd); struct scsi_disk *sdkp = scsi_disk(rq->q->disk); unsigned int zno = blk_rq_zone_no(rq); - enum req_opf op = req_op(rq); + enum req_op op = req_op(rq); unsigned long flags; /* diff --git a/drivers/ufs/core/ufshpb.c b/drivers/ufs/core/ufshpb.c index de2bb8401bc4..24f1ee82c215 100644 --- a/drivers/ufs/core/ufshpb.c +++ b/drivers/ufs/core/ufshpb.c @@ -433,9 +433,8 @@ int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) return 0; } -static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb, - int rgn_idx, enum req_opf dir, - bool atomic) +static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb, int rgn_idx, + enum req_op dir, bool atomic) { struct ufshpb_req *rq; struct request *req; diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 9c0eef1ff32a..a221ddb12da6 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -60,8 +60,7 @@ static void zonefs_account_active(struct inode *inode) } } -static inline int zonefs_zone_mgmt(struct inode *inode, - enum req_opf op) +static inline int zonefs_zone_mgmt(struct inode *inode, enum req_op op) { struct zonefs_inode_info *zi = ZONEFS_I(inode); int ret; @@ -525,7 +524,7 @@ static int zonefs_file_truncate(struct inode *inode, loff_t isize) { struct zonefs_inode_info *zi = ZONEFS_I(inode); loff_t old_isize; - enum req_opf op; + enum req_op op; int ret = 0; /* diff --git a/fs/zonefs/trace.h b/fs/zonefs/trace.h index f369d7d50303..21501da764bd 100644 --- a/fs/zonefs/trace.h +++ b/fs/zonefs/trace.h @@ -20,7 +20,7 @@ #define show_dev(dev) MAJOR(dev), MINOR(dev) TRACE_EVENT(zonefs_zone_mgmt, - TP_PROTO(struct inode *inode, enum req_opf op), + TP_PROTO(struct inode *inode, enum req_op op), TP_ARGS(inode, op), TP_STRUCT__entry( __field(dev_t, dev) diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index a24d4078fb21..0e6a2af7ed3d 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -337,8 +337,12 @@ enum { typedef __u32 __bitwise blk_mq_req_flags_t; -/* - * Operations and flags common to the bio and request structures. +#define REQ_OP_BITS 8 +#define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1) +#define REQ_FLAG_BITS 24 + +/** + * enum req_op - Operations common to the bio and request structures. * We use 8 bits for encoding the operation, and the remaining 24 for flags. * * The least significant bit of the operation number indicates the data @@ -350,11 +354,7 @@ typedef __u32 __bitwise blk_mq_req_flags_t; * If a operation does not transfer data the least significant bit has no * meaning. */ -#define REQ_OP_BITS 8 -#define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1) -#define REQ_FLAG_BITS 24 - -enum req_opf { +enum req_op { /* read sectors from the device */ REQ_OP_READ = 0, /* write sectors to the device */ @@ -509,7 +509,7 @@ static inline bool op_is_discard(unsigned int op) * due to its different handling in the block layer and device response in * case of command failure. */ -static inline bool op_is_zone_mgmt(enum req_opf op) +static inline bool op_is_zone_mgmt(enum req_op op) { switch (op & REQ_OP_MASK) { case REQ_OP_ZONE_RESET: diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2775763c51b9..ec072a5129bf 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -322,7 +322,7 @@ void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model); int blkdev_report_zones(struct block_device *bdev, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data); unsigned int bdev_nr_zones(struct block_device *bdev); -extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op, +extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, sector_t sectors, sector_t nr_sectors, gfp_t gfp_mask); int blk_revalidate_disk_zones(struct gendisk *disk, -- cgit v1.2.3 From 77e7ffd7ad3952909be6a9c599b7d164c8866fec Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:28 -0700 Subject: block: Use enum req_op where appropriate Change the type of the arguments that are used to pass a REQ_OP_* value from int or unsigned int into enum req_op to improve static type checking. Cc: Christoph Hellwig Cc: Ming Lei Cc: Hannes Reinecke Cc: Damien Le Moal Cc: Johannes Thumshirn Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-3-bvanassche@acm.org Signed-off-by: Jens Axboe --- block/blk-core.c | 6 +++--- block/blk-mq-debugfs.c | 2 +- block/blk-throttle.c | 7 ++++--- block/blk-wbt.c | 2 +- block/blk.h | 2 +- include/linux/blk_types.h | 2 +- include/linux/blkdev.h | 6 +++--- 7 files changed, 14 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 8365996a8ef8..67b8bcfa27f0 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -136,7 +136,7 @@ static const char *const blk_op_name[] = { * string format. Useful in the debugging and tracing bio or request. For * invalid REQ_OP_XXX it returns string "UNKNOWN". */ -inline const char *blk_op_str(unsigned int op) +inline const char *blk_op_str(enum req_op op) { const char *op_str = "UNKNOWN"; @@ -953,7 +953,7 @@ again: } unsigned long bdev_start_io_acct(struct block_device *bdev, - unsigned int sectors, unsigned int op, + unsigned int sectors, enum req_op op, unsigned long start_time) { const int sgrp = op_stat_group(op); @@ -994,7 +994,7 @@ unsigned long bio_start_io_acct(struct bio *bio) } EXPORT_SYMBOL_GPL(bio_start_io_acct); -void bdev_end_io_acct(struct block_device *bdev, unsigned int op, +void bdev_end_io_acct(struct block_device *bdev, enum req_op op, unsigned long start_time) { const int sgrp = op_stat_group(op); diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 7ee1b13380d0..6cc2411e2d26 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -304,7 +304,7 @@ static const char *blk_mq_rq_state_name(enum mq_rq_state rq_state) int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq) { const struct blk_mq_ops *const mq_ops = rq->q->mq_ops; - const unsigned int op = req_op(rq); + const enum req_op op = req_op(rq); const char *op_str = blk_op_str(op); seq_printf(m, "%p {.op=", rq); diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 139b2d7a99e2..9f5fe62afff9 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -2203,8 +2203,9 @@ out_unlock: #ifdef CONFIG_BLK_DEV_THROTTLING_LOW static void throtl_track_latency(struct throtl_data *td, sector_t size, - int op, unsigned long time) + enum req_op op, unsigned long time) { + const bool rw = op_is_write(op); struct latency_bucket *latency; int index; @@ -2215,10 +2216,10 @@ static void throtl_track_latency(struct throtl_data *td, sector_t size, index = request_bucket_index(size); - latency = get_cpu_ptr(td->latency_buckets[op]); + latency = get_cpu_ptr(td->latency_buckets[rw]); latency[index].total_latency += time; latency[index].samples++; - put_cpu_ptr(td->latency_buckets[op]); + put_cpu_ptr(td->latency_buckets[rw]); } void blk_throtl_stat_add(struct request *rq, u64 time_ns) diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 0c119be0e813..7bf09ae06577 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -670,7 +670,7 @@ u64 wbt_default_latency_nsec(struct request_queue *q) static int wbt_data_dir(const struct request *rq) { - const int op = req_op(rq); + const enum req_op op = req_op(rq); if (op == REQ_OP_READ) return READ; diff --git a/block/blk.h b/block/blk.h index b71e22c97d77..c4b084bfe87c 100644 --- a/block/blk.h +++ b/block/blk.h @@ -160,7 +160,7 @@ static inline bool blk_discard_mergable(struct request *req) } static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q, - int op) + enum req_op op) { if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) return min(q->limits.max_discard_sectors, diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 0e6a2af7ed3d..cce8768bc00b 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -522,7 +522,7 @@ static inline bool op_is_zone_mgmt(enum req_op op) } } -static inline int op_stat_group(unsigned int op) +static inline int op_stat_group(enum req_op op) { if (op_is_discard(op)) return STAT_DISCARD; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index ec072a5129bf..2f13f0062192 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -872,7 +872,7 @@ extern void blk_queue_exit(struct request_queue *q); extern void blk_sync_queue(struct request_queue *q); /* Helper to convert REQ_OP_XXX to its string format XXX */ -extern const char *blk_op_str(unsigned int op); +extern const char *blk_op_str(enum req_op op); int blk_status_to_errno(blk_status_t status); blk_status_t errno_to_blk_status(int errno); @@ -1434,9 +1434,9 @@ static inline void blk_wake_io_task(struct task_struct *waiter) } unsigned long bdev_start_io_acct(struct block_device *bdev, - unsigned int sectors, unsigned int op, + unsigned int sectors, enum req_op op, unsigned long start_time); -void bdev_end_io_acct(struct block_device *bdev, unsigned int op, +void bdev_end_io_acct(struct block_device *bdev, enum req_op op, unsigned long start_time); void bio_start_io_acct_time(struct bio *bio, unsigned long start_time); -- cgit v1.2.3 From 86947df3a9236481276e8baadde50a403b02b4d4 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:29 -0700 Subject: block: Change the type of the last .rw_page() argument All .rw_page() callers pass an enum req_op value as last argument. Make this explicit by changing the type of the last argument into enum req_op. See also commit 3f289dcb4b26 ("block: make bdev_ops->rw_page() take a REQ_OP instead of bool"). Cc: Tejun Heo Cc: Minchan Kim Cc: Dan Williams Cc: Hannes Reinecke Cc: Damien Le Moal Cc: Johannes Thumshirn Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-4-bvanassche@acm.org Signed-off-by: Jens Axboe --- drivers/block/brd.c | 2 +- drivers/block/zram/zram_drv.c | 2 +- drivers/nvdimm/btt.c | 2 +- drivers/nvdimm/pmem.c | 2 +- include/linux/blkdev.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/brd.c b/drivers/block/brd.c index 9e26d5e769f3..7b82876af36e 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -310,7 +310,7 @@ static void brd_submit_bio(struct bio *bio) } static int brd_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, unsigned int op) + struct page *page, enum req_op op) { struct brd_device *brd = bdev->bd_disk->private_data; int err; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index e5233c911e43..a35b86c58aa2 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1631,7 +1631,7 @@ static void zram_slot_free_notify(struct block_device *bdev, } static int zram_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, unsigned int op) + struct page *page, enum req_op op) { int offset, ret; u32 index; diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 5e622c0d4b66..dfbf73145d16 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1483,7 +1483,7 @@ static void btt_submit_bio(struct bio *bio) } static int btt_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, unsigned int op) + struct page *page, enum req_op op) { struct btt *btt = bdev->bd_disk->private_data; int rc; diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index a72b81fa3242..f36efcc11f67 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -239,7 +239,7 @@ static void pmem_submit_bio(struct bio *bio) } static int pmem_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, unsigned int op) + struct page *page, enum req_op op) { struct pmem_device *pmem = bdev->bd_disk->private_data; blk_status_t rc; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2f13f0062192..ca2ff113ea00 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1381,7 +1381,7 @@ struct block_device_operations { unsigned int flags); int (*open) (struct block_device *, fmode_t); void (*release) (struct gendisk *, fmode_t); - int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int); + int (*rw_page)(struct block_device *, sector_t, struct page *, enum req_op); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); unsigned int (*check_events) (struct gendisk *disk, -- cgit v1.2.3 From 2d9b02be73ba8efba406b399a722b4e33614dd0e Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:30 -0700 Subject: block: Change the type of req_op() and bio_op() into enum req_op Improve static type checking by changing the type of the value returned by req_op() and bio_op() from unsigned int into enum req_op. Insert 'default: break;' in switch statements on the enum req_op type to prevent that the compiler warns about these switch statements. Cc: Christoph Hellwig Cc: Ming Lei Cc: Hannes Reinecke Cc: Damien Le Moal Cc: Johannes Thumshirn Cc: Tim Waugh Cc: Alasdair Kergon Cc: Mike Snitzer Cc: Mikulas Patocka Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-5-bvanassche@acm.org Signed-off-by: Jens Axboe --- block/blk-merge.c | 2 ++ drivers/block/paride/pd.c | 2 ++ drivers/md/dm.c | 2 ++ include/linux/blk-mq.h | 6 ++++-- include/linux/blk_types.h | 6 ++++-- 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/block/blk-merge.c b/block/blk-merge.c index 5abf5aa5a5f0..de178a8b4c82 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -405,6 +405,8 @@ unsigned int blk_recalc_rq_segments(struct request *rq) return 1; case REQ_OP_WRITE_ZEROES: return 0; + default: + break; } rq_for_each_bvec(bv, rq, iter) diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index c8c14c6f5c3a..f8a75bc90f70 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -501,6 +501,8 @@ static enum action do_pd_io_start(void) return do_pd_read_start(); else return do_pd_write_start(); + default: + break; } return Fail; } diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 33d3799bb66e..6c21922b87d0 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1542,6 +1542,8 @@ static blk_status_t __process_abnormal_io(struct clone_info *ci, case REQ_OP_WRITE_ZEROES: num_bios = ti->num_write_zeroes_bios; break; + default: + break; } /* diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index d74f6a6b7e69..677195de0663 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -198,8 +198,10 @@ struct request { void *end_io_data; }; -#define req_op(req) \ - ((req)->cmd_flags & REQ_OP_MASK) +static inline enum req_op req_op(const struct request *req) +{ + return req->cmd_flags & REQ_OP_MASK; +} static inline bool blk_rq_is_passthrough(struct request *rq) { diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index cce8768bc00b..e66cbe377ae8 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -463,8 +463,10 @@ enum stat_group { NR_STAT_GROUPS }; -#define bio_op(bio) \ - ((bio)->bi_opf & REQ_OP_MASK) +static inline enum req_op bio_op(const struct bio *bio) +{ + return bio->bi_opf & REQ_OP_MASK; +} /* obsolete, don't use in new code */ static inline void bio_set_op_attrs(struct bio *bio, unsigned op, -- cgit v1.2.3 From 342a72a334073f163da924b69c3d3fb4685eb33a Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:31 -0700 Subject: block: Introduce the type blk_opf_t Introduce the type blk_opf_t for the request operation and flags (REQ_OP_* and REQ_*). This type will be used to improve documentation of the block layer code and also to allow sparse to verify whether request flags are used correctly. Cc: Christoph Hellwig Cc: Ming Lei Cc: Hannes Reinecke Cc: Damien Le Moal Cc: Johannes Thumshirn Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-6-bvanassche@acm.org Signed-off-by: Jens Axboe --- include/linux/blk_types.h | 97 +++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index e66cbe377ae8..1ef99790f6ed 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -240,6 +240,8 @@ static inline void bio_issue_init(struct bio_issue *issue, ((u64)size << BIO_ISSUE_SIZE_SHIFT)); } +typedef __u32 __bitwise blk_opf_t; + typedef unsigned int blk_qc_t; #define BLK_QC_T_NONE -1U @@ -250,7 +252,7 @@ typedef unsigned int blk_qc_t; struct bio { struct bio *bi_next; /* request queue link */ struct block_device *bi_bdev; - unsigned int bi_opf; /* bottom bits REQ_OP, top bits + blk_opf_t bi_opf; /* bottom bits REQ_OP, top bits * req_flags. */ unsigned short bi_flags; /* BIO_* below */ @@ -338,7 +340,7 @@ enum { typedef __u32 __bitwise blk_mq_req_flags_t; #define REQ_OP_BITS 8 -#define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1) +#define REQ_OP_MASK (__force blk_opf_t)((1 << REQ_OP_BITS) - 1) #define REQ_FLAG_BITS 24 /** @@ -356,35 +358,35 @@ typedef __u32 __bitwise blk_mq_req_flags_t; */ enum req_op { /* read sectors from the device */ - REQ_OP_READ = 0, + REQ_OP_READ = (__force blk_opf_t)0, /* write sectors to the device */ - REQ_OP_WRITE = 1, + REQ_OP_WRITE = (__force blk_opf_t)1, /* flush the volatile write cache */ - REQ_OP_FLUSH = 2, + REQ_OP_FLUSH = (__force blk_opf_t)2, /* discard sectors */ - REQ_OP_DISCARD = 3, + REQ_OP_DISCARD = (__force blk_opf_t)3, /* securely erase sectors */ - REQ_OP_SECURE_ERASE = 5, + REQ_OP_SECURE_ERASE = (__force blk_opf_t)5, /* write the zero filled sector many times */ - REQ_OP_WRITE_ZEROES = 9, + REQ_OP_WRITE_ZEROES = (__force blk_opf_t)9, /* Open a zone */ - REQ_OP_ZONE_OPEN = 10, + REQ_OP_ZONE_OPEN = (__force blk_opf_t)10, /* Close a zone */ - REQ_OP_ZONE_CLOSE = 11, + REQ_OP_ZONE_CLOSE = (__force blk_opf_t)11, /* Transition a zone to full */ - REQ_OP_ZONE_FINISH = 12, + REQ_OP_ZONE_FINISH = (__force blk_opf_t)12, /* write data at the current zone write pointer */ - REQ_OP_ZONE_APPEND = 13, + REQ_OP_ZONE_APPEND = (__force blk_opf_t)13, /* reset a zone write pointer */ - REQ_OP_ZONE_RESET = 15, + REQ_OP_ZONE_RESET = (__force blk_opf_t)15, /* reset all the zone present on the device */ - REQ_OP_ZONE_RESET_ALL = 17, + REQ_OP_ZONE_RESET_ALL = (__force blk_opf_t)17, /* Driver private requests */ - REQ_OP_DRV_IN = 34, - REQ_OP_DRV_OUT = 35, + REQ_OP_DRV_IN = (__force blk_opf_t)34, + REQ_OP_DRV_OUT = (__force blk_opf_t)35, - REQ_OP_LAST, + REQ_OP_LAST = (__force blk_opf_t)36, }; enum req_flag_bits { @@ -425,28 +427,31 @@ enum req_flag_bits { __REQ_NR_BITS, /* stops here */ }; -#define REQ_FAILFAST_DEV (1ULL << __REQ_FAILFAST_DEV) -#define REQ_FAILFAST_TRANSPORT (1ULL << __REQ_FAILFAST_TRANSPORT) -#define REQ_FAILFAST_DRIVER (1ULL << __REQ_FAILFAST_DRIVER) -#define REQ_SYNC (1ULL << __REQ_SYNC) -#define REQ_META (1ULL << __REQ_META) -#define REQ_PRIO (1ULL << __REQ_PRIO) -#define REQ_NOMERGE (1ULL << __REQ_NOMERGE) -#define REQ_IDLE (1ULL << __REQ_IDLE) -#define REQ_INTEGRITY (1ULL << __REQ_INTEGRITY) -#define REQ_FUA (1ULL << __REQ_FUA) -#define REQ_PREFLUSH (1ULL << __REQ_PREFLUSH) -#define REQ_RAHEAD (1ULL << __REQ_RAHEAD) -#define REQ_BACKGROUND (1ULL << __REQ_BACKGROUND) -#define REQ_NOWAIT (1ULL << __REQ_NOWAIT) -#define REQ_CGROUP_PUNT (1ULL << __REQ_CGROUP_PUNT) - -#define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP) -#define REQ_POLLED (1ULL << __REQ_POLLED) -#define REQ_ALLOC_CACHE (1ULL << __REQ_ALLOC_CACHE) - -#define REQ_DRV (1ULL << __REQ_DRV) -#define REQ_SWAP (1ULL << __REQ_SWAP) +#define REQ_FAILFAST_DEV \ + (__force blk_opf_t)(1ULL << __REQ_FAILFAST_DEV) +#define REQ_FAILFAST_TRANSPORT \ + (__force blk_opf_t)(1ULL << __REQ_FAILFAST_TRANSPORT) +#define REQ_FAILFAST_DRIVER \ + (__force blk_opf_t)(1ULL << __REQ_FAILFAST_DRIVER) +#define REQ_SYNC (__force blk_opf_t)(1ULL << __REQ_SYNC) +#define REQ_META (__force blk_opf_t)(1ULL << __REQ_META) +#define REQ_PRIO (__force blk_opf_t)(1ULL << __REQ_PRIO) +#define REQ_NOMERGE (__force blk_opf_t)(1ULL << __REQ_NOMERGE) +#define REQ_IDLE (__force blk_opf_t)(1ULL << __REQ_IDLE) +#define REQ_INTEGRITY (__force blk_opf_t)(1ULL << __REQ_INTEGRITY) +#define REQ_FUA (__force blk_opf_t)(1ULL << __REQ_FUA) +#define REQ_PREFLUSH (__force blk_opf_t)(1ULL << __REQ_PREFLUSH) +#define REQ_RAHEAD (__force blk_opf_t)(1ULL << __REQ_RAHEAD) +#define REQ_BACKGROUND (__force blk_opf_t)(1ULL << __REQ_BACKGROUND) +#define REQ_NOWAIT (__force blk_opf_t)(1ULL << __REQ_NOWAIT) +#define REQ_CGROUP_PUNT (__force blk_opf_t)(1ULL << __REQ_CGROUP_PUNT) + +#define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP) +#define REQ_POLLED (__force blk_opf_t)(1ULL << __REQ_POLLED) +#define REQ_ALLOC_CACHE (__force blk_opf_t)(1ULL << __REQ_ALLOC_CACHE) + +#define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV) +#define REQ_SWAP (__force blk_opf_t)(1ULL << __REQ_SWAP) #define REQ_FAILFAST_MASK \ (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER) @@ -469,22 +474,22 @@ static inline enum req_op bio_op(const struct bio *bio) } /* obsolete, don't use in new code */ -static inline void bio_set_op_attrs(struct bio *bio, unsigned op, - unsigned op_flags) +static inline void bio_set_op_attrs(struct bio *bio, enum req_op op, + blk_opf_t op_flags) { bio->bi_opf = op | op_flags; } -static inline bool op_is_write(unsigned int op) +static inline bool op_is_write(blk_opf_t op) { - return (op & 1); + return !!(op & (__force blk_opf_t)1); } /* * Check if the bio or request is one that needs special treatment in the * flush state machine. */ -static inline bool op_is_flush(unsigned int op) +static inline bool op_is_flush(blk_opf_t op) { return op & (REQ_FUA | REQ_PREFLUSH); } @@ -494,13 +499,13 @@ static inline bool op_is_flush(unsigned int op) * PREFLUSH flag. Other operations may be marked as synchronous using the * REQ_SYNC flag. */ -static inline bool op_is_sync(unsigned int op) +static inline bool op_is_sync(blk_opf_t op) { return (op & REQ_OP_MASK) == REQ_OP_READ || (op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH)); } -static inline bool op_is_discard(unsigned int op) +static inline bool op_is_discard(blk_opf_t op) { return (op & REQ_OP_MASK) == REQ_OP_DISCARD; } -- cgit v1.2.3 From 16458cf3bd15e5624205df6e8a76b9a5363555f3 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:32 -0700 Subject: block: Use the new blk_opf_t type Use the new blk_opf_t type for arguments and variables that represent request flags or a bitwise combination of a request operation and request flags. Rename the function arguments and also a structure member that hold a request operation and flags from 'rw' into 'opf'. This patch does not change any functionality. Cc: Christoph Hellwig Cc: Ming Lei Cc: Hannes Reinecke Cc: Damien Le Moal Cc: Johannes Thumshirn Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-7-bvanassche@acm.org Signed-off-by: Jens Axboe --- block/bio.c | 10 +++++----- block/blk-cgroup-rwstat.h | 8 ++++---- block/blk-core.c | 2 +- block/blk-flush.c | 6 +++--- block/blk-merge.c | 6 +++--- block/blk-mq-debugfs.c | 4 ++-- block/blk-mq.c | 15 ++++++++------- block/blk-mq.h | 6 +++--- block/blk-wbt.c | 16 ++++++++-------- block/elevator.h | 2 +- block/fops.c | 12 ++++++------ include/linux/bio.h | 10 +++++----- include/linux/blk-mq.h | 6 +++--- include/linux/blkdev.h | 2 +- 14 files changed, 53 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index 888ee81ea303..6f9f883f9a65 100644 --- a/block/bio.c +++ b/block/bio.c @@ -239,7 +239,7 @@ static void bio_free(struct bio *bio) * when IO has completed, or when the bio is released. */ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, - unsigned short max_vecs, unsigned int opf) + unsigned short max_vecs, blk_opf_t opf) { bio->bi_next = NULL; bio->bi_bdev = bdev; @@ -292,7 +292,7 @@ EXPORT_SYMBOL(bio_init); * preserved are the ones that are initialized by bio_alloc_bioset(). See * comment in struct bio. */ -void bio_reset(struct bio *bio, struct block_device *bdev, unsigned int opf) +void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf) { bio_uninit(bio); memset(bio, 0, BIO_RESET_BYTES); @@ -341,7 +341,7 @@ void bio_chain(struct bio *bio, struct bio *parent) EXPORT_SYMBOL(bio_chain); struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, - unsigned int nr_pages, unsigned int opf, gfp_t gfp) + unsigned int nr_pages, blk_opf_t opf, gfp_t gfp) { struct bio *new = bio_alloc(bdev, nr_pages, opf, gfp); @@ -409,7 +409,7 @@ static void punt_bios_to_rescuer(struct bio_set *bs) } static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, - unsigned short nr_vecs, unsigned int opf, gfp_t gfp, + unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp, struct bio_set *bs) { struct bio_alloc_cache *cache; @@ -468,7 +468,7 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, * Returns: Pointer to new bio on success, NULL on failure. */ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, - unsigned int opf, gfp_t gfp_mask, + blk_opf_t opf, gfp_t gfp_mask, struct bio_set *bs) { gfp_t saved_gfp = gfp_mask; diff --git a/block/blk-cgroup-rwstat.h b/block/blk-cgroup-rwstat.h index 9f2723b34b75..022527b0b043 100644 --- a/block/blk-cgroup-rwstat.h +++ b/block/blk-cgroup-rwstat.h @@ -59,20 +59,20 @@ void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, * caller is responsible for synchronizing calls to this function. */ static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat, - unsigned int op, uint64_t val) + blk_opf_t opf, uint64_t val) { struct percpu_counter *cnt; - if (op_is_discard(op)) + if (op_is_discard(opf)) cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD]; - else if (op_is_write(op)) + else if (op_is_write(opf)) cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE]; else cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ]; percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH); - if (op_is_sync(op)) + if (op_is_sync(opf)) cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC]; else cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC]; diff --git a/block/blk-core.c b/block/blk-core.c index 67b8bcfa27f0..123468b9d2e4 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1203,7 +1203,7 @@ EXPORT_SYMBOL_GPL(blk_io_schedule); int __init blk_dev_init(void) { - BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS)); + BUILD_BUG_ON((__force u32)REQ_OP_LAST >= (1 << REQ_OP_BITS)); BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * sizeof_field(struct request, cmd_flags)); BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * diff --git a/block/blk-flush.c b/block/blk-flush.c index c68968724870..d20a0c6b2c66 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -94,7 +94,7 @@ enum { }; static void blk_kick_flush(struct request_queue *q, - struct blk_flush_queue *fq, unsigned int flags); + struct blk_flush_queue *fq, blk_opf_t flags); static inline struct blk_flush_queue * blk_get_flush_queue(struct request_queue *q, struct blk_mq_ctx *ctx) @@ -173,7 +173,7 @@ static void blk_flush_complete_seq(struct request *rq, { struct request_queue *q = rq->q; struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; - unsigned int cmd_flags; + blk_opf_t cmd_flags; BUG_ON(rq->flush.seq & seq); rq->flush.seq |= seq; @@ -290,7 +290,7 @@ bool is_flush_rq(struct request *rq) * */ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, - unsigned int flags) + blk_opf_t flags) { struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; struct request *first_rq = diff --git a/block/blk-merge.c b/block/blk-merge.c index de178a8b4c82..3c3f785f558a 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -712,7 +712,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, */ void blk_rq_set_mixed_merge(struct request *rq) { - unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; + blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK; struct bio *bio; if (rq->rq_flags & RQF_MIXED_MERGE) @@ -928,7 +928,7 @@ enum bio_merge_status { static enum bio_merge_status bio_attempt_back_merge(struct request *req, struct bio *bio, unsigned int nr_segs) { - const int ff = bio->bi_opf & REQ_FAILFAST_MASK; + const blk_opf_t ff = bio->bi_opf & REQ_FAILFAST_MASK; if (!ll_back_merge_fn(req, bio, nr_segs)) return BIO_MERGE_FAILED; @@ -952,7 +952,7 @@ static enum bio_merge_status bio_attempt_back_merge(struct request *req, static enum bio_merge_status bio_attempt_front_merge(struct request *req, struct bio *bio, unsigned int nr_segs) { - const int ff = bio->bi_opf & REQ_FAILFAST_MASK; + const blk_opf_t ff = bio->bi_opf & REQ_FAILFAST_MASK; if (!ll_front_merge_fn(req, bio, nr_segs)) return BIO_MERGE_FAILED; diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 6cc2411e2d26..8559cea7f300 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -313,8 +313,8 @@ int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq) else seq_printf(m, "%s", op_str); seq_puts(m, ", .cmd_flags="); - blk_flags_show(m, rq->cmd_flags & ~REQ_OP_MASK, cmd_flag_name, - ARRAY_SIZE(cmd_flag_name)); + blk_flags_show(m, (__force unsigned int)(rq->cmd_flags & ~REQ_OP_MASK), + cmd_flag_name, ARRAY_SIZE(cmd_flag_name)); seq_puts(m, ", .rq_flags="); blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name, ARRAY_SIZE(rqf_name)); diff --git a/block/blk-mq.c b/block/blk-mq.c index f1b84e20b1a9..d716b7f3763f 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -510,13 +510,13 @@ retry: alloc_time_ns); } -struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, +struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, blk_mq_req_flags_t flags) { struct blk_mq_alloc_data data = { .q = q, .flags = flags, - .cmd_flags = op, + .cmd_flags = opf, .nr_tags = 1, }; struct request *rq; @@ -540,12 +540,12 @@ out_queue_exit: EXPORT_SYMBOL(blk_mq_alloc_request); struct request *blk_mq_alloc_request_hctx(struct request_queue *q, - unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx) + blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx) { struct blk_mq_alloc_data data = { .q = q, .flags = flags, - .cmd_flags = op, + .cmd_flags = opf, .nr_tags = 1, }; u64 alloc_time_ns = 0; @@ -660,7 +660,7 @@ void blk_dump_rq_flags(struct request *rq, char *msg) { printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg, rq->q->disk ? rq->q->disk->disk_name : "?", - (unsigned long long) rq->cmd_flags); + (__force unsigned long long) rq->cmd_flags); printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", (unsigned long long)blk_rq_pos(rq), @@ -713,8 +713,9 @@ static void blk_print_req_error(struct request *req, blk_status_t status) "phys_seg %u prio class %u\n", blk_status_to_str(status), req->q->disk ? req->q->disk->disk_name : "?", - blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)), - req->cmd_flags & ~REQ_OP_MASK, + blk_rq_pos(req), (__force u32)req_op(req), + blk_op_str(req_op(req)), + (__force u32)(req->cmd_flags & ~REQ_OP_MASK), req->nr_phys_segments, IOPRIO_PRIO_CLASS(req->ioprio)); } diff --git a/block/blk-mq.h b/block/blk-mq.h index e694ec67d646..8ca453ac243d 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -86,7 +86,7 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue * return xa_load(&q->hctx_table, q->tag_set->map[type].mq_map[cpu]); } -static inline enum hctx_type blk_mq_get_hctx_type(unsigned int opf) +static inline enum hctx_type blk_mq_get_hctx_type(blk_opf_t opf) { enum hctx_type type = HCTX_TYPE_DEFAULT; @@ -107,7 +107,7 @@ static inline enum hctx_type blk_mq_get_hctx_type(unsigned int opf) * @ctx: software queue cpu ctx */ static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, - unsigned int opf, + blk_opf_t opf, struct blk_mq_ctx *ctx) { return ctx->hctxs[blk_mq_get_hctx_type(opf)]; @@ -152,7 +152,7 @@ struct blk_mq_alloc_data { struct request_queue *q; blk_mq_req_flags_t flags; unsigned int shallow_depth; - unsigned int cmd_flags; + blk_opf_t cmd_flags; req_flags_t rq_flags; /* allocate multiple requests/tags in one go */ diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 7bf09ae06577..f2e4bf1dca47 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -451,7 +451,7 @@ static bool close_io(struct rq_wb *rwb) #define REQ_HIPRIO (REQ_SYNC | REQ_META | REQ_PRIO) -static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) +static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf) { unsigned int limit; @@ -462,7 +462,7 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) if (!rwb_enabled(rwb)) return UINT_MAX; - if ((rw & REQ_OP_MASK) == REQ_OP_DISCARD) + if ((opf & REQ_OP_MASK) == REQ_OP_DISCARD) return rwb->wb_background; /* @@ -473,9 +473,9 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) * the idle limit, or go to normal if we haven't had competing * IO for a bit. */ - if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd()) + if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd()) limit = rwb->rq_depth.max_depth; - else if ((rw & REQ_BACKGROUND) || close_io(rwb)) { + else if ((opf & REQ_BACKGROUND) || close_io(rwb)) { /* * If less than 100ms since we completed unrelated IO, * limit us to half the depth for background writeback. @@ -490,13 +490,13 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) struct wbt_wait_data { struct rq_wb *rwb; enum wbt_flags wb_acct; - unsigned long rw; + blk_opf_t opf; }; static bool wbt_inflight_cb(struct rq_wait *rqw, void *private_data) { struct wbt_wait_data *data = private_data; - return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw)); + return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf)); } static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data) @@ -510,13 +510,13 @@ static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data) * the timer to kick off queuing again. */ static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, - unsigned long rw) + blk_opf_t opf) { struct rq_wait *rqw = get_rq_wait(rwb, wb_acct); struct wbt_wait_data data = { .rwb = rwb, .wb_acct = wb_acct, - .rw = rw, + .opf = opf, }; rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb); diff --git a/block/elevator.h b/block/elevator.h index 16cd8bdedb7e..3f0593b3bf9d 100644 --- a/block/elevator.h +++ b/block/elevator.h @@ -34,7 +34,7 @@ struct elevator_mq_ops { int (*request_merge)(struct request_queue *q, struct request **, struct bio *); void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); void (*requests_merged)(struct request_queue *, struct request *, struct request *); - void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *); + void (*limit_depth)(blk_opf_t, struct blk_mq_alloc_data *); void (*prepare_request)(struct request *); void (*finish_request)(struct request *); void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); diff --git a/block/fops.c b/block/fops.c index 86d3cab9bf93..29066ac5a2fa 100644 --- a/block/fops.c +++ b/block/fops.c @@ -32,14 +32,14 @@ static int blkdev_get_block(struct inode *inode, sector_t iblock, return 0; } -static unsigned int dio_bio_write_op(struct kiocb *iocb) +static blk_opf_t dio_bio_write_op(struct kiocb *iocb) { - unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; + blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; /* avoid the need for a I/O completion work item */ if (iocb->ki_flags & IOCB_DSYNC) - op |= REQ_FUA; - return op; + opf |= REQ_FUA; + return opf; } static bool blkdev_dio_unaligned(struct block_device *bdev, loff_t pos, @@ -175,7 +175,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, struct blkdev_dio *dio; struct bio *bio; bool is_read = (iov_iter_rw(iter) == READ), is_sync; - unsigned int opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb); + blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb); loff_t pos = iocb->ki_pos; int ret = 0; @@ -297,7 +297,7 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, { struct block_device *bdev = iocb->ki_filp->private_data; bool is_read = iov_iter_rw(iter) == READ; - unsigned int opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb); + blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb); struct blkdev_dio *dio; struct bio *bio; loff_t pos = iocb->ki_pos; diff --git a/include/linux/bio.h b/include/linux/bio.h index 992ee987f273..ca22b06700a9 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -405,7 +405,7 @@ extern void bioset_exit(struct bio_set *); extern int biovec_init_pool(mempool_t *pool, int pool_entries); struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, - unsigned int opf, gfp_t gfp_mask, + blk_opf_t opf, gfp_t gfp_mask, struct bio_set *bs); struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask); extern void bio_put(struct bio *); @@ -418,7 +418,7 @@ int bio_init_clone(struct block_device *bdev, struct bio *bio, extern struct bio_set fs_bio_set; static inline struct bio *bio_alloc(struct block_device *bdev, - unsigned short nr_vecs, unsigned int opf, gfp_t gfp_mask) + unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask) { return bio_alloc_bioset(bdev, nr_vecs, opf, gfp_mask, &fs_bio_set); } @@ -456,9 +456,9 @@ struct request_queue; extern int submit_bio_wait(struct bio *bio); void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, - unsigned short max_vecs, unsigned int opf); + unsigned short max_vecs, blk_opf_t opf); extern void bio_uninit(struct bio *); -void bio_reset(struct bio *bio, struct block_device *bdev, unsigned int opf); +void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf); void bio_chain(struct bio *, struct bio *); int bio_add_page(struct bio *, struct page *, unsigned len, unsigned off); @@ -789,6 +789,6 @@ static inline void bio_clear_polled(struct bio *bio) } struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, - unsigned int nr_pages, unsigned int opf, gfp_t gfp); + unsigned int nr_pages, blk_opf_t opf, gfp_t gfp); #endif /* __LINUX_BIO_H */ diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 677195de0663..effee1dc715a 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -80,7 +80,7 @@ struct request { struct blk_mq_ctx *mq_ctx; struct blk_mq_hw_ctx *mq_hctx; - unsigned int cmd_flags; /* op and common flags */ + blk_opf_t cmd_flags; /* op and common flags */ req_flags_t rq_flags; int tag; @@ -715,10 +715,10 @@ enum { BLK_MQ_REQ_PM = (__force blk_mq_req_flags_t)(1 << 2), }; -struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, +struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, blk_mq_req_flags_t flags); struct request *blk_mq_alloc_request_hctx(struct request_queue *q, - unsigned int op, blk_mq_req_flags_t flags, + blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx); /* diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index ca2ff113ea00..d04bdf549efa 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -250,7 +250,7 @@ static inline int blk_validate_block_size(unsigned long bsize) return 0; } -static inline bool blk_op_is_passthrough(unsigned int op) +static inline bool blk_op_is_passthrough(blk_opf_t op) { op &= REQ_OP_MASK; return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT; -- cgit v1.2.3 From 919dbca8670d0f7828dfbb2f9b434ac22dca8d2e Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:37 -0700 Subject: blktrace: Use the new blk_opf_t type Improve static type checking by using the new blk_opf_t type for a function argument that represents a combination of a request operation and request flags. Rename that argument from 'op' into 'opf' to make its role more clear. Cc: Christoph Hellwig Cc: Steven Rostedt Cc: Li Zefan Cc: Chaitanya Kulkarni Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-12-bvanassche@acm.org Signed-off-by: Jens Axboe --- include/linux/blktrace_api.h | 3 ++- kernel/trace/blktrace.c | 51 ++++++++++++++++++++++---------------------- 2 files changed, 27 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h index f6f9b544365a..cfbda114348c 100644 --- a/include/linux/blktrace_api.h +++ b/include/linux/blktrace_api.h @@ -7,6 +7,7 @@ #include #include #include +#include #if defined(CONFIG_BLK_DEV_IO_TRACE) @@ -105,7 +106,7 @@ struct compat_blk_user_trace_setup { #endif -void blk_fill_rwbs(char *rwbs, unsigned int op); +void blk_fill_rwbs(char *rwbs, blk_opf_t opf); static inline sector_t blk_rq_trace_sector(struct request *rq) { diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index 4327b51da403..150058f5daa9 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -205,7 +205,7 @@ static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), #define BLK_TC_PREFLUSH BLK_TC_FLUSH /* The ilog2() calls fall out because they're constant */ -#define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \ +#define MASK_TC_BIT(rw, __name) ((__force u32)(rw & REQ_ ## __name) << \ (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name)) /* @@ -213,8 +213,8 @@ static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), * blk_io_trace structure and places it in a per-cpu subbuffer. */ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, - int op, int op_flags, u32 what, int error, int pdu_len, - void *pdu_data, u64 cgid) + const blk_opf_t opf, u32 what, int error, + int pdu_len, void *pdu_data, u64 cgid) { struct task_struct *tsk = current; struct ring_buffer_event *event = NULL; @@ -227,16 +227,17 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, int cpu; bool blk_tracer = blk_tracer_enabled; ssize_t cgid_len = cgid ? sizeof(cgid) : 0; + const enum req_op op = opf & REQ_OP_MASK; if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer)) return; what |= ddir_act[op_is_write(op) ? WRITE : READ]; - what |= MASK_TC_BIT(op_flags, SYNC); - what |= MASK_TC_BIT(op_flags, RAHEAD); - what |= MASK_TC_BIT(op_flags, META); - what |= MASK_TC_BIT(op_flags, PREFLUSH); - what |= MASK_TC_BIT(op_flags, FUA); + what |= MASK_TC_BIT(opf, SYNC); + what |= MASK_TC_BIT(opf, RAHEAD); + what |= MASK_TC_BIT(opf, META); + what |= MASK_TC_BIT(opf, PREFLUSH); + what |= MASK_TC_BIT(opf, FUA); if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE) what |= BLK_TC_ACT(BLK_TC_DISCARD); if (op == REQ_OP_FLUSH) @@ -842,9 +843,8 @@ static void blk_add_trace_rq(struct request *rq, blk_status_t error, else what |= BLK_TC_ACT(BLK_TC_FS); - __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq), - rq->cmd_flags, what, blk_status_to_errno(error), 0, - NULL, cgid); + __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, rq->cmd_flags, + what, blk_status_to_errno(error), 0, NULL, cgid); rcu_read_unlock(); } @@ -903,7 +903,7 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, } __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, - bio_op(bio), bio->bi_opf, what, error, 0, NULL, + bio->bi_opf, what, error, 0, NULL, blk_trace_bio_get_cgid(q, bio)); rcu_read_unlock(); } @@ -949,7 +949,7 @@ static void blk_add_trace_plug(void *ignore, struct request_queue *q) rcu_read_lock(); bt = rcu_dereference(q->blk_trace); if (bt) - __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); + __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); rcu_read_unlock(); } @@ -969,7 +969,7 @@ static void blk_add_trace_unplug(void *ignore, struct request_queue *q, else what = BLK_TA_UNPLUG_TIMER; - __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); + __blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); } rcu_read_unlock(); } @@ -985,8 +985,7 @@ static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) __be64 rpdu = cpu_to_be64(pdu); __blk_add_trace(bt, bio->bi_iter.bi_sector, - bio->bi_iter.bi_size, bio_op(bio), bio->bi_opf, - BLK_TA_SPLIT, + bio->bi_iter.bi_size, bio->bi_opf, BLK_TA_SPLIT, blk_status_to_errno(bio->bi_status), sizeof(rpdu), &rpdu, blk_trace_bio_get_cgid(q, bio)); @@ -1022,7 +1021,7 @@ static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, r.sector_from = cpu_to_be64(from); __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, - bio_op(bio), bio->bi_opf, BLK_TA_REMAP, + bio->bi_opf, BLK_TA_REMAP, blk_status_to_errno(bio->bi_status), sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); rcu_read_unlock(); @@ -1058,7 +1057,7 @@ static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, r.sector_from = cpu_to_be64(from); __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), - req_op(rq), rq->cmd_flags, BLK_TA_REMAP, 0, + rq->cmd_flags, BLK_TA_REMAP, 0, sizeof(r), &r, blk_trace_request_get_cgid(rq)); rcu_read_unlock(); } @@ -1084,7 +1083,7 @@ void blk_add_driver_data(struct request *rq, void *data, size_t len) return; } - __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0, + __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, BLK_TA_DRV_DATA, 0, len, data, blk_trace_request_get_cgid(rq)); rcu_read_unlock(); @@ -1881,14 +1880,14 @@ out: * caller with resulting string. * **/ -void blk_fill_rwbs(char *rwbs, unsigned int op) +void blk_fill_rwbs(char *rwbs, blk_opf_t opf) { int i = 0; - if (op & REQ_PREFLUSH) + if (opf & REQ_PREFLUSH) rwbs[i++] = 'F'; - switch (op & REQ_OP_MASK) { + switch (opf & REQ_OP_MASK) { case REQ_OP_WRITE: rwbs[i++] = 'W'; break; @@ -1909,13 +1908,13 @@ void blk_fill_rwbs(char *rwbs, unsigned int op) rwbs[i++] = 'N'; } - if (op & REQ_FUA) + if (opf & REQ_FUA) rwbs[i++] = 'F'; - if (op & REQ_RAHEAD) + if (opf & REQ_RAHEAD) rwbs[i++] = 'A'; - if (op & REQ_SYNC) + if (opf & REQ_SYNC) rwbs[i++] = 'S'; - if (op & REQ_META) + if (opf & REQ_META) rwbs[i++] = 'M'; rwbs[i] = '\0'; -- cgit v1.2.3 From 581075e4f6475bb97c73ecccf68636a9453a31fd Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:06:47 -0700 Subject: dm/core: Reduce the size of struct dm_io_request Combine the bi_op and bi_op_flags into the bi_opf member. Use the new blk_opf_t type to improve static type checking. This patch does not change any functionality. Cc: Alasdair Kergon Cc: Mike Snitzer Cc: Mikulas Patocka Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-22-bvanassche@acm.org Signed-off-by: Jens Axboe --- drivers/md/dm-bufio.c | 9 +++------ drivers/md/dm-integrity.c | 15 +++++---------- drivers/md/dm-io.c | 10 ++++++---- drivers/md/dm-kcopyd.c | 3 +-- drivers/md/dm-log.c | 6 ++---- drivers/md/dm-raid1.c | 12 +++++------- drivers/md/dm-snap-persistent.c | 3 +-- drivers/md/dm-writecache.c | 12 ++++-------- include/linux/dm-io.h | 4 ++-- 9 files changed, 29 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 5ffa1dcf84cf..1b7acda45c78 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -582,8 +582,7 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t sector, { int r; struct dm_io_request io_req = { - .bi_op = rw, - .bi_op_flags = 0, + .bi_opf = rw, .notify.fn = dmio_complete, .notify.context = b, .client = b->c->dm_io, @@ -1341,8 +1340,7 @@ EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers); int dm_bufio_issue_flush(struct dm_bufio_client *c) { struct dm_io_request io_req = { - .bi_op = REQ_OP_WRITE, - .bi_op_flags = REQ_PREFLUSH | REQ_SYNC, + .bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC, .mem.type = DM_IO_KMEM, .mem.ptr.addr = NULL, .client = c->dm_io, @@ -1365,8 +1363,7 @@ EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count) { struct dm_io_request io_req = { - .bi_op = REQ_OP_DISCARD, - .bi_op_flags = REQ_SYNC, + .bi_opf = REQ_OP_DISCARD | REQ_SYNC, .mem.type = DM_IO_KMEM, .mem.ptr.addr = NULL, .client = c->dm_io, diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 148978ad03a8..2ccc103dea1e 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -557,8 +557,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags) struct dm_io_region io_loc; int r; - io_req.bi_op = op; - io_req.bi_op_flags = op_flags; + io_req.bi_opf = op | op_flags; io_req.mem.type = DM_IO_KMEM; io_req.mem.ptr.addr = ic->sb; io_req.notify.fn = NULL; @@ -1067,8 +1066,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, int op, int op_flags, pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); - io_req.bi_op = op; - io_req.bi_op_flags = op_flags; + io_req.bi_opf = op | op_flags; io_req.mem.type = DM_IO_PAGE_LIST; if (ic->journal_io) io_req.mem.ptr.pl = &ic->journal_io[pl_index]; @@ -1188,8 +1186,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsig pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); - io_req.bi_op = REQ_OP_WRITE; - io_req.bi_op_flags = 0; + io_req.bi_opf = REQ_OP_WRITE; io_req.mem.type = DM_IO_PAGE_LIST; io_req.mem.ptr.pl = &ic->journal[pl_index]; io_req.mem.offset = pl_offset; @@ -1516,8 +1513,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat if (!ic->meta_dev) flush_data = false; if (flush_data) { - fr.io_req.bi_op = REQ_OP_WRITE, - fr.io_req.bi_op_flags = REQ_PREFLUSH | REQ_SYNC, + fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC, fr.io_req.mem.type = DM_IO_KMEM, fr.io_req.mem.ptr.addr = NULL, fr.io_req.notify.fn = flush_notify, @@ -2706,8 +2702,7 @@ next_chunk: if (unlikely(dm_integrity_failed(ic))) goto err; - io_req.bi_op = REQ_OP_READ; - io_req.bi_op_flags = 0; + io_req.bi_opf = REQ_OP_READ; io_req.mem.type = DM_IO_VMA; io_req.mem.ptr.addr = ic->recalc_buffer; io_req.notify.fn = NULL; diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index e4b95eaeec8c..0606e00d1817 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -489,7 +489,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, case DM_IO_VMA: flush_kernel_vmap_range(io_req->mem.ptr.vma, size); - if (io_req->bi_op == REQ_OP_READ) { + if ((io_req->bi_opf & REQ_OP_MASK) == REQ_OP_READ) { dp->vma_invalidate_address = io_req->mem.ptr.vma; dp->vma_invalidate_size = size; } @@ -519,11 +519,13 @@ int dm_io(struct dm_io_request *io_req, unsigned num_regions, if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, - io_req->bi_op, io_req->bi_op_flags, &dp, + io_req->bi_opf & REQ_OP_MASK, + io_req->bi_opf & ~REQ_OP_MASK, &dp, sync_error_bits); - return async_io(io_req->client, num_regions, where, io_req->bi_op, - io_req->bi_op_flags, &dp, io_req->notify.fn, + return async_io(io_req->client, num_regions, where, + io_req->bi_opf & REQ_OP_MASK, + io_req->bi_opf & ~REQ_OP_MASK, &dp, io_req->notify.fn, io_req->notify.context); } EXPORT_SYMBOL(dm_io); diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 37b03ab7e5c9..a99b994e2b62 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -549,8 +549,7 @@ static int run_io_job(struct kcopyd_job *job) { int r; struct dm_io_request io_req = { - .bi_op = job->rw, - .bi_op_flags = 0, + .bi_opf = job->rw, .mem.type = DM_IO_PAGE_LIST, .mem.ptr.pl = job->pages, .mem.offset = 0, diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 0c6620e7b7bf..56ad13f9347b 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -293,8 +293,7 @@ static void header_from_disk(struct log_header_core *core, struct log_header_dis static int rw_header(struct log_c *lc, int op) { - lc->io_req.bi_op = op; - lc->io_req.bi_op_flags = 0; + lc->io_req.bi_opf = op; return dm_io(&lc->io_req, 1, &lc->header_location, NULL); } @@ -307,8 +306,7 @@ static int flush_header(struct log_c *lc) .count = 0, }; - lc->io_req.bi_op = REQ_OP_WRITE; - lc->io_req.bi_op_flags = REQ_PREFLUSH; + lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; return dm_io(&lc->io_req, 1, &null_location, NULL); } diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index 8811d484fdd1..06a38dc32025 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -260,8 +260,7 @@ static int mirror_flush(struct dm_target *ti) struct dm_io_region io[MAX_NR_MIRRORS]; struct mirror *m; struct dm_io_request io_req = { - .bi_op = REQ_OP_WRITE, - .bi_op_flags = REQ_PREFLUSH | REQ_SYNC, + .bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC, .mem.type = DM_IO_KMEM, .mem.ptr.addr = NULL, .client = ms->io_client, @@ -535,8 +534,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio) { struct dm_io_region io; struct dm_io_request io_req = { - .bi_op = REQ_OP_READ, - .bi_op_flags = 0, + .bi_opf = REQ_OP_READ, .mem.type = DM_IO_BIO, .mem.ptr.bio = bio, .notify.fn = read_callback, @@ -648,9 +646,9 @@ static void do_write(struct mirror_set *ms, struct bio *bio) unsigned int i; struct dm_io_region io[MAX_NR_MIRRORS], *dest = io; struct mirror *m; + blk_opf_t op_flags = bio->bi_opf & (REQ_FUA | REQ_PREFLUSH); struct dm_io_request io_req = { - .bi_op = REQ_OP_WRITE, - .bi_op_flags = bio->bi_opf & (REQ_FUA | REQ_PREFLUSH), + .bi_opf = REQ_OP_WRITE | op_flags, .mem.type = DM_IO_BIO, .mem.ptr.bio = bio, .notify.fn = write_callback, @@ -659,7 +657,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio) }; if (bio_op(bio) == REQ_OP_DISCARD) { - io_req.bi_op = REQ_OP_DISCARD; + io_req.bi_opf = REQ_OP_DISCARD | op_flags; io_req.mem.type = DM_IO_KMEM; io_req.mem.ptr.addr = NULL; } diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c index 3bb5cff5d6fc..eaf969de3d3a 100644 --- a/drivers/md/dm-snap-persistent.c +++ b/drivers/md/dm-snap-persistent.c @@ -235,8 +235,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int op, .count = ps->store->chunk_size, }; struct dm_io_request io_req = { - .bi_op = op, - .bi_op_flags = op_flags, + .bi_opf = op | op_flags, .mem.type = DM_IO_VMA, .mem.ptr.vma = area, .client = ps->io_client, diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index d74c5a7a0ab4..2b994b3e22a7 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -523,8 +523,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) region.sector += wc->start_sector; atomic_inc(&endio.count); - req.bi_op = REQ_OP_WRITE; - req.bi_op_flags = REQ_SYNC; + req.bi_opf = REQ_OP_WRITE | REQ_SYNC; req.mem.type = DM_IO_VMA; req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY; req.client = wc->dm_io; @@ -562,8 +561,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc) region.sector += wc->start_sector; - req.bi_op = REQ_OP_WRITE; - req.bi_op_flags = REQ_SYNC | REQ_FUA; + req.bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_FUA; req.mem.type = DM_IO_VMA; req.mem.ptr.vma = (char *)wc->memory_map; req.client = wc->dm_io; @@ -592,8 +590,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) region.bdev = dev->bdev; region.sector = 0; region.count = 0; - req.bi_op = REQ_OP_WRITE; - req.bi_op_flags = REQ_PREFLUSH; + req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; req.mem.type = DM_IO_KMEM; req.mem.ptr.addr = NULL; req.client = wc->dm_io; @@ -981,8 +978,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors region.bdev = wc->ssd_dev->bdev; region.sector = wc->start_sector; region.count = n_sectors; - req.bi_op = REQ_OP_READ; - req.bi_op_flags = REQ_SYNC; + req.bi_opf = REQ_OP_READ | REQ_SYNC; req.mem.type = DM_IO_VMA; req.mem.ptr.vma = (char *)wc->memory_map; req.client = wc->dm_io; diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index a52c6580cc9a..8e1c4ab5df04 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -13,6 +13,7 @@ #ifdef __KERNEL__ #include +#include struct dm_io_region { struct block_device *bdev; @@ -57,8 +58,7 @@ struct dm_io_notify { */ struct dm_io_client; struct dm_io_request { - int bi_op; /* REQ_OP */ - int bi_op_flags; /* req_flag_bits */ + blk_opf_t bi_opf; /* Request type and flags */ struct dm_io_memory mem; /* Memory to use for io */ struct dm_io_notify notify; /* Synchronous if notify.fn is NULL */ struct dm_io_client *client; /* Client memory handler */ -- cgit v1.2.3 From f8e6e4bd9fd8c452565f3eaeb358e3cc08d880f4 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:07:11 -0700 Subject: mm: Use the new blk_opf_t type Improve static type checking by using the new blk_opf_t type for block layer request flags. Cc: Andrew Morton Cc: Christoph Hellwig Cc: Jan Kara Cc: Stefan Roesch Cc: NeilBrown Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-46-bvanassche@acm.org Signed-off-by: Jens Axboe --- include/linux/writeback.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/writeback.h b/include/linux/writeback.h index da21d63f70e2..e91bea371b18 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -101,9 +101,9 @@ struct writeback_control { #endif }; -static inline int wbc_to_write_flags(struct writeback_control *wbc) +static inline blk_opf_t wbc_to_write_flags(struct writeback_control *wbc) { - int flags = 0; + blk_opf_t flags = 0; if (wbc->punt_to_cgroup) flags = REQ_CGROUP_PUNT; -- cgit v1.2.3 From 3ae7286943ae6f6bfecfe0a3da9d1a4c64f5531f Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:07:12 -0700 Subject: fs/buffer: Use the new blk_opf_t type Improve static type checking by using the new blk_opf_t type for block layer request flags. Change WRITE into REQ_OP_WRITE. This patch does not change any functionality since REQ_OP_WRITE == WRITE == 1. Reviewed-by: Jan Kara Cc: Al Viro Cc: Christoph Hellwig Cc: Matthew Wilcox Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-47-bvanassche@acm.org Signed-off-by: Jens Axboe --- fs/buffer.c | 21 +++++++++++---------- include/linux/buffer_head.h | 9 +++++---- 2 files changed, 16 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/buffer.c b/fs/buffer.c index 898c7f301b1b..4a00b61f35ec 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -52,8 +52,8 @@ #include "internal.h" static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); -static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, - struct writeback_control *wbc); +static int submit_bh_wbc(enum req_op op, blk_opf_t op_flags, + struct buffer_head *bh, struct writeback_control *wbc); #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) @@ -1716,7 +1716,7 @@ int __block_write_full_page(struct inode *inode, struct page *page, struct buffer_head *bh, *head; unsigned int blocksize, bbits; int nr_underway = 0; - int write_flags = wbc_to_write_flags(wbc); + blk_opf_t write_flags = wbc_to_write_flags(wbc); head = create_page_buffers(page, inode, (1 << BH_Dirty)|(1 << BH_Uptodate)); @@ -2994,8 +2994,8 @@ static void end_bio_bh_io_sync(struct bio *bio) bio_put(bio); } -static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, - struct writeback_control *wbc) +static int submit_bh_wbc(enum req_op op, blk_opf_t op_flags, + struct buffer_head *bh, struct writeback_control *wbc) { struct bio *bio; @@ -3040,7 +3040,7 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, return 0; } -int submit_bh(int op, int op_flags, struct buffer_head *bh) +int submit_bh(enum req_op op, blk_opf_t op_flags, struct buffer_head *bh) { return submit_bh_wbc(op, op_flags, bh, NULL); } @@ -3072,7 +3072,8 @@ EXPORT_SYMBOL(submit_bh); * All of the buffers must be for the same device, and must also be a * multiple of the current approved size for the device. */ -void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[]) +void ll_rw_block(enum req_op op, blk_opf_t op_flags, int nr, + struct buffer_head *bhs[]) { int i; @@ -3081,7 +3082,7 @@ void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[]) if (!trylock_buffer(bh)) continue; - if (op == WRITE) { + if (op == REQ_OP_WRITE) { if (test_clear_buffer_dirty(bh)) { bh->b_end_io = end_buffer_write_sync; get_bh(bh); @@ -3101,7 +3102,7 @@ void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[]) } EXPORT_SYMBOL(ll_rw_block); -void write_dirty_buffer(struct buffer_head *bh, int op_flags) +void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) { lock_buffer(bh); if (!test_clear_buffer_dirty(bh)) { @@ -3119,7 +3120,7 @@ EXPORT_SYMBOL(write_dirty_buffer); * and then start new I/O and then wait upon it. The caller must have a ref on * the buffer_head. */ -int __sync_dirty_buffer(struct buffer_head *bh, int op_flags) +int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) { int ret = 0; diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index c9d1463bb20f..9795df9400bd 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -9,6 +9,7 @@ #define _LINUX_BUFFER_HEAD_H #include +#include #include #include #include @@ -201,11 +202,11 @@ struct buffer_head *alloc_buffer_head(gfp_t gfp_flags); void free_buffer_head(struct buffer_head * bh); void unlock_buffer(struct buffer_head *bh); void __lock_buffer(struct buffer_head *bh); -void ll_rw_block(int, int, int, struct buffer_head * bh[]); +void ll_rw_block(enum req_op, blk_opf_t, int, struct buffer_head * bh[]); int sync_dirty_buffer(struct buffer_head *bh); -int __sync_dirty_buffer(struct buffer_head *bh, int op_flags); -void write_dirty_buffer(struct buffer_head *bh, int op_flags); -int submit_bh(int, int, struct buffer_head *); +int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); +void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); +int submit_bh(enum req_op, blk_opf_t, struct buffer_head *); void write_boundary_block(struct block_device *bdev, sector_t bblock, unsigned blocksize); int bh_uptodate_or_lock(struct buffer_head *bh); -- cgit v1.2.3 From 1420c4a549bf28ffddbed827d61fb3d4d2132ddb Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:07:13 -0700 Subject: fs/buffer: Combine two submit_bh() and ll_rw_block() arguments Both submit_bh() and ll_rw_block() accept a request operation type and request flags as their first two arguments. Micro-optimize these two functions by combining these first two arguments into a single argument. This patch does not change the behavior of any of the modified code. Cc: Alexander Viro Cc: Jan Kara Acked-by: Song Liu (for the md changes) Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-48-bvanassche@acm.org Signed-off-by: Jens Axboe --- drivers/md/md-bitmap.c | 4 ++-- fs/buffer.c | 53 +++++++++++++++++++++++---------------------- fs/ext4/fast_commit.c | 2 +- fs/ext4/mmp.c | 2 +- fs/ext4/super.c | 6 ++--- fs/gfs2/bmap.c | 5 ++--- fs/gfs2/dir.c | 5 ++--- fs/gfs2/meta_io.c | 9 ++++---- fs/gfs2/quota.c | 2 +- fs/isofs/compress.c | 2 +- fs/jbd2/commit.c | 8 +++---- fs/jbd2/journal.c | 4 ++-- fs/jbd2/recovery.c | 4 ++-- fs/nilfs2/btnode.c | 2 +- fs/nilfs2/gcinode.c | 2 +- fs/nilfs2/mdt.c | 2 +- fs/ntfs/aops.c | 6 ++--- fs/ntfs/compress.c | 2 +- fs/ntfs/file.c | 2 +- fs/ntfs/logfile.c | 2 +- fs/ntfs/mft.c | 4 ++-- fs/ntfs3/file.c | 2 +- fs/ntfs3/inode.c | 2 +- fs/ocfs2/aops.c | 2 +- fs/ocfs2/buffer_head_io.c | 8 +++---- fs/ocfs2/super.c | 2 +- fs/reiserfs/inode.c | 4 ++-- fs/reiserfs/journal.c | 12 +++++----- fs/reiserfs/stree.c | 4 ++-- fs/reiserfs/super.c | 2 +- fs/udf/dir.c | 2 +- fs/udf/directory.c | 2 +- fs/udf/inode.c | 2 +- fs/ufs/balloc.c | 2 +- include/linux/buffer_head.h | 4 ++-- 35 files changed, 88 insertions(+), 90 deletions(-) (limited to 'include/linux') diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 0a21b8317103..bf6dffadbe6f 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -302,7 +302,7 @@ static void write_page(struct bitmap *bitmap, struct page *page, int wait) atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(REQ_OP_WRITE, REQ_SYNC, bh); + submit_bh(REQ_OP_WRITE | REQ_SYNC, bh); bh = bh->b_this_page; } @@ -394,7 +394,7 @@ static int read_page(struct file *file, unsigned long index, atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); } blk_cur++; bh = bh->b_this_page; diff --git a/fs/buffer.c b/fs/buffer.c index 4a00b61f35ec..af53569930bb 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -52,8 +52,8 @@ #include "internal.h" static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); -static int submit_bh_wbc(enum req_op op, blk_opf_t op_flags, - struct buffer_head *bh, struct writeback_control *wbc); +static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, + struct writeback_control *wbc); #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) @@ -562,7 +562,7 @@ void write_boundary_block(struct block_device *bdev, struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize); if (bh) { if (buffer_dirty(bh)) - ll_rw_block(REQ_OP_WRITE, 0, 1, &bh); + ll_rw_block(REQ_OP_WRITE, 1, &bh); put_bh(bh); } } @@ -1174,7 +1174,7 @@ static struct buffer_head *__bread_slow(struct buffer_head *bh) } else { get_bh(bh); bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); wait_on_buffer(bh); if (buffer_uptodate(bh)) return bh; @@ -1342,7 +1342,7 @@ void __breadahead(struct block_device *bdev, sector_t block, unsigned size) { struct buffer_head *bh = __getblk(bdev, block, size); if (likely(bh)) { - ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh); + ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh); brelse(bh); } } @@ -1353,7 +1353,7 @@ void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size, { struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp); if (likely(bh)) { - ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh); + ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh); brelse(bh); } } @@ -1804,7 +1804,7 @@ int __block_write_full_page(struct inode *inode, struct page *page, do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, wbc); + submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc); nr_underway++; } bh = next; @@ -1858,7 +1858,7 @@ recover: struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { clear_buffer_dirty(bh); - submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, wbc); + submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc); nr_underway++; } bh = next; @@ -2033,7 +2033,7 @@ int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len, if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh) && (block_start < from || block_end > to)) { - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); *wait_bh++=bh; } } @@ -2334,7 +2334,7 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) if (buffer_uptodate(bh)) end_buffer_async_read(bh, 1); else - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); } return 0; } @@ -2665,7 +2665,7 @@ int nobh_write_begin(struct address_space *mapping, loff_t pos, unsigned len, if (block_start < from || block_end > to) { lock_buffer(bh); bh->b_end_io = end_buffer_read_nobh; - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); nr_reads++; } } @@ -2915,7 +2915,7 @@ int block_truncate_page(struct address_space *mapping, if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) { err = -EIO; - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); wait_on_buffer(bh); /* Uhhuh. Read error. Complain and punt. */ if (!buffer_uptodate(bh)) @@ -2994,9 +2994,10 @@ static void end_bio_bh_io_sync(struct bio *bio) bio_put(bio); } -static int submit_bh_wbc(enum req_op op, blk_opf_t op_flags, - struct buffer_head *bh, struct writeback_control *wbc) +static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, + struct writeback_control *wbc) { + const enum req_op op = opf & REQ_OP_MASK; struct bio *bio; BUG_ON(!buffer_locked(bh)); @@ -3012,11 +3013,11 @@ static int submit_bh_wbc(enum req_op op, blk_opf_t op_flags, clear_buffer_write_io_error(bh); if (buffer_meta(bh)) - op_flags |= REQ_META; + opf |= REQ_META; if (buffer_prio(bh)) - op_flags |= REQ_PRIO; + opf |= REQ_PRIO; - bio = bio_alloc(bh->b_bdev, 1, op | op_flags, GFP_NOIO); + bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); @@ -3040,9 +3041,9 @@ static int submit_bh_wbc(enum req_op op, blk_opf_t op_flags, return 0; } -int submit_bh(enum req_op op, blk_opf_t op_flags, struct buffer_head *bh) +int submit_bh(blk_opf_t opf, struct buffer_head *bh) { - return submit_bh_wbc(op, op_flags, bh, NULL); + return submit_bh_wbc(opf, bh, NULL); } EXPORT_SYMBOL(submit_bh); @@ -3072,9 +3073,9 @@ EXPORT_SYMBOL(submit_bh); * All of the buffers must be for the same device, and must also be a * multiple of the current approved size for the device. */ -void ll_rw_block(enum req_op op, blk_opf_t op_flags, int nr, - struct buffer_head *bhs[]) +void ll_rw_block(const blk_opf_t opf, int nr, struct buffer_head *bhs[]) { + const enum req_op op = opf & REQ_OP_MASK; int i; for (i = 0; i < nr; i++) { @@ -3086,14 +3087,14 @@ void ll_rw_block(enum req_op op, blk_opf_t op_flags, int nr, if (test_clear_buffer_dirty(bh)) { bh->b_end_io = end_buffer_write_sync; get_bh(bh); - submit_bh(op, op_flags, bh); + submit_bh(opf, bh); continue; } } else { if (!buffer_uptodate(bh)) { bh->b_end_io = end_buffer_read_sync; get_bh(bh); - submit_bh(op, op_flags, bh); + submit_bh(opf, bh); continue; } } @@ -3111,7 +3112,7 @@ void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) } bh->b_end_io = end_buffer_write_sync; get_bh(bh); - submit_bh(REQ_OP_WRITE, op_flags, bh); + submit_bh(REQ_OP_WRITE | op_flags, bh); } EXPORT_SYMBOL(write_dirty_buffer); @@ -3138,7 +3139,7 @@ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) get_bh(bh); bh->b_end_io = end_buffer_write_sync; - ret = submit_bh(REQ_OP_WRITE, op_flags, bh); + ret = submit_bh(REQ_OP_WRITE | op_flags, bh); wait_on_buffer(bh); if (!ret && !buffer_uptodate(bh)) ret = -EIO; @@ -3366,7 +3367,7 @@ int bh_submit_read(struct buffer_head *bh) get_bh(bh); bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); wait_on_buffer(bh); if (buffer_uptodate(bh)) return 0; diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 795a60ad1897..0df5482c6c1c 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -668,7 +668,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) set_buffer_dirty(bh); set_buffer_uptodate(bh); bh->b_end_io = ext4_end_buffer_io_sync; - submit_bh(REQ_OP_WRITE, write_flags, bh); + submit_bh(REQ_OP_WRITE | write_flags, bh); EXT4_SB(sb)->s_fc_bh = NULL; } diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c index b221f313ded6..9af68a7ecdcf 100644 --- a/fs/ext4/mmp.c +++ b/fs/ext4/mmp.c @@ -52,7 +52,7 @@ static int write_mmp_block(struct super_block *sb, struct buffer_head *bh) lock_buffer(bh); bh->b_end_io = end_buffer_write_sync; get_bh(bh); - submit_bh(REQ_OP_WRITE, REQ_SYNC | REQ_META | REQ_PRIO, bh); + submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, bh); wait_on_buffer(bh); sb_end_write(sb); if (unlikely(!buffer_uptodate(bh))) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 845f2f8aee5f..24922184b622 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -171,7 +171,7 @@ static inline void __ext4_read_bh(struct buffer_head *bh, int op_flags, bh->b_end_io = end_io ? end_io : end_buffer_read_sync; get_bh(bh); - submit_bh(REQ_OP_READ, op_flags, bh); + submit_bh(REQ_OP_READ | op_flags, bh); } void ext4_read_bh_nowait(struct buffer_head *bh, int op_flags, @@ -5939,8 +5939,8 @@ static int ext4_commit_super(struct super_block *sb) /* Clear potential dirty bit if it was journalled update */ clear_buffer_dirty(sbh); sbh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE, - REQ_SYNC | (test_opt(sb, BARRIER) ? REQ_FUA : 0), sbh); + submit_bh(REQ_OP_WRITE | REQ_SYNC | + (test_opt(sb, BARRIER) ? REQ_FUA : 0), sbh); wait_on_buffer(sbh); if (buffer_write_io_error(sbh)) { ext4_msg(sb, KERN_ERR, "I/O error while writing " diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index b6697333bb2b..3bdb2c668a71 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -310,9 +310,8 @@ static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end) if (trylock_buffer(rabh)) { if (!buffer_uptodate(rabh)) { rabh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, - REQ_RAHEAD | REQ_META | REQ_PRIO, - rabh); + submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META | + REQ_PRIO, rabh); continue; } unlock_buffer(rabh); diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c index 42b7dfffb5e7..a0562dd1bada 100644 --- a/fs/gfs2/dir.c +++ b/fs/gfs2/dir.c @@ -1508,9 +1508,8 @@ static void gfs2_dir_readahead(struct inode *inode, unsigned hsize, u32 index, continue; } bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, - REQ_RAHEAD | REQ_META | REQ_PRIO, - bh); + submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META | + REQ_PRIO, bh); continue; } brelse(bh); diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index 868dcc71b581..3570739f005d 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -75,7 +75,7 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh(REQ_OP_WRITE, write_flags, bh); + submit_bh(REQ_OP_WRITE | write_flags, bh); nr_underway++; } bh = next; @@ -527,7 +527,7 @@ struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen) if (buffer_uptodate(first_bh)) goto out; if (!buffer_locked(first_bh)) - ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &first_bh); + ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &first_bh); dblock++; extlen--; @@ -536,9 +536,8 @@ struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen) bh = gfs2_getbuf(gl, dblock, CREATE); if (!buffer_uptodate(bh) && !buffer_locked(bh)) - ll_rw_block(REQ_OP_READ, - REQ_RAHEAD | REQ_META | REQ_PRIO, - 1, &bh); + ll_rw_block(REQ_OP_READ | REQ_RAHEAD | REQ_META | + REQ_PRIO, 1, &bh); brelse(bh); dblock++; extlen--; diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index 59d727a4ae2c..c98a7faa67d3 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -746,7 +746,7 @@ static int gfs2_write_buf_to_page(struct gfs2_inode *ip, unsigned long index, if (PageUptodate(page)) set_buffer_uptodate(bh); if (!buffer_uptodate(bh)) { - ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh); + ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &bh); wait_on_buffer(bh); if (!buffer_uptodate(bh)) goto unlock_out; diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index 95a19f25d61c..b466172eec25 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -82,7 +82,7 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, return 0; } haveblocks = isofs_get_blocks(inode, blocknum, bhs, needblocks); - ll_rw_block(REQ_OP_READ, 0, haveblocks, bhs); + ll_rw_block(REQ_OP_READ, haveblocks, bhs); curbh = 0; curpage = 0; diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index eb315e81f1a6..890b5543a1c5 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -155,10 +155,10 @@ static int journal_submit_commit_record(journal_t *journal, if (journal->j_flags & JBD2_BARRIER && !jbd2_has_feature_async_commit(journal)) - ret = submit_bh(REQ_OP_WRITE, - REQ_SYNC | REQ_PREFLUSH | REQ_FUA, bh); + ret = submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH | + REQ_FUA, bh); else - ret = submit_bh(REQ_OP_WRITE, REQ_SYNC, bh); + ret = submit_bh(REQ_OP_WRITE | REQ_SYNC, bh); *cbh = bh; return ret; @@ -763,7 +763,7 @@ start_journal_io: clear_buffer_dirty(bh); set_buffer_uptodate(bh); bh->b_end_io = journal_end_buffer_io_sync; - submit_bh(REQ_OP_WRITE, REQ_SYNC, bh); + submit_bh(REQ_OP_WRITE | REQ_SYNC, bh); } cond_resched(); diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 9015f5fa2862..07e6aaf7e213 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -1638,7 +1638,7 @@ static int jbd2_write_superblock(journal_t *journal, int write_flags) sb->s_checksum = jbd2_superblock_csum(journal, sb); get_bh(bh); bh->b_end_io = end_buffer_write_sync; - ret = submit_bh(REQ_OP_WRITE, write_flags, bh); + ret = submit_bh(REQ_OP_WRITE | write_flags, bh); wait_on_buffer(bh); if (buffer_write_io_error(bh)) { clear_buffer_write_io_error(bh); @@ -1900,7 +1900,7 @@ static int journal_get_superblock(journal_t *journal) J_ASSERT(bh != NULL); if (!buffer_uptodate(bh)) { - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); wait_on_buffer(bh); if (!buffer_uptodate(bh)) { printk(KERN_ERR diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c index 8ca3527189f8..e699d6ab2c0e 100644 --- a/fs/jbd2/recovery.c +++ b/fs/jbd2/recovery.c @@ -100,7 +100,7 @@ static int do_readahead(journal_t *journal, unsigned int start) if (!buffer_uptodate(bh) && !buffer_locked(bh)) { bufs[nbufs++] = bh; if (nbufs == MAXBUF) { - ll_rw_block(REQ_OP_READ, 0, nbufs, bufs); + ll_rw_block(REQ_OP_READ, nbufs, bufs); journal_brelse_array(bufs, nbufs); nbufs = 0; } @@ -109,7 +109,7 @@ static int do_readahead(journal_t *journal, unsigned int start) } if (nbufs) - ll_rw_block(REQ_OP_READ, 0, nbufs, bufs); + ll_rw_block(REQ_OP_READ, nbufs, bufs); err = 0; failed: diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index ca611ac09f7c..5c39efbf733f 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c @@ -122,7 +122,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, bh->b_blocknr = pblocknr; /* set block address for read */ bh->b_end_io = end_buffer_read_sync; get_bh(bh); - submit_bh(mode, mode_flags, bh); + submit_bh(mode | mode_flags, bh); bh->b_blocknr = blocknr; /* set back to the given block address */ *submit_ptr = pblocknr; err = 0; diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c index 04fdd420eae7..847def8af315 100644 --- a/fs/nilfs2/gcinode.c +++ b/fs/nilfs2/gcinode.c @@ -92,7 +92,7 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff, bh->b_blocknr = pbn; bh->b_end_io = end_buffer_read_sync; get_bh(bh); - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); if (vbn) bh->b_blocknr = vbn; out: diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c index d29a0f2b9c16..66e8811c2528 100644 --- a/fs/nilfs2/mdt.c +++ b/fs/nilfs2/mdt.c @@ -148,7 +148,7 @@ nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, bh->b_end_io = end_buffer_read_sync; get_bh(bh); - submit_bh(mode, mode_flags, bh); + submit_bh(mode | mode_flags, bh); ret = 0; trace_nilfs2_mdt_submit_block(inode, inode->i_ino, blkoff, mode); diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index 9e3964ea2ea0..b5765fdb3a47 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c @@ -342,7 +342,7 @@ handle_zblock: for (i = 0; i < nr; i++) { tbh = arr[i]; if (likely(!buffer_uptodate(tbh))) - submit_bh(REQ_OP_READ, 0, tbh); + submit_bh(REQ_OP_READ, tbh); else ntfs_end_buffer_async_read(tbh, 1); } @@ -859,7 +859,7 @@ lock_retry_remap: do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); need_end_writeback = false; } bh = next; @@ -1187,7 +1187,7 @@ lock_retry_remap: BUG_ON(!buffer_mapped(tbh)); get_bh(tbh); tbh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE, 0, tbh); + submit_bh(REQ_OP_WRITE, tbh); } /* Synchronize the mft mirror now if not @sync. */ if (is_mft && !sync) diff --git a/fs/ntfs/compress.c b/fs/ntfs/compress.c index a60f543e7557..587e9b187873 100644 --- a/fs/ntfs/compress.c +++ b/fs/ntfs/compress.c @@ -658,7 +658,7 @@ lock_retry_remap: } get_bh(tbh); tbh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, 0, tbh); + submit_bh(REQ_OP_READ, tbh); } /* Wait for io completion on all buffer heads. */ diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index a8abe2296514..46ed69b86c33 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -537,7 +537,7 @@ static inline int ntfs_submit_bh_for_read(struct buffer_head *bh) lock_buffer(bh); get_bh(bh); bh->b_end_io = end_buffer_read_sync; - return submit_bh(REQ_OP_READ, 0, bh); + return submit_bh(REQ_OP_READ, bh); } /** diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c index bc1bf217b38e..6ce60ffc6ac0 100644 --- a/fs/ntfs/logfile.c +++ b/fs/ntfs/logfile.c @@ -807,7 +807,7 @@ map_vcn: * completed ignore errors afterwards as we can assume * that if one buffer worked all of them will work. */ - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); if (should_wait) { should_wait = false; wait_on_buffer(bh); diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c index 0d62cd5bb7f8..f7bf5ce960cc 100644 --- a/fs/ntfs/mft.c +++ b/fs/ntfs/mft.c @@ -583,7 +583,7 @@ int ntfs_sync_mft_mirror(ntfs_volume *vol, const unsigned long mft_no, clear_buffer_dirty(tbh); get_bh(tbh); tbh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE, 0, tbh); + submit_bh(REQ_OP_WRITE, tbh); } /* Wait on i/o completion of buffers. */ for (i_bhs = 0; i_bhs < nr_bhs; i_bhs++) { @@ -780,7 +780,7 @@ int write_mft_record_nolock(ntfs_inode *ni, MFT_RECORD *m, int sync) clear_buffer_dirty(tbh); get_bh(tbh); tbh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE, 0, tbh); + submit_bh(REQ_OP_WRITE, tbh); } /* Synchronize the mft mirror now if not @sync. */ if (!sync && ni->mft_no < vol->mftmirr_size) diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 8e9d2b35175f..4a21745711fe 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -242,7 +242,7 @@ static int ntfs_zero_range(struct inode *inode, u64 vbo, u64 vbo_to) lock_buffer(bh); bh->b_end_io = end_buffer_read_sync; get_bh(bh); - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); wait_on_buffer(bh); if (!buffer_uptodate(bh)) { diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index be4ebdd8048b..d100a063def2 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -629,7 +629,7 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, bh->b_size = block_size; off = vbo & (PAGE_SIZE - 1); set_bh_page(bh, page, off); - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); wait_on_buffer(bh); if (!buffer_uptodate(bh)) { err = -EIO; diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 35d40a67204c..304ed2be1b83 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -638,7 +638,7 @@ int ocfs2_map_page_blocks(struct page *page, u64 *p_blkno, !buffer_new(bh) && ocfs2_should_read_blk(inode, page, block_start) && (block_start < from || block_end > to)) { - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); *wait_bh++=bh; } diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c index e7758778abef..196638a22b48 100644 --- a/fs/ocfs2/buffer_head_io.c +++ b/fs/ocfs2/buffer_head_io.c @@ -64,7 +64,7 @@ int ocfs2_write_block(struct ocfs2_super *osb, struct buffer_head *bh, get_bh(bh); /* for end_buffer_write_sync() */ bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); wait_on_buffer(bh); @@ -147,7 +147,7 @@ int ocfs2_read_blocks_sync(struct ocfs2_super *osb, u64 block, get_bh(bh); /* for end_buffer_read_sync() */ bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); } read_failure: @@ -328,7 +328,7 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, if (validate) set_buffer_needs_validate(bh); bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, 0, bh); + submit_bh(REQ_OP_READ, bh); continue; } } @@ -449,7 +449,7 @@ int ocfs2_write_super_or_backup(struct ocfs2_super *osb, get_bh(bh); /* for end_buffer_write_sync() */ bh->b_end_io = end_buffer_write_sync; ocfs2_compute_meta_ecc(osb->sb, bh->b_data, &di->i_check); - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); wait_on_buffer(bh); diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index f7298816d8d9..e68807196076 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1785,7 +1785,7 @@ static int ocfs2_get_sector(struct super_block *sb, if (!buffer_dirty(*bh)) clear_buffer_uptodate(*bh); unlock_buffer(*bh); - ll_rw_block(REQ_OP_READ, 0, 1, bh); + ll_rw_block(REQ_OP_READ, 1, bh); wait_on_buffer(*bh); if (!buffer_uptodate(*bh)) { mlog_errno(-EIO); diff --git a/fs/reiserfs/inode.c b/fs/reiserfs/inode.c index 0cffe054b78e..23f542d1748b 100644 --- a/fs/reiserfs/inode.c +++ b/fs/reiserfs/inode.c @@ -2664,7 +2664,7 @@ static int reiserfs_write_full_page(struct page *page, do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); nr++; } put_bh(bh); @@ -2724,7 +2724,7 @@ fail: struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { clear_buffer_dirty(bh); - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); nr++; } put_bh(bh); diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c index d8cc9a366124..94addfcefede 100644 --- a/fs/reiserfs/journal.c +++ b/fs/reiserfs/journal.c @@ -650,7 +650,7 @@ static void submit_logged_buffer(struct buffer_head *bh) BUG(); if (!buffer_uptodate(bh)) BUG(); - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); } static void submit_ordered_buffer(struct buffer_head *bh) @@ -660,7 +660,7 @@ static void submit_ordered_buffer(struct buffer_head *bh) clear_buffer_dirty(bh); if (!buffer_uptodate(bh)) BUG(); - submit_bh(REQ_OP_WRITE, 0, bh); + submit_bh(REQ_OP_WRITE, bh); } #define CHUNK_SIZE 32 @@ -868,7 +868,7 @@ loop_next: */ if (buffer_dirty(bh) && unlikely(bh->b_page->mapping == NULL)) { spin_unlock(lock); - ll_rw_block(REQ_OP_WRITE, 0, 1, &bh); + ll_rw_block(REQ_OP_WRITE, 1, &bh); spin_lock(lock); } put_bh(bh); @@ -1054,7 +1054,7 @@ static int flush_commit_list(struct super_block *s, if (tbh) { if (buffer_dirty(tbh)) { depth = reiserfs_write_unlock_nested(s); - ll_rw_block(REQ_OP_WRITE, 0, 1, &tbh); + ll_rw_block(REQ_OP_WRITE, 1, &tbh); reiserfs_write_lock_nested(s, depth); } put_bh(tbh) ; @@ -2240,7 +2240,7 @@ abort_replay: } } /* read in the log blocks, memcpy to the corresponding real block */ - ll_rw_block(REQ_OP_READ, 0, get_desc_trans_len(desc), log_blocks); + ll_rw_block(REQ_OP_READ, get_desc_trans_len(desc), log_blocks); for (i = 0; i < get_desc_trans_len(desc); i++) { wait_on_buffer(log_blocks[i]); @@ -2342,7 +2342,7 @@ static struct buffer_head *reiserfs_breada(struct block_device *dev, } else bhlist[j++] = bh; } - ll_rw_block(REQ_OP_READ, 0, j, bhlist); + ll_rw_block(REQ_OP_READ, j, bhlist); for (i = 1; i < j; i++) brelse(bhlist[i]); bh = bhlist[0]; diff --git a/fs/reiserfs/stree.c b/fs/reiserfs/stree.c index ef42729216d1..9a293609a022 100644 --- a/fs/reiserfs/stree.c +++ b/fs/reiserfs/stree.c @@ -579,7 +579,7 @@ static int search_by_key_reada(struct super_block *s, if (!buffer_uptodate(bh[j])) { if (depth == -1) depth = reiserfs_write_unlock_nested(s); - ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, bh + j); + ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, bh + j); } brelse(bh[j]); } @@ -685,7 +685,7 @@ int search_by_key(struct super_block *sb, const struct cpu_key *key, if (!buffer_uptodate(bh) && depth == -1) depth = reiserfs_write_unlock_nested(sb); - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); wait_on_buffer(bh); if (depth != -1) diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c index cfb7c44c7366..c88cd2ce0665 100644 --- a/fs/reiserfs/super.c +++ b/fs/reiserfs/super.c @@ -1702,7 +1702,7 @@ static int read_super_block(struct super_block *s, int offset) /* after journal replay, reread all bitmap and super blocks */ static int reread_meta_blocks(struct super_block *s) { - ll_rw_block(REQ_OP_READ, 0, 1, &SB_BUFFER_WITH_SB(s)); + ll_rw_block(REQ_OP_READ, 1, &SB_BUFFER_WITH_SB(s)); wait_on_buffer(SB_BUFFER_WITH_SB(s)); if (!buffer_uptodate(SB_BUFFER_WITH_SB(s))) { reiserfs_warning(s, "reiserfs-2504", "error reading the super"); diff --git a/fs/udf/dir.c b/fs/udf/dir.c index 42e3e551fa4c..cad3772f9dbe 100644 --- a/fs/udf/dir.c +++ b/fs/udf/dir.c @@ -130,7 +130,7 @@ static int udf_readdir(struct file *file, struct dir_context *ctx) brelse(tmp); } if (num) { - ll_rw_block(REQ_OP_READ, REQ_RAHEAD, num, bha); + ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha); for (i = 0; i < num; i++) brelse(bha[i]); } diff --git a/fs/udf/directory.c b/fs/udf/directory.c index 73720320f0ab..a2adf6293093 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -89,7 +89,7 @@ struct fileIdentDesc *udf_fileident_read(struct inode *dir, loff_t *nf_pos, brelse(tmp); } if (num) { - ll_rw_block(REQ_OP_READ, REQ_RAHEAD, num, bha); + ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha); for (i = 0; i < num; i++) brelse(bha[i]); } diff --git a/fs/udf/inode.c b/fs/udf/inode.c index edc88716751a..8d06daed549f 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -1214,7 +1214,7 @@ struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, if (buffer_uptodate(bh)) return bh; - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); wait_on_buffer(bh); if (buffer_uptodate(bh)) diff --git a/fs/ufs/balloc.c b/fs/ufs/balloc.c index 075d3d9114c8..bd810d8239f2 100644 --- a/fs/ufs/balloc.c +++ b/fs/ufs/balloc.c @@ -296,7 +296,7 @@ static void ufs_change_blocknr(struct inode *inode, sector_t beg, if (!buffer_mapped(bh)) map_bh(bh, inode->i_sb, oldb + pos); if (!buffer_uptodate(bh)) { - ll_rw_block(REQ_OP_READ, 0, 1, &bh); + ll_rw_block(REQ_OP_READ, 1, &bh); wait_on_buffer(bh); if (!buffer_uptodate(bh)) { ufs_error(inode->i_sb, __func__, diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 9795df9400bd..bb68eb6407da 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -202,11 +202,11 @@ struct buffer_head *alloc_buffer_head(gfp_t gfp_flags); void free_buffer_head(struct buffer_head * bh); void unlock_buffer(struct buffer_head *bh); void __lock_buffer(struct buffer_head *bh); -void ll_rw_block(enum req_op, blk_opf_t, int, struct buffer_head * bh[]); +void ll_rw_block(blk_opf_t, int, struct buffer_head * bh[]); int sync_dirty_buffer(struct buffer_head *bh); int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); -int submit_bh(enum req_op, blk_opf_t, struct buffer_head *); +int submit_bh(blk_opf_t, struct buffer_head *); void write_boundary_block(struct block_device *bdev, sector_t bblock, unsigned blocksize); int bh_uptodate_or_lock(struct buffer_head *bh); -- cgit v1.2.3 From 6669797b0dd41ced457760b6e1014fdda8ce19ce Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 14 Jul 2022 11:07:22 -0700 Subject: fs/jbd2: Fix the documentation of the jbd2_write_superblock() callers Commit 2a222ca992c3 ("fs: have submit_bh users pass in op and flags separately") renamed the jbd2_write_superblock() 'write_op' argument into 'write_flags'. Propagate this change to the jbd2_write_superblock() callers. Additionally, change the type of 'write_flags' into blk_opf_t. Cc: Mike Christie Cc: Theodore Ts'o Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20220714180729.1065367-57-bvanassche@acm.org Signed-off-by: Jens Axboe --- fs/jbd2/journal.c | 15 ++++++++------- include/linux/jbd2.h | 2 +- include/trace/events/jbd2.h | 12 ++++++------ 3 files changed, 15 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 07e6aaf7e213..2a1b9da7c3e3 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -1604,7 +1604,7 @@ static int journal_reset(journal_t *journal) * This function expects that the caller will have locked the journal * buffer head, and will return with it unlocked */ -static int jbd2_write_superblock(journal_t *journal, int write_flags) +static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) { struct buffer_head *bh = journal->j_sb_buffer; journal_superblock_t *sb = journal->j_superblock; @@ -1661,13 +1661,14 @@ static int jbd2_write_superblock(journal_t *journal, int write_flags) * @journal: The journal to update. * @tail_tid: TID of the new transaction at the tail of the log * @tail_block: The first block of the transaction at the tail of the log - * @write_op: With which operation should we write the journal sb + * @write_flags: Flags for the journal sb write operation * * Update a journal's superblock information about log tail and write it to * disk, waiting for the IO to complete. */ int jbd2_journal_update_sb_log_tail(journal_t *journal, tid_t tail_tid, - unsigned long tail_block, int write_op) + unsigned long tail_block, + blk_opf_t write_flags) { journal_superblock_t *sb = journal->j_superblock; int ret; @@ -1687,7 +1688,7 @@ int jbd2_journal_update_sb_log_tail(journal_t *journal, tid_t tail_tid, sb->s_sequence = cpu_to_be32(tail_tid); sb->s_start = cpu_to_be32(tail_block); - ret = jbd2_write_superblock(journal, write_op); + ret = jbd2_write_superblock(journal, write_flags); if (ret) goto out; @@ -1704,12 +1705,12 @@ out: /** * jbd2_mark_journal_empty() - Mark on disk journal as empty. * @journal: The journal to update. - * @write_op: With which operation should we write the journal sb + * @write_flags: Flags for the journal sb write operation * * Update a journal's dynamic superblock fields to show that journal is empty. * Write updated superblock to disk waiting for IO to complete. */ -static void jbd2_mark_journal_empty(journal_t *journal, int write_op) +static void jbd2_mark_journal_empty(journal_t *journal, blk_opf_t write_flags) { journal_superblock_t *sb = journal->j_superblock; bool had_fast_commit = false; @@ -1735,7 +1736,7 @@ static void jbd2_mark_journal_empty(journal_t *journal, int write_op) had_fast_commit = true; } - jbd2_write_superblock(journal, write_op); + jbd2_write_superblock(journal, write_flags); if (had_fast_commit) jbd2_set_feature_fast_commit(journal); diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index e79d6e0b14e8..dc1724131300 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -1557,7 +1557,7 @@ extern int jbd2_journal_wipe (journal_t *, int); extern int jbd2_journal_skip_recovery (journal_t *); extern void jbd2_journal_update_sb_errno(journal_t *); extern int jbd2_journal_update_sb_log_tail (journal_t *, tid_t, - unsigned long, int); + unsigned long, blk_opf_t); extern void jbd2_journal_abort (journal_t *, int); extern int jbd2_journal_errno (journal_t *); extern void jbd2_journal_ack_err (journal_t *); diff --git a/include/trace/events/jbd2.h b/include/trace/events/jbd2.h index a4dfe005983d..99f783c384bb 100644 --- a/include/trace/events/jbd2.h +++ b/include/trace/events/jbd2.h @@ -355,22 +355,22 @@ TRACE_EVENT(jbd2_update_log_tail, TRACE_EVENT(jbd2_write_superblock, - TP_PROTO(journal_t *journal, int write_op), + TP_PROTO(journal_t *journal, blk_opf_t write_flags), - TP_ARGS(journal, write_op), + TP_ARGS(journal, write_flags), TP_STRUCT__entry( __field( dev_t, dev ) - __field( int, write_op ) + __field( blk_opf_t, write_flags ) ), TP_fast_assign( __entry->dev = journal->j_fs_dev->bd_dev; - __entry->write_op = write_op; + __entry->write_flags = write_flags; ), - TP_printk("dev %d,%d write_op %x", MAJOR(__entry->dev), - MINOR(__entry->dev), __entry->write_op) + TP_printk("dev %d,%d write_flags %x", MAJOR(__entry->dev), + MINOR(__entry->dev), (__force u32)__entry->write_flags) ); TRACE_EVENT(jbd2_lock_buffer_stall, -- cgit v1.2.3 From b644c95598adfe2b968e97daee3a890dd2fda84d Mon Sep 17 00:00:00 2001 From: PaddyKP_Yao Date: Mon, 11 Jul 2022 19:51:25 +0800 Subject: platform/x86: asus-wmi: Add mic-mute LED classdev support In some new ASUS devices, hotkey Fn+F13 is used for mic mute. If mic-mute LED is present by checking WMI ASUS_WMI_DEVID_MICMUTE_LED, we will add a mic-mute LED classdev, asus::micmute, in the asus-wmi driver to control it. The binding of mic-mute LED controls will be swithched with LED trigger. Signed-off-by: PaddyKP_Yao Link: https://lore.kernel.org/r/20220711115125.2072508-1-PaddyKP_Yao@asus.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/Kconfig | 2 ++ drivers/platform/x86/asus-wmi.c | 25 +++++++++++++++++++++++++ include/linux/platform_data/x86/asus-wmi.h | 1 + 3 files changed, 28 insertions(+) (limited to 'include/linux') diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 7fa88efeef4d..6a33c862452b 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -273,6 +273,8 @@ config ASUS_WMI select INPUT_SPARSEKMAP select LEDS_CLASS select NEW_LEDS + select LEDS_TRIGGERS + select LEDS_TRIGGER_AUDIO select ACPI_PLATFORM_PROFILE help Say Y here if you have a WMI aware Asus laptop (like Eee PCs or new diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index 62ce198a3463..89b604e04d7f 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -208,6 +208,7 @@ struct asus_wmi { int kbd_led_wk; struct led_classdev lightbar_led; int lightbar_led_wk; + struct led_classdev micmute_led; struct workqueue_struct *led_workqueue; struct work_struct tpd_led_work; struct work_struct wlan_led_work; @@ -1028,12 +1029,23 @@ static enum led_brightness lightbar_led_get(struct led_classdev *led_cdev) return result & ASUS_WMI_DSTS_LIGHTBAR_MASK; } +static int micmute_led_set(struct led_classdev *led_cdev, + enum led_brightness brightness) +{ + int state = brightness != LED_OFF; + int err; + + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_MICMUTE_LED, state, NULL); + return err < 0 ? err : 0; +} + static void asus_wmi_led_exit(struct asus_wmi *asus) { led_classdev_unregister(&asus->kbd_led); led_classdev_unregister(&asus->tpd_led); led_classdev_unregister(&asus->wlan_led); led_classdev_unregister(&asus->lightbar_led); + led_classdev_unregister(&asus->micmute_led); if (asus->led_workqueue) destroy_workqueue(asus->led_workqueue); @@ -1105,6 +1117,19 @@ static int asus_wmi_led_init(struct asus_wmi *asus) &asus->lightbar_led); } + if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_MICMUTE_LED)) { + asus->micmute_led.name = "asus::micmute"; + asus->micmute_led.max_brightness = 1; + asus->micmute_led.brightness = ledtrig_audio_get(LED_AUDIO_MICMUTE); + asus->micmute_led.brightness_set_blocking = micmute_led_set; + asus->micmute_led.default_trigger = "audio-micmute"; + + rv = led_classdev_register(&asus->platform_device->dev, + &asus->micmute_led); + if (rv) + goto error; + } + error: if (rv) asus_wmi_led_exit(asus); diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index a571b47ff362..98f2b2f20f3e 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -49,6 +49,7 @@ #define ASUS_WMI_DEVID_LED4 0x00020014 #define ASUS_WMI_DEVID_LED5 0x00020015 #define ASUS_WMI_DEVID_LED6 0x00020016 +#define ASUS_WMI_DEVID_MICMUTE_LED 0x00040017 /* Backlight and Brightness */ #define ASUS_WMI_DEVID_ALS_ENABLE 0x00050001 /* Ambient Light Sensor */ -- cgit v1.2.3 From cd89edda4002b7fb3c0a6765c3a60a60d5b1dc16 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Thu, 14 Jul 2022 20:42:12 +0800 Subject: PCI: loongson: Add ACPI init support Loongson PCH (LS7A chipset) will be used by both MIPS-based and LoongArch- based Loongson processors. MIPS-based Loongson uses FDT, while LoongArch- based Loongson uses ACPI. Add ACPI init support for the driver in pci-loongson.c because it is currently FDT-only. LoongArch is a new RISC ISA, mainline support will come soon, and documentations are here (in translation): https://github.com/loongson/LoongArch-Documentation Link: https://lore.kernel.org/r/20220714124216.1489304-4-chenhuacai@loongson.cn Signed-off-by: Huacai Chen Signed-off-by: Bjorn Helgaas --- drivers/acpi/pci_mcfg.c | 10 ++++ drivers/pci/controller/Kconfig | 2 +- drivers/pci/controller/pci-loongson.c | 94 ++++++++++++++++++++++++++++------- include/linux/pci-ecam.h | 1 + 4 files changed, 87 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index 63b98eae5e75..860014b89b8e 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -172,6 +172,16 @@ static struct mcfg_fixup mcfg_quirks[] = { ALTRA_ECAM_QUIRK(1, 14), ALTRA_ECAM_QUIRK(1, 15), #endif /* ARM64 */ + +#ifdef CONFIG_LOONGARCH +#define LOONGSON_ECAM_MCFG(table_id, seg) \ + { "LOONGS", table_id, 1, seg, MCFG_BUS_ANY, &loongson_pci_ecam_ops } + + LOONGSON_ECAM_MCFG("\0", 0), + LOONGSON_ECAM_MCFG("LOONGSON", 0), + LOONGSON_ECAM_MCFG("\0", 1), + LOONGSON_ECAM_MCFG("LOONGSON", 1), +#endif /* LOONGARCH */ }; static char mcfg_oem_id[ACPI_OEM_ID_SIZE]; diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index b8d96d38064d..9dbd73898b47 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -293,7 +293,7 @@ config PCI_HYPERV_INTERFACE config PCI_LOONGSON bool "LOONGSON PCI Controller" depends on MACH_LOONGSON64 || COMPILE_TEST - depends on OF + depends on OF || ACPI depends on PCI_QUIRKS default MACH_LOONGSON64 help diff --git a/drivers/pci/controller/pci-loongson.c b/drivers/pci/controller/pci-loongson.c index 565453882ffe..cd29800974e7 100644 --- a/drivers/pci/controller/pci-loongson.c +++ b/drivers/pci/controller/pci-loongson.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "../pci.h" @@ -97,39 +99,53 @@ static void loongson_mrrs_quirk(struct pci_dev *dev) } DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, loongson_mrrs_quirk); -static void __iomem *cfg1_map(struct loongson_pci *priv, int bus, - unsigned int devfn, int where) +static struct loongson_pci *pci_bus_to_loongson_pci(struct pci_bus *bus) { - unsigned long addroff = 0x0; + struct pci_config_window *cfg; - if (bus != 0) - addroff |= BIT(28); /* Type 1 Access */ - addroff |= (where & 0xff) | ((where & 0xf00) << 16); - addroff |= (bus << 16) | (devfn << 8); - return priv->cfg1_base + addroff; + if (acpi_disabled) + return (struct loongson_pci *)(bus->sysdata); + + cfg = bus->sysdata; + return (struct loongson_pci *)(cfg->priv); } -static void __iomem *cfg0_map(struct loongson_pci *priv, int bus, - unsigned int devfn, int where) +static void __iomem *cfg0_map(struct loongson_pci *priv, struct pci_bus *bus, + unsigned int devfn, int where) { unsigned long addroff = 0x0; + unsigned char busnum = bus->number; - if (bus != 0) + if (!pci_is_root_bus(bus)) { addroff |= BIT(24); /* Type 1 Access */ - addroff |= (bus << 16) | (devfn << 8) | where; + addroff |= (busnum << 16); + } + addroff |= (devfn << 8) | where; return priv->cfg0_base + addroff; } -static void __iomem *pci_loongson_map_bus(struct pci_bus *bus, unsigned int devfn, - int where) +static void __iomem *cfg1_map(struct loongson_pci *priv, struct pci_bus *bus, + unsigned int devfn, int where) { + unsigned long addroff = 0x0; unsigned char busnum = bus->number; - struct pci_host_bridge *bridge = pci_find_host_bridge(bus); - struct loongson_pci *priv = pci_host_bridge_priv(bridge); + + if (!pci_is_root_bus(bus)) { + addroff |= BIT(28); /* Type 1 Access */ + addroff |= (busnum << 16); + } + addroff |= (devfn << 8) | (where & 0xff) | ((where & 0xf00) << 16); + return priv->cfg1_base + addroff; +} + +static void __iomem *pci_loongson_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct loongson_pci *priv = pci_bus_to_loongson_pci(bus); /* * Do not read more than one device on the bus other than - * the host bus. For our hardware the root bus is always bus 0. + * the host bus. */ if (priv->data->flags & FLAG_DEV_FIX && !pci_is_root_bus(bus) && PCI_SLOT(devfn) > 0) @@ -137,15 +153,17 @@ static void __iomem *pci_loongson_map_bus(struct pci_bus *bus, unsigned int devf /* CFG0 can only access standard space */ if (where < PCI_CFG_SPACE_SIZE && priv->cfg0_base) - return cfg0_map(priv, busnum, devfn, where); + return cfg0_map(priv, bus, devfn, where); /* CFG1 can access extended space */ if (where < PCI_CFG_SPACE_EXP_SIZE && priv->cfg1_base) - return cfg1_map(priv, busnum, devfn, where); + return cfg1_map(priv, bus, devfn, where); return NULL; } +#ifdef CONFIG_OF + static int loongson_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { int irq; @@ -259,3 +277,41 @@ static struct platform_driver loongson_pci_driver = { .probe = loongson_pci_probe, }; builtin_platform_driver(loongson_pci_driver); + +#endif + +#ifdef CONFIG_ACPI + +static int loongson_pci_ecam_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct loongson_pci *priv; + struct loongson_pci_data *data; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + cfg->priv = priv; + data->flags = FLAG_CFG1; + priv->data = data; + priv->cfg1_base = cfg->win - (cfg->busr.start << 16); + + return 0; +} + +const struct pci_ecam_ops loongson_pci_ecam_ops = { + .bus_shift = 16, + .init = loongson_pci_ecam_init, + .pci_ops = { + .map_bus = pci_loongson_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +#endif diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h index adea5a4771cf..6b1301e2498e 100644 --- a/include/linux/pci-ecam.h +++ b/include/linux/pci-ecam.h @@ -87,6 +87,7 @@ extern const struct pci_ecam_ops xgene_v1_pcie_ecam_ops; /* APM X-Gene PCIe v1 * extern const struct pci_ecam_ops xgene_v2_pcie_ecam_ops; /* APM X-Gene PCIe v2.x */ extern const struct pci_ecam_ops al_pcie_ops; /* Amazon Annapurna Labs PCIe */ extern const struct pci_ecam_ops tegra194_pcie_ops; /* Tegra194 PCIe */ +extern const struct pci_ecam_ops loongson_pci_ecam_ops; /* Loongson PCIe */ #endif #if IS_ENABLED(CONFIG_PCI_HOST_COMMON) -- cgit v1.2.3 From e2863a78593d638d3924a6f67900c4820034f349 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 1 Jul 2022 05:54:24 -0700 Subject: lib/bitmap: change return types to bool where appropriate Some bitmap functions return boolean results in int variables. Fix it by changing return types to bool. Signed-off-by: Yury Norov --- include/linux/bitmap.h | 8 ++++---- lib/bitmap.c | 4 ++-- tools/include/linux/bitmap.h | 8 ++++---- tools/lib/bitmap.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index c91638e507f2..e1a438bdda52 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -148,13 +148,13 @@ void __bitmap_shift_left(unsigned long *dst, const unsigned long *src, unsigned int shift, unsigned int nbits); void bitmap_cut(unsigned long *dst, const unsigned long *src, unsigned int first, unsigned int cut, unsigned int nbits); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); -int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); void __bitmap_replace(unsigned long *dst, const unsigned long *old, const unsigned long *new, @@ -315,7 +315,7 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits); bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits)) #endif -static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, +static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) @@ -341,7 +341,7 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, __bitmap_xor(dst, src1, src2, nbits); } -static inline int bitmap_andnot(unsigned long *dst, const unsigned long *src1, +static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) diff --git a/lib/bitmap.c b/lib/bitmap.c index e903e13c62e1..9bc80f5bf149 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -237,7 +237,7 @@ void bitmap_cut(unsigned long *dst, const unsigned long *src, } EXPORT_SYMBOL(bitmap_cut); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; @@ -275,7 +275,7 @@ void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_xor); -int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; diff --git a/tools/include/linux/bitmap.h b/tools/include/linux/bitmap.h index afdf93bebaaf..2ae7ab8ed7d1 100644 --- a/tools/include/linux/bitmap.h +++ b/tools/include/linux/bitmap.h @@ -14,7 +14,7 @@ int __bitmap_weight(const unsigned long *bitmap, int bits); void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits); bool __bitmap_equal(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits); @@ -45,7 +45,7 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits); } -static inline int bitmap_empty(const unsigned long *src, unsigned nbits) +static inline bool bitmap_empty(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); @@ -53,7 +53,7 @@ static inline int bitmap_empty(const unsigned long *src, unsigned nbits) return find_first_bit(src, nbits) == nbits; } -static inline int bitmap_full(const unsigned long *src, unsigned int nbits) +static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); @@ -146,7 +146,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, * @src2: operand 2 * @nbits: size of bitmap */ -static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, +static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) diff --git a/tools/lib/bitmap.c b/tools/lib/bitmap.c index 354f8cdc0880..2e351d63fdba 100644 --- a/tools/lib/bitmap.c +++ b/tools/lib/bitmap.c @@ -57,7 +57,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, return ret; } -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; -- cgit v1.2.3 From 3a06ed80265fa62eecaf519d92f1633e4f9510c7 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Fri, 8 Jul 2022 17:57:14 +0800 Subject: extcon: Add EXTCON_DISP_CVBS and EXTCON_DISP_EDP Add EXTCON_DISP_CVBS for Composite Video Broadcast Signal. Add EXTCON_DISP_EDP for Embedded Display Port [1] https://en.wikipedia.org/wiki/Composite_video [2] https://en.wikipedia.org/wiki/DisplayPort#eDP Signed-off-by: Michael Wu Signed-off-by: Chanwoo Choi --- drivers/extcon/extcon.c | 10 ++++++++++ include/linux/extcon.h | 2 ++ 2 files changed, 12 insertions(+) (limited to 'include/linux') diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c index 0e40418ad287..e1c71359b605 100644 --- a/drivers/extcon/extcon.c +++ b/drivers/extcon/extcon.c @@ -167,6 +167,16 @@ static const struct __extcon_info { .id = EXTCON_DISP_HMD, .name = "HMD", }, + [EXTCON_DISP_CVBS] = { + .type = EXTCON_TYPE_DISP, + .id = EXTCON_DISP_CVBS, + .name = "CVBS", + }, + [EXTCON_DISP_EDP] = { + .type = EXTCON_TYPE_DISP, + .id = EXTCON_DISP_EDP, + .name = "EDP", + }, /* Miscellaneous external connector */ [EXTCON_DOCK] = { diff --git a/include/linux/extcon.h b/include/linux/extcon.h index 685401d94d39..3c45c3846fe9 100644 --- a/include/linux/extcon.h +++ b/include/linux/extcon.h @@ -76,6 +76,8 @@ #define EXTCON_DISP_VGA 43 /* Video Graphics Array */ #define EXTCON_DISP_DP 44 /* Display Port */ #define EXTCON_DISP_HMD 45 /* Head-Mounted Display */ +#define EXTCON_DISP_CVBS 46 /* Composite Video Broadcast Signal */ +#define EXTCON_DISP_EDP 47 /* Embedded Display Port */ /* Miscellaneous external connector */ #define EXTCON_DOCK 60 -- cgit v1.2.3 From 309c56e84602d894e7ca424b0b13837117568fca Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 8 Jul 2022 10:06:13 +0200 Subject: iommu: remove the unused dev_has_feat method This method is never actually called. Signed-off-by: Christoph Hellwig Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/20220708080616.238833-2-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 1 - include/linux/iommu.h | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d9c1623ec1a9..1b6c17dd81ee 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2853,7 +2853,6 @@ static struct iommu_ops arm_smmu_ops = { .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, - .dev_has_feat = arm_smmu_dev_has_feature, .dev_feat_enabled = arm_smmu_dev_feature_enabled, .dev_enable_feat = arm_smmu_dev_enable_feature, .dev_disable_feat = arm_smmu_dev_disable_feature, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e6abd998dbe7..a3acdb46b939 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -164,8 +164,7 @@ struct iommu_iort_rmr_data { * supported, this feature must be enabled before and * disabled after %IOMMU_DEV_FEAT_SVA. * - * Device drivers query whether a feature is supported using - * iommu_dev_has_feature(), and enable it using iommu_dev_enable_feature(). + * Device drivers enable a feature using iommu_dev_enable_feature(). */ enum iommu_dev_features { IOMMU_DEV_FEAT_SVA, @@ -248,7 +247,6 @@ struct iommu_ops { bool (*is_attach_deferred)(struct device *dev); /* Per device IOMMU features */ - bool (*dev_has_feat)(struct device *dev, enum iommu_dev_features f); bool (*dev_feat_enabled)(struct device *dev, enum iommu_dev_features f); int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); -- cgit v1.2.3 From a871765d5588531e1972902d1ae077b8be306c94 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 8 Jul 2022 10:06:14 +0200 Subject: iommu: remove iommu_dev_feature_enabled Remove the unused iommu_dev_feature_enabled function. Signed-off-by: Christoph Hellwig Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/20220708080616.238833-3-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 1 - drivers/iommu/iommu.c | 13 ------------- include/linux/iommu.h | 9 --------- 3 files changed, 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1b6c17dd81ee..4d30a8d2bc23 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2853,7 +2853,6 @@ static struct iommu_ops arm_smmu_ops = { .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, - .dev_feat_enabled = arm_smmu_dev_feature_enabled, .dev_enable_feat = arm_smmu_dev_enable_feature, .dev_disable_feat = arm_smmu_dev_disable_feature, .sva_bind = arm_smmu_sva_bind, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 0aa141646bdf..1bb016a6a2aa 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2760,19 +2760,6 @@ int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) } EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); -bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) -{ - if (dev->iommu && dev->iommu->iommu_dev) { - const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; - - if (ops->dev_feat_enabled) - return ops->dev_feat_enabled(dev, feat); - } - - return false; -} -EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled); - /** * iommu_sva_bind_device() - Bind a process address space to a device * @dev: the device diff --git a/include/linux/iommu.h b/include/linux/iommu.h index a3acdb46b939..0bc2eb14b026 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -215,7 +215,6 @@ struct iommu_iotlb_gather { * driver init to device driver init (default no) * @dev_has/enable/disable_feat: per device entries to check/enable/disable * iommu specific features. - * @dev_feat_enabled: check enabled feature * @sva_bind: Bind process address space to device * @sva_unbind: Unbind process address space from device * @sva_get_pasid: Get PASID associated to a SVA handle @@ -247,7 +246,6 @@ struct iommu_ops { bool (*is_attach_deferred)(struct device *dev); /* Per device IOMMU features */ - bool (*dev_feat_enabled)(struct device *dev, enum iommu_dev_features f); int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); @@ -670,7 +668,6 @@ void iommu_release_device(struct device *dev); int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f); -bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f); struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, @@ -997,12 +994,6 @@ const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) return NULL; } -static inline bool -iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) -{ - return false; -} - static inline int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) { -- cgit v1.2.3 From ae3ff39a51a0f5843960487962e110339f321b0f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 8 Jul 2022 10:06:15 +0200 Subject: iommu: remove the put_resv_regions method All drivers that implement get_resv_regions just use generic_put_resv_regions to implement the put side. Remove the indirections and document the allocations constraints. Signed-off-by: Christoph Hellwig Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/20220708080616.238833-4-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 1 - drivers/iommu/apple-dart.c | 1 - drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 1 - drivers/iommu/arm/arm-smmu/arm-smmu.c | 1 - drivers/iommu/intel/iommu.c | 1 - drivers/iommu/iommu.c | 21 ++++----------------- drivers/iommu/mtk_iommu.c | 1 - drivers/iommu/virtio-iommu.c | 5 ++--- include/linux/iommu.h | 4 ---- 9 files changed, 6 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 840831d5d2ad..e66e071e8c3b 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2280,7 +2280,6 @@ const struct iommu_ops amd_iommu_ops = { .probe_finalize = amd_iommu_probe_finalize, .device_group = amd_iommu_device_group, .get_resv_regions = amd_iommu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, .pgsize_bitmap = AMD_IOMMU_PGSIZES, .def_domain_type = amd_iommu_def_domain_type, diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index e87d3cf54ed6..1b1725759262 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -768,7 +768,6 @@ static const struct iommu_ops apple_dart_iommu_ops = { .of_xlate = apple_dart_of_xlate, .def_domain_type = apple_dart_def_domain_type, .get_resv_regions = apple_dart_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, .pgsize_bitmap = -1UL, /* Restricted during dart probe */ .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4d30a8d2bc23..4a5e435567f1 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2852,7 +2852,6 @@ static struct iommu_ops arm_smmu_ops = { .device_group = arm_smmu_device_group, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, .dev_enable_feat = arm_smmu_dev_enable_feature, .dev_disable_feat = arm_smmu_dev_disable_feature, .sva_bind = arm_smmu_sva_bind, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 588929bed1bc..2d4129a4ccfc 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1584,7 +1584,6 @@ static struct iommu_ops arm_smmu_ops = { .device_group = arm_smmu_device_group, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, .def_domain_type = arm_smmu_def_domain_type, .pgsize_bitmap = -1UL, /* Restricted during device attach */ .owner = THIS_MODULE, diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 44016594831d..49d616aa2148 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4911,7 +4911,6 @@ const struct iommu_ops intel_iommu_ops = { .probe_finalize = intel_iommu_probe_finalize, .release_device = intel_iommu_release_device, .get_resv_regions = intel_iommu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, .device_group = intel_iommu_device_group, .dev_enable_feat = intel_iommu_dev_enable_feat, .dev_disable_feat = intel_iommu_dev_disable_feat, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 1bb016a6a2aa..f53f8b2d27a5 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2579,27 +2579,14 @@ void iommu_get_resv_regions(struct device *dev, struct list_head *list) ops->get_resv_regions(dev, list); } -void iommu_put_resv_regions(struct device *dev, struct list_head *list) -{ - const struct iommu_ops *ops = dev_iommu_ops(dev); - - if (ops->put_resv_regions) - ops->put_resv_regions(dev, list); -} - /** - * generic_iommu_put_resv_regions - Reserved region driver helper + * iommu_put_resv_regions - release resered regions * @dev: device for which to free reserved regions * @list: reserved region list for device * - * IOMMU drivers can use this to implement their .put_resv_regions() callback - * for simple reservations. If a per region callback is provided that will be - * used to free all memory allocations associated with the reserved region or - * else just free up the memory for the regions. If an IOMMU driver allocates - * additional resources per region, it is going to have to implement a custom - * callback. + * This releases a reserved region list acquired by iommu_get_resv_regions(). */ -void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list) +void iommu_put_resv_regions(struct device *dev, struct list_head *list) { struct iommu_resv_region *entry, *next; @@ -2610,7 +2597,7 @@ void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list) kfree(entry); } } -EXPORT_SYMBOL(generic_iommu_put_resv_regions); +EXPORT_SYMBOL(iommu_put_resv_regions); struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 5c3d9366c25c..95fd21c7207a 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -928,7 +928,6 @@ static const struct iommu_ops mtk_iommu_ops = { .device_group = mtk_iommu_device_group, .of_xlate = mtk_iommu_of_xlate, .get_resv_regions = mtk_iommu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 55337796a5f8..feeb5fde72a3 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -964,7 +964,7 @@ static struct iommu_device *viommu_probe_device(struct device *dev) return &viommu->iommu; err_free_dev: - generic_iommu_put_resv_regions(dev, &vdev->resv_regions); + iommu_put_resv_regions(dev, &vdev->resv_regions); kfree(vdev); return ERR_PTR(ret); @@ -983,7 +983,7 @@ static void viommu_release_device(struct device *dev) { struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); - generic_iommu_put_resv_regions(dev, &vdev->resv_regions); + iommu_put_resv_regions(dev, &vdev->resv_regions); kfree(vdev); } @@ -1007,7 +1007,6 @@ static struct iommu_ops viommu_ops = { .release_device = viommu_release_device, .device_group = viommu_device_group, .get_resv_regions = viommu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, .of_xlate = viommu_of_xlate, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0bc2eb14b026..ea30f00dc145 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -209,7 +209,6 @@ struct iommu_iotlb_gather { * group and attached to the groups domain * @device_group: find iommu group for a particular device * @get_resv_regions: Request list of reserved regions for a device - * @put_resv_regions: Free list of reserved regions for a device * @of_xlate: add OF master IDs to iommu grouping * @is_attach_deferred: Check if domain attach should be deferred from iommu * driver init to device driver init (default no) @@ -240,7 +239,6 @@ struct iommu_ops { /* Request/Free a list of reserved regions for a device */ void (*get_resv_regions)(struct device *dev, struct list_head *list); - void (*put_resv_regions)(struct device *dev, struct list_head *list); int (*of_xlate)(struct device *dev, struct of_phandle_args *args); bool (*is_attach_deferred)(struct device *dev); @@ -454,8 +452,6 @@ extern void iommu_set_fault_handler(struct iommu_domain *domain, extern void iommu_get_resv_regions(struct device *dev, struct list_head *list); extern void iommu_put_resv_regions(struct device *dev, struct list_head *list); -extern void generic_iommu_put_resv_regions(struct device *dev, - struct list_head *list); extern void iommu_set_default_passthrough(bool cmd_line); extern void iommu_set_default_translated(bool cmd_line); extern bool iommu_default_passthrough(void); -- cgit v1.2.3 From f9903555dd05a4096d8fef4eb823c0b94f710982 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 12 Jul 2022 08:08:46 +0800 Subject: iommu/vt-d: Remove unnecessary exported symbol The exported symbol intel_iommu_gfx_mapped is not used anywhere in the tree. Remove it to avoid dead code. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Reviewed-by: Jason Gunthorpe Reviewed-by: Steve Wahl Link: https://lore.kernel.org/r/20220514014322.2927339-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 6 ------ include/linux/intel-iommu.h | 1 - 2 files changed, 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 5c0dce78586a..6564af7ec0dd 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -314,9 +314,6 @@ static int iommu_skip_te_disable; #define IDENTMAP_GFX 2 #define IDENTMAP_AZALIA 4 -int intel_iommu_gfx_mapped; -EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); - DEFINE_SPINLOCK(device_domain_lock); static LIST_HEAD(device_domain_list); @@ -4093,9 +4090,6 @@ int __init intel_iommu_init(void) if (list_empty(&dmar_satc_units)) pr_info("No SATC found\n"); - if (dmar_map_gfx) - intel_iommu_gfx_mapped = 1; - init_no_remapping_devices(); ret = init_dmars(); diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 5fcf89faa31a..4ebf3c4da45e 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -787,7 +787,6 @@ extern int iommu_calculate_agaw(struct intel_iommu *iommu); extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); extern int dmar_disabled; extern int intel_iommu_enabled; -extern int intel_iommu_gfx_mapped; #else static inline int iommu_calculate_agaw(struct intel_iommu *iommu) { -- cgit v1.2.3 From 853788b9a66ff089053df3b4ed7d166e61def449 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 12 Jul 2022 08:08:49 +0800 Subject: x86/boot/tboot: Move tboot_force_iommu() to Intel IOMMU tboot_force_iommu() is only called by the Intel IOMMU driver. Move the helper into that driver. No functional change intended. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Reviewed-by: Jason Gunthorpe Reviewed-by: Steve Wahl Link: https://lore.kernel.org/r/20220514014322.2927339-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- arch/x86/kernel/tboot.c | 15 --------------- drivers/iommu/intel/iommu.c | 14 ++++++++++++++ include/linux/tboot.h | 2 -- 3 files changed, 14 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index 0c1154a1c403..3bacd935f840 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -6,7 +6,6 @@ * Copyright (c) 2006-2009, Intel Corporation */ -#include #include #include #include @@ -516,17 +515,3 @@ struct acpi_table_header *tboot_get_dmar_table(struct acpi_table_header *dmar_tb return dmar_tbl; } - -int tboot_force_iommu(void) -{ - if (!tboot_enabled()) - return 0; - - if (no_iommu || dmar_disabled) - pr_warn("Forcing Intel-IOMMU to enabled\n"); - - dmar_disabled = 0; - no_iommu = 0; - - return 1; -} diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 6564af7ec0dd..c70948021e8e 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4019,6 +4019,20 @@ static int __init probe_acpi_namespace_devices(void) return 0; } +static __init int tboot_force_iommu(void) +{ + if (!tboot_enabled()) + return 0; + + if (no_iommu || dmar_disabled) + pr_warn("Forcing Intel-IOMMU to enabled\n"); + + dmar_disabled = 0; + no_iommu = 0; + + return 1; +} + int __init intel_iommu_init(void) { int ret = -ENODEV; diff --git a/include/linux/tboot.h b/include/linux/tboot.h index 5146d2574e85..d2279160ef39 100644 --- a/include/linux/tboot.h +++ b/include/linux/tboot.h @@ -126,7 +126,6 @@ extern void tboot_probe(void); extern void tboot_shutdown(u32 shutdown_type); extern struct acpi_table_header *tboot_get_dmar_table( struct acpi_table_header *dmar_tbl); -extern int tboot_force_iommu(void); #else @@ -136,7 +135,6 @@ extern int tboot_force_iommu(void); #define tboot_sleep(sleep_state, pm1a_control, pm1b_control) \ do { } while (0) #define tboot_get_dmar_table(dmar_tbl) (dmar_tbl) -#define tboot_force_iommu() 0 #endif /* !CONFIG_INTEL_TXT */ -- cgit v1.2.3 From 2585a2790e7fdb3dadfe309c9220bbc03704f84f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 12 Jul 2022 08:08:50 +0800 Subject: iommu/vt-d: Move include/linux/intel-iommu.h under iommu This header file is private to the Intel IOMMU driver. Move it to the driver folder. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Reviewed-by: Jason Gunthorpe Reviewed-by: Steve Wahl Link: https://lore.kernel.org/r/20220514014322.2927339-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- MAINTAINERS | 1 - drivers/iommu/intel/cap_audit.c | 2 +- drivers/iommu/intel/debugfs.c | 2 +- drivers/iommu/intel/dmar.c | 2 +- drivers/iommu/intel/iommu.c | 2 +- drivers/iommu/intel/iommu.h | 831 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intel/irq_remapping.c | 2 +- drivers/iommu/intel/pasid.c | 2 +- drivers/iommu/intel/perf.c | 2 +- drivers/iommu/intel/svm.c | 2 +- drivers/iommu/intel/trace.h | 3 +- include/linux/intel-iommu.h | 831 ------------------------------------ 12 files changed, 841 insertions(+), 841 deletions(-) create mode 100644 drivers/iommu/intel/iommu.h delete mode 100644 include/linux/intel-iommu.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index f679152bdbad..8f9ed151ad4c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10077,7 +10077,6 @@ L: iommu@lists.linux.dev S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git F: drivers/iommu/intel/ -F: include/linux/intel-iommu.h F: include/linux/intel-svm.h INTEL IOP-ADMA DMA DRIVER diff --git a/drivers/iommu/intel/cap_audit.c b/drivers/iommu/intel/cap_audit.c index 71596fc62822..3ee68393122f 100644 --- a/drivers/iommu/intel/cap_audit.c +++ b/drivers/iommu/intel/cap_audit.c @@ -10,7 +10,7 @@ #define pr_fmt(fmt) "DMAR: " fmt -#include +#include "iommu.h" #include "cap_audit.h" static u64 intel_iommu_cap_sanity; diff --git a/drivers/iommu/intel/debugfs.c b/drivers/iommu/intel/debugfs.c index ed796eea4581..d927ef10641b 100644 --- a/drivers/iommu/intel/debugfs.c +++ b/drivers/iommu/intel/debugfs.c @@ -10,11 +10,11 @@ #include #include -#include #include #include +#include "iommu.h" #include "pasid.h" #include "perf.h" diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index f91b45be1d92..2a5e0f91e647 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -31,6 +30,7 @@ #include #include +#include "iommu.h" #include "../irq_remapping.h" #include "perf.h" #include "trace.h" diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index c70948021e8e..10bda4bec8fe 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -26,6 +25,7 @@ #include #include +#include "iommu.h" #include "../irq_remapping.h" #include "../iommu-sva-lib.h" #include "pasid.h" diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h new file mode 100644 index 000000000000..4ebf3c4da45e --- /dev/null +++ b/drivers/iommu/intel/iommu.h @@ -0,0 +1,831 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright © 2006-2015, Intel Corporation. + * + * Authors: Ashok Raj + * Anil S Keshavamurthy + * David Woodhouse + */ + +#ifndef _INTEL_IOMMU_H_ +#define _INTEL_IOMMU_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* + * VT-d hardware uses 4KiB page size regardless of host page size. + */ +#define VTD_PAGE_SHIFT (12) +#define VTD_PAGE_SIZE (1UL << VTD_PAGE_SHIFT) +#define VTD_PAGE_MASK (((u64)-1) << VTD_PAGE_SHIFT) +#define VTD_PAGE_ALIGN(addr) (((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK) + +#define VTD_STRIDE_SHIFT (9) +#define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT) + +#define DMA_PTE_READ BIT_ULL(0) +#define DMA_PTE_WRITE BIT_ULL(1) +#define DMA_PTE_LARGE_PAGE BIT_ULL(7) +#define DMA_PTE_SNP BIT_ULL(11) + +#define DMA_FL_PTE_PRESENT BIT_ULL(0) +#define DMA_FL_PTE_US BIT_ULL(2) +#define DMA_FL_PTE_ACCESS BIT_ULL(5) +#define DMA_FL_PTE_DIRTY BIT_ULL(6) +#define DMA_FL_PTE_XD BIT_ULL(63) + +#define ADDR_WIDTH_5LEVEL (57) +#define ADDR_WIDTH_4LEVEL (48) + +#define CONTEXT_TT_MULTI_LEVEL 0 +#define CONTEXT_TT_DEV_IOTLB 1 +#define CONTEXT_TT_PASS_THROUGH 2 +#define CONTEXT_PASIDE BIT_ULL(3) + +/* + * Intel IOMMU register specification per version 1.0 public spec. + */ +#define DMAR_VER_REG 0x0 /* Arch version supported by this IOMMU */ +#define DMAR_CAP_REG 0x8 /* Hardware supported capabilities */ +#define DMAR_ECAP_REG 0x10 /* Extended capabilities supported */ +#define DMAR_GCMD_REG 0x18 /* Global command register */ +#define DMAR_GSTS_REG 0x1c /* Global status register */ +#define DMAR_RTADDR_REG 0x20 /* Root entry table */ +#define DMAR_CCMD_REG 0x28 /* Context command reg */ +#define DMAR_FSTS_REG 0x34 /* Fault Status register */ +#define DMAR_FECTL_REG 0x38 /* Fault control register */ +#define DMAR_FEDATA_REG 0x3c /* Fault event interrupt data register */ +#define DMAR_FEADDR_REG 0x40 /* Fault event interrupt addr register */ +#define DMAR_FEUADDR_REG 0x44 /* Upper address register */ +#define DMAR_AFLOG_REG 0x58 /* Advanced Fault control */ +#define DMAR_PMEN_REG 0x64 /* Enable Protected Memory Region */ +#define DMAR_PLMBASE_REG 0x68 /* PMRR Low addr */ +#define DMAR_PLMLIMIT_REG 0x6c /* PMRR low limit */ +#define DMAR_PHMBASE_REG 0x70 /* pmrr high base addr */ +#define DMAR_PHMLIMIT_REG 0x78 /* pmrr high limit */ +#define DMAR_IQH_REG 0x80 /* Invalidation queue head register */ +#define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */ +#define DMAR_IQ_SHIFT 4 /* Invalidation queue head/tail shift */ +#define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */ +#define DMAR_ICS_REG 0x9c /* Invalidation complete status register */ +#define DMAR_IQER_REG 0xb0 /* Invalidation queue error record register */ +#define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */ +#define DMAR_PQH_REG 0xc0 /* Page request queue head register */ +#define DMAR_PQT_REG 0xc8 /* Page request queue tail register */ +#define DMAR_PQA_REG 0xd0 /* Page request queue address register */ +#define DMAR_PRS_REG 0xdc /* Page request status register */ +#define DMAR_PECTL_REG 0xe0 /* Page request event control register */ +#define DMAR_PEDATA_REG 0xe4 /* Page request event interrupt data register */ +#define DMAR_PEADDR_REG 0xe8 /* Page request event interrupt addr register */ +#define DMAR_PEUADDR_REG 0xec /* Page request event Upper address register */ +#define DMAR_MTRRCAP_REG 0x100 /* MTRR capability register */ +#define DMAR_MTRRDEF_REG 0x108 /* MTRR default type register */ +#define DMAR_MTRR_FIX64K_00000_REG 0x120 /* MTRR Fixed range registers */ +#define DMAR_MTRR_FIX16K_80000_REG 0x128 +#define DMAR_MTRR_FIX16K_A0000_REG 0x130 +#define DMAR_MTRR_FIX4K_C0000_REG 0x138 +#define DMAR_MTRR_FIX4K_C8000_REG 0x140 +#define DMAR_MTRR_FIX4K_D0000_REG 0x148 +#define DMAR_MTRR_FIX4K_D8000_REG 0x150 +#define DMAR_MTRR_FIX4K_E0000_REG 0x158 +#define DMAR_MTRR_FIX4K_E8000_REG 0x160 +#define DMAR_MTRR_FIX4K_F0000_REG 0x168 +#define DMAR_MTRR_FIX4K_F8000_REG 0x170 +#define DMAR_MTRR_PHYSBASE0_REG 0x180 /* MTRR Variable range registers */ +#define DMAR_MTRR_PHYSMASK0_REG 0x188 +#define DMAR_MTRR_PHYSBASE1_REG 0x190 +#define DMAR_MTRR_PHYSMASK1_REG 0x198 +#define DMAR_MTRR_PHYSBASE2_REG 0x1a0 +#define DMAR_MTRR_PHYSMASK2_REG 0x1a8 +#define DMAR_MTRR_PHYSBASE3_REG 0x1b0 +#define DMAR_MTRR_PHYSMASK3_REG 0x1b8 +#define DMAR_MTRR_PHYSBASE4_REG 0x1c0 +#define DMAR_MTRR_PHYSMASK4_REG 0x1c8 +#define DMAR_MTRR_PHYSBASE5_REG 0x1d0 +#define DMAR_MTRR_PHYSMASK5_REG 0x1d8 +#define DMAR_MTRR_PHYSBASE6_REG 0x1e0 +#define DMAR_MTRR_PHYSMASK6_REG 0x1e8 +#define DMAR_MTRR_PHYSBASE7_REG 0x1f0 +#define DMAR_MTRR_PHYSMASK7_REG 0x1f8 +#define DMAR_MTRR_PHYSBASE8_REG 0x200 +#define DMAR_MTRR_PHYSMASK8_REG 0x208 +#define DMAR_MTRR_PHYSBASE9_REG 0x210 +#define DMAR_MTRR_PHYSMASK9_REG 0x218 +#define DMAR_VCCAP_REG 0xe30 /* Virtual command capability register */ +#define DMAR_VCMD_REG 0xe00 /* Virtual command register */ +#define DMAR_VCRSP_REG 0xe10 /* Virtual command response register */ + +#define DMAR_IQER_REG_IQEI(reg) FIELD_GET(GENMASK_ULL(3, 0), reg) +#define DMAR_IQER_REG_ITESID(reg) FIELD_GET(GENMASK_ULL(47, 32), reg) +#define DMAR_IQER_REG_ICESID(reg) FIELD_GET(GENMASK_ULL(63, 48), reg) + +#define OFFSET_STRIDE (9) + +#define dmar_readq(a) readq(a) +#define dmar_writeq(a,v) writeq(v,a) +#define dmar_readl(a) readl(a) +#define dmar_writel(a, v) writel(v, a) + +#define DMAR_VER_MAJOR(v) (((v) & 0xf0) >> 4) +#define DMAR_VER_MINOR(v) ((v) & 0x0f) + +/* + * Decoding Capability Register + */ +#define cap_5lp_support(c) (((c) >> 60) & 1) +#define cap_pi_support(c) (((c) >> 59) & 1) +#define cap_fl1gp_support(c) (((c) >> 56) & 1) +#define cap_read_drain(c) (((c) >> 55) & 1) +#define cap_write_drain(c) (((c) >> 54) & 1) +#define cap_max_amask_val(c) (((c) >> 48) & 0x3f) +#define cap_num_fault_regs(c) ((((c) >> 40) & 0xff) + 1) +#define cap_pgsel_inv(c) (((c) >> 39) & 1) + +#define cap_super_page_val(c) (((c) >> 34) & 0xf) +#define cap_super_offset(c) (((find_first_bit(&cap_super_page_val(c), 4)) \ + * OFFSET_STRIDE) + 21) + +#define cap_fault_reg_offset(c) ((((c) >> 24) & 0x3ff) * 16) +#define cap_max_fault_reg_offset(c) \ + (cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16) + +#define cap_zlr(c) (((c) >> 22) & 1) +#define cap_isoch(c) (((c) >> 23) & 1) +#define cap_mgaw(c) ((((c) >> 16) & 0x3f) + 1) +#define cap_sagaw(c) (((c) >> 8) & 0x1f) +#define cap_caching_mode(c) (((c) >> 7) & 1) +#define cap_phmr(c) (((c) >> 6) & 1) +#define cap_plmr(c) (((c) >> 5) & 1) +#define cap_rwbf(c) (((c) >> 4) & 1) +#define cap_afl(c) (((c) >> 3) & 1) +#define cap_ndoms(c) (((unsigned long)1) << (4 + 2 * ((c) & 0x7))) +/* + * Extended Capability Register + */ + +#define ecap_rps(e) (((e) >> 49) & 0x1) +#define ecap_smpwc(e) (((e) >> 48) & 0x1) +#define ecap_flts(e) (((e) >> 47) & 0x1) +#define ecap_slts(e) (((e) >> 46) & 0x1) +#define ecap_slads(e) (((e) >> 45) & 0x1) +#define ecap_vcs(e) (((e) >> 44) & 0x1) +#define ecap_smts(e) (((e) >> 43) & 0x1) +#define ecap_dit(e) (((e) >> 41) & 0x1) +#define ecap_pds(e) (((e) >> 42) & 0x1) +#define ecap_pasid(e) (((e) >> 40) & 0x1) +#define ecap_pss(e) (((e) >> 35) & 0x1f) +#define ecap_eafs(e) (((e) >> 34) & 0x1) +#define ecap_nwfs(e) (((e) >> 33) & 0x1) +#define ecap_srs(e) (((e) >> 31) & 0x1) +#define ecap_ers(e) (((e) >> 30) & 0x1) +#define ecap_prs(e) (((e) >> 29) & 0x1) +#define ecap_broken_pasid(e) (((e) >> 28) & 0x1) +#define ecap_dis(e) (((e) >> 27) & 0x1) +#define ecap_nest(e) (((e) >> 26) & 0x1) +#define ecap_mts(e) (((e) >> 25) & 0x1) +#define ecap_ecs(e) (((e) >> 24) & 0x1) +#define ecap_iotlb_offset(e) ((((e) >> 8) & 0x3ff) * 16) +#define ecap_max_iotlb_offset(e) (ecap_iotlb_offset(e) + 16) +#define ecap_coherent(e) ((e) & 0x1) +#define ecap_qis(e) ((e) & 0x2) +#define ecap_pass_through(e) (((e) >> 6) & 0x1) +#define ecap_eim_support(e) (((e) >> 4) & 0x1) +#define ecap_ir_support(e) (((e) >> 3) & 0x1) +#define ecap_dev_iotlb_support(e) (((e) >> 2) & 0x1) +#define ecap_max_handle_mask(e) (((e) >> 20) & 0xf) +#define ecap_sc_support(e) (((e) >> 7) & 0x1) /* Snooping Control */ + +/* Virtual command interface capability */ +#define vccap_pasid(v) (((v) & DMA_VCS_PAS)) /* PASID allocation */ + +/* IOTLB_REG */ +#define DMA_TLB_FLUSH_GRANU_OFFSET 60 +#define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60) +#define DMA_TLB_DSI_FLUSH (((u64)2) << 60) +#define DMA_TLB_PSI_FLUSH (((u64)3) << 60) +#define DMA_TLB_IIRG(type) ((type >> 60) & 3) +#define DMA_TLB_IAIG(val) (((val) >> 57) & 3) +#define DMA_TLB_READ_DRAIN (((u64)1) << 49) +#define DMA_TLB_WRITE_DRAIN (((u64)1) << 48) +#define DMA_TLB_DID(id) (((u64)((id) & 0xffff)) << 32) +#define DMA_TLB_IVT (((u64)1) << 63) +#define DMA_TLB_IH_NONLEAF (((u64)1) << 6) +#define DMA_TLB_MAX_SIZE (0x3f) + +/* INVALID_DESC */ +#define DMA_CCMD_INVL_GRANU_OFFSET 61 +#define DMA_ID_TLB_GLOBAL_FLUSH (((u64)1) << 4) +#define DMA_ID_TLB_DSI_FLUSH (((u64)2) << 4) +#define DMA_ID_TLB_PSI_FLUSH (((u64)3) << 4) +#define DMA_ID_TLB_READ_DRAIN (((u64)1) << 7) +#define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6) +#define DMA_ID_TLB_DID(id) (((u64)((id & 0xffff) << 16))) +#define DMA_ID_TLB_IH_NONLEAF (((u64)1) << 6) +#define DMA_ID_TLB_ADDR(addr) (addr) +#define DMA_ID_TLB_ADDR_MASK(mask) (mask) + +/* PMEN_REG */ +#define DMA_PMEN_EPM (((u32)1)<<31) +#define DMA_PMEN_PRS (((u32)1)<<0) + +/* GCMD_REG */ +#define DMA_GCMD_TE (((u32)1) << 31) +#define DMA_GCMD_SRTP (((u32)1) << 30) +#define DMA_GCMD_SFL (((u32)1) << 29) +#define DMA_GCMD_EAFL (((u32)1) << 28) +#define DMA_GCMD_WBF (((u32)1) << 27) +#define DMA_GCMD_QIE (((u32)1) << 26) +#define DMA_GCMD_SIRTP (((u32)1) << 24) +#define DMA_GCMD_IRE (((u32) 1) << 25) +#define DMA_GCMD_CFI (((u32) 1) << 23) + +/* GSTS_REG */ +#define DMA_GSTS_TES (((u32)1) << 31) +#define DMA_GSTS_RTPS (((u32)1) << 30) +#define DMA_GSTS_FLS (((u32)1) << 29) +#define DMA_GSTS_AFLS (((u32)1) << 28) +#define DMA_GSTS_WBFS (((u32)1) << 27) +#define DMA_GSTS_QIES (((u32)1) << 26) +#define DMA_GSTS_IRTPS (((u32)1) << 24) +#define DMA_GSTS_IRES (((u32)1) << 25) +#define DMA_GSTS_CFIS (((u32)1) << 23) + +/* DMA_RTADDR_REG */ +#define DMA_RTADDR_RTT (((u64)1) << 11) +#define DMA_RTADDR_SMT (((u64)1) << 10) + +/* CCMD_REG */ +#define DMA_CCMD_ICC (((u64)1) << 63) +#define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61) +#define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61) +#define DMA_CCMD_DEVICE_INVL (((u64)3) << 61) +#define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32) +#define DMA_CCMD_MASK_NOBIT 0 +#define DMA_CCMD_MASK_1BIT 1 +#define DMA_CCMD_MASK_2BIT 2 +#define DMA_CCMD_MASK_3BIT 3 +#define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16) +#define DMA_CCMD_DID(d) ((u64)((d) & 0xffff)) + +/* FECTL_REG */ +#define DMA_FECTL_IM (((u32)1) << 31) + +/* FSTS_REG */ +#define DMA_FSTS_PFO (1 << 0) /* Primary Fault Overflow */ +#define DMA_FSTS_PPF (1 << 1) /* Primary Pending Fault */ +#define DMA_FSTS_IQE (1 << 4) /* Invalidation Queue Error */ +#define DMA_FSTS_ICE (1 << 5) /* Invalidation Completion Error */ +#define DMA_FSTS_ITE (1 << 6) /* Invalidation Time-out Error */ +#define DMA_FSTS_PRO (1 << 7) /* Page Request Overflow */ +#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff) + +/* FRCD_REG, 32 bits access */ +#define DMA_FRCD_F (((u32)1) << 31) +#define dma_frcd_type(d) ((d >> 30) & 1) +#define dma_frcd_fault_reason(c) (c & 0xff) +#define dma_frcd_source_id(c) (c & 0xffff) +#define dma_frcd_pasid_value(c) (((c) >> 8) & 0xfffff) +#define dma_frcd_pasid_present(c) (((c) >> 31) & 1) +/* low 64 bit */ +#define dma_frcd_page_addr(d) (d & (((u64)-1) << PAGE_SHIFT)) + +/* PRS_REG */ +#define DMA_PRS_PPR ((u32)1) +#define DMA_PRS_PRO ((u32)2) + +#define DMA_VCS_PAS ((u64)1) + +#define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \ +do { \ + cycles_t start_time = get_cycles(); \ + while (1) { \ + sts = op(iommu->reg + offset); \ + if (cond) \ + break; \ + if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start_time))\ + panic("DMAR hardware is malfunctioning\n"); \ + cpu_relax(); \ + } \ +} while (0) + +#define QI_LENGTH 256 /* queue length */ + +enum { + QI_FREE, + QI_IN_USE, + QI_DONE, + QI_ABORT +}; + +#define QI_CC_TYPE 0x1 +#define QI_IOTLB_TYPE 0x2 +#define QI_DIOTLB_TYPE 0x3 +#define QI_IEC_TYPE 0x4 +#define QI_IWD_TYPE 0x5 +#define QI_EIOTLB_TYPE 0x6 +#define QI_PC_TYPE 0x7 +#define QI_DEIOTLB_TYPE 0x8 +#define QI_PGRP_RESP_TYPE 0x9 +#define QI_PSTRM_RESP_TYPE 0xa + +#define QI_IEC_SELECTIVE (((u64)1) << 4) +#define QI_IEC_IIDEX(idx) (((u64)(idx & 0xffff) << 32)) +#define QI_IEC_IM(m) (((u64)(m & 0x1f) << 27)) + +#define QI_IWD_STATUS_DATA(d) (((u64)d) << 32) +#define QI_IWD_STATUS_WRITE (((u64)1) << 5) +#define QI_IWD_FENCE (((u64)1) << 6) +#define QI_IWD_PRQ_DRAIN (((u64)1) << 7) + +#define QI_IOTLB_DID(did) (((u64)did) << 16) +#define QI_IOTLB_DR(dr) (((u64)dr) << 7) +#define QI_IOTLB_DW(dw) (((u64)dw) << 6) +#define QI_IOTLB_GRAN(gran) (((u64)gran) >> (DMA_TLB_FLUSH_GRANU_OFFSET-4)) +#define QI_IOTLB_ADDR(addr) (((u64)addr) & VTD_PAGE_MASK) +#define QI_IOTLB_IH(ih) (((u64)ih) << 6) +#define QI_IOTLB_AM(am) (((u8)am) & 0x3f) + +#define QI_CC_FM(fm) (((u64)fm) << 48) +#define QI_CC_SID(sid) (((u64)sid) << 32) +#define QI_CC_DID(did) (((u64)did) << 16) +#define QI_CC_GRAN(gran) (((u64)gran) >> (DMA_CCMD_INVL_GRANU_OFFSET-4)) + +#define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32) +#define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16) +#define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) +#define QI_DEV_IOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \ + ((u64)((pfsid >> 4) & 0xfff) << 52)) +#define QI_DEV_IOTLB_SIZE 1 +#define QI_DEV_IOTLB_MAX_INVS 32 + +#define QI_PC_PASID(pasid) (((u64)pasid) << 32) +#define QI_PC_DID(did) (((u64)did) << 16) +#define QI_PC_GRAN(gran) (((u64)gran) << 4) + +/* PASID cache invalidation granu */ +#define QI_PC_ALL_PASIDS 0 +#define QI_PC_PASID_SEL 1 +#define QI_PC_GLOBAL 3 + +#define QI_EIOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) +#define QI_EIOTLB_IH(ih) (((u64)ih) << 6) +#define QI_EIOTLB_AM(am) (((u64)am) & 0x3f) +#define QI_EIOTLB_PASID(pasid) (((u64)pasid) << 32) +#define QI_EIOTLB_DID(did) (((u64)did) << 16) +#define QI_EIOTLB_GRAN(gran) (((u64)gran) << 4) + +/* QI Dev-IOTLB inv granu */ +#define QI_DEV_IOTLB_GRAN_ALL 1 +#define QI_DEV_IOTLB_GRAN_PASID_SEL 0 + +#define QI_DEV_EIOTLB_ADDR(a) ((u64)(a) & VTD_PAGE_MASK) +#define QI_DEV_EIOTLB_SIZE (((u64)1) << 11) +#define QI_DEV_EIOTLB_PASID(p) ((u64)((p) & 0xfffff) << 32) +#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) +#define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) +#define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \ + ((u64)((pfsid >> 4) & 0xfff) << 52)) +#define QI_DEV_EIOTLB_MAX_INVS 32 + +/* Page group response descriptor QW0 */ +#define QI_PGRP_PASID_P(p) (((u64)(p)) << 4) +#define QI_PGRP_PDP(p) (((u64)(p)) << 5) +#define QI_PGRP_RESP_CODE(res) (((u64)(res)) << 12) +#define QI_PGRP_DID(rid) (((u64)(rid)) << 16) +#define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) + +/* Page group response descriptor QW1 */ +#define QI_PGRP_LPIG(x) (((u64)(x)) << 2) +#define QI_PGRP_IDX(idx) (((u64)(idx)) << 3) + + +#define QI_RESP_SUCCESS 0x0 +#define QI_RESP_INVALID 0x1 +#define QI_RESP_FAILURE 0xf + +#define QI_GRAN_NONG_PASID 2 +#define QI_GRAN_PSI_PASID 3 + +#define qi_shift(iommu) (DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap)) + +struct qi_desc { + u64 qw0; + u64 qw1; + u64 qw2; + u64 qw3; +}; + +struct q_inval { + raw_spinlock_t q_lock; + void *desc; /* invalidation queue */ + int *desc_status; /* desc status */ + int free_head; /* first free entry */ + int free_tail; /* last free entry */ + int free_cnt; +}; + +struct dmar_pci_notify_info; + +#ifdef CONFIG_IRQ_REMAP +/* 1MB - maximum possible interrupt remapping table size */ +#define INTR_REMAP_PAGE_ORDER 8 +#define INTR_REMAP_TABLE_REG_SIZE 0xf +#define INTR_REMAP_TABLE_REG_SIZE_MASK 0xf + +#define INTR_REMAP_TABLE_ENTRIES 65536 + +struct irq_domain; + +struct ir_table { + struct irte *base; + unsigned long *bitmap; +}; + +void intel_irq_remap_add_device(struct dmar_pci_notify_info *info); +#else +static inline void +intel_irq_remap_add_device(struct dmar_pci_notify_info *info) { } +#endif + +struct iommu_flush { + void (*flush_context)(struct intel_iommu *iommu, u16 did, u16 sid, + u8 fm, u64 type); + void (*flush_iotlb)(struct intel_iommu *iommu, u16 did, u64 addr, + unsigned int size_order, u64 type); +}; + +enum { + SR_DMAR_FECTL_REG, + SR_DMAR_FEDATA_REG, + SR_DMAR_FEADDR_REG, + SR_DMAR_FEUADDR_REG, + MAX_SR_DMAR_REGS +}; + +#define VTD_FLAG_TRANS_PRE_ENABLED (1 << 0) +#define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1) +#define VTD_FLAG_SVM_CAPABLE (1 << 2) + +extern int intel_iommu_sm; +extern spinlock_t device_domain_lock; + +#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap)) +#define pasid_supported(iommu) (sm_supported(iommu) && \ + ecap_pasid((iommu)->ecap)) + +struct pasid_entry; +struct pasid_state_entry; +struct page_req_dsc; + +/* + * 0: Present + * 1-11: Reserved + * 12-63: Context Ptr (12 - (haw-1)) + * 64-127: Reserved + */ +struct root_entry { + u64 lo; + u64 hi; +}; + +/* + * low 64 bits: + * 0: present + * 1: fault processing disable + * 2-3: translation type + * 12-63: address space root + * high 64 bits: + * 0-2: address width + * 3-6: aval + * 8-23: domain id + */ +struct context_entry { + u64 lo; + u64 hi; +}; + +/* + * When VT-d works in the scalable mode, it allows DMA translation to + * happen through either first level or second level page table. This + * bit marks that the DMA translation for the domain goes through the + * first level page table, otherwise, it goes through the second level. + */ +#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(1) + +struct dmar_domain { + int nid; /* node id */ + + unsigned int iommu_refcnt[DMAR_UNITS_SUPPORTED]; + /* Refcount of devices per iommu */ + + + u16 iommu_did[DMAR_UNITS_SUPPORTED]; + /* Domain ids per IOMMU. Use u16 since + * domain ids are 16 bit wide according + * to VT-d spec, section 9.3 */ + + u8 has_iotlb_device: 1; + u8 iommu_coherency: 1; /* indicate coherency of iommu access */ + u8 force_snooping : 1; /* Create IOPTEs with snoop control */ + u8 set_pte_snp:1; + + struct list_head devices; /* all devices' list */ + struct iova_domain iovad; /* iova's that belong to this domain */ + + struct dma_pte *pgd; /* virtual address */ + int gaw; /* max guest address width */ + + /* adjusted guest address width, 0 is level 2 30-bit */ + int agaw; + + int flags; /* flags to find out type of domain */ + int iommu_superpage;/* Level of superpages supported: + 0 == 4KiB (no superpages), 1 == 2MiB, + 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ + u64 max_addr; /* maximum mapped address */ + + struct iommu_domain domain; /* generic domain data structure for + iommu core */ +}; + +struct intel_iommu { + void __iomem *reg; /* Pointer to hardware regs, virtual addr */ + u64 reg_phys; /* physical address of hw register set */ + u64 reg_size; /* size of hw register set */ + u64 cap; + u64 ecap; + u64 vccap; + u32 gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */ + raw_spinlock_t register_lock; /* protect register handling */ + int seq_id; /* sequence id of the iommu */ + int agaw; /* agaw of this iommu */ + int msagaw; /* max sagaw of this iommu */ + unsigned int irq, pr_irq; + u16 segment; /* PCI segment# */ + unsigned char name[13]; /* Device Name */ + +#ifdef CONFIG_INTEL_IOMMU + unsigned long *domain_ids; /* bitmap of domains */ + spinlock_t lock; /* protect context, domain ids */ + struct root_entry *root_entry; /* virtual address */ + + struct iommu_flush flush; +#endif +#ifdef CONFIG_INTEL_IOMMU_SVM + struct page_req_dsc *prq; + unsigned char prq_name[16]; /* Name for PRQ interrupt */ + struct completion prq_complete; + struct ioasid_allocator_ops pasid_allocator; /* Custom allocator for PASIDs */ +#endif + struct iopf_queue *iopf_queue; + unsigned char iopfq_name[16]; + struct q_inval *qi; /* Queued invalidation info */ + u32 *iommu_state; /* Store iommu states between suspend and resume.*/ + +#ifdef CONFIG_IRQ_REMAP + struct ir_table *ir_table; /* Interrupt remapping info */ + struct irq_domain *ir_domain; + struct irq_domain *ir_msi_domain; +#endif + struct iommu_device iommu; /* IOMMU core code handle */ + int node; + u32 flags; /* Software defined flags */ + + struct dmar_drhd_unit *drhd; + void *perf_statistic; +}; + +/* PCI domain-device relationship */ +struct device_domain_info { + struct list_head link; /* link to domain siblings */ + struct list_head global; /* link to global list */ + u32 segment; /* PCI segment number */ + u8 bus; /* PCI bus number */ + u8 devfn; /* PCI devfn number */ + u16 pfsid; /* SRIOV physical function source ID */ + u8 pasid_supported:3; + u8 pasid_enabled:1; + u8 pri_supported:1; + u8 pri_enabled:1; + u8 ats_supported:1; + u8 ats_enabled:1; + u8 ats_qdep; + struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ + struct intel_iommu *iommu; /* IOMMU used by this device */ + struct dmar_domain *domain; /* pointer to domain */ + struct pasid_table *pasid_table; /* pasid table */ +}; + +static inline void __iommu_flush_cache( + struct intel_iommu *iommu, void *addr, int size) +{ + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(addr, size); +} + +/* Convert generic struct iommu_domain to private struct dmar_domain */ +static inline struct dmar_domain *to_dmar_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct dmar_domain, domain); +} + +/* + * 0: readable + * 1: writable + * 2-6: reserved + * 7: super page + * 8-10: available + * 11: snoop behavior + * 12-63: Host physical address + */ +struct dma_pte { + u64 val; +}; + +static inline void dma_clear_pte(struct dma_pte *pte) +{ + pte->val = 0; +} + +static inline u64 dma_pte_addr(struct dma_pte *pte) +{ +#ifdef CONFIG_64BIT + return pte->val & VTD_PAGE_MASK & (~DMA_FL_PTE_XD); +#else + /* Must have a full atomic 64-bit read */ + return __cmpxchg64(&pte->val, 0ULL, 0ULL) & + VTD_PAGE_MASK & (~DMA_FL_PTE_XD); +#endif +} + +static inline bool dma_pte_present(struct dma_pte *pte) +{ + return (pte->val & 3) != 0; +} + +static inline bool dma_pte_superpage(struct dma_pte *pte) +{ + return (pte->val & DMA_PTE_LARGE_PAGE); +} + +static inline bool first_pte_in_page(struct dma_pte *pte) +{ + return IS_ALIGNED((unsigned long)pte, VTD_PAGE_SIZE); +} + +static inline int nr_pte_to_next_page(struct dma_pte *pte) +{ + return first_pte_in_page(pte) ? BIT_ULL(VTD_STRIDE_SHIFT) : + (struct dma_pte *)ALIGN((unsigned long)pte, VTD_PAGE_SIZE) - pte; +} + +extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); + +extern int dmar_enable_qi(struct intel_iommu *iommu); +extern void dmar_disable_qi(struct intel_iommu *iommu); +extern int dmar_reenable_qi(struct intel_iommu *iommu); +extern void qi_global_iec(struct intel_iommu *iommu); + +extern void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, + u8 fm, u64 type); +extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, + unsigned int size_order, u64 type); +extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, + u16 qdep, u64 addr, unsigned mask); + +void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, + unsigned long npages, bool ih); + +void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, + u32 pasid, u16 qdep, u64 addr, + unsigned int size_order); +void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu, + u32 pasid); + +int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc, + unsigned int count, unsigned long options); +/* + * Options used in qi_submit_sync: + * QI_OPT_WAIT_DRAIN - Wait for PRQ drain completion, spec 6.5.2.8. + */ +#define QI_OPT_WAIT_DRAIN BIT(0) + +extern int dmar_ir_support(void); + +void *alloc_pgtable_page(int node); +void free_pgtable_page(void *vaddr); +struct intel_iommu *domain_get_iommu(struct dmar_domain *domain); +void iommu_flush_write_buffer(struct intel_iommu *iommu); +int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev); +struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn); + +#ifdef CONFIG_INTEL_IOMMU_SVM +extern void intel_svm_check(struct intel_iommu *iommu); +extern int intel_svm_enable_prq(struct intel_iommu *iommu); +extern int intel_svm_finish_prq(struct intel_iommu *iommu); +struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, + void *drvdata); +void intel_svm_unbind(struct iommu_sva *handle); +u32 intel_svm_get_pasid(struct iommu_sva *handle); +int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, + struct iommu_page_response *msg); + +struct intel_svm_dev { + struct list_head list; + struct rcu_head rcu; + struct device *dev; + struct intel_iommu *iommu; + struct iommu_sva sva; + unsigned long prq_seq_number; + u32 pasid; + int users; + u16 did; + u16 dev_iotlb:1; + u16 sid, qdep; +}; + +struct intel_svm { + struct mmu_notifier notifier; + struct mm_struct *mm; + + unsigned int flags; + u32 pasid; + struct list_head devs; +}; +#else +static inline void intel_svm_check(struct intel_iommu *iommu) {} +#endif + +#ifdef CONFIG_INTEL_IOMMU_DEBUGFS +void intel_iommu_debugfs_init(void); +#else +static inline void intel_iommu_debugfs_init(void) {} +#endif /* CONFIG_INTEL_IOMMU_DEBUGFS */ + +extern const struct attribute_group *intel_iommu_groups[]; +bool context_present(struct context_entry *context); +struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, + u8 devfn, int alloc); + +extern const struct iommu_ops intel_iommu_ops; + +#ifdef CONFIG_INTEL_IOMMU +extern int iommu_calculate_agaw(struct intel_iommu *iommu); +extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); +extern int dmar_disabled; +extern int intel_iommu_enabled; +#else +static inline int iommu_calculate_agaw(struct intel_iommu *iommu) +{ + return 0; +} +static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) +{ + return 0; +} +#define dmar_disabled (1) +#define intel_iommu_enabled (0) +#endif + +static inline const char *decode_prq_descriptor(char *str, size_t size, + u64 dw0, u64 dw1, u64 dw2, u64 dw3) +{ + char *buf = str; + int bytes; + + bytes = snprintf(buf, size, + "rid=0x%llx addr=0x%llx %c%c%c%c%c pasid=0x%llx index=0x%llx", + FIELD_GET(GENMASK_ULL(31, 16), dw0), + FIELD_GET(GENMASK_ULL(63, 12), dw1), + dw1 & BIT_ULL(0) ? 'r' : '-', + dw1 & BIT_ULL(1) ? 'w' : '-', + dw0 & BIT_ULL(52) ? 'x' : '-', + dw0 & BIT_ULL(53) ? 'p' : '-', + dw1 & BIT_ULL(2) ? 'l' : '-', + FIELD_GET(GENMASK_ULL(51, 32), dw0), + FIELD_GET(GENMASK_ULL(11, 3), dw1)); + + /* Private Data */ + if (dw0 & BIT_ULL(9)) { + size -= bytes; + buf += bytes; + snprintf(buf, size, " private=0x%llx/0x%llx\n", dw2, dw3); + } + + return str; +} + +#endif diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index a67319597884..2e9683e970f8 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -21,6 +20,7 @@ #include #include +#include "iommu.h" #include "../irq_remapping.h" #include "cap_audit.h" diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 17cad7c1f62d..43f090381ec7 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -12,13 +12,13 @@ #include #include #include -#include #include #include #include #include #include +#include "iommu.h" #include "pasid.h" /* diff --git a/drivers/iommu/intel/perf.c b/drivers/iommu/intel/perf.c index 0e8e03252d92..94ee70ac38e3 100644 --- a/drivers/iommu/intel/perf.c +++ b/drivers/iommu/intel/perf.c @@ -9,8 +9,8 @@ */ #include -#include +#include "iommu.h" #include "perf.h" static DEFINE_SPINLOCK(latency_lock); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 70b40d007a52..580713aa9e07 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -5,7 +5,6 @@ * Authors: David Woodhouse */ -#include #include #include #include @@ -22,6 +21,7 @@ #include #include +#include "iommu.h" #include "pasid.h" #include "perf.h" #include "../iommu-sva-lib.h" diff --git a/drivers/iommu/intel/trace.h b/drivers/iommu/intel/trace.h index 25cb7f88e1a2..93d96f93a89b 100644 --- a/drivers/iommu/intel/trace.h +++ b/drivers/iommu/intel/trace.h @@ -13,7 +13,8 @@ #define _TRACE_INTEL_IOMMU_H #include -#include + +#include "iommu.h" #define MSG_MAX 256 diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h deleted file mode 100644 index 4ebf3c4da45e..000000000000 --- a/include/linux/intel-iommu.h +++ /dev/null @@ -1,831 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright © 2006-2015, Intel Corporation. - * - * Authors: Ashok Raj - * Anil S Keshavamurthy - * David Woodhouse - */ - -#ifndef _INTEL_IOMMU_H_ -#define _INTEL_IOMMU_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* - * VT-d hardware uses 4KiB page size regardless of host page size. - */ -#define VTD_PAGE_SHIFT (12) -#define VTD_PAGE_SIZE (1UL << VTD_PAGE_SHIFT) -#define VTD_PAGE_MASK (((u64)-1) << VTD_PAGE_SHIFT) -#define VTD_PAGE_ALIGN(addr) (((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK) - -#define VTD_STRIDE_SHIFT (9) -#define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT) - -#define DMA_PTE_READ BIT_ULL(0) -#define DMA_PTE_WRITE BIT_ULL(1) -#define DMA_PTE_LARGE_PAGE BIT_ULL(7) -#define DMA_PTE_SNP BIT_ULL(11) - -#define DMA_FL_PTE_PRESENT BIT_ULL(0) -#define DMA_FL_PTE_US BIT_ULL(2) -#define DMA_FL_PTE_ACCESS BIT_ULL(5) -#define DMA_FL_PTE_DIRTY BIT_ULL(6) -#define DMA_FL_PTE_XD BIT_ULL(63) - -#define ADDR_WIDTH_5LEVEL (57) -#define ADDR_WIDTH_4LEVEL (48) - -#define CONTEXT_TT_MULTI_LEVEL 0 -#define CONTEXT_TT_DEV_IOTLB 1 -#define CONTEXT_TT_PASS_THROUGH 2 -#define CONTEXT_PASIDE BIT_ULL(3) - -/* - * Intel IOMMU register specification per version 1.0 public spec. - */ -#define DMAR_VER_REG 0x0 /* Arch version supported by this IOMMU */ -#define DMAR_CAP_REG 0x8 /* Hardware supported capabilities */ -#define DMAR_ECAP_REG 0x10 /* Extended capabilities supported */ -#define DMAR_GCMD_REG 0x18 /* Global command register */ -#define DMAR_GSTS_REG 0x1c /* Global status register */ -#define DMAR_RTADDR_REG 0x20 /* Root entry table */ -#define DMAR_CCMD_REG 0x28 /* Context command reg */ -#define DMAR_FSTS_REG 0x34 /* Fault Status register */ -#define DMAR_FECTL_REG 0x38 /* Fault control register */ -#define DMAR_FEDATA_REG 0x3c /* Fault event interrupt data register */ -#define DMAR_FEADDR_REG 0x40 /* Fault event interrupt addr register */ -#define DMAR_FEUADDR_REG 0x44 /* Upper address register */ -#define DMAR_AFLOG_REG 0x58 /* Advanced Fault control */ -#define DMAR_PMEN_REG 0x64 /* Enable Protected Memory Region */ -#define DMAR_PLMBASE_REG 0x68 /* PMRR Low addr */ -#define DMAR_PLMLIMIT_REG 0x6c /* PMRR low limit */ -#define DMAR_PHMBASE_REG 0x70 /* pmrr high base addr */ -#define DMAR_PHMLIMIT_REG 0x78 /* pmrr high limit */ -#define DMAR_IQH_REG 0x80 /* Invalidation queue head register */ -#define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */ -#define DMAR_IQ_SHIFT 4 /* Invalidation queue head/tail shift */ -#define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */ -#define DMAR_ICS_REG 0x9c /* Invalidation complete status register */ -#define DMAR_IQER_REG 0xb0 /* Invalidation queue error record register */ -#define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */ -#define DMAR_PQH_REG 0xc0 /* Page request queue head register */ -#define DMAR_PQT_REG 0xc8 /* Page request queue tail register */ -#define DMAR_PQA_REG 0xd0 /* Page request queue address register */ -#define DMAR_PRS_REG 0xdc /* Page request status register */ -#define DMAR_PECTL_REG 0xe0 /* Page request event control register */ -#define DMAR_PEDATA_REG 0xe4 /* Page request event interrupt data register */ -#define DMAR_PEADDR_REG 0xe8 /* Page request event interrupt addr register */ -#define DMAR_PEUADDR_REG 0xec /* Page request event Upper address register */ -#define DMAR_MTRRCAP_REG 0x100 /* MTRR capability register */ -#define DMAR_MTRRDEF_REG 0x108 /* MTRR default type register */ -#define DMAR_MTRR_FIX64K_00000_REG 0x120 /* MTRR Fixed range registers */ -#define DMAR_MTRR_FIX16K_80000_REG 0x128 -#define DMAR_MTRR_FIX16K_A0000_REG 0x130 -#define DMAR_MTRR_FIX4K_C0000_REG 0x138 -#define DMAR_MTRR_FIX4K_C8000_REG 0x140 -#define DMAR_MTRR_FIX4K_D0000_REG 0x148 -#define DMAR_MTRR_FIX4K_D8000_REG 0x150 -#define DMAR_MTRR_FIX4K_E0000_REG 0x158 -#define DMAR_MTRR_FIX4K_E8000_REG 0x160 -#define DMAR_MTRR_FIX4K_F0000_REG 0x168 -#define DMAR_MTRR_FIX4K_F8000_REG 0x170 -#define DMAR_MTRR_PHYSBASE0_REG 0x180 /* MTRR Variable range registers */ -#define DMAR_MTRR_PHYSMASK0_REG 0x188 -#define DMAR_MTRR_PHYSBASE1_REG 0x190 -#define DMAR_MTRR_PHYSMASK1_REG 0x198 -#define DMAR_MTRR_PHYSBASE2_REG 0x1a0 -#define DMAR_MTRR_PHYSMASK2_REG 0x1a8 -#define DMAR_MTRR_PHYSBASE3_REG 0x1b0 -#define DMAR_MTRR_PHYSMASK3_REG 0x1b8 -#define DMAR_MTRR_PHYSBASE4_REG 0x1c0 -#define DMAR_MTRR_PHYSMASK4_REG 0x1c8 -#define DMAR_MTRR_PHYSBASE5_REG 0x1d0 -#define DMAR_MTRR_PHYSMASK5_REG 0x1d8 -#define DMAR_MTRR_PHYSBASE6_REG 0x1e0 -#define DMAR_MTRR_PHYSMASK6_REG 0x1e8 -#define DMAR_MTRR_PHYSBASE7_REG 0x1f0 -#define DMAR_MTRR_PHYSMASK7_REG 0x1f8 -#define DMAR_MTRR_PHYSBASE8_REG 0x200 -#define DMAR_MTRR_PHYSMASK8_REG 0x208 -#define DMAR_MTRR_PHYSBASE9_REG 0x210 -#define DMAR_MTRR_PHYSMASK9_REG 0x218 -#define DMAR_VCCAP_REG 0xe30 /* Virtual command capability register */ -#define DMAR_VCMD_REG 0xe00 /* Virtual command register */ -#define DMAR_VCRSP_REG 0xe10 /* Virtual command response register */ - -#define DMAR_IQER_REG_IQEI(reg) FIELD_GET(GENMASK_ULL(3, 0), reg) -#define DMAR_IQER_REG_ITESID(reg) FIELD_GET(GENMASK_ULL(47, 32), reg) -#define DMAR_IQER_REG_ICESID(reg) FIELD_GET(GENMASK_ULL(63, 48), reg) - -#define OFFSET_STRIDE (9) - -#define dmar_readq(a) readq(a) -#define dmar_writeq(a,v) writeq(v,a) -#define dmar_readl(a) readl(a) -#define dmar_writel(a, v) writel(v, a) - -#define DMAR_VER_MAJOR(v) (((v) & 0xf0) >> 4) -#define DMAR_VER_MINOR(v) ((v) & 0x0f) - -/* - * Decoding Capability Register - */ -#define cap_5lp_support(c) (((c) >> 60) & 1) -#define cap_pi_support(c) (((c) >> 59) & 1) -#define cap_fl1gp_support(c) (((c) >> 56) & 1) -#define cap_read_drain(c) (((c) >> 55) & 1) -#define cap_write_drain(c) (((c) >> 54) & 1) -#define cap_max_amask_val(c) (((c) >> 48) & 0x3f) -#define cap_num_fault_regs(c) ((((c) >> 40) & 0xff) + 1) -#define cap_pgsel_inv(c) (((c) >> 39) & 1) - -#define cap_super_page_val(c) (((c) >> 34) & 0xf) -#define cap_super_offset(c) (((find_first_bit(&cap_super_page_val(c), 4)) \ - * OFFSET_STRIDE) + 21) - -#define cap_fault_reg_offset(c) ((((c) >> 24) & 0x3ff) * 16) -#define cap_max_fault_reg_offset(c) \ - (cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16) - -#define cap_zlr(c) (((c) >> 22) & 1) -#define cap_isoch(c) (((c) >> 23) & 1) -#define cap_mgaw(c) ((((c) >> 16) & 0x3f) + 1) -#define cap_sagaw(c) (((c) >> 8) & 0x1f) -#define cap_caching_mode(c) (((c) >> 7) & 1) -#define cap_phmr(c) (((c) >> 6) & 1) -#define cap_plmr(c) (((c) >> 5) & 1) -#define cap_rwbf(c) (((c) >> 4) & 1) -#define cap_afl(c) (((c) >> 3) & 1) -#define cap_ndoms(c) (((unsigned long)1) << (4 + 2 * ((c) & 0x7))) -/* - * Extended Capability Register - */ - -#define ecap_rps(e) (((e) >> 49) & 0x1) -#define ecap_smpwc(e) (((e) >> 48) & 0x1) -#define ecap_flts(e) (((e) >> 47) & 0x1) -#define ecap_slts(e) (((e) >> 46) & 0x1) -#define ecap_slads(e) (((e) >> 45) & 0x1) -#define ecap_vcs(e) (((e) >> 44) & 0x1) -#define ecap_smts(e) (((e) >> 43) & 0x1) -#define ecap_dit(e) (((e) >> 41) & 0x1) -#define ecap_pds(e) (((e) >> 42) & 0x1) -#define ecap_pasid(e) (((e) >> 40) & 0x1) -#define ecap_pss(e) (((e) >> 35) & 0x1f) -#define ecap_eafs(e) (((e) >> 34) & 0x1) -#define ecap_nwfs(e) (((e) >> 33) & 0x1) -#define ecap_srs(e) (((e) >> 31) & 0x1) -#define ecap_ers(e) (((e) >> 30) & 0x1) -#define ecap_prs(e) (((e) >> 29) & 0x1) -#define ecap_broken_pasid(e) (((e) >> 28) & 0x1) -#define ecap_dis(e) (((e) >> 27) & 0x1) -#define ecap_nest(e) (((e) >> 26) & 0x1) -#define ecap_mts(e) (((e) >> 25) & 0x1) -#define ecap_ecs(e) (((e) >> 24) & 0x1) -#define ecap_iotlb_offset(e) ((((e) >> 8) & 0x3ff) * 16) -#define ecap_max_iotlb_offset(e) (ecap_iotlb_offset(e) + 16) -#define ecap_coherent(e) ((e) & 0x1) -#define ecap_qis(e) ((e) & 0x2) -#define ecap_pass_through(e) (((e) >> 6) & 0x1) -#define ecap_eim_support(e) (((e) >> 4) & 0x1) -#define ecap_ir_support(e) (((e) >> 3) & 0x1) -#define ecap_dev_iotlb_support(e) (((e) >> 2) & 0x1) -#define ecap_max_handle_mask(e) (((e) >> 20) & 0xf) -#define ecap_sc_support(e) (((e) >> 7) & 0x1) /* Snooping Control */ - -/* Virtual command interface capability */ -#define vccap_pasid(v) (((v) & DMA_VCS_PAS)) /* PASID allocation */ - -/* IOTLB_REG */ -#define DMA_TLB_FLUSH_GRANU_OFFSET 60 -#define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60) -#define DMA_TLB_DSI_FLUSH (((u64)2) << 60) -#define DMA_TLB_PSI_FLUSH (((u64)3) << 60) -#define DMA_TLB_IIRG(type) ((type >> 60) & 3) -#define DMA_TLB_IAIG(val) (((val) >> 57) & 3) -#define DMA_TLB_READ_DRAIN (((u64)1) << 49) -#define DMA_TLB_WRITE_DRAIN (((u64)1) << 48) -#define DMA_TLB_DID(id) (((u64)((id) & 0xffff)) << 32) -#define DMA_TLB_IVT (((u64)1) << 63) -#define DMA_TLB_IH_NONLEAF (((u64)1) << 6) -#define DMA_TLB_MAX_SIZE (0x3f) - -/* INVALID_DESC */ -#define DMA_CCMD_INVL_GRANU_OFFSET 61 -#define DMA_ID_TLB_GLOBAL_FLUSH (((u64)1) << 4) -#define DMA_ID_TLB_DSI_FLUSH (((u64)2) << 4) -#define DMA_ID_TLB_PSI_FLUSH (((u64)3) << 4) -#define DMA_ID_TLB_READ_DRAIN (((u64)1) << 7) -#define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6) -#define DMA_ID_TLB_DID(id) (((u64)((id & 0xffff) << 16))) -#define DMA_ID_TLB_IH_NONLEAF (((u64)1) << 6) -#define DMA_ID_TLB_ADDR(addr) (addr) -#define DMA_ID_TLB_ADDR_MASK(mask) (mask) - -/* PMEN_REG */ -#define DMA_PMEN_EPM (((u32)1)<<31) -#define DMA_PMEN_PRS (((u32)1)<<0) - -/* GCMD_REG */ -#define DMA_GCMD_TE (((u32)1) << 31) -#define DMA_GCMD_SRTP (((u32)1) << 30) -#define DMA_GCMD_SFL (((u32)1) << 29) -#define DMA_GCMD_EAFL (((u32)1) << 28) -#define DMA_GCMD_WBF (((u32)1) << 27) -#define DMA_GCMD_QIE (((u32)1) << 26) -#define DMA_GCMD_SIRTP (((u32)1) << 24) -#define DMA_GCMD_IRE (((u32) 1) << 25) -#define DMA_GCMD_CFI (((u32) 1) << 23) - -/* GSTS_REG */ -#define DMA_GSTS_TES (((u32)1) << 31) -#define DMA_GSTS_RTPS (((u32)1) << 30) -#define DMA_GSTS_FLS (((u32)1) << 29) -#define DMA_GSTS_AFLS (((u32)1) << 28) -#define DMA_GSTS_WBFS (((u32)1) << 27) -#define DMA_GSTS_QIES (((u32)1) << 26) -#define DMA_GSTS_IRTPS (((u32)1) << 24) -#define DMA_GSTS_IRES (((u32)1) << 25) -#define DMA_GSTS_CFIS (((u32)1) << 23) - -/* DMA_RTADDR_REG */ -#define DMA_RTADDR_RTT (((u64)1) << 11) -#define DMA_RTADDR_SMT (((u64)1) << 10) - -/* CCMD_REG */ -#define DMA_CCMD_ICC (((u64)1) << 63) -#define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61) -#define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61) -#define DMA_CCMD_DEVICE_INVL (((u64)3) << 61) -#define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32) -#define DMA_CCMD_MASK_NOBIT 0 -#define DMA_CCMD_MASK_1BIT 1 -#define DMA_CCMD_MASK_2BIT 2 -#define DMA_CCMD_MASK_3BIT 3 -#define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16) -#define DMA_CCMD_DID(d) ((u64)((d) & 0xffff)) - -/* FECTL_REG */ -#define DMA_FECTL_IM (((u32)1) << 31) - -/* FSTS_REG */ -#define DMA_FSTS_PFO (1 << 0) /* Primary Fault Overflow */ -#define DMA_FSTS_PPF (1 << 1) /* Primary Pending Fault */ -#define DMA_FSTS_IQE (1 << 4) /* Invalidation Queue Error */ -#define DMA_FSTS_ICE (1 << 5) /* Invalidation Completion Error */ -#define DMA_FSTS_ITE (1 << 6) /* Invalidation Time-out Error */ -#define DMA_FSTS_PRO (1 << 7) /* Page Request Overflow */ -#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff) - -/* FRCD_REG, 32 bits access */ -#define DMA_FRCD_F (((u32)1) << 31) -#define dma_frcd_type(d) ((d >> 30) & 1) -#define dma_frcd_fault_reason(c) (c & 0xff) -#define dma_frcd_source_id(c) (c & 0xffff) -#define dma_frcd_pasid_value(c) (((c) >> 8) & 0xfffff) -#define dma_frcd_pasid_present(c) (((c) >> 31) & 1) -/* low 64 bit */ -#define dma_frcd_page_addr(d) (d & (((u64)-1) << PAGE_SHIFT)) - -/* PRS_REG */ -#define DMA_PRS_PPR ((u32)1) -#define DMA_PRS_PRO ((u32)2) - -#define DMA_VCS_PAS ((u64)1) - -#define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \ -do { \ - cycles_t start_time = get_cycles(); \ - while (1) { \ - sts = op(iommu->reg + offset); \ - if (cond) \ - break; \ - if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start_time))\ - panic("DMAR hardware is malfunctioning\n"); \ - cpu_relax(); \ - } \ -} while (0) - -#define QI_LENGTH 256 /* queue length */ - -enum { - QI_FREE, - QI_IN_USE, - QI_DONE, - QI_ABORT -}; - -#define QI_CC_TYPE 0x1 -#define QI_IOTLB_TYPE 0x2 -#define QI_DIOTLB_TYPE 0x3 -#define QI_IEC_TYPE 0x4 -#define QI_IWD_TYPE 0x5 -#define QI_EIOTLB_TYPE 0x6 -#define QI_PC_TYPE 0x7 -#define QI_DEIOTLB_TYPE 0x8 -#define QI_PGRP_RESP_TYPE 0x9 -#define QI_PSTRM_RESP_TYPE 0xa - -#define QI_IEC_SELECTIVE (((u64)1) << 4) -#define QI_IEC_IIDEX(idx) (((u64)(idx & 0xffff) << 32)) -#define QI_IEC_IM(m) (((u64)(m & 0x1f) << 27)) - -#define QI_IWD_STATUS_DATA(d) (((u64)d) << 32) -#define QI_IWD_STATUS_WRITE (((u64)1) << 5) -#define QI_IWD_FENCE (((u64)1) << 6) -#define QI_IWD_PRQ_DRAIN (((u64)1) << 7) - -#define QI_IOTLB_DID(did) (((u64)did) << 16) -#define QI_IOTLB_DR(dr) (((u64)dr) << 7) -#define QI_IOTLB_DW(dw) (((u64)dw) << 6) -#define QI_IOTLB_GRAN(gran) (((u64)gran) >> (DMA_TLB_FLUSH_GRANU_OFFSET-4)) -#define QI_IOTLB_ADDR(addr) (((u64)addr) & VTD_PAGE_MASK) -#define QI_IOTLB_IH(ih) (((u64)ih) << 6) -#define QI_IOTLB_AM(am) (((u8)am) & 0x3f) - -#define QI_CC_FM(fm) (((u64)fm) << 48) -#define QI_CC_SID(sid) (((u64)sid) << 32) -#define QI_CC_DID(did) (((u64)did) << 16) -#define QI_CC_GRAN(gran) (((u64)gran) >> (DMA_CCMD_INVL_GRANU_OFFSET-4)) - -#define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32) -#define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16) -#define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) -#define QI_DEV_IOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \ - ((u64)((pfsid >> 4) & 0xfff) << 52)) -#define QI_DEV_IOTLB_SIZE 1 -#define QI_DEV_IOTLB_MAX_INVS 32 - -#define QI_PC_PASID(pasid) (((u64)pasid) << 32) -#define QI_PC_DID(did) (((u64)did) << 16) -#define QI_PC_GRAN(gran) (((u64)gran) << 4) - -/* PASID cache invalidation granu */ -#define QI_PC_ALL_PASIDS 0 -#define QI_PC_PASID_SEL 1 -#define QI_PC_GLOBAL 3 - -#define QI_EIOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) -#define QI_EIOTLB_IH(ih) (((u64)ih) << 6) -#define QI_EIOTLB_AM(am) (((u64)am) & 0x3f) -#define QI_EIOTLB_PASID(pasid) (((u64)pasid) << 32) -#define QI_EIOTLB_DID(did) (((u64)did) << 16) -#define QI_EIOTLB_GRAN(gran) (((u64)gran) << 4) - -/* QI Dev-IOTLB inv granu */ -#define QI_DEV_IOTLB_GRAN_ALL 1 -#define QI_DEV_IOTLB_GRAN_PASID_SEL 0 - -#define QI_DEV_EIOTLB_ADDR(a) ((u64)(a) & VTD_PAGE_MASK) -#define QI_DEV_EIOTLB_SIZE (((u64)1) << 11) -#define QI_DEV_EIOTLB_PASID(p) ((u64)((p) & 0xfffff) << 32) -#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) -#define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) -#define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \ - ((u64)((pfsid >> 4) & 0xfff) << 52)) -#define QI_DEV_EIOTLB_MAX_INVS 32 - -/* Page group response descriptor QW0 */ -#define QI_PGRP_PASID_P(p) (((u64)(p)) << 4) -#define QI_PGRP_PDP(p) (((u64)(p)) << 5) -#define QI_PGRP_RESP_CODE(res) (((u64)(res)) << 12) -#define QI_PGRP_DID(rid) (((u64)(rid)) << 16) -#define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) - -/* Page group response descriptor QW1 */ -#define QI_PGRP_LPIG(x) (((u64)(x)) << 2) -#define QI_PGRP_IDX(idx) (((u64)(idx)) << 3) - - -#define QI_RESP_SUCCESS 0x0 -#define QI_RESP_INVALID 0x1 -#define QI_RESP_FAILURE 0xf - -#define QI_GRAN_NONG_PASID 2 -#define QI_GRAN_PSI_PASID 3 - -#define qi_shift(iommu) (DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap)) - -struct qi_desc { - u64 qw0; - u64 qw1; - u64 qw2; - u64 qw3; -}; - -struct q_inval { - raw_spinlock_t q_lock; - void *desc; /* invalidation queue */ - int *desc_status; /* desc status */ - int free_head; /* first free entry */ - int free_tail; /* last free entry */ - int free_cnt; -}; - -struct dmar_pci_notify_info; - -#ifdef CONFIG_IRQ_REMAP -/* 1MB - maximum possible interrupt remapping table size */ -#define INTR_REMAP_PAGE_ORDER 8 -#define INTR_REMAP_TABLE_REG_SIZE 0xf -#define INTR_REMAP_TABLE_REG_SIZE_MASK 0xf - -#define INTR_REMAP_TABLE_ENTRIES 65536 - -struct irq_domain; - -struct ir_table { - struct irte *base; - unsigned long *bitmap; -}; - -void intel_irq_remap_add_device(struct dmar_pci_notify_info *info); -#else -static inline void -intel_irq_remap_add_device(struct dmar_pci_notify_info *info) { } -#endif - -struct iommu_flush { - void (*flush_context)(struct intel_iommu *iommu, u16 did, u16 sid, - u8 fm, u64 type); - void (*flush_iotlb)(struct intel_iommu *iommu, u16 did, u64 addr, - unsigned int size_order, u64 type); -}; - -enum { - SR_DMAR_FECTL_REG, - SR_DMAR_FEDATA_REG, - SR_DMAR_FEADDR_REG, - SR_DMAR_FEUADDR_REG, - MAX_SR_DMAR_REGS -}; - -#define VTD_FLAG_TRANS_PRE_ENABLED (1 << 0) -#define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1) -#define VTD_FLAG_SVM_CAPABLE (1 << 2) - -extern int intel_iommu_sm; -extern spinlock_t device_domain_lock; - -#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap)) -#define pasid_supported(iommu) (sm_supported(iommu) && \ - ecap_pasid((iommu)->ecap)) - -struct pasid_entry; -struct pasid_state_entry; -struct page_req_dsc; - -/* - * 0: Present - * 1-11: Reserved - * 12-63: Context Ptr (12 - (haw-1)) - * 64-127: Reserved - */ -struct root_entry { - u64 lo; - u64 hi; -}; - -/* - * low 64 bits: - * 0: present - * 1: fault processing disable - * 2-3: translation type - * 12-63: address space root - * high 64 bits: - * 0-2: address width - * 3-6: aval - * 8-23: domain id - */ -struct context_entry { - u64 lo; - u64 hi; -}; - -/* - * When VT-d works in the scalable mode, it allows DMA translation to - * happen through either first level or second level page table. This - * bit marks that the DMA translation for the domain goes through the - * first level page table, otherwise, it goes through the second level. - */ -#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(1) - -struct dmar_domain { - int nid; /* node id */ - - unsigned int iommu_refcnt[DMAR_UNITS_SUPPORTED]; - /* Refcount of devices per iommu */ - - - u16 iommu_did[DMAR_UNITS_SUPPORTED]; - /* Domain ids per IOMMU. Use u16 since - * domain ids are 16 bit wide according - * to VT-d spec, section 9.3 */ - - u8 has_iotlb_device: 1; - u8 iommu_coherency: 1; /* indicate coherency of iommu access */ - u8 force_snooping : 1; /* Create IOPTEs with snoop control */ - u8 set_pte_snp:1; - - struct list_head devices; /* all devices' list */ - struct iova_domain iovad; /* iova's that belong to this domain */ - - struct dma_pte *pgd; /* virtual address */ - int gaw; /* max guest address width */ - - /* adjusted guest address width, 0 is level 2 30-bit */ - int agaw; - - int flags; /* flags to find out type of domain */ - int iommu_superpage;/* Level of superpages supported: - 0 == 4KiB (no superpages), 1 == 2MiB, - 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ - u64 max_addr; /* maximum mapped address */ - - struct iommu_domain domain; /* generic domain data structure for - iommu core */ -}; - -struct intel_iommu { - void __iomem *reg; /* Pointer to hardware regs, virtual addr */ - u64 reg_phys; /* physical address of hw register set */ - u64 reg_size; /* size of hw register set */ - u64 cap; - u64 ecap; - u64 vccap; - u32 gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */ - raw_spinlock_t register_lock; /* protect register handling */ - int seq_id; /* sequence id of the iommu */ - int agaw; /* agaw of this iommu */ - int msagaw; /* max sagaw of this iommu */ - unsigned int irq, pr_irq; - u16 segment; /* PCI segment# */ - unsigned char name[13]; /* Device Name */ - -#ifdef CONFIG_INTEL_IOMMU - unsigned long *domain_ids; /* bitmap of domains */ - spinlock_t lock; /* protect context, domain ids */ - struct root_entry *root_entry; /* virtual address */ - - struct iommu_flush flush; -#endif -#ifdef CONFIG_INTEL_IOMMU_SVM - struct page_req_dsc *prq; - unsigned char prq_name[16]; /* Name for PRQ interrupt */ - struct completion prq_complete; - struct ioasid_allocator_ops pasid_allocator; /* Custom allocator for PASIDs */ -#endif - struct iopf_queue *iopf_queue; - unsigned char iopfq_name[16]; - struct q_inval *qi; /* Queued invalidation info */ - u32 *iommu_state; /* Store iommu states between suspend and resume.*/ - -#ifdef CONFIG_IRQ_REMAP - struct ir_table *ir_table; /* Interrupt remapping info */ - struct irq_domain *ir_domain; - struct irq_domain *ir_msi_domain; -#endif - struct iommu_device iommu; /* IOMMU core code handle */ - int node; - u32 flags; /* Software defined flags */ - - struct dmar_drhd_unit *drhd; - void *perf_statistic; -}; - -/* PCI domain-device relationship */ -struct device_domain_info { - struct list_head link; /* link to domain siblings */ - struct list_head global; /* link to global list */ - u32 segment; /* PCI segment number */ - u8 bus; /* PCI bus number */ - u8 devfn; /* PCI devfn number */ - u16 pfsid; /* SRIOV physical function source ID */ - u8 pasid_supported:3; - u8 pasid_enabled:1; - u8 pri_supported:1; - u8 pri_enabled:1; - u8 ats_supported:1; - u8 ats_enabled:1; - u8 ats_qdep; - struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ - struct intel_iommu *iommu; /* IOMMU used by this device */ - struct dmar_domain *domain; /* pointer to domain */ - struct pasid_table *pasid_table; /* pasid table */ -}; - -static inline void __iommu_flush_cache( - struct intel_iommu *iommu, void *addr, int size) -{ - if (!ecap_coherent(iommu->ecap)) - clflush_cache_range(addr, size); -} - -/* Convert generic struct iommu_domain to private struct dmar_domain */ -static inline struct dmar_domain *to_dmar_domain(struct iommu_domain *dom) -{ - return container_of(dom, struct dmar_domain, domain); -} - -/* - * 0: readable - * 1: writable - * 2-6: reserved - * 7: super page - * 8-10: available - * 11: snoop behavior - * 12-63: Host physical address - */ -struct dma_pte { - u64 val; -}; - -static inline void dma_clear_pte(struct dma_pte *pte) -{ - pte->val = 0; -} - -static inline u64 dma_pte_addr(struct dma_pte *pte) -{ -#ifdef CONFIG_64BIT - return pte->val & VTD_PAGE_MASK & (~DMA_FL_PTE_XD); -#else - /* Must have a full atomic 64-bit read */ - return __cmpxchg64(&pte->val, 0ULL, 0ULL) & - VTD_PAGE_MASK & (~DMA_FL_PTE_XD); -#endif -} - -static inline bool dma_pte_present(struct dma_pte *pte) -{ - return (pte->val & 3) != 0; -} - -static inline bool dma_pte_superpage(struct dma_pte *pte) -{ - return (pte->val & DMA_PTE_LARGE_PAGE); -} - -static inline bool first_pte_in_page(struct dma_pte *pte) -{ - return IS_ALIGNED((unsigned long)pte, VTD_PAGE_SIZE); -} - -static inline int nr_pte_to_next_page(struct dma_pte *pte) -{ - return first_pte_in_page(pte) ? BIT_ULL(VTD_STRIDE_SHIFT) : - (struct dma_pte *)ALIGN((unsigned long)pte, VTD_PAGE_SIZE) - pte; -} - -extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); - -extern int dmar_enable_qi(struct intel_iommu *iommu); -extern void dmar_disable_qi(struct intel_iommu *iommu); -extern int dmar_reenable_qi(struct intel_iommu *iommu); -extern void qi_global_iec(struct intel_iommu *iommu); - -extern void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, - u8 fm, u64 type); -extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, - unsigned int size_order, u64 type); -extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, - u16 qdep, u64 addr, unsigned mask); - -void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, - unsigned long npages, bool ih); - -void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, - u32 pasid, u16 qdep, u64 addr, - unsigned int size_order); -void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu, - u32 pasid); - -int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc, - unsigned int count, unsigned long options); -/* - * Options used in qi_submit_sync: - * QI_OPT_WAIT_DRAIN - Wait for PRQ drain completion, spec 6.5.2.8. - */ -#define QI_OPT_WAIT_DRAIN BIT(0) - -extern int dmar_ir_support(void); - -void *alloc_pgtable_page(int node); -void free_pgtable_page(void *vaddr); -struct intel_iommu *domain_get_iommu(struct dmar_domain *domain); -void iommu_flush_write_buffer(struct intel_iommu *iommu); -int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev); -struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn); - -#ifdef CONFIG_INTEL_IOMMU_SVM -extern void intel_svm_check(struct intel_iommu *iommu); -extern int intel_svm_enable_prq(struct intel_iommu *iommu); -extern int intel_svm_finish_prq(struct intel_iommu *iommu); -struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, - void *drvdata); -void intel_svm_unbind(struct iommu_sva *handle); -u32 intel_svm_get_pasid(struct iommu_sva *handle); -int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, - struct iommu_page_response *msg); - -struct intel_svm_dev { - struct list_head list; - struct rcu_head rcu; - struct device *dev; - struct intel_iommu *iommu; - struct iommu_sva sva; - unsigned long prq_seq_number; - u32 pasid; - int users; - u16 did; - u16 dev_iotlb:1; - u16 sid, qdep; -}; - -struct intel_svm { - struct mmu_notifier notifier; - struct mm_struct *mm; - - unsigned int flags; - u32 pasid; - struct list_head devs; -}; -#else -static inline void intel_svm_check(struct intel_iommu *iommu) {} -#endif - -#ifdef CONFIG_INTEL_IOMMU_DEBUGFS -void intel_iommu_debugfs_init(void); -#else -static inline void intel_iommu_debugfs_init(void) {} -#endif /* CONFIG_INTEL_IOMMU_DEBUGFS */ - -extern const struct attribute_group *intel_iommu_groups[]; -bool context_present(struct context_entry *context); -struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, - u8 devfn, int alloc); - -extern const struct iommu_ops intel_iommu_ops; - -#ifdef CONFIG_INTEL_IOMMU -extern int iommu_calculate_agaw(struct intel_iommu *iommu); -extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); -extern int dmar_disabled; -extern int intel_iommu_enabled; -#else -static inline int iommu_calculate_agaw(struct intel_iommu *iommu) -{ - return 0; -} -static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) -{ - return 0; -} -#define dmar_disabled (1) -#define intel_iommu_enabled (0) -#endif - -static inline const char *decode_prq_descriptor(char *str, size_t size, - u64 dw0, u64 dw1, u64 dw2, u64 dw3) -{ - char *buf = str; - int bytes; - - bytes = snprintf(buf, size, - "rid=0x%llx addr=0x%llx %c%c%c%c%c pasid=0x%llx index=0x%llx", - FIELD_GET(GENMASK_ULL(31, 16), dw0), - FIELD_GET(GENMASK_ULL(63, 12), dw1), - dw1 & BIT_ULL(0) ? 'r' : '-', - dw1 & BIT_ULL(1) ? 'w' : '-', - dw0 & BIT_ULL(52) ? 'x' : '-', - dw0 & BIT_ULL(53) ? 'p' : '-', - dw1 & BIT_ULL(2) ? 'l' : '-', - FIELD_GET(GENMASK_ULL(51, 32), dw0), - FIELD_GET(GENMASK_ULL(11, 3), dw1)); - - /* Private Data */ - if (dw0 & BIT_ULL(9)) { - size -= bytes; - buf += bytes; - snprintf(buf, size, " private=0x%llx/0x%llx\n", dw2, dw3); - } - - return str; -} - -#endif -- cgit v1.2.3 From 25357900f4e67094dd09b7dcb8a5f47c3f5a159d Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 12 Jul 2022 08:09:08 +0800 Subject: iommu/vt-d: Make DMAR_UNITS_SUPPORTED default 1024 If the available hardware exceeds DMAR_UNITS_SUPPORTED (previously set to MAX_IO_APICS, or 128), it causes these messages: "DMAR: Failed to allocate seq_id", "DMAR: Parse DMAR table failure.", and "x2apic: IRQ remapping doesn't support X2APIC mode x2apic disabled"; and the system fails to boot properly. To support up to 64 sockets with 10 DMAR units each (640), make the value of DMAR_UNITS_SUPPORTED default 1024. Signed-off-by: Steve Wahl Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Steve Wahl Link: https://lore.kernel.org/linux-iommu/20220615183650.32075-1-steve.wahl@hpe.com/ Link: https://lore.kernel.org/r/20220702015610.2849494-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- include/linux/dmar.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dmar.h b/include/linux/dmar.h index cbd714a198a0..d81a51978d01 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -18,11 +18,7 @@ struct acpi_dmar_header; -#ifdef CONFIG_X86 -# define DMAR_UNITS_SUPPORTED MAX_IO_APICS -#else -# define DMAR_UNITS_SUPPORTED 64 -#endif +#define DMAR_UNITS_SUPPORTED 1024 /* DMAR Flags */ #define DMAR_INTR_REMAP 0x1 -- cgit v1.2.3 From fb2accadaa9427a374fa9f482ea24ca731f675ba Mon Sep 17 00:00:00 2001 From: Brijesh Singh Date: Wed, 13 Jul 2022 17:56:48 -0500 Subject: iommu/amd: Introduce function to check and enable SNP To support SNP, IOMMU needs to be enabled, and prohibits IOMMU configurations where DTE[Mode]=0, which means it cannot be supported with IOMMU passthrough domain (a.k.a IOMMU_DOMAIN_IDENTITY), and when AMD IOMMU driver is configured to not use the IOMMU host (v1) page table. Otherwise, RMP table initialization could cause the system to crash. The request to enable SNP support in IOMMU must be done before PCI initialization state of the IOMMU driver because enabling SNP affects how IOMMU driver sets up IOMMU data structures (i.e. DTE). Unlike other IOMMU features, SNP feature does not have an enable bit in the IOMMU control register. Instead, the IOMMU driver introduces an amd_iommu_snp_en variable to track enabling state of SNP. Introduce amd_iommu_snp_enable() for other drivers to request enabling the SNP support in IOMMU, which checks all prerequisites and determines if the feature can be safely enabled. Please see the IOMMU spec section 2.12 for further details. Reviewed-by: Robin Murphy Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Brijesh Singh Link: https://lore.kernel.org/r/20220713225651.20758-7-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 ++ drivers/iommu/amd/init.c | 42 ++++++++++++++++++++++++++++++++++++++++++ include/linux/amd-iommu.h | 4 ++++ 3 files changed, 48 insertions(+) (limited to 'include/linux') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 1b945d47741c..84e5bb1bf01b 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -140,4 +140,6 @@ extern struct dev_table_entry *get_dev_table(struct amd_iommu *iommu); extern u64 amd_iommu_efr; extern u64 amd_iommu_efr2; + +extern bool amd_iommu_snp_en; #endif diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 7c6e61451ff7..cddcb6f7e3c9 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -168,6 +168,10 @@ static int amd_iommu_target_ivhd_type; u64 amd_iommu_efr; u64 amd_iommu_efr2; +/* SNP is enabled on the system? */ +bool amd_iommu_snp_en; +EXPORT_SYMBOL(amd_iommu_snp_en); + LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */ LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */ @@ -3556,3 +3560,41 @@ int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true); } + +#ifdef CONFIG_AMD_MEM_ENCRYPT +int amd_iommu_snp_enable(void) +{ + /* + * The SNP support requires that IOMMU must be enabled, and is + * not configured in the passthrough mode. + */ + if (no_iommu || iommu_default_passthrough()) { + pr_err("SNP: IOMMU is disabled or configured in passthrough mode, SNP cannot be supported"); + return -EINVAL; + } + + /* + * Prevent enabling SNP after IOMMU_ENABLED state because this process + * affect how IOMMU driver sets up data structures and configures + * IOMMU hardware. + */ + if (init_state > IOMMU_ENABLED) { + pr_err("SNP: Too late to enable SNP for IOMMU.\n"); + return -EINVAL; + } + + amd_iommu_snp_en = check_feature_on_all_iommus(FEATURE_SNP); + if (!amd_iommu_snp_en) + return -EINVAL; + + pr_info("SNP enabled\n"); + + /* Enforce IOMMU v1 pagetable when SNP is enabled. */ + if (amd_iommu_pgtable != AMD_IOMMU_V1) { + pr_warn("Force to using AMD IOMMU v1 page table due to SNP\n"); + amd_iommu_pgtable = AMD_IOMMU_V1; + } + + return 0; +} +#endif diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index 58e6c3806c09..953e6f12fa1c 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -206,4 +206,8 @@ int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value); struct amd_iommu *get_amd_iommu(unsigned int idx); +#ifdef CONFIG_AMD_MEM_ENCRYPT +int amd_iommu_snp_enable(void); +#endif + #endif /* _ASM_X86_AMD_IOMMU_H */ -- cgit v1.2.3 From 1e0b3b0b6cb5e04bcb25fbe4164180d64e3ace07 Mon Sep 17 00:00:00 2001 From: Ilan Peer Date: Wed, 27 Apr 2022 18:02:10 +0300 Subject: wifi: mac80211: Align with Draft P802.11be_D1.5 Align the mac80211 implementation with P802.11be_D1.5. Signed-off-by: Ilan Peer Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 52 ++++++++++++++++++--------- net/mac80211/ieee80211_i.h | 4 +++ net/mac80211/mlme.c | 32 +++++++++++++++++ net/mac80211/util.c | 87 +++++++++++++++++++++++++++++----------------- 4 files changed, 128 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index f386f9ed41f3..e75c73ca11ec 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2046,25 +2046,38 @@ struct ieee80211_eht_cap_elem { u8 optional[]; } __packed; +#define IEEE80211_EHT_OPER_INFO_PRESENT 0x1 +#define IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT 0x2 + /** * struct ieee80211_eht_operation - eht operation element * * This structure is the "EHT Operation Element" fields as - * described in P802.11be_D1.4 section 9.4.2.311 + * described in P802.11be_D1.5 section 9.4.2.311 * - * FIXME: The spec is unclear how big the fields are, and doesn't - * indicate the "Disabled Subchannel Bitmap Present" in the - * structure (Figure 9-1002a) at all ... + * @params: EHT operation element parameters. See &IEEE80211_EHT_OPER_* + * @optional: optional parts */ struct ieee80211_eht_operation { - u8 chan_width; - u8 ccfs; - u8 present_bm; - - u8 disable_subchannel_bitmap[]; + u8 params; + u8 optional[]; } __packed; -#define IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT 0x1 +/** + * struct ieee80211_eht_operation_info - eht operation information + * + * @control: EHT operation information control. + * @ccfs0: defines a channel center frequency for a 20, 40, 80, 160, or 320 MHz + * EHT BSS. + * @ccfs1: defines a channel center frequency for a 160 or 320 MHz EHT BSS. + * @optional: optional parts + */ +struct ieee80211_eht_operation_info { + u8 control; + u8 ccfs0; + u8 ccfs1; + u8 optional[]; +} __packed; /* 802.11ac VHT Capabilities */ #define IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895 0x00000000 @@ -2773,10 +2786,12 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) #define IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2 0x08 #define IEEE80211_EHT_MAC_CAP0_RESTRICTED_TWT 0x10 #define IEEE80211_EHT_MAC_CAP0_SCS_TRAFFIC_DESC 0x20 -#define IEEE80211_EHT_MAC_CAP0_MAX_AMPDU_LEN_MASK 0xc0 -#define IEEE80211_EHT_MAC_CAP0_MAX_AMPDU_LEN_3895 0 -#define IEEE80211_EHT_MAC_CAP0_MAX_AMPDU_LEN_7991 1 -#define IEEE80211_EHT_MAC_CAP0_MAX_AMPDU_LEN_11454 2 +#define IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK 0xc0 +#define IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_3895 0 +#define IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_7991 1 +#define IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_11454 2 + +#define IEEE80211_EHT_MAC_CAP1_MAX_AMPDU_LEN_MASK 0x01 /* EHT PHY capabilities as defined in P802.11be_D1.4 section 9.4.2.313.3 */ #define IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ 0x02 @@ -2949,8 +2964,13 @@ ieee80211_eht_oper_size_ok(const u8 *data, u8 len) if (len < needed) return false; - if (elem->present_bm & IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT) - needed += 2; + if (elem->params & IEEE80211_EHT_OPER_INFO_PRESENT) { + needed += 3; + + if (elem->params & + IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT) + needed += 2; + } return len >= needed; } diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index f6791b47f78f..aa1e438ee61e 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -2326,6 +2326,10 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info, const struct ieee80211_vht_operation *oper, const struct ieee80211_ht_operation *htop, struct cfg80211_chan_def *chandef); +void ieee80211_chandef_eht_oper(struct ieee80211_sub_if_data *sdata, + const struct ieee80211_eht_operation *eht_oper, + bool support_160, bool support_320, + struct cfg80211_chan_def *chandef); bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, const struct ieee80211_he_operation *he_oper, const struct ieee80211_eht_operation *eht_oper, diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index e5c058db451d..39f13f3c29bf 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -303,6 +303,38 @@ ieee80211_determine_chantype(struct ieee80211_sub_if_data *sdata, *chandef = vht_chandef; + /* + * handle the case that the EHT operation indicates that it holds EHT + * operation information (in case that the channel width differs from + * the channel width reported in HT/VHT/HE). + */ + if (eht_oper && (eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) { + struct cfg80211_chan_def eht_chandef = *chandef; + + ieee80211_chandef_eht_oper(sdata, eht_oper, + eht_chandef.width == + NL80211_CHAN_WIDTH_160, + false, &eht_chandef); + + if (!cfg80211_chandef_valid(&eht_chandef)) { + if (!(ifmgd->flags & IEEE80211_STA_DISABLE_EHT)) + sdata_info(sdata, + "AP EHT information is invalid, disabling EHT\n"); + ret = IEEE80211_STA_DISABLE_EHT; + goto out; + } + + if (!cfg80211_chandef_compatible(chandef, &eht_chandef)) { + if (!(ifmgd->flags & IEEE80211_STA_DISABLE_EHT)) + sdata_info(sdata, + "AP EHT information is incompatible, disabling EHT\n"); + ret = IEEE80211_STA_DISABLE_EHT; + goto out; + } + + *chandef = eht_chandef; + } + ret = 0; out: diff --git a/net/mac80211/util.c b/net/mac80211/util.c index 3e29ef1f81ad..924192238042 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -3459,6 +3459,58 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info, return true; } +void ieee80211_chandef_eht_oper(struct ieee80211_sub_if_data *sdata, + const struct ieee80211_eht_operation *eht_oper, + bool support_160, bool support_320, + struct cfg80211_chan_def *chandef) +{ + struct ieee80211_eht_operation_info *info = (void *)eht_oper->optional; + + chandef->center_freq1 = + ieee80211_channel_to_frequency(info->ccfs0, + chandef->chan->band); + + switch (u8_get_bits(info->control, + IEEE80211_EHT_OPER_CHAN_WIDTH)) { + case IEEE80211_EHT_OPER_CHAN_WIDTH_20MHZ: + chandef->width = NL80211_CHAN_WIDTH_20; + break; + case IEEE80211_EHT_OPER_CHAN_WIDTH_40MHZ: + chandef->width = NL80211_CHAN_WIDTH_40; + break; + case IEEE80211_EHT_OPER_CHAN_WIDTH_80MHZ: + chandef->width = NL80211_CHAN_WIDTH_80; + break; + case IEEE80211_EHT_OPER_CHAN_WIDTH_160MHZ: + if (support_160) { + chandef->width = NL80211_CHAN_WIDTH_160; + chandef->center_freq1 = + ieee80211_channel_to_frequency(info->ccfs1, + chandef->chan->band); + } else { + chandef->width = NL80211_CHAN_WIDTH_80; + } + break; + case IEEE80211_EHT_OPER_CHAN_WIDTH_320MHZ: + if (support_320) { + chandef->width = NL80211_CHAN_WIDTH_320; + chandef->center_freq1 = + ieee80211_channel_to_frequency(info->ccfs1, + chandef->chan->band); + } else if (support_160) { + chandef->width = NL80211_CHAN_WIDTH_160; + } else { + chandef->width = NL80211_CHAN_WIDTH_80; + + if (chandef->center_freq1 > chandef->chan->center_freq) + chandef->center_freq1 -= 40; + else + chandef->center_freq1 += 40; + } + break; + } +} + bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, const struct ieee80211_he_operation *he_oper, const struct ieee80211_eht_operation *eht_oper, @@ -3539,7 +3591,8 @@ bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, break; } - if (!eht_oper) { + if (!eht_oper || + !(eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) { switch (u8_get_bits(he_6ghz_oper->control, IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH)) { case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_20MHZ: @@ -3583,36 +3636,8 @@ bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, support_320 = eht_phy_cap & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; - switch (u8_get_bits(eht_oper->chan_width, - IEEE80211_EHT_OPER_CHAN_WIDTH)) { - case IEEE80211_EHT_OPER_CHAN_WIDTH_20MHZ: - he_chandef.width = NL80211_CHAN_WIDTH_20; - break; - case IEEE80211_EHT_OPER_CHAN_WIDTH_40MHZ: - he_chandef.width = NL80211_CHAN_WIDTH_40; - break; - case IEEE80211_EHT_OPER_CHAN_WIDTH_80MHZ: - he_chandef.width = NL80211_CHAN_WIDTH_80; - break; - case IEEE80211_EHT_OPER_CHAN_WIDTH_160MHZ: - if (support_160) - he_chandef.width = NL80211_CHAN_WIDTH_160; - else - he_chandef.width = NL80211_CHAN_WIDTH_80; - break; - case IEEE80211_EHT_OPER_CHAN_WIDTH_320MHZ: - if (support_320) - he_chandef.width = NL80211_CHAN_WIDTH_320; - else if (support_160) - he_chandef.width = NL80211_CHAN_WIDTH_160; - else - he_chandef.width = NL80211_CHAN_WIDTH_80; - break; - } - - he_chandef.center_freq1 = - ieee80211_channel_to_frequency(eht_oper->ccfs, - NL80211_BAND_6GHZ); + ieee80211_chandef_eht_oper(sdata, eht_oper, support_160, + support_320, &he_chandef); } if (!cfg80211_chandef_valid(&he_chandef)) { -- cgit v1.2.3 From 062e8e02dfd43c94b0d601c071e8a5c50bed830e Mon Sep 17 00:00:00 2001 From: Ilan Peer Date: Wed, 1 Jun 2022 17:43:34 +0300 Subject: wifi: mac80211: Align with Draft P802.11be_D2.0 Align the mac80211 implementation with P802.11be_D2.0. Signed-off-by: Ilan Peer Signed-off-by: Johannes Berg --- drivers/net/wireless/mac80211_hwsim.c | 6 +++--- include/linux/ieee80211.h | 23 +++++++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c index 737f8ed56d94..f437d8a65268 100644 --- a/drivers/net/wireless/mac80211_hwsim.c +++ b/drivers/net/wireless/mac80211_hwsim.c @@ -3118,7 +3118,7 @@ static const struct ieee80211_sband_iftype_data sband_capa_2ghz[] = { .has_eht = true, .eht_cap_elem = { .mac_cap_info[0] = - IEEE80211_EHT_MAC_CAP0_NSEP_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | IEEE80211_EHT_MAC_CAP0_OM_CONTROL | IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1, .phy_cap_info[0] = @@ -3271,7 +3271,7 @@ static const struct ieee80211_sband_iftype_data sband_capa_5ghz[] = { .has_eht = true, .eht_cap_elem = { .mac_cap_info[0] = - IEEE80211_EHT_MAC_CAP0_NSEP_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | IEEE80211_EHT_MAC_CAP0_OM_CONTROL | IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1, .phy_cap_info[0] = @@ -3453,7 +3453,7 @@ static const struct ieee80211_sband_iftype_data sband_capa_6ghz[] = { .has_eht = true, .eht_cap_elem = { .mac_cap_info[0] = - IEEE80211_EHT_MAC_CAP0_NSEP_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | IEEE80211_EHT_MAC_CAP0_OM_CONTROL | IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1, .phy_cap_info[0] = diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index e75c73ca11ec..cca564372d16 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2020,7 +2020,7 @@ struct ieee80211_eht_mcs_nss_supp_bw { * struct ieee80211_eht_cap_elem_fixed - EHT capabilities fixed data * * This structure is the "EHT Capabilities element" fixed fields as - * described in P802.11be_D1.4 section 9.4.2.313. + * described in P802.11be_D2.0 section 9.4.2.313. * * @mac_cap_info: MAC capabilities, see IEEE80211_EHT_MAC_CAP* * @phy_cap_info: PHY capabilities, see IEEE80211_EHT_PHY_CAP* @@ -2046,20 +2046,27 @@ struct ieee80211_eht_cap_elem { u8 optional[]; } __packed; -#define IEEE80211_EHT_OPER_INFO_PRESENT 0x1 -#define IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT 0x2 +#define IEEE80211_EHT_OPER_INFO_PRESENT 0x01 +#define IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT 0x02 +#define IEEE80211_EHT_OPER_EHT_DEF_PE_DURATION 0x04 +#define IEEE80211_EHT_OPER_GROUP_ADDRESSED_BU_IND_LIMIT 0x08 +#define IEEE80211_EHT_OPER_GROUP_ADDRESSED_BU_IND_EXP_MASK 0x30 /** * struct ieee80211_eht_operation - eht operation element * * This structure is the "EHT Operation Element" fields as - * described in P802.11be_D1.5 section 9.4.2.311 + * described in P802.11be_D2.0 section 9.4.2.311 * * @params: EHT operation element parameters. See &IEEE80211_EHT_OPER_* + * @basic_mcs_nss: indicates the EHT-MCSs for each number of spatial streams in + * EHT PPDUs that are supported by all EHT STAs in the BSS in transmit and + * receive. * @optional: optional parts */ struct ieee80211_eht_operation { u8 params; + __le32 basic_mcs_nss; u8 optional[]; } __packed; @@ -2779,8 +2786,8 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) #define S1G_OPER_CH_WIDTH_PRIMARY_1MHZ BIT(0) #define S1G_OPER_CH_WIDTH_OPER GENMASK(4, 1) -/* EHT MAC capabilities as defined in P802.11be_D1.4 section 9.4.2.313.2 */ -#define IEEE80211_EHT_MAC_CAP0_NSEP_PRIO_ACCESS 0x01 +/* EHT MAC capabilities as defined in P802.11be_D2.0 section 9.4.2.313.2 */ +#define IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS 0x01 #define IEEE80211_EHT_MAC_CAP0_OM_CONTROL 0x02 #define IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1 0x04 #define IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2 0x08 @@ -2793,7 +2800,7 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) #define IEEE80211_EHT_MAC_CAP1_MAX_AMPDU_LEN_MASK 0x01 -/* EHT PHY capabilities as defined in P802.11be_D1.4 section 9.4.2.313.3 */ +/* EHT PHY capabilities as defined in P802.11be_D2.0 section 9.4.2.313.3 */ #define IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ 0x02 #define IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ 0x04 #define IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI 0x08 @@ -2858,7 +2865,7 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) #define IEEE80211_EHT_PHY_CAP8_RX_4096QAM_WIDER_BW_DL_OFDMA 0x02 /* - * EHT operation channel width as defined in P802.11be_D1.4 section 9.4.2.311 + * EHT operation channel width as defined in P802.11be_D2.0 section 9.4.2.311 */ #define IEEE80211_EHT_OPER_CHAN_WIDTH 0x7 #define IEEE80211_EHT_OPER_CHAN_WIDTH_20MHZ 0 -- cgit v1.2.3 From 93064e15c8a3a8394319a11b8037666e4b7d653d Mon Sep 17 00:00:00 2001 From: Stefan Binding Date: Thu, 7 Jul 2022 16:10:36 +0100 Subject: ACPI: utils: Add api to read _SUB from ACPI Add a wrapper function to read the _SUB string from ACPI. Signed-off-by: Stefan Binding Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20220707151037.3901050-2-sbinding@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/acpi/utils.c | 38 ++++++++++++++++++++++++++++++++++++++ include/linux/acpi.h | 6 ++++++ 2 files changed, 44 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c index 3a9773a09e19..5a7b8065e77f 100644 --- a/drivers/acpi/utils.c +++ b/drivers/acpi/utils.c @@ -291,6 +291,44 @@ int acpi_get_local_address(acpi_handle handle, u32 *addr) } EXPORT_SYMBOL(acpi_get_local_address); +#define ACPI_MAX_SUB_BUF_SIZE 9 + +const char *acpi_get_subsystem_id(acpi_handle handle) +{ + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *obj; + acpi_status status; + const char *sub; + size_t len; + + status = acpi_evaluate_object(handle, METHOD_NAME__SUB, NULL, &buffer); + if (ACPI_FAILURE(status)) { + acpi_handle_debug(handle, "Reading ACPI _SUB failed: %#x\n", status); + return ERR_PTR(-ENODATA); + } + + obj = buffer.pointer; + if (obj->type == ACPI_TYPE_STRING) { + len = strlen(obj->string.pointer); + if (len < ACPI_MAX_SUB_BUF_SIZE && len > 0) { + sub = kstrdup(obj->string.pointer, GFP_KERNEL); + if (!sub) + sub = ERR_PTR(-ENOMEM); + } else { + acpi_handle_err(handle, "ACPI _SUB Length %zu is Invalid\n", len); + sub = ERR_PTR(-ENODATA); + } + } else { + acpi_handle_warn(handle, "Warning ACPI _SUB did not return a string\n"); + sub = ERR_PTR(-ENODATA); + } + + acpi_os_free(buffer.pointer); + + return sub; +} +EXPORT_SYMBOL_GPL(acpi_get_subsystem_id); + acpi_status acpi_evaluate_reference(acpi_handle handle, acpi_string pathname, diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 4f82a5bc6d98..968a187f14f0 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -762,6 +762,7 @@ static inline u64 acpi_arch_get_root_pointer(void) #endif int acpi_get_local_address(acpi_handle handle, u32 *addr); +const char *acpi_get_subsystem_id(acpi_handle handle); #else /* !CONFIG_ACPI */ @@ -1023,6 +1024,11 @@ static inline int acpi_get_local_address(acpi_handle handle, u32 *addr) return -ENODEV; } +static inline const char *acpi_get_subsystem_id(acpi_handle handle) +{ + return ERR_PTR(-ENODEV); +} + static inline int acpi_register_wakeup_handler(int wake_irq, bool (*wakeup)(void *context), void *context) { -- cgit v1.2.3 From 28a6ed8e39f77f6ac613ec9b7461aa75e85fa79a Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 11 Jul 2022 07:22:57 +0000 Subject: platform/chrome: Add Type-C mux set command definitions Copy EC header definitions for the USB Type-C Mux control command from the EC code base. Also pull in "TBT_UFP_REPLY" definitions, since that is the prior entry in the enum. These headers are already present in the EC code base. [1] [1] https://chromium.googlesource.com/chromiumos/platform/ec/+/b80f85a94a423273c1638ef7b662c56931a138dd/include/ec_commands.h Signed-off-by: Prashant Malani Link: https://lore.kernel.org/r/20220711072333.2064341-4-pmalani@chromium.org Signed-off-by: Greg Kroah-Hartman --- include/linux/platform_data/cros_ec_commands.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux') diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 8cfa8cfca77e..a3945c5e7f50 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -5722,8 +5722,21 @@ enum typec_control_command { TYPEC_CONTROL_COMMAND_EXIT_MODES, TYPEC_CONTROL_COMMAND_CLEAR_EVENTS, TYPEC_CONTROL_COMMAND_ENTER_MODE, + TYPEC_CONTROL_COMMAND_TBT_UFP_REPLY, + TYPEC_CONTROL_COMMAND_USB_MUX_SET, }; +/* Replies the AP may specify to the TBT EnterMode command as a UFP */ +enum typec_tbt_ufp_reply { + TYPEC_TBT_UFP_REPLY_NAK, + TYPEC_TBT_UFP_REPLY_ACK, +}; + +struct typec_usb_mux_set { + uint8_t mux_index; /* Index of the mux to set in the chain */ + uint8_t mux_flags; /* USB_PD_MUX_*-encoded USB mux state to set */ +} __ec_align1; + struct ec_params_typec_control { uint8_t port; uint8_t command; /* enum typec_control_command */ @@ -5737,6 +5750,8 @@ struct ec_params_typec_control { union { uint32_t clear_events_mask; uint8_t mode_to_enter; /* enum typec_mode */ + uint8_t tbt_ufp_reply; /* enum typec_tbt_ufp_reply */ + struct typec_usb_mux_set mux_params; uint8_t placeholder[128]; }; } __ec_align1; @@ -5815,6 +5830,9 @@ enum tcpc_cc_polarity { #define PD_STATUS_EVENT_SOP_DISC_DONE BIT(0) #define PD_STATUS_EVENT_SOP_PRIME_DISC_DONE BIT(1) #define PD_STATUS_EVENT_HARD_RESET BIT(2) +#define PD_STATUS_EVENT_DISCONNECTED BIT(3) +#define PD_STATUS_EVENT_MUX_0_SET_DONE BIT(4) +#define PD_STATUS_EVENT_MUX_1_SET_DONE BIT(5) struct ec_params_typec_status { uint8_t port; -- cgit v1.2.3 From 4dea97f8636d0514befc9fc5cf342b351b7d0e20 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 1 Jul 2022 05:54:25 -0700 Subject: lib/bitmap: change type of bitmap_weight to unsigned long bitmap_weight() doesn't return negative values, so change it's type to unsigned long. It may help compiler to generate better code and catch bugs. Signed-off-by: Yury Norov --- drivers/dma/ti/k3-udma.c | 6 +++--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/i915/display/intel_display_power.c | 2 +- drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c | 2 +- drivers/iommu/intel/iommu.c | 2 +- drivers/net/ethernet/mellanox/mlx4/fw.c | 2 +- drivers/net/wireless/ath/ath9k/htc_drv_debug.c | 2 +- drivers/net/wireless/ath/carl9170/debug.c | 2 +- include/linux/bitmap.h | 5 +++-- lib/bitmap.c | 5 ++--- tools/include/linux/bitmap.h | 4 ++-- tools/lib/bitmap.c | 4 ++-- 12 files changed, 19 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 2f0d2c68c93c..07cb48db76ba 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -4997,7 +4997,7 @@ static int setup_resources(struct udma_dev *ud) switch (ud->match_data->type) { case DMA_TYPE_UDMA: dev_info(dev, - "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", + "Channels: %d (tchan: %lu, rchan: %lu, gp-rflow: %lu)\n", ch_count, ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), @@ -5008,7 +5008,7 @@ static int setup_resources(struct udma_dev *ud) break; case DMA_TYPE_BCDMA: dev_info(dev, - "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n", + "Channels: %d (bchan: %lu, tchan: %lu, rchan: %lu)\n", ch_count, ud->bchan_cnt - bitmap_weight(ud->bchan_map, ud->bchan_cnt), @@ -5019,7 +5019,7 @@ static int setup_resources(struct udma_dev *ud) break; case DMA_TYPE_PKTDMA: dev_info(dev, - "Channels: %d (tchan: %u, rchan: %u)\n", + "Channels: %d (tchan: %lu, rchan: %lu)\n", ch_count, ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index 16699158e00d..1098506ba1aa 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -195,7 +195,7 @@ void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) set_bit(i, adev->gfx.mec.queue_bitmap); } - dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); + dev_dbg(adev->dev, "mec queue bitmap weight=%ld\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); } void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 949edc983a16..4204e8a0e1a6 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -378,7 +378,7 @@ static void print_power_domains(struct i915_power_domains *power_domains, power_domains); enum intel_display_power_domain domain; - drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM)); + drm_dbg(&i915->drm, "%s (%ld):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM)); for_each_power_domain(domain, mask) drm_dbg(&i915->drm, "%s use_count %d\n", intel_display_power_domain_str(domain), diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c index 56a3063545ec..3d35b3982a60 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c @@ -363,7 +363,7 @@ void mdp5_smp_dump(struct mdp5_smp *smp, struct drm_printer *p) } drm_printf(p, "TOTAL:\t%d\t(of %d)\n", total, smp->blk_cnt); - drm_printf(p, "AVAIL:\t%d\n", smp->blk_cnt - + drm_printf(p, "AVAIL:\t%ld\n", smp->blk_cnt - bitmap_weight(state->state, smp->blk_cnt)); if (drm_can_sleep()) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 44016594831d..2f1b09f34706 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3946,7 +3946,7 @@ static ssize_t domains_used_show(struct device *dev, struct device_attribute *attr, char *buf) { struct intel_iommu *iommu = dev_to_intel_iommu(dev); - return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids, + return sprintf(buf, "%ld\n", bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap))); } static DEVICE_ATTR_RO(domains_used); diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c index 42c96c9d7fb1..af054a3808ca 100644 --- a/drivers/net/ethernet/mellanox/mlx4/fw.c +++ b/drivers/net/ethernet/mellanox/mlx4/fw.c @@ -463,7 +463,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, field = min( bitmap_weight(actv_ports.ports, dev->caps.num_ports), - dev->caps.num_ports); + (unsigned long)dev->caps.num_ports); MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); size = dev->caps.function_caps; /* set PF behaviours */ diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_debug.c b/drivers/net/wireless/ath/ath9k/htc_drv_debug.c index b3ed65e5c4da..d32056c4ee1d 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_debug.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_debug.c @@ -296,7 +296,7 @@ static ssize_t read_file_slot(struct file *file, char __user *user_buf, spin_lock_bh(&priv->tx.tx_lock); len = scnprintf(buf, sizeof(buf), "TX slot bitmap : %*pb\n" - "Used slots : %d\n", + "Used slots : %ld\n", MAX_TX_BUF_NUM, priv->tx.tx_slot, bitmap_weight(priv->tx.tx_slot, MAX_TX_BUF_NUM)); spin_unlock_bh(&priv->tx.tx_lock); diff --git a/drivers/net/wireless/ath/carl9170/debug.c b/drivers/net/wireless/ath/carl9170/debug.c index bb40889d7c72..7e88882b87d9 100644 --- a/drivers/net/wireless/ath/carl9170/debug.c +++ b/drivers/net/wireless/ath/carl9170/debug.c @@ -221,7 +221,7 @@ static char *carl9170_debugfs_mem_usage_read(struct ar9170 *ar, char *buf, ADD(buf, *len, bufsize, "jar: [%*pb]\n", ar->fw.mem_blocks, ar->mem_bitmap); - ADD(buf, *len, bufsize, "cookies: used:%3d / total:%3d, allocs:%d\n", + ADD(buf, *len, bufsize, "cookies: used:%3ld / total:%3d, allocs:%d\n", bitmap_weight(ar->mem_bitmap, ar->fw.mem_blocks), ar->fw.mem_blocks, atomic_read(&ar->mem_allocs)); diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index e1a438bdda52..035d4ac66641 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -163,7 +163,7 @@ bool __bitmap_intersects(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); -int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); void __bitmap_set(unsigned long *map, unsigned int start, int len); void __bitmap_clear(unsigned long *map, unsigned int start, int len); @@ -431,7 +431,8 @@ static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) return find_first_zero_bit(src, nbits) == nbits; } -static __always_inline int bitmap_weight(const unsigned long *src, unsigned int nbits) +static __always_inline +unsigned long bitmap_weight(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); diff --git a/lib/bitmap.c b/lib/bitmap.c index 9bc80f5bf149..2b67cd657692 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -333,10 +333,9 @@ bool __bitmap_subset(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_subset); -int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int bits) { - unsigned int k, lim = bits/BITS_PER_LONG; - int w = 0; + unsigned long k, w = 0, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; k++) w += hweight_long(bitmap[k]); diff --git a/tools/include/linux/bitmap.h b/tools/include/linux/bitmap.h index 2ae7ab8ed7d1..ae1852e39142 100644 --- a/tools/include/linux/bitmap.h +++ b/tools/include/linux/bitmap.h @@ -11,7 +11,7 @@ #define DECLARE_BITMAP(name,bits) \ unsigned long name[BITS_TO_LONGS(bits)] -int __bitmap_weight(const unsigned long *bitmap, int bits); +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int bits); void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, @@ -61,7 +61,7 @@ static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) return find_first_zero_bit(src, nbits) == nbits; } -static inline int bitmap_weight(const unsigned long *src, unsigned int nbits) +static inline unsigned long bitmap_weight(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); diff --git a/tools/lib/bitmap.c b/tools/lib/bitmap.c index 2e351d63fdba..e1fafc131a49 100644 --- a/tools/lib/bitmap.c +++ b/tools/lib/bitmap.c @@ -5,9 +5,9 @@ */ #include -int __bitmap_weight(const unsigned long *bitmap, int bits) +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int bits) { - int k, w = 0, lim = bits/BITS_PER_LONG; + unsigned long k, w = 0, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; k++) w += hweight_long(bitmap[k]); -- cgit v1.2.3 From cb32c285cc10e428589194e30233d673e7c23c72 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 1 Jul 2022 05:54:26 -0700 Subject: cpumask: change return types to bool where appropriate Some cpumask functions have integer return types where return values are naturally booleans. Signed-off-by: Yury Norov --- include/linux/cpumask.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index fe29ac7cc469..b54e27d9da6b 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -372,9 +372,9 @@ static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns 1 if @cpu is set in @cpumask, else returns 0 + * Returns true if @cpu is set in @cpumask, else returns false */ -static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) +static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) { return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); } @@ -384,11 +384,11 @@ static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpuma * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0 + * Returns true if @cpu is set in old bitmap of @cpumask, else returns false * * test_and_set_bit wrapper for cpumasks. */ -static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) +static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) { return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -398,11 +398,11 @@ static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpu * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0 + * Returns true if @cpu is set in old bitmap of @cpumask, else returns false * * test_and_clear_bit wrapper for cpumasks. */ -static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) +static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) { return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -431,9 +431,9 @@ static inline void cpumask_clear(struct cpumask *dstp) * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns 0, else returns 1 + * If *@dstp is empty, returns false, else returns true */ -static inline int cpumask_and(struct cpumask *dstp, +static inline bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { @@ -474,9 +474,9 @@ static inline void cpumask_xor(struct cpumask *dstp, * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns 0, else returns 1 + * If *@dstp is empty, returns false, else returns true */ -static inline int cpumask_andnot(struct cpumask *dstp, +static inline bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { @@ -539,9 +539,9 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input * - * Returns 1 if *@src1p is a subset of *@src2p, else returns 0 + * Returns true if *@src1p is a subset of *@src2p, else returns false */ -static inline int cpumask_subset(const struct cpumask *src1p, +static inline bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), -- cgit v1.2.3 From 8b6b795d9bfc031a8953c40fac8d3cf67e1a4d3d Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 1 Jul 2022 05:54:27 -0700 Subject: lib/cpumask: change return types to unsigned where appropriate Switch return types to unsigned int where return values cannot be negative. Signed-off-by: Yury Norov --- include/linux/cpumask.h | 14 +++++++------- lib/cpumask.c | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index b54e27d9da6b..760022bcb925 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -176,12 +176,12 @@ static inline unsigned int cpumask_local_spread(unsigned int i, int node) return 0; } -static inline int cpumask_any_and_distribute(const struct cpumask *src1p, +static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) { return cpumask_first_and(src1p, src2p); } -static inline int cpumask_any_distribute(const struct cpumask *srcp) +static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp) { return cpumask_first(srcp); } @@ -258,12 +258,12 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); } -int __pure cpumask_next_and(int n, const struct cpumask *, const struct cpumask *); -int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu); +unsigned int __pure cpumask_next_and(int n, const struct cpumask *, const struct cpumask *); +unsigned int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu); unsigned int cpumask_local_spread(unsigned int i, int node); -int cpumask_any_and_distribute(const struct cpumask *src1p, +unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p); -int cpumask_any_distribute(const struct cpumask *srcp); +unsigned int cpumask_any_distribute(const struct cpumask *srcp); /** * for_each_cpu - iterate over every cpu in a mask @@ -289,7 +289,7 @@ int cpumask_any_distribute(const struct cpumask *srcp); (cpu) = cpumask_next_zero((cpu), (mask)), \ (cpu) < nr_cpu_ids;) -extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); +unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); /** * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location diff --git a/lib/cpumask.c b/lib/cpumask.c index a971a82d2f43..da68f6bbde44 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -31,7 +31,7 @@ EXPORT_SYMBOL(cpumask_next); * * Returns >= nr_cpu_ids if no further cpus set in both. */ -int cpumask_next_and(int n, const struct cpumask *src1p, +unsigned int cpumask_next_and(int n, const struct cpumask *src1p, const struct cpumask *src2p) { /* -1 is a legal arg here. */ @@ -50,7 +50,7 @@ EXPORT_SYMBOL(cpumask_next_and); * Often used to find any cpu but smp_processor_id() in a mask. * Returns >= nr_cpu_ids if no cpus set. */ -int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) +unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) { unsigned int i; @@ -74,9 +74,9 @@ EXPORT_SYMBOL(cpumask_any_but); * Note: the @wrap argument is required for the start condition when * we cannot assume @start is set in @mask. */ -int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) +unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) { - int next; + unsigned int next; again: next = cpumask_next(n, mask); @@ -205,7 +205,7 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask) */ unsigned int cpumask_local_spread(unsigned int i, int node) { - int cpu; + unsigned int cpu; /* Wrap: we always want a cpu. */ i %= num_online_cpus(); @@ -243,10 +243,10 @@ static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); * * Returns >= nr_cpu_ids if the intersection is empty. */ -int cpumask_any_and_distribute(const struct cpumask *src1p, +unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) { - int next, prev; + unsigned int next, prev; /* NOTE: our first selection will skip 0. */ prev = __this_cpu_read(distribute_cpu_mask_prev); @@ -262,9 +262,9 @@ int cpumask_any_and_distribute(const struct cpumask *src1p, } EXPORT_SYMBOL(cpumask_any_and_distribute); -int cpumask_any_distribute(const struct cpumask *srcp) +unsigned int cpumask_any_distribute(const struct cpumask *srcp) { - int next, prev; + unsigned int next, prev; /* NOTE: our first selection will skip 0. */ prev = __this_cpu_read(distribute_cpu_mask_prev); -- cgit v1.2.3 From 9b2e70860ef2f0d74b6d9e57929d57b14481b9c9 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 1 Jul 2022 05:54:28 -0700 Subject: lib/cpumask: move trivial wrappers around find_bit to the header To avoid circular dependencies, cpumask keeps simple (almost) one-line wrappers around find_bit() in a c-file. Commit 47d8c15615c0a2 ("include: move find.h from asm_generic to linux") moved find.h header out of asm_generic include path, and it helped to fix many circular dependencies, including some in cpumask.h. This patch moves those one-liners to header files. Signed-off-by: Yury Norov --- include/linux/cpumask.h | 57 ++++++++++++++++++++++++++++++++++++++++++++++--- lib/cpumask.c | 55 ----------------------------------------------- 2 files changed, 54 insertions(+), 58 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 760022bcb925..ea3de2c2c180 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -241,7 +241,21 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits); } -unsigned int __pure cpumask_next(int n, const struct cpumask *srcp); +/** + * cpumask_next - get the next cpu in a cpumask + * @n: the cpu prior to the place to search (ie. return will be > @n) + * @srcp: the cpumask pointer + * + * Returns >= nr_cpu_ids if no further cpus set. + */ +static inline +unsigned int cpumask_next(int n, const struct cpumask *srcp) +{ + /* -1 is a legal arg here. */ + if (n != -1) + cpumask_check(n); + return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); +} /** * cpumask_next_zero - get the next unset cpu in a cpumask @@ -258,8 +272,25 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); } -unsigned int __pure cpumask_next_and(int n, const struct cpumask *, const struct cpumask *); -unsigned int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu); +/** + * cpumask_next_and - get the next cpu in *src1p & *src2p + * @n: the cpu prior to the place to search (ie. return will be > @n) + * @src1p: the first cpumask pointer + * @src2p: the second cpumask pointer + * + * Returns >= nr_cpu_ids if no further cpus set in both. + */ +static inline +unsigned int cpumask_next_and(int n, const struct cpumask *src1p, + const struct cpumask *src2p) +{ + /* -1 is a legal arg here. */ + if (n != -1) + cpumask_check(n); + return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), + nr_cpumask_bits, n + 1); +} + unsigned int cpumask_local_spread(unsigned int i, int node); unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p); @@ -324,6 +355,26 @@ unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, boo for ((cpu) = -1; \ (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ (cpu) < nr_cpu_ids;) + +/** + * cpumask_any_but - return a "random" in a cpumask, but not this one. + * @mask: the cpumask to search + * @cpu: the cpu to ignore. + * + * Often used to find any cpu but smp_processor_id() in a mask. + * Returns >= nr_cpu_ids if no cpus set. + */ +static inline +unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) +{ + unsigned int i; + + cpumask_check(cpu); + for_each_cpu(i, mask) + if (i != cpu) + break; + return i; +} #endif /* SMP */ #define CPU_BITS_NONE \ diff --git a/lib/cpumask.c b/lib/cpumask.c index da68f6bbde44..cb7262ff8633 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -7,61 +7,6 @@ #include #include -/** - * cpumask_next - get the next cpu in a cpumask - * @n: the cpu prior to the place to search (ie. return will be > @n) - * @srcp: the cpumask pointer - * - * Returns >= nr_cpu_ids if no further cpus set. - */ -unsigned int cpumask_next(int n, const struct cpumask *srcp) -{ - /* -1 is a legal arg here. */ - if (n != -1) - cpumask_check(n); - return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); -} -EXPORT_SYMBOL(cpumask_next); - -/** - * cpumask_next_and - get the next cpu in *src1p & *src2p - * @n: the cpu prior to the place to search (ie. return will be > @n) - * @src1p: the first cpumask pointer - * @src2p: the second cpumask pointer - * - * Returns >= nr_cpu_ids if no further cpus set in both. - */ -unsigned int cpumask_next_and(int n, const struct cpumask *src1p, - const struct cpumask *src2p) -{ - /* -1 is a legal arg here. */ - if (n != -1) - cpumask_check(n); - return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), - nr_cpumask_bits, n + 1); -} -EXPORT_SYMBOL(cpumask_next_and); - -/** - * cpumask_any_but - return a "random" in a cpumask, but not this one. - * @mask: the cpumask to search - * @cpu: the cpu to ignore. - * - * Often used to find any cpu but smp_processor_id() in a mask. - * Returns >= nr_cpu_ids if no cpus set. - */ -unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) -{ - unsigned int i; - - cpumask_check(cpu); - for_each_cpu(i, mask) - if (i != cpu) - break; - return i; -} -EXPORT_SYMBOL(cpumask_any_but); - /** * cpumask_next_wrap - helper to implement for_each_cpu_wrap * @n: the cpu prior to the place to search -- cgit v1.2.3 From db96b0c5f9db22d908ab5f7cd75904adba4b28ca Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 10 Jun 2021 10:56:49 +0200 Subject: headers/deps: mm: Optimize header dependencies There's a couple of superfluous inclusions here - remove them before doing bigger changes. Signed-off-by: Ingo Molnar Signed-off-by: Yury Norov --- include/linux/gfp.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 2d2ccae933c2..52f2c873a7d4 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -2,10 +2,7 @@ #ifndef __LINUX_GFP_H #define __LINUX_GFP_H -#include #include -#include -#include #include /* The typedef is in types.h but we want the documentation here */ -- cgit v1.2.3 From cb5a065b4ea9c062a18143c8a14e831179687f54 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 14 Apr 2022 18:42:28 +0200 Subject: headers/deps: mm: Split out of This is a much smaller header. Signed-off-by: Ingo Molnar Signed-off-by: Yury Norov --- include/linux/gfp.h | 345 +-------------------------------------------- include/linux/gfp_types.h | 348 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 350 insertions(+), 343 deletions(-) create mode 100644 include/linux/gfp_types.h (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 52f2c873a7d4..f314be58fa77 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -2,354 +2,13 @@ #ifndef __LINUX_GFP_H #define __LINUX_GFP_H +#include + #include #include -/* The typedef is in types.h but we want the documentation here */ -#if 0 -/** - * typedef gfp_t - Memory allocation flags. - * - * GFP flags are commonly used throughout Linux to indicate how memory - * should be allocated. The GFP acronym stands for get_free_pages(), - * the underlying memory allocation function. Not every GFP flag is - * supported by every function which may allocate memory. Most users - * will want to use a plain ``GFP_KERNEL``. - */ -typedef unsigned int __bitwise gfp_t; -#endif - struct vm_area_struct; -/* - * In case of changes, please don't forget to update - * include/trace/events/mmflags.h and tools/perf/builtin-kmem.c - */ - -/* Plain integer GFP bitmasks. Do not use this directly. */ -#define ___GFP_DMA 0x01u -#define ___GFP_HIGHMEM 0x02u -#define ___GFP_DMA32 0x04u -#define ___GFP_MOVABLE 0x08u -#define ___GFP_RECLAIMABLE 0x10u -#define ___GFP_HIGH 0x20u -#define ___GFP_IO 0x40u -#define ___GFP_FS 0x80u -#define ___GFP_ZERO 0x100u -#define ___GFP_ATOMIC 0x200u -#define ___GFP_DIRECT_RECLAIM 0x400u -#define ___GFP_KSWAPD_RECLAIM 0x800u -#define ___GFP_WRITE 0x1000u -#define ___GFP_NOWARN 0x2000u -#define ___GFP_RETRY_MAYFAIL 0x4000u -#define ___GFP_NOFAIL 0x8000u -#define ___GFP_NORETRY 0x10000u -#define ___GFP_MEMALLOC 0x20000u -#define ___GFP_COMP 0x40000u -#define ___GFP_NOMEMALLOC 0x80000u -#define ___GFP_HARDWALL 0x100000u -#define ___GFP_THISNODE 0x200000u -#define ___GFP_ACCOUNT 0x400000u -#define ___GFP_ZEROTAGS 0x800000u -#ifdef CONFIG_KASAN_HW_TAGS -#define ___GFP_SKIP_ZERO 0x1000000u -#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u -#define ___GFP_SKIP_KASAN_POISON 0x4000000u -#else -#define ___GFP_SKIP_ZERO 0 -#define ___GFP_SKIP_KASAN_UNPOISON 0 -#define ___GFP_SKIP_KASAN_POISON 0 -#endif -#ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x8000000u -#else -#define ___GFP_NOLOCKDEP 0 -#endif -/* If the above are modified, __GFP_BITS_SHIFT may need updating */ - -/* - * Physical address zone modifiers (see linux/mmzone.h - low four bits) - * - * Do not put any conditional on these. If necessary modify the definitions - * without the underscores and use them consistently. The definitions here may - * be used in bit comparisons. - */ -#define __GFP_DMA ((__force gfp_t)___GFP_DMA) -#define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) -#define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) -#define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ -#define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) - -/** - * DOC: Page mobility and placement hints - * - * Page mobility and placement hints - * --------------------------------- - * - * These flags provide hints about how mobile the page is. Pages with similar - * mobility are placed within the same pageblocks to minimise problems due - * to external fragmentation. - * - * %__GFP_MOVABLE (also a zone modifier) indicates that the page can be - * moved by page migration during memory compaction or can be reclaimed. - * - * %__GFP_RECLAIMABLE is used for slab allocations that specify - * SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers. - * - * %__GFP_WRITE indicates the caller intends to dirty the page. Where possible, - * these pages will be spread between local zones to avoid all the dirty - * pages being in one zone (fair zone allocation policy). - * - * %__GFP_HARDWALL enforces the cpuset memory allocation policy. - * - * %__GFP_THISNODE forces the allocation to be satisfied from the requested - * node with no fallbacks or placement policy enforcements. - * - * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - */ -#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) -#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) -#define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) -#define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) -#define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) - -/** - * DOC: Watermark modifiers - * - * Watermark modifiers -- controls access to emergency reserves - * ------------------------------------------------------------ - * - * %__GFP_HIGH indicates that the caller is high-priority and that granting - * the request is necessary before the system can make forward progress. - * For example, creating an IO context to clean pages. - * - * %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is - * high priority. Users are typically interrupt handlers. This may be - * used in conjunction with %__GFP_HIGH - * - * %__GFP_MEMALLOC allows access to all memory. This should only be used when - * the caller guarantees the allocation will allow more memory to be freed - * very shortly e.g. process exiting or swapping. Users either should - * be the MM or co-ordinating closely with the VM (e.g. swap over NFS). - * Users of this flag have to be extremely careful to not deplete the reserve - * completely and implement a throttling mechanism which controls the - * consumption of the reserve based on the amount of freed memory. - * Usage of a pre-allocated pool (e.g. mempool) should be always considered - * before using this flag. - * - * %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves. - * This takes precedence over the %__GFP_MEMALLOC flag if both are set. - */ -#define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC) -#define __GFP_HIGH ((__force gfp_t)___GFP_HIGH) -#define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC) -#define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC) - -/** - * DOC: Reclaim modifiers - * - * Reclaim modifiers - * ----------------- - * Please note that all the following flags are only applicable to sleepable - * allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them). - * - * %__GFP_IO can start physical IO. - * - * %__GFP_FS can call down to the low-level FS. Clearing the flag avoids the - * allocator recursing into the filesystem which might already be holding - * locks. - * - * %__GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim. - * This flag can be cleared to avoid unnecessary delays when a fallback - * option is available. - * - * %__GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when - * the low watermark is reached and have it reclaim pages until the high - * watermark is reached. A caller may wish to clear this flag when fallback - * options are available and the reclaim is likely to disrupt the system. The - * canonical example is THP allocation where a fallback is cheap but - * reclaim/compaction may cause indirect stalls. - * - * %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim. - * - * The default allocator behavior depends on the request size. We have a concept - * of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER). - * !costly allocations are too essential to fail so they are implicitly - * non-failing by default (with some exceptions like OOM victims might fail so - * the caller still has to check for failures) while costly requests try to be - * not disruptive and back off even without invoking the OOM killer. - * The following three modifiers might be used to override some of these - * implicit rules - * - * %__GFP_NORETRY: The VM implementation will try only very lightweight - * memory direct reclaim to get some memory under memory pressure (thus - * it can sleep). It will avoid disruptive actions like OOM killer. The - * caller must handle the failure which is quite likely to happen under - * heavy memory pressure. The flag is suitable when failure can easily be - * handled at small cost, such as reduced throughput - * - * %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim - * procedures that have previously failed if there is some indication - * that progress has been made else where. It can wait for other - * tasks to attempt high level approaches to freeing memory such as - * compaction (which removes fragmentation) and page-out. - * There is still a definite limit to the number of retries, but it is - * a larger limit than with %__GFP_NORETRY. - * Allocations with this flag may fail, but only when there is - * genuinely little unused memory. While these allocations do not - * directly trigger the OOM killer, their failure indicates that - * the system is likely to need to use the OOM killer soon. The - * caller must handle failure, but can reasonably do so by failing - * a higher-level request, or completing it only in a much less - * efficient manner. - * If the allocation does fail, and the caller is in a position to - * free some non-essential memory, doing so could benefit the system - * as a whole. - * - * %__GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller - * cannot handle allocation failures. The allocation could block - * indefinitely but will never return with failure. Testing for - * failure is pointless. - * New users should be evaluated carefully (and the flag should be - * used only when there is no reasonable failure policy) but it is - * definitely preferable to use the flag rather than opencode endless - * loop around allocator. - * Using this flag for costly allocations is _highly_ discouraged. - */ -#define __GFP_IO ((__force gfp_t)___GFP_IO) -#define __GFP_FS ((__force gfp_t)___GFP_FS) -#define __GFP_DIRECT_RECLAIM ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */ -#define __GFP_KSWAPD_RECLAIM ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */ -#define __GFP_RECLAIM ((__force gfp_t)(___GFP_DIRECT_RECLAIM|___GFP_KSWAPD_RECLAIM)) -#define __GFP_RETRY_MAYFAIL ((__force gfp_t)___GFP_RETRY_MAYFAIL) -#define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL) -#define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY) - -/** - * DOC: Action modifiers - * - * Action modifiers - * ---------------- - * - * %__GFP_NOWARN suppresses allocation failure reports. - * - * %__GFP_COMP address compound page metadata. - * - * %__GFP_ZERO returns a zeroed page on success. - * - * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself - * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that - * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting - * memory tags at the same time as zeroing memory has minimal additional - * performace impact. - * - * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation. - * Only effective in HW_TAGS mode. - * - * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation. - * Typically, used for userspace pages. Only effective in HW_TAGS mode. - */ -#define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) -#define __GFP_COMP ((__force gfp_t)___GFP_COMP) -#define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) -#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) -#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) -#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON) -#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) - -/* Disable lockdep for GFP context tracking */ -#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) - -/* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) -#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) - -/** - * DOC: Useful GFP flag combinations - * - * Useful GFP flag combinations - * ---------------------------- - * - * Useful GFP flag combinations that are commonly used. It is recommended - * that subsystems start with one of these combinations and then set/clear - * %__GFP_FOO flags as necessary. - * - * %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower - * watermark is applied to allow access to "atomic reserves". - * The current implementation doesn't support NMI and few other strict - * non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT. - * - * %GFP_KERNEL is typical for kernel-internal allocations. The caller requires - * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. - * - * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is - * accounted to kmemcg. - * - * %GFP_NOWAIT is for kernel allocations that should not stall for direct - * reclaim, start physical IO or use any filesystem callback. - * - * %GFP_NOIO will use direct reclaim to discard clean pages or slab pages - * that do not require the starting of any physical IO. - * Please try to avoid using this flag directly and instead use - * memalloc_noio_{save,restore} to mark the whole scope which cannot - * perform any IO with a short explanation why. All allocation requests - * will inherit GFP_NOIO implicitly. - * - * %GFP_NOFS will use direct reclaim but will not use any filesystem interfaces. - * Please try to avoid using this flag directly and instead use - * memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't - * recurse into the FS layer with a short explanation why. All allocation - * requests will inherit GFP_NOFS implicitly. - * - * %GFP_USER is for userspace allocations that also need to be directly - * accessibly by the kernel or hardware. It is typically used by hardware - * for buffers that are mapped to userspace (e.g. graphics) that hardware - * still must DMA to. cpuset limits are enforced for these allocations. - * - * %GFP_DMA exists for historical reasons and should be avoided where possible. - * The flags indicates that the caller requires that the lowest zone be - * used (%ZONE_DMA or 16M on x86-64). Ideally, this would be removed but - * it would require careful auditing as some users really require it and - * others use the flag to avoid lowmem reserves in %ZONE_DMA and treat the - * lowest zone as a type of emergency reserve. - * - * %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit - * address. Note that kmalloc(..., GFP_DMA32) does not return DMA32 memory - * because the DMA32 kmalloc cache array is not implemented. - * (Reason: there is no such user in kernel). - * - * %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, - * do not need to be directly accessible by the kernel but that cannot - * move once in use. An example may be a hardware allocation that maps - * data directly into userspace but has no addressing limitations. - * - * %GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not - * need direct access to but can use kmap() when access is required. They - * are expected to be movable via page reclaim or page migration. Typically, - * pages on the LRU would also be allocated with %GFP_HIGHUSER_MOVABLE. - * - * %GFP_TRANSHUGE and %GFP_TRANSHUGE_LIGHT are used for THP allocations. They - * are compound allocations that will generally fail quickly if memory is not - * available and will not wake kswapd/kcompactd on failure. The _LIGHT - * version does not attempt reclaim/compaction at all and is by default used - * in page fault path, while the non-light is used by khugepaged. - */ -#define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) -#define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) -#define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT) -#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) -#define GFP_NOIO (__GFP_RECLAIM) -#define GFP_NOFS (__GFP_RECLAIM | __GFP_IO) -#define GFP_USER (__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL) -#define GFP_DMA __GFP_DMA -#define GFP_DMA32 __GFP_DMA32 -#define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) -#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \ - __GFP_SKIP_KASAN_POISON) -#define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ - __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) -#define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) - /* Convert GFP flags to their corresponding migrate type */ #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) #define GFP_MOVABLE_SHIFT 3 diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h new file mode 100644 index 000000000000..06fc85cee23f --- /dev/null +++ b/include/linux/gfp_types.h @@ -0,0 +1,348 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_GFP_TYPES_H +#define __LINUX_GFP_TYPES_H + +/* The typedef is in types.h but we want the documentation here */ +#if 0 +/** + * typedef gfp_t - Memory allocation flags. + * + * GFP flags are commonly used throughout Linux to indicate how memory + * should be allocated. The GFP acronym stands for get_free_pages(), + * the underlying memory allocation function. Not every GFP flag is + * supported by every function which may allocate memory. Most users + * will want to use a plain ``GFP_KERNEL``. + */ +typedef unsigned int __bitwise gfp_t; +#endif + +/* + * In case of changes, please don't forget to update + * include/trace/events/mmflags.h and tools/perf/builtin-kmem.c + */ + +/* Plain integer GFP bitmasks. Do not use this directly. */ +#define ___GFP_DMA 0x01u +#define ___GFP_HIGHMEM 0x02u +#define ___GFP_DMA32 0x04u +#define ___GFP_MOVABLE 0x08u +#define ___GFP_RECLAIMABLE 0x10u +#define ___GFP_HIGH 0x20u +#define ___GFP_IO 0x40u +#define ___GFP_FS 0x80u +#define ___GFP_ZERO 0x100u +#define ___GFP_ATOMIC 0x200u +#define ___GFP_DIRECT_RECLAIM 0x400u +#define ___GFP_KSWAPD_RECLAIM 0x800u +#define ___GFP_WRITE 0x1000u +#define ___GFP_NOWARN 0x2000u +#define ___GFP_RETRY_MAYFAIL 0x4000u +#define ___GFP_NOFAIL 0x8000u +#define ___GFP_NORETRY 0x10000u +#define ___GFP_MEMALLOC 0x20000u +#define ___GFP_COMP 0x40000u +#define ___GFP_NOMEMALLOC 0x80000u +#define ___GFP_HARDWALL 0x100000u +#define ___GFP_THISNODE 0x200000u +#define ___GFP_ACCOUNT 0x400000u +#define ___GFP_ZEROTAGS 0x800000u +#ifdef CONFIG_KASAN_HW_TAGS +#define ___GFP_SKIP_ZERO 0x1000000u +#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u +#define ___GFP_SKIP_KASAN_POISON 0x4000000u +#else +#define ___GFP_SKIP_ZERO 0 +#define ___GFP_SKIP_KASAN_UNPOISON 0 +#define ___GFP_SKIP_KASAN_POISON 0 +#endif +#ifdef CONFIG_LOCKDEP +#define ___GFP_NOLOCKDEP 0x8000000u +#else +#define ___GFP_NOLOCKDEP 0 +#endif +/* If the above are modified, __GFP_BITS_SHIFT may need updating */ + +/* + * Physical address zone modifiers (see linux/mmzone.h - low four bits) + * + * Do not put any conditional on these. If necessary modify the definitions + * without the underscores and use them consistently. The definitions here may + * be used in bit comparisons. + */ +#define __GFP_DMA ((__force gfp_t)___GFP_DMA) +#define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) +#define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) +#define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ +#define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) + +/** + * DOC: Page mobility and placement hints + * + * Page mobility and placement hints + * --------------------------------- + * + * These flags provide hints about how mobile the page is. Pages with similar + * mobility are placed within the same pageblocks to minimise problems due + * to external fragmentation. + * + * %__GFP_MOVABLE (also a zone modifier) indicates that the page can be + * moved by page migration during memory compaction or can be reclaimed. + * + * %__GFP_RECLAIMABLE is used for slab allocations that specify + * SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers. + * + * %__GFP_WRITE indicates the caller intends to dirty the page. Where possible, + * these pages will be spread between local zones to avoid all the dirty + * pages being in one zone (fair zone allocation policy). + * + * %__GFP_HARDWALL enforces the cpuset memory allocation policy. + * + * %__GFP_THISNODE forces the allocation to be satisfied from the requested + * node with no fallbacks or placement policy enforcements. + * + * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. + */ +#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) +#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) +#define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) +#define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) +#define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) + +/** + * DOC: Watermark modifiers + * + * Watermark modifiers -- controls access to emergency reserves + * ------------------------------------------------------------ + * + * %__GFP_HIGH indicates that the caller is high-priority and that granting + * the request is necessary before the system can make forward progress. + * For example, creating an IO context to clean pages. + * + * %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is + * high priority. Users are typically interrupt handlers. This may be + * used in conjunction with %__GFP_HIGH + * + * %__GFP_MEMALLOC allows access to all memory. This should only be used when + * the caller guarantees the allocation will allow more memory to be freed + * very shortly e.g. process exiting or swapping. Users either should + * be the MM or co-ordinating closely with the VM (e.g. swap over NFS). + * Users of this flag have to be extremely careful to not deplete the reserve + * completely and implement a throttling mechanism which controls the + * consumption of the reserve based on the amount of freed memory. + * Usage of a pre-allocated pool (e.g. mempool) should be always considered + * before using this flag. + * + * %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves. + * This takes precedence over the %__GFP_MEMALLOC flag if both are set. + */ +#define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC) +#define __GFP_HIGH ((__force gfp_t)___GFP_HIGH) +#define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC) +#define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC) + +/** + * DOC: Reclaim modifiers + * + * Reclaim modifiers + * ----------------- + * Please note that all the following flags are only applicable to sleepable + * allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them). + * + * %__GFP_IO can start physical IO. + * + * %__GFP_FS can call down to the low-level FS. Clearing the flag avoids the + * allocator recursing into the filesystem which might already be holding + * locks. + * + * %__GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim. + * This flag can be cleared to avoid unnecessary delays when a fallback + * option is available. + * + * %__GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when + * the low watermark is reached and have it reclaim pages until the high + * watermark is reached. A caller may wish to clear this flag when fallback + * options are available and the reclaim is likely to disrupt the system. The + * canonical example is THP allocation where a fallback is cheap but + * reclaim/compaction may cause indirect stalls. + * + * %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim. + * + * The default allocator behavior depends on the request size. We have a concept + * of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER). + * !costly allocations are too essential to fail so they are implicitly + * non-failing by default (with some exceptions like OOM victims might fail so + * the caller still has to check for failures) while costly requests try to be + * not disruptive and back off even without invoking the OOM killer. + * The following three modifiers might be used to override some of these + * implicit rules + * + * %__GFP_NORETRY: The VM implementation will try only very lightweight + * memory direct reclaim to get some memory under memory pressure (thus + * it can sleep). It will avoid disruptive actions like OOM killer. The + * caller must handle the failure which is quite likely to happen under + * heavy memory pressure. The flag is suitable when failure can easily be + * handled at small cost, such as reduced throughput + * + * %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim + * procedures that have previously failed if there is some indication + * that progress has been made else where. It can wait for other + * tasks to attempt high level approaches to freeing memory such as + * compaction (which removes fragmentation) and page-out. + * There is still a definite limit to the number of retries, but it is + * a larger limit than with %__GFP_NORETRY. + * Allocations with this flag may fail, but only when there is + * genuinely little unused memory. While these allocations do not + * directly trigger the OOM killer, their failure indicates that + * the system is likely to need to use the OOM killer soon. The + * caller must handle failure, but can reasonably do so by failing + * a higher-level request, or completing it only in a much less + * efficient manner. + * If the allocation does fail, and the caller is in a position to + * free some non-essential memory, doing so could benefit the system + * as a whole. + * + * %__GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller + * cannot handle allocation failures. The allocation could block + * indefinitely but will never return with failure. Testing for + * failure is pointless. + * New users should be evaluated carefully (and the flag should be + * used only when there is no reasonable failure policy) but it is + * definitely preferable to use the flag rather than opencode endless + * loop around allocator. + * Using this flag for costly allocations is _highly_ discouraged. + */ +#define __GFP_IO ((__force gfp_t)___GFP_IO) +#define __GFP_FS ((__force gfp_t)___GFP_FS) +#define __GFP_DIRECT_RECLAIM ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */ +#define __GFP_KSWAPD_RECLAIM ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */ +#define __GFP_RECLAIM ((__force gfp_t)(___GFP_DIRECT_RECLAIM|___GFP_KSWAPD_RECLAIM)) +#define __GFP_RETRY_MAYFAIL ((__force gfp_t)___GFP_RETRY_MAYFAIL) +#define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL) +#define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY) + +/** + * DOC: Action modifiers + * + * Action modifiers + * ---------------- + * + * %__GFP_NOWARN suppresses allocation failure reports. + * + * %__GFP_COMP address compound page metadata. + * + * %__GFP_ZERO returns a zeroed page on success. + * + * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself + * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that + * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting + * memory tags at the same time as zeroing memory has minimal additional + * performace impact. + * + * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation. + * Only effective in HW_TAGS mode. + * + * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation. + * Typically, used for userspace pages. Only effective in HW_TAGS mode. + */ +#define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) +#define __GFP_COMP ((__force gfp_t)___GFP_COMP) +#define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) +#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) +#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) +#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON) +#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) + +/* Disable lockdep for GFP context tracking */ +#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) + +/* Room for N __GFP_FOO bits */ +#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) + +/** + * DOC: Useful GFP flag combinations + * + * Useful GFP flag combinations + * ---------------------------- + * + * Useful GFP flag combinations that are commonly used. It is recommended + * that subsystems start with one of these combinations and then set/clear + * %__GFP_FOO flags as necessary. + * + * %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower + * watermark is applied to allow access to "atomic reserves". + * The current implementation doesn't support NMI and few other strict + * non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT. + * + * %GFP_KERNEL is typical for kernel-internal allocations. The caller requires + * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. + * + * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is + * accounted to kmemcg. + * + * %GFP_NOWAIT is for kernel allocations that should not stall for direct + * reclaim, start physical IO or use any filesystem callback. + * + * %GFP_NOIO will use direct reclaim to discard clean pages or slab pages + * that do not require the starting of any physical IO. + * Please try to avoid using this flag directly and instead use + * memalloc_noio_{save,restore} to mark the whole scope which cannot + * perform any IO with a short explanation why. All allocation requests + * will inherit GFP_NOIO implicitly. + * + * %GFP_NOFS will use direct reclaim but will not use any filesystem interfaces. + * Please try to avoid using this flag directly and instead use + * memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't + * recurse into the FS layer with a short explanation why. All allocation + * requests will inherit GFP_NOFS implicitly. + * + * %GFP_USER is for userspace allocations that also need to be directly + * accessibly by the kernel or hardware. It is typically used by hardware + * for buffers that are mapped to userspace (e.g. graphics) that hardware + * still must DMA to. cpuset limits are enforced for these allocations. + * + * %GFP_DMA exists for historical reasons and should be avoided where possible. + * The flags indicates that the caller requires that the lowest zone be + * used (%ZONE_DMA or 16M on x86-64). Ideally, this would be removed but + * it would require careful auditing as some users really require it and + * others use the flag to avoid lowmem reserves in %ZONE_DMA and treat the + * lowest zone as a type of emergency reserve. + * + * %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit + * address. Note that kmalloc(..., GFP_DMA32) does not return DMA32 memory + * because the DMA32 kmalloc cache array is not implemented. + * (Reason: there is no such user in kernel). + * + * %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, + * do not need to be directly accessible by the kernel but that cannot + * move once in use. An example may be a hardware allocation that maps + * data directly into userspace but has no addressing limitations. + * + * %GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not + * need direct access to but can use kmap() when access is required. They + * are expected to be movable via page reclaim or page migration. Typically, + * pages on the LRU would also be allocated with %GFP_HIGHUSER_MOVABLE. + * + * %GFP_TRANSHUGE and %GFP_TRANSHUGE_LIGHT are used for THP allocations. They + * are compound allocations that will generally fail quickly if memory is not + * available and will not wake kswapd/kcompactd on failure. The _LIGHT + * version does not attempt reclaim/compaction at all and is by default used + * in page fault path, while the non-light is used by khugepaged. + */ +#define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) +#define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) +#define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT) +#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) +#define GFP_NOIO (__GFP_RECLAIM) +#define GFP_NOFS (__GFP_RECLAIM | __GFP_IO) +#define GFP_USER (__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL) +#define GFP_DMA __GFP_DMA +#define GFP_DMA32 __GFP_DMA32 +#define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) +#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \ + __GFP_SKIP_KASAN_POISON) +#define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ + __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) +#define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) + +#endif /* __LINUX_GFP_TYPES_H */ -- cgit v1.2.3 From f0dd891dd5a1d6dc6c9d486333aac4f433f17d17 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 1 Jul 2022 05:54:30 -0700 Subject: lib/cpumask: move some one-line wrappers to header file After moving gfp flags to a separate header, it's possible to move some cpumask allocators into headers, and avoid creating real functions. Signed-off-by: Yury Norov --- include/linux/cpumask.h | 34 +++++++++++++++++++++++++++++++--- lib/cpumask.c | 28 ---------------------------- 2 files changed, 31 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index ea3de2c2c180..80627362c774 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -12,6 +12,8 @@ #include #include #include +#include +#include /* Don't assign or return these: may not be this big! */ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; @@ -794,9 +796,35 @@ typedef struct cpumask *cpumask_var_t; #define __cpumask_var_read_mostly __read_mostly bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); -bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags); -bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); -bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags); + +static inline +bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) +{ + return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); +} + +/** + * alloc_cpumask_var - allocate a struct cpumask + * @mask: pointer to cpumask_var_t where the cpumask is returned + * @flags: GFP_ flags + * + * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is + * a nop returning a constant 1 (in ). + * + * See alloc_cpumask_var_node. + */ +static inline +bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); +} + +static inline +bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return alloc_cpumask_var(mask, flags | __GFP_ZERO); +} + void alloc_bootmem_cpumask_var(cpumask_var_t *mask); void free_cpumask_var(cpumask_var_t mask); void free_bootmem_cpumask_var(cpumask_var_t mask); diff --git a/lib/cpumask.c b/lib/cpumask.c index cb7262ff8633..f0ae119be8c4 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -70,34 +70,6 @@ bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) } EXPORT_SYMBOL(alloc_cpumask_var_node); -bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) -{ - return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); -} -EXPORT_SYMBOL(zalloc_cpumask_var_node); - -/** - * alloc_cpumask_var - allocate a struct cpumask - * @mask: pointer to cpumask_var_t where the cpumask is returned - * @flags: GFP_ flags - * - * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is - * a nop returning a constant 1 (in ). - * - * See alloc_cpumask_var_node. - */ -bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) -{ - return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); -} -EXPORT_SYMBOL(alloc_cpumask_var); - -bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) -{ - return alloc_cpumask_var(mask, flags | __GFP_ZERO); -} -EXPORT_SYMBOL(zalloc_cpumask_var); - /** * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena. * @mask: pointer to cpumask_var_t where the cpumask is returned -- cgit v1.2.3 From a8755e9bdd6a416331e286cc25cddbd93b2c064a Mon Sep 17 00:00:00 2001 From: Dinh Nguyen Date: Fri, 15 Jul 2022 10:03:49 -0500 Subject: firmware: stratix10-svc: fix kernel-doc warning include/linux/firmware/intel/stratix10-svc-client.h:55: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Flag bit for COMMAND_RECONFIG Fixes: 4a4709d470e6 ("firmware: stratix10-svc: add new FCS commands") Signed-off-by: Dinh Nguyen Reported-by: Stephen Rothwell Link: https://lore.kernel.org/r/20220715150349.2413994-1-dinguyen@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/firmware/intel/stratix10-svc-client.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index a776a787ac75..0c16037fd08d 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -51,7 +51,8 @@ #define SVC_STATUS_ERROR 5 #define SVC_STATUS_NO_SUPPORT 6 #define SVC_STATUS_INVALID_PARAM 7 -/** + +/* * Flag bit for COMMAND_RECONFIG * * COMMAND_RECONFIG_FLAG_PARTIAL: -- cgit v1.2.3 From 7ee951acd31a88f941fd6535fbdee3a1567f1d63 Mon Sep 17 00:00:00 2001 From: Phil Auld Date: Fri, 15 Jul 2022 09:49:24 -0400 Subject: drivers/base: fix userspace break from using bin_attributes for cpumap and cpulist Using bin_attributes with a 0 size causes fstat and friends to return that 0 size. This breaks userspace code that retrieves the size before reading the file. Rather than reverting 75bd50fa841 ("drivers/base/node.c: use bin_attribute to break the size limitation of cpumap ABI") let's put in a size value at compile time. For cpulist the maximum size is on the order of NR_CPUS * (ceil(log10(NR_CPUS)) + 1)/2 which for 8192 is 20480 (8192 * 5)/2. In order to get near that you'd need a system with every other CPU on one node. For example: (0,2,4,8, ... ). To simplify the math and support larger NR_CPUS in the future we are using (NR_CPUS * 7)/2. We also set it to a min of PAGE_SIZE to retain the older behavior for smaller NR_CPUS. The cpumap file the size works out to be NR_CPUS/4 + NR_CPUS/32 - 1 (or NR_CPUS * 9/32 - 1) including the ","s. Add a set of macros for these values to cpumask.h so they can be used in multiple places. Apply these to the handful of such files in drivers/base/topology.c as well as node.c. As an example, on an 80 cpu 4-node system (NR_CPUS == 8192): before: -r--r--r--. 1 root root 0 Jul 12 14:08 system/node/node0/cpulist -r--r--r--. 1 root root 0 Jul 11 17:25 system/node/node0/cpumap after: -r--r--r--. 1 root root 28672 Jul 13 11:32 system/node/node0/cpulist -r--r--r--. 1 root root 4096 Jul 13 11:31 system/node/node0/cpumap CONFIG_NR_CPUS = 16384 -r--r--r--. 1 root root 57344 Jul 13 14:03 system/node/node0/cpulist -r--r--r--. 1 root root 4607 Jul 13 14:02 system/node/node0/cpumap The actual number of cpus doesn't matter for the reported size since they are based on NR_CPUS. Fixes: 75bd50fa841d ("drivers/base/node.c: use bin_attribute to break the size limitation of cpumap ABI") Fixes: bb9ec13d156e ("topology: use bin_attribute to break the size limitation of cpumap ABI") Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Cc: Yury Norov Cc: stable@vger.kernel.org Acked-by: Yury Norov (for include/linux/cpumask.h) Signed-off-by: Phil Auld Link: https://lore.kernel.org/r/20220715134924.3466194-1-pauld@redhat.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/node.c | 4 ++-- drivers/base/topology.c | 32 ++++++++++++++++---------------- include/linux/cpumask.h | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/node.c b/drivers/base/node.c index 0ac6376ef7a1..eb0f43784c2b 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -45,7 +45,7 @@ static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj, return n; } -static BIN_ATTR_RO(cpumap, 0); +static BIN_ATTR_RO(cpumap, CPUMAP_FILE_MAX_BYTES); static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj, struct bin_attribute *attr, char *buf, @@ -66,7 +66,7 @@ static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj, return n; } -static BIN_ATTR_RO(cpulist, 0); +static BIN_ATTR_RO(cpulist, CPULIST_FILE_MAX_BYTES); /** * struct node_access_nodes - Access class device to hold user visible diff --git a/drivers/base/topology.c b/drivers/base/topology.c index ac6ad9ab67f9..89f98be5c5b9 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -62,47 +62,47 @@ define_id_show_func(ppin, "0x%llx"); static DEVICE_ATTR_ADMIN_RO(ppin); define_siblings_read_func(thread_siblings, sibling_cpumask); -static BIN_ATTR_RO(thread_siblings, 0); -static BIN_ATTR_RO(thread_siblings_list, 0); +static BIN_ATTR_RO(thread_siblings, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(thread_siblings_list, CPULIST_FILE_MAX_BYTES); define_siblings_read_func(core_cpus, sibling_cpumask); -static BIN_ATTR_RO(core_cpus, 0); -static BIN_ATTR_RO(core_cpus_list, 0); +static BIN_ATTR_RO(core_cpus, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(core_cpus_list, CPULIST_FILE_MAX_BYTES); define_siblings_read_func(core_siblings, core_cpumask); -static BIN_ATTR_RO(core_siblings, 0); -static BIN_ATTR_RO(core_siblings_list, 0); +static BIN_ATTR_RO(core_siblings, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(core_siblings_list, CPULIST_FILE_MAX_BYTES); #ifdef TOPOLOGY_CLUSTER_SYSFS define_siblings_read_func(cluster_cpus, cluster_cpumask); -static BIN_ATTR_RO(cluster_cpus, 0); -static BIN_ATTR_RO(cluster_cpus_list, 0); +static BIN_ATTR_RO(cluster_cpus, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(cluster_cpus_list, CPULIST_FILE_MAX_BYTES); #endif #ifdef TOPOLOGY_DIE_SYSFS define_siblings_read_func(die_cpus, die_cpumask); -static BIN_ATTR_RO(die_cpus, 0); -static BIN_ATTR_RO(die_cpus_list, 0); +static BIN_ATTR_RO(die_cpus, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(die_cpus_list, CPULIST_FILE_MAX_BYTES); #endif define_siblings_read_func(package_cpus, core_cpumask); -static BIN_ATTR_RO(package_cpus, 0); -static BIN_ATTR_RO(package_cpus_list, 0); +static BIN_ATTR_RO(package_cpus, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(package_cpus_list, CPULIST_FILE_MAX_BYTES); #ifdef TOPOLOGY_BOOK_SYSFS define_id_show_func(book_id, "%d"); static DEVICE_ATTR_RO(book_id); define_siblings_read_func(book_siblings, book_cpumask); -static BIN_ATTR_RO(book_siblings, 0); -static BIN_ATTR_RO(book_siblings_list, 0); +static BIN_ATTR_RO(book_siblings, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(book_siblings_list, CPULIST_FILE_MAX_BYTES); #endif #ifdef TOPOLOGY_DRAWER_SYSFS define_id_show_func(drawer_id, "%d"); static DEVICE_ATTR_RO(drawer_id); define_siblings_read_func(drawer_siblings, drawer_cpumask); -static BIN_ATTR_RO(drawer_siblings, 0); -static BIN_ATTR_RO(drawer_siblings_list, 0); +static BIN_ATTR_RO(drawer_siblings, CPUMAP_FILE_MAX_BYTES); +static BIN_ATTR_RO(drawer_siblings_list, CPULIST_FILE_MAX_BYTES); #endif static struct bin_attribute *bin_attrs[] = { diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index fe29ac7cc469..4592d0845941 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -1071,4 +1071,22 @@ cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, [0] = 1UL \ } } +/* + * Provide a valid theoretical max size for cpumap and cpulist sysfs files + * to avoid breaking userspace which may allocate a buffer based on the size + * reported by e.g. fstat. + * + * for cpumap NR_CPUS * 9/32 - 1 should be an exact length. + * + * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up + * to 2 orders of magnitude larger than 8192. And then we divide by 2 to + * cover a worst-case of every other cpu being on one of two nodes for a + * very large NR_CPUS. + * + * Use PAGE_SIZE as a minimum for smaller configurations. + */ +#define CPUMAP_FILE_MAX_BYTES ((((NR_CPUS * 9)/32 - 1) > PAGE_SIZE) \ + ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE) +#define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE) + #endif /* __LINUX_CPUMASK_H */ -- cgit v1.2.3 From 65d9a9a60fd71be964effb2e94747a6acb6e7015 Mon Sep 17 00:00:00 2001 From: "Naveen N. Rao" Date: Fri, 1 Jul 2022 13:04:04 +0530 Subject: kexec_file: drop weak attribute from functions As requested (http://lkml.kernel.org/r/87ee0q7b92.fsf@email.froward.int.ebiederm.org), this series converts weak functions in kexec to use the #ifdef approach. Quoting the 3e35142ef99fe ("kexec_file: drop weak attribute from arch_kexec_apply_relocations[_add]") changelog: : Since commit d1bcae833b32f1 ("ELF: Don't generate unused section symbols") : [1], binutils (v2.36+) started dropping section symbols that it thought : were unused. This isn't an issue in general, but with kexec_file.c, gcc : is placing kexec_arch_apply_relocations[_add] into a separate : .text.unlikely section and the section symbol ".text.unlikely" is being : dropped. Due to this, recordmcount is unable to find a non-weak symbol in : .text.unlikely to generate a relocation record against. This patch (of 2); Drop __weak attribute from functions in kexec_file.c: - arch_kexec_kernel_image_probe() - arch_kimage_file_post_load_cleanup() - arch_kexec_kernel_image_load() - arch_kexec_locate_mem_hole() - arch_kexec_kernel_verify_sig() arch_kexec_kernel_image_load() calls into kexec_image_load_default(), so drop the static attribute for the latter. arch_kexec_kernel_verify_sig() is not overridden by any architecture, so drop the __weak attribute. Link: https://lkml.kernel.org/r/cover.1656659357.git.naveen.n.rao@linux.vnet.ibm.com Link: https://lkml.kernel.org/r/2cd7ca1fe4d6bb6ca38e3283c717878388ed6788.1656659357.git.naveen.n.rao@linux.vnet.ibm.com Signed-off-by: Naveen N. Rao Suggested-by: Eric Biederman Signed-off-by: Andrew Morton Signed-off-by: Mimi Zohar --- arch/arm64/include/asm/kexec.h | 4 +++- arch/powerpc/include/asm/kexec.h | 9 ++++++++ arch/s390/include/asm/kexec.h | 3 +++ arch/x86/include/asm/kexec.h | 6 ++++++ include/linux/kexec.h | 44 ++++++++++++++++++++++++++++++++++------ kernel/kexec_file.c | 35 ++------------------------------ 6 files changed, 61 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 9839bfc163d7..78d272b26ebd 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -115,7 +115,9 @@ extern const struct kexec_file_ops kexec_image_ops; struct kimage; -extern int arch_kimage_file_post_load_cleanup(struct kimage *image); +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup + extern int load_other_segments(struct kimage *image, unsigned long kernel_load_addr, unsigned long kernel_size, char *initrd, unsigned long initrd_len, diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 2aefe14e1442..1e5e9b6ec78d 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -120,6 +120,15 @@ int setup_purgatory(struct kimage *image, const void *slave_code, #ifdef CONFIG_PPC64 struct kexec_buf; +int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, unsigned long buf_len); +#define arch_kexec_kernel_image_probe arch_kexec_kernel_image_probe + +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup + +int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf); +#define arch_kexec_locate_mem_hole arch_kexec_locate_mem_hole + int load_crashdump_segments_ppc64(struct kimage *image, struct kexec_buf *kbuf); int setup_purgatory_ppc64(struct kimage *image, const void *slave_code, diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h index 649ecdcc8734..8886aadc11a3 100644 --- a/arch/s390/include/asm/kexec.h +++ b/arch/s390/include/asm/kexec.h @@ -92,5 +92,8 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, const Elf_Shdr *relsec, const Elf_Shdr *symtab); #define arch_kexec_apply_relocations_add arch_kexec_apply_relocations_add + +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup #endif #endif /*_S390_KEXEC_H */ diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index 6ad8d946cd3e..5ec359c1b50c 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -193,6 +193,12 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, const Elf_Shdr *relsec, const Elf_Shdr *symtab); #define arch_kexec_apply_relocations_add arch_kexec_apply_relocations_add + +void *arch_kexec_kernel_image_load(struct kimage *image); +#define arch_kexec_kernel_image_load arch_kexec_kernel_image_load + +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup #endif #endif diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 475683cd67f1..6958c6b471f4 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -188,21 +188,53 @@ int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name, void *buf, unsigned int size, bool get_value); void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name); +void *kexec_image_load_default(struct kimage *image); + +#ifndef arch_kexec_kernel_image_probe +static inline int +arch_kexec_kernel_image_probe(struct kimage *image, void *buf, unsigned long buf_len) +{ + return kexec_image_probe_default(image, buf, buf_len); +} +#endif + +#ifndef arch_kimage_file_post_load_cleanup +static inline int arch_kimage_file_post_load_cleanup(struct kimage *image) +{ + return kexec_image_post_load_cleanup_default(image); +} +#endif + +#ifndef arch_kexec_kernel_image_load +static inline void *arch_kexec_kernel_image_load(struct kimage *image) +{ + return kexec_image_load_default(image); +} +#endif -/* Architectures may override the below functions */ -int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, - unsigned long buf_len); -void *arch_kexec_kernel_image_load(struct kimage *image); -int arch_kimage_file_post_load_cleanup(struct kimage *image); #ifdef CONFIG_KEXEC_SIG int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, unsigned long buf_len); #endif -int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf); extern int kexec_add_buffer(struct kexec_buf *kbuf); int kexec_locate_mem_hole(struct kexec_buf *kbuf); +#ifndef arch_kexec_locate_mem_hole +/** + * arch_kexec_locate_mem_hole - Find free memory to place the segments. + * @kbuf: Parameters for the memory search. + * + * On success, kbuf->mem will have the start address of the memory region found. + * + * Return: 0 on success, negative errno on error. + */ +static inline int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf) +{ + return kexec_locate_mem_hole(kbuf); +} +#endif + /* Alignment required for elf header segment */ #define ELF_CORE_HEADER_ALIGN 4096 diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index f9261c07b048..0c27c81351ee 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -62,14 +62,7 @@ int kexec_image_probe_default(struct kimage *image, void *buf, return ret; } -/* Architectures can provide this probe function */ -int __weak arch_kexec_kernel_image_probe(struct kimage *image, void *buf, - unsigned long buf_len) -{ - return kexec_image_probe_default(image, buf, buf_len); -} - -static void *kexec_image_load_default(struct kimage *image) +void *kexec_image_load_default(struct kimage *image) { if (!image->fops || !image->fops->load) return ERR_PTR(-ENOEXEC); @@ -80,11 +73,6 @@ static void *kexec_image_load_default(struct kimage *image) image->cmdline_buf_len); } -void * __weak arch_kexec_kernel_image_load(struct kimage *image) -{ - return kexec_image_load_default(image); -} - int kexec_image_post_load_cleanup_default(struct kimage *image) { if (!image->fops || !image->fops->cleanup) @@ -93,11 +81,6 @@ int kexec_image_post_load_cleanup_default(struct kimage *image) return image->fops->cleanup(image->image_loader_data); } -int __weak arch_kimage_file_post_load_cleanup(struct kimage *image) -{ - return kexec_image_post_load_cleanup_default(image); -} - #ifdef CONFIG_KEXEC_SIG static int kexec_image_verify_sig_default(struct kimage *image, void *buf, unsigned long buf_len) @@ -110,8 +93,7 @@ static int kexec_image_verify_sig_default(struct kimage *image, void *buf, return image->fops->verify_sig(buf, buf_len); } -int __weak arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, - unsigned long buf_len) +int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, unsigned long buf_len) { return kexec_image_verify_sig_default(image, buf, buf_len); } @@ -621,19 +603,6 @@ int kexec_locate_mem_hole(struct kexec_buf *kbuf) return ret == 1 ? 0 : -EADDRNOTAVAIL; } -/** - * arch_kexec_locate_mem_hole - Find free memory to place the segments. - * @kbuf: Parameters for the memory search. - * - * On success, kbuf->mem will have the start address of the memory region found. - * - * Return: 0 on success, negative errno on error. - */ -int __weak arch_kexec_locate_mem_hole(struct kexec_buf *kbuf) -{ - return kexec_locate_mem_hole(kbuf); -} - /** * kexec_add_buffer - place a buffer in a kexec segment * @kbuf: Buffer contents and memory parameters. -- cgit v1.2.3 From 0738eceb6201691534df07e0928d0a6168a35787 Mon Sep 17 00:00:00 2001 From: "Naveen N. Rao" Date: Fri, 1 Jul 2022 13:04:05 +0530 Subject: kexec: drop weak attribute from functions Drop __weak attribute from functions in kexec_core.c: - machine_kexec_post_load() - arch_kexec_protect_crashkres() - arch_kexec_unprotect_crashkres() - crash_free_reserved_phys_range() Link: https://lkml.kernel.org/r/c0f6219e03cb399d166d518ab505095218a902dd.1656659357.git.naveen.n.rao@linux.vnet.ibm.com Signed-off-by: Naveen N. Rao Suggested-by: Eric Biederman Signed-off-by: Andrew Morton Signed-off-by: Mimi Zohar --- arch/arm64/include/asm/kexec.h | 16 ++++++++++++++-- arch/powerpc/include/asm/kexec.h | 5 +++++ arch/s390/include/asm/kexec.h | 11 +++++++++++ arch/x86/include/asm/kexec.h | 6 ++++++ include/linux/kexec.h | 32 ++++++++++++++++++++++++++++---- kernel/kexec_core.c | 27 --------------------------- 6 files changed, 64 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 78d272b26ebd..559bfae26715 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -84,16 +84,30 @@ static inline void crash_setup_regs(struct pt_regs *newregs, extern bool crash_is_nosave(unsigned long pfn); extern void crash_prepare_suspend(void); extern void crash_post_resume(void); + +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); +#define crash_free_reserved_phys_range crash_free_reserved_phys_range #else static inline bool crash_is_nosave(unsigned long pfn) {return false; } static inline void crash_prepare_suspend(void) {} static inline void crash_post_resume(void) {} #endif +struct kimage; + #if defined(CONFIG_KEXEC_CORE) void cpu_soft_restart(unsigned long el2_switch, unsigned long entry, unsigned long arg0, unsigned long arg1, unsigned long arg2); + +int machine_kexec_post_load(struct kimage *image); +#define machine_kexec_post_load machine_kexec_post_load + +void arch_kexec_protect_crashkres(void); +#define arch_kexec_protect_crashkres arch_kexec_protect_crashkres + +void arch_kexec_unprotect_crashkres(void); +#define arch_kexec_unprotect_crashkres arch_kexec_unprotect_crashkres #endif #define ARCH_HAS_KIMAGE_ARCH @@ -113,8 +127,6 @@ struct kimage_arch { #ifdef CONFIG_KEXEC_FILE extern const struct kexec_file_ops kexec_image_ops; -struct kimage; - int arch_kimage_file_post_load_cleanup(struct kimage *image); #define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 1e5e9b6ec78d..d6f4edfe4737 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -98,6 +98,11 @@ void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_co void kexec_copy_flush(struct kimage *image); +#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_PPC_RTAS) +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); +#define crash_free_reserved_phys_range crash_free_reserved_phys_range +#endif + #ifdef CONFIG_KEXEC_FILE extern const struct kexec_file_ops kexec_elf64_ops; diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h index 8886aadc11a3..1bd08eb56d5f 100644 --- a/arch/s390/include/asm/kexec.h +++ b/arch/s390/include/asm/kexec.h @@ -85,6 +85,17 @@ struct kimage_arch { extern const struct kexec_file_ops s390_kexec_image_ops; extern const struct kexec_file_ops s390_kexec_elf_ops; +#ifdef CONFIG_CRASH_DUMP +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); +#define crash_free_reserved_phys_range crash_free_reserved_phys_range + +void arch_kexec_protect_crashkres(void); +#define arch_kexec_protect_crashkres arch_kexec_protect_crashkres + +void arch_kexec_unprotect_crashkres(void); +#define arch_kexec_unprotect_crashkres arch_kexec_unprotect_crashkres +#endif + #ifdef CONFIG_KEXEC_FILE struct purgatory_info; int arch_kexec_apply_relocations_add(struct purgatory_info *pi, diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index 5ec359c1b50c..a3760ca796aa 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -186,6 +186,12 @@ extern int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, extern void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages); #define arch_kexec_pre_free_pages arch_kexec_pre_free_pages +void arch_kexec_protect_crashkres(void); +#define arch_kexec_protect_crashkres arch_kexec_protect_crashkres + +void arch_kexec_unprotect_crashkres(void); +#define arch_kexec_unprotect_crashkres arch_kexec_unprotect_crashkres + #ifdef CONFIG_KEXEC_FILE struct purgatory_info; int arch_kexec_apply_relocations_add(struct purgatory_info *pi, diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 6958c6b471f4..8107606ad1e8 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -390,7 +390,10 @@ extern void machine_kexec_cleanup(struct kimage *image); extern int kernel_kexec(void); extern struct page *kimage_alloc_control_pages(struct kimage *image, unsigned int order); -int machine_kexec_post_load(struct kimage *image); + +#ifndef machine_kexec_post_load +static inline int machine_kexec_post_load(struct kimage *image) { return 0; } +#endif extern void __crash_kexec(struct pt_regs *); extern void crash_kexec(struct pt_regs *); @@ -423,10 +426,21 @@ extern bool kexec_in_progress; int crash_shrink_memory(unsigned long new_size); size_t crash_get_memory_size(void); -void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); -void arch_kexec_protect_crashkres(void); -void arch_kexec_unprotect_crashkres(void); +#ifndef arch_kexec_protect_crashkres +/* + * Protection mechanism for crashkernel reserved memory after + * the kdump kernel is loaded. + * + * Provide an empty default implementation here -- architecture + * code may override this + */ +static inline void arch_kexec_protect_crashkres(void) { } +#endif + +#ifndef arch_kexec_unprotect_crashkres +static inline void arch_kexec_unprotect_crashkres(void) { } +#endif #ifndef page_to_boot_pfn static inline unsigned long page_to_boot_pfn(struct page *page) @@ -456,6 +470,16 @@ static inline phys_addr_t boot_phys_to_phys(unsigned long boot_phys) } #endif +#ifndef crash_free_reserved_phys_range +static inline void crash_free_reserved_phys_range(unsigned long begin, unsigned long end) +{ + unsigned long addr; + + for (addr = begin; addr < end; addr += PAGE_SIZE) + free_reserved_page(boot_pfn_to_page(addr >> PAGE_SHIFT)); +} +#endif + static inline unsigned long virt_to_boot_phys(void *addr) { return phys_to_boot_phys(__pa((unsigned long)addr)); diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 4d34c78334ce..acd029b307e4 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -591,11 +591,6 @@ static void kimage_free_extra_pages(struct kimage *image) } -int __weak machine_kexec_post_load(struct kimage *image) -{ - return 0; -} - void kimage_terminate(struct kimage *image) { if (*image->entry != 0) @@ -1020,15 +1015,6 @@ size_t crash_get_memory_size(void) return size; } -void __weak crash_free_reserved_phys_range(unsigned long begin, - unsigned long end) -{ - unsigned long addr; - - for (addr = begin; addr < end; addr += PAGE_SIZE) - free_reserved_page(boot_pfn_to_page(addr >> PAGE_SHIFT)); -} - int crash_shrink_memory(unsigned long new_size) { int ret = 0; @@ -1225,16 +1211,3 @@ int kernel_kexec(void) mutex_unlock(&kexec_mutex); return error; } - -/* - * Protection mechanism for crashkernel reserved memory after - * the kdump kernel is loaded. - * - * Provide an empty default implementation here -- architecture - * code may override this - */ -void __weak arch_kexec_protect_crashkres(void) -{} - -void __weak arch_kexec_unprotect_crashkres(void) -{} -- cgit v1.2.3 From 689a71493bd2f31c024f8c0395f85a1fd4b2138e Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Thu, 14 Jul 2022 21:40:24 +0800 Subject: kexec: clean up arch_kexec_kernel_verify_sig Before commit 105e10e2cf1c ("kexec_file: drop weak attribute from functions"), there was already no arch-specific implementation of arch_kexec_kernel_verify_sig. With weak attribute dropped by that commit, arch_kexec_kernel_verify_sig is completely useless. So clean it up. Note later patches are dependent on this patch so it should be backported to the stable tree as well. Cc: stable@vger.kernel.org Suggested-by: Eric W. Biederman Reviewed-by: Michal Suchanek Acked-by: Baoquan He Signed-off-by: Coiby Xu [zohar@linux.ibm.com: reworded patch description "Note"] Link: https://lore.kernel.org/linux-integrity/20220714134027.394370-1-coxu@redhat.com/ Signed-off-by: Mimi Zohar --- include/linux/kexec.h | 5 ----- kernel/kexec_file.c | 33 +++++++++++++-------------------- 2 files changed, 13 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 8107606ad1e8..7f710fb3712b 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -212,11 +212,6 @@ static inline void *arch_kexec_kernel_image_load(struct kimage *image) } #endif -#ifdef CONFIG_KEXEC_SIG -int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, - unsigned long buf_len); -#endif - extern int kexec_add_buffer(struct kexec_buf *kbuf); int kexec_locate_mem_hole(struct kexec_buf *kbuf); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 0c27c81351ee..6dc1294c90fc 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -81,24 +81,6 @@ int kexec_image_post_load_cleanup_default(struct kimage *image) return image->fops->cleanup(image->image_loader_data); } -#ifdef CONFIG_KEXEC_SIG -static int kexec_image_verify_sig_default(struct kimage *image, void *buf, - unsigned long buf_len) -{ - if (!image->fops || !image->fops->verify_sig) { - pr_debug("kernel loader does not support signature verification.\n"); - return -EKEYREJECTED; - } - - return image->fops->verify_sig(buf, buf_len); -} - -int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, unsigned long buf_len) -{ - return kexec_image_verify_sig_default(image, buf, buf_len); -} -#endif - /* * Free up memory used by kernel, initrd, and command line. This is temporary * memory allocation which is not needed any more after these buffers have @@ -141,13 +123,24 @@ void kimage_file_post_load_cleanup(struct kimage *image) } #ifdef CONFIG_KEXEC_SIG +static int kexec_image_verify_sig(struct kimage *image, void *buf, + unsigned long buf_len) +{ + if (!image->fops || !image->fops->verify_sig) { + pr_debug("kernel loader does not support signature verification.\n"); + return -EKEYREJECTED; + } + + return image->fops->verify_sig(buf, buf_len); +} + static int kimage_validate_signature(struct kimage *image) { int ret; - ret = arch_kexec_kernel_verify_sig(image, image->kernel_buf, - image->kernel_buf_len); + ret = kexec_image_verify_sig(image, image->kernel_buf, + image->kernel_buf_len); if (ret) { if (sig_enforce) { -- cgit v1.2.3 From c903dae8941deb55043ee46ded29e84e97cd84bb Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Thu, 14 Jul 2022 21:40:25 +0800 Subject: kexec, KEYS: make the code in bzImage64_verify_sig generic commit 278311e417be ("kexec, KEYS: Make use of platform keyring for signature verify") adds platform keyring support on x86 kexec but not arm64. The code in bzImage64_verify_sig uses the keys on the .builtin_trusted_keys, .machine, if configured and enabled, .secondary_trusted_keys, also if configured, and .platform keyrings to verify the signed kernel image as PE file. Cc: kexec@lists.infradead.org Cc: keyrings@vger.kernel.org Cc: linux-security-module@vger.kernel.org Reviewed-by: Michal Suchanek Signed-off-by: Coiby Xu Signed-off-by: Mimi Zohar --- arch/x86/kernel/kexec-bzimage64.c | 20 +------------------- include/linux/kexec.h | 7 +++++++ kernel/kexec_file.c | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 170d0fd68b1f..f299b48f9c9f 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include @@ -528,28 +527,11 @@ static int bzImage64_cleanup(void *loader_data) return 0; } -#ifdef CONFIG_KEXEC_BZIMAGE_VERIFY_SIG -static int bzImage64_verify_sig(const char *kernel, unsigned long kernel_len) -{ - int ret; - - ret = verify_pefile_signature(kernel, kernel_len, - VERIFY_USE_SECONDARY_KEYRING, - VERIFYING_KEXEC_PE_SIGNATURE); - if (ret == -ENOKEY && IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING)) { - ret = verify_pefile_signature(kernel, kernel_len, - VERIFY_USE_PLATFORM_KEYRING, - VERIFYING_KEXEC_PE_SIGNATURE); - } - return ret; -} -#endif - const struct kexec_file_ops kexec_bzImage64_ops = { .probe = bzImage64_probe, .load = bzImage64_load, .cleanup = bzImage64_cleanup, #ifdef CONFIG_KEXEC_BZIMAGE_VERIFY_SIG - .verify_sig = bzImage64_verify_sig, + .verify_sig = kexec_kernel_verify_pe_sig, #endif }; diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 7f710fb3712b..13e6c4b58f07 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -19,6 +19,7 @@ #include #include +#include /* Location of a reserved region to hold the crash kernel. */ @@ -212,6 +213,12 @@ static inline void *arch_kexec_kernel_image_load(struct kimage *image) } #endif +#ifdef CONFIG_KEXEC_SIG +#ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION +int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len); +#endif +#endif + extern int kexec_add_buffer(struct kexec_buf *kbuf); int kexec_locate_mem_hole(struct kexec_buf *kbuf); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 6dc1294c90fc..a7b411c22f19 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -123,6 +123,23 @@ void kimage_file_post_load_cleanup(struct kimage *image) } #ifdef CONFIG_KEXEC_SIG +#ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION +int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len) +{ + int ret; + + ret = verify_pefile_signature(kernel, kernel_len, + VERIFY_USE_SECONDARY_KEYRING, + VERIFYING_KEXEC_PE_SIGNATURE); + if (ret == -ENOKEY && IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING)) { + ret = verify_pefile_signature(kernel, kernel_len, + VERIFY_USE_PLATFORM_KEYRING, + VERIFYING_KEXEC_PE_SIGNATURE); + } + return ret; +} +#endif + static int kexec_image_verify_sig(struct kimage *image, void *buf, unsigned long buf_len) { -- cgit v1.2.3 From ae6ccaa650380d243cf43d31c864c5ced2fd4612 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 7 Jul 2022 08:15:52 +0100 Subject: PM: EM: convert power field to micro-Watts precision and align drivers The milli-Watts precision causes rounding errors while calculating efficiency cost for each OPP. This is especially visible in the 'simple' Energy Model (EM), where the power for each OPP is provided from OPP framework. This can cause some OPPs to be marked inefficient, while using micro-Watts precision that might not happen. Update all EM users which access 'power' field and assume the value is in milli-Watts. Solve also an issue with potential overflow in calculation of energy estimation on 32bit machine. It's needed now since the power value (thus the 'cost' as well) are higher. Example calculation which shows the rounding error and impact: power = 'dyn-power-coeff' * volt_mV * volt_mV * freq_MHz power_a_uW = (100 * 600mW * 600mW * 500MHz) / 10^6 = 18000 power_a_mW = (100 * 600mW * 600mW * 500MHz) / 10^9 = 18 power_b_uW = (100 * 605mW * 605mW * 600MHz) / 10^6 = 21961 power_b_mW = (100 * 605mW * 605mW * 600MHz) / 10^9 = 21 max_freq = 2000MHz cost_a_mW = 18 * 2000MHz/500MHz = 72 cost_a_uW = 18000 * 2000MHz/500MHz = 72000 cost_b_mW = 21 * 2000MHz/600MHz = 70 // <- artificially better cost_b_uW = 21961 * 2000MHz/600MHz = 73203 The 'cost_b_mW' (which is based on old milli-Watts) is misleadingly better that the 'cost_b_uW' (this patch uses micro-Watts) and such would have impact on the 'inefficient OPPs' information in the Cpufreq framework. This patch set removes the rounding issue. Signed-off-by: Lukasz Luba Acked-by: Daniel Lezcano Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/mediatek-cpufreq-hw.c | 7 +++-- drivers/cpufreq/scmi-cpufreq.c | 6 ++++ drivers/opp/of.c | 15 +++++----- drivers/powercap/dtpm_cpu.c | 5 ++-- drivers/thermal/cpufreq_cooling.c | 13 +++++++-- drivers/thermal/devfreq_cooling.c | 19 +++++++++--- include/linux/energy_model.h | 54 ++++++++++++++++++++++++----------- kernel/power/energy_model.c | 24 ++++++++++------ 8 files changed, 100 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index 813cccbfe934..f0e0a35c7f21 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -51,7 +51,7 @@ static const u16 cpufreq_mtk_offsets[REG_ARRAY_SIZE] = { }; static int __maybe_unused -mtk_cpufreq_get_cpu_power(struct device *cpu_dev, unsigned long *mW, +mtk_cpufreq_get_cpu_power(struct device *cpu_dev, unsigned long *uW, unsigned long *KHz) { struct mtk_cpufreq_data *data; @@ -71,8 +71,9 @@ mtk_cpufreq_get_cpu_power(struct device *cpu_dev, unsigned long *mW, i--; *KHz = data->table[i].frequency; - *mW = readl_relaxed(data->reg_bases[REG_EM_POWER_TBL] + - i * LUT_ROW_SIZE) / 1000; + /* Provide micro-Watts value to the Energy Model */ + *uW = readl_relaxed(data->reg_bases[REG_EM_POWER_TBL] + + i * LUT_ROW_SIZE); return 0; } diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index 6d2a4cf46db7..bfd35583d653 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -19,6 +19,7 @@ #include #include #include +#include struct scmi_data { int domain_id; @@ -99,6 +100,7 @@ static int __maybe_unused scmi_get_cpu_power(struct device *cpu_dev, unsigned long *power, unsigned long *KHz) { + bool power_scale_mw = perf_ops->power_scale_mw_get(ph); unsigned long Hz; int ret, domain; @@ -112,6 +114,10 @@ scmi_get_cpu_power(struct device *cpu_dev, unsigned long *power, if (ret) return ret; + /* Provide bigger resolution power to the Energy Model */ + if (power_scale_mw) + *power *= MICROWATT_PER_MILLIWATT; + /* The EM framework specifies the frequency in KHz. */ *KHz = Hz / 1000; diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 30394929d700..eb89c9a75985 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -1443,12 +1443,12 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_of_node); * It provides the power used by @dev at @kHz if it is the frequency of an * existing OPP, or at the frequency of the first OPP above @kHz otherwise * (see dev_pm_opp_find_freq_ceil()). This function updates @kHz to the ceiled - * frequency and @mW to the associated power. + * frequency and @uW to the associated power. * * Returns 0 on success or a proper -EINVAL value in case of error. */ static int __maybe_unused -_get_dt_power(struct device *dev, unsigned long *mW, unsigned long *kHz) +_get_dt_power(struct device *dev, unsigned long *uW, unsigned long *kHz) { struct dev_pm_opp *opp; unsigned long opp_freq, opp_power; @@ -1465,7 +1465,7 @@ _get_dt_power(struct device *dev, unsigned long *mW, unsigned long *kHz) return -EINVAL; *kHz = opp_freq / 1000; - *mW = opp_power / 1000; + *uW = opp_power; return 0; } @@ -1475,14 +1475,14 @@ _get_dt_power(struct device *dev, unsigned long *mW, unsigned long *kHz) * This computes the power estimated by @dev at @kHz if it is the frequency * of an existing OPP, or at the frequency of the first OPP above @kHz otherwise * (see dev_pm_opp_find_freq_ceil()). This function updates @kHz to the ceiled - * frequency and @mW to the associated power. The power is estimated as + * frequency and @uW to the associated power. The power is estimated as * P = C * V^2 * f with C being the device's capacitance and V and f * respectively the voltage and frequency of the OPP. * * Returns -EINVAL if the power calculation failed because of missing * parameters, 0 otherwise. */ -static int __maybe_unused _get_power(struct device *dev, unsigned long *mW, +static int __maybe_unused _get_power(struct device *dev, unsigned long *uW, unsigned long *kHz) { struct dev_pm_opp *opp; @@ -1512,9 +1512,10 @@ static int __maybe_unused _get_power(struct device *dev, unsigned long *mW, return -EINVAL; tmp = (u64)cap * mV * mV * (Hz / 1000000); - do_div(tmp, 1000000000); + /* Provide power in micro-Watts */ + do_div(tmp, 1000000); - *mW = (unsigned long)tmp; + *uW = (unsigned long)tmp; *kHz = Hz / 1000; return 0; diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c index f5eced0842b3..61c5ff80bd30 100644 --- a/drivers/powercap/dtpm_cpu.c +++ b/drivers/powercap/dtpm_cpu.c @@ -53,7 +53,7 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) for (i = 0; i < pd->nr_perf_states; i++) { - power = pd->table[i].power * MICROWATT_PER_MILLIWATT * nr_cpus; + power = pd->table[i].power * nr_cpus; if (power > power_limit) break; @@ -63,8 +63,7 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) freq_qos_update_request(&dtpm_cpu->qos_req, freq); - power_limit = pd->table[i - 1].power * - MICROWATT_PER_MILLIWATT * nr_cpus; + power_limit = pd->table[i - 1].power * nr_cpus; return power_limit; } diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index b8151d95a806..dc19e7c80751 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -101,6 +102,7 @@ static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, u32 freq) { + unsigned long power_mw; int i; for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { @@ -108,16 +110,23 @@ static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, break; } - return cpufreq_cdev->em->table[i + 1].power; + power_mw = cpufreq_cdev->em->table[i + 1].power; + power_mw /= MICROWATT_PER_MILLIWATT; + + return power_mw; } static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev, u32 power) { + unsigned long em_power_mw; int i; for (i = cpufreq_cdev->max_level; i > 0; i--) { - if (power >= cpufreq_cdev->em->table[i].power) + /* Convert EM power to milli-Watts to make safe comparison */ + em_power_mw = cpufreq_cdev->em->table[i].power; + em_power_mw /= MICROWATT_PER_MILLIWATT; + if (power >= em_power_mw) break; } diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c index 8c76f9655e57..8d1260f65061 100644 --- a/drivers/thermal/devfreq_cooling.c +++ b/drivers/thermal/devfreq_cooling.c @@ -200,7 +200,11 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd res = dfc->power_ops->get_real_power(df, power, freq, voltage); if (!res) { state = dfc->capped_state; + + /* Convert EM power into milli-Watts first */ dfc->res_util = dfc->em_pd->table[state].power; + dfc->res_util /= MICROWATT_PER_MILLIWATT; + dfc->res_util *= SCALE_ERROR_MITIGATION; if (*power > 1) @@ -218,8 +222,10 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd _normalize_load(&status); - /* Scale power for utilization */ + /* Convert EM power into milli-Watts first */ *power = dfc->em_pd->table[perf_idx].power; + *power /= MICROWATT_PER_MILLIWATT; + /* Scale power for utilization */ *power *= status.busy_time; *power >>= 10; } @@ -244,6 +250,7 @@ static int devfreq_cooling_state2power(struct thermal_cooling_device *cdev, perf_idx = dfc->max_state - state; *power = dfc->em_pd->table[perf_idx].power; + *power /= MICROWATT_PER_MILLIWATT; return 0; } @@ -254,7 +261,7 @@ static int devfreq_cooling_power2state(struct thermal_cooling_device *cdev, struct devfreq_cooling_device *dfc = cdev->devdata; struct devfreq *df = dfc->devfreq; struct devfreq_dev_status status; - unsigned long freq; + unsigned long freq, em_power_mw; s32 est_power; int i; @@ -279,9 +286,13 @@ static int devfreq_cooling_power2state(struct thermal_cooling_device *cdev, * Find the first cooling state that is within the power * budget. The EM power table is sorted ascending. */ - for (i = dfc->max_state; i > 0; i--) - if (est_power >= dfc->em_pd->table[i].power) + for (i = dfc->max_state; i > 0; i--) { + /* Convert EM power to milli-Watts to make safe comparison */ + em_power_mw = dfc->em_pd->table[i].power; + em_power_mw /= MICROWATT_PER_MILLIWATT; + if (est_power >= em_power_mw) break; + } *state = dfc->max_state - i; dfc->capped_state = *state; diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 8419bffb4398..b9caa01dfac4 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -62,7 +62,7 @@ struct em_perf_domain { /* * em_perf_domain flags: * - * EM_PERF_DOMAIN_MILLIWATTS: The power values are in milli-Watts or some + * EM_PERF_DOMAIN_MICROWATTS: The power values are in micro-Watts or some * other scale. * * EM_PERF_DOMAIN_SKIP_INEFFICIENCIES: Skip inefficient states when estimating @@ -71,7 +71,7 @@ struct em_perf_domain { * EM_PERF_DOMAIN_ARTIFICIAL: The power values are artificial and might be * created by platform missing real power information */ -#define EM_PERF_DOMAIN_MILLIWATTS BIT(0) +#define EM_PERF_DOMAIN_MICROWATTS BIT(0) #define EM_PERF_DOMAIN_SKIP_INEFFICIENCIES BIT(1) #define EM_PERF_DOMAIN_ARTIFICIAL BIT(2) @@ -79,22 +79,44 @@ struct em_perf_domain { #define em_is_artificial(em) ((em)->flags & EM_PERF_DOMAIN_ARTIFICIAL) #ifdef CONFIG_ENERGY_MODEL -#define EM_MAX_POWER 0xFFFF +/* + * The max power value in micro-Watts. The limit of 64 Watts is set as + * a safety net to not overflow multiplications on 32bit platforms. The + * 32bit value limit for total Perf Domain power implies a limit of + * maximum CPUs in such domain to 64. + */ +#define EM_MAX_POWER (64000000) /* 64 Watts */ + +/* + * To avoid possible energy estimation overflow on 32bit machines add + * limits to number of CPUs in the Perf. Domain. + * We are safe on 64bit machine, thus some big number. + */ +#ifdef CONFIG_64BIT +#define EM_MAX_NUM_CPUS 4096 +#else +#define EM_MAX_NUM_CPUS 16 +#endif /* - * Increase resolution of energy estimation calculations for 64-bit - * architectures. The extra resolution improves decision made by EAS for the - * task placement when two Performance Domains might provide similar energy - * estimation values (w/o better resolution the values could be equal). + * To avoid an overflow on 32bit machines while calculating the energy + * use a different order in the operation. First divide by the 'cpu_scale' + * which would reduce big value stored in the 'cost' field, then multiply by + * the 'sum_util'. This would allow to handle existing platforms, which have + * e.g. power ~1.3 Watt at max freq, so the 'cost' value > 1mln micro-Watts. + * In such scenario, where there are 4 CPUs in the Perf. Domain the 'sum_util' + * could be 4096, then multiplication: 'cost' * 'sum_util' would overflow. + * This reordering of operations has some limitations, we lose small + * precision in the estimation (comparing to 64bit platform w/o reordering). * - * We increase resolution only if we have enough bits to allow this increased - * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit - * are pretty high and the returns do not justify the increased costs. + * We are safe on 64bit machine. */ #ifdef CONFIG_64BIT -#define em_scale_power(p) ((p) * 1000) +#define em_estimate_energy(cost, sum_util, scale_cpu) \ + (((cost) * (sum_util)) / (scale_cpu)) #else -#define em_scale_power(p) (p) +#define em_estimate_energy(cost, sum_util, scale_cpu) \ + (((cost) / (scale_cpu)) * (sum_util)) #endif struct em_data_callback { @@ -112,7 +134,7 @@ struct em_data_callback { * and frequency. * * In case of CPUs, the power is the one of a single CPU in the domain, - * expressed in milli-Watts or an abstract scale. It is expected to + * expressed in micro-Watts or an abstract scale. It is expected to * fit in the [0, EM_MAX_POWER] range. * * Return 0 on success. @@ -148,7 +170,7 @@ struct em_perf_domain *em_cpu_get(int cpu); struct em_perf_domain *em_pd_get(struct device *dev); int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *span, - bool milliwatts); + bool microwatts); void em_dev_unregister_perf_domain(struct device *dev); /** @@ -273,7 +295,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * pd_nrg = ------------------------ (4) * scale_cpu */ - return ps->cost * sum_util / scale_cpu; + return em_estimate_energy(ps->cost, sum_util, scale_cpu); } /** @@ -297,7 +319,7 @@ struct em_data_callback {}; static inline int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *span, - bool milliwatts) + bool microwatts) { return -EINVAL; } diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 6c373f2960e7..f82111837b8d 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -145,7 +145,7 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, /* * The power returned by active_state() is expected to be - * positive and to fit into 16 bits. + * positive and be in range. */ if (!power || power > EM_MAX_POWER) { dev_err(dev, "EM: invalid power: %lu\n", @@ -170,7 +170,7 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, goto free_ps_table; } } else { - power_res = em_scale_power(table[i].power); + power_res = table[i].power; cost = div64_u64(fmax * power_res, table[i].frequency); } @@ -201,9 +201,17 @@ static int em_create_pd(struct device *dev, int nr_states, { struct em_perf_domain *pd; struct device *cpu_dev; - int cpu, ret; + int cpu, ret, num_cpus; if (_is_cpu_device(dev)) { + num_cpus = cpumask_weight(cpus); + + /* Prevent max possible energy calculation to not overflow */ + if (num_cpus > EM_MAX_NUM_CPUS) { + dev_err(dev, "EM: too many CPUs, overflow possible\n"); + return -EINVAL; + } + pd = kzalloc(sizeof(*pd) + cpumask_size(), GFP_KERNEL); if (!pd) return -ENOMEM; @@ -314,13 +322,13 @@ EXPORT_SYMBOL_GPL(em_cpu_get); * @cpus : Pointer to cpumask_t, which in case of a CPU device is * obligatory. It can be taken from i.e. 'policy->cpus'. For other * type of devices this should be set to NULL. - * @milliwatts : Flag indicating that the power values are in milliWatts or + * @microwatts : Flag indicating that the power values are in micro-Watts or * in some other scale. It must be set properly. * * Create Energy Model tables for a performance domain using the callbacks * defined in cb. * - * The @milliwatts is important to set with correct value. Some kernel + * The @microwatts is important to set with correct value. Some kernel * sub-systems might rely on this flag and check if all devices in the EM are * using the same scale. * @@ -331,7 +339,7 @@ EXPORT_SYMBOL_GPL(em_cpu_get); */ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *cpus, - bool milliwatts) + bool microwatts) { unsigned long cap, prev_cap = 0; unsigned long flags = 0; @@ -381,8 +389,8 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, } } - if (milliwatts) - flags |= EM_PERF_DOMAIN_MILLIWATTS; + if (microwatts) + flags |= EM_PERF_DOMAIN_MICROWATTS; else if (cb->get_cost) flags |= EM_PERF_DOMAIN_ARTIFICIAL; -- cgit v1.2.3 From 5e0fd2026cdd474a85b3135c312912321e60f47a Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 7 Jul 2022 08:15:54 +0100 Subject: firmware: arm_scmi: Get detailed power scale from perf In SCMI v3.1 the power scale can be in micro-Watts. The upper layers, e.g. cpufreq and EM should handle received power values properly (upscale when needed). Thus, provide an interface which allows to check what is the scale for power values. The old interface allowed to distinguish between bogo-Watts and milli-Watts only (which was good for older SCMI spec). Acked-by: Sudeep Holla Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/firmware/arm_scmi/perf.c | 18 +++++++++++------- include/linux/scmi_protocol.h | 8 +++++++- 2 files changed, 18 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index bbb0331801ff..92414e53f908 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -170,8 +170,7 @@ struct perf_dom_info { struct scmi_perf_info { u32 version; int num_domains; - bool power_scale_mw; - bool power_scale_uw; + enum scmi_power_scale power_scale; u64 stats_addr; u32 stats_size; struct perf_dom_info *dom_info; @@ -201,9 +200,13 @@ static int scmi_perf_attributes_get(const struct scmi_protocol_handle *ph, u16 flags = le16_to_cpu(attr->flags); pi->num_domains = le16_to_cpu(attr->num_domains); - pi->power_scale_mw = POWER_SCALE_IN_MILLIWATT(flags); + + if (POWER_SCALE_IN_MILLIWATT(flags)) + pi->power_scale = SCMI_POWER_MILLIWATTS; if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3) - pi->power_scale_uw = POWER_SCALE_IN_MICROWATT(flags); + if (POWER_SCALE_IN_MICROWATT(flags)) + pi->power_scale = SCMI_POWER_MICROWATTS; + pi->stats_addr = le32_to_cpu(attr->stats_addr_low) | (u64)le32_to_cpu(attr->stats_addr_high) << 32; pi->stats_size = le32_to_cpu(attr->stats_size); @@ -792,11 +795,12 @@ static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph, return dom->fc_info && dom->fc_info->level_set_addr; } -static bool scmi_power_scale_mw_get(const struct scmi_protocol_handle *ph) +static enum scmi_power_scale +scmi_power_scale_get(const struct scmi_protocol_handle *ph) { struct scmi_perf_info *pi = ph->get_priv(ph); - return pi->power_scale_mw; + return pi->power_scale; } static const struct scmi_perf_proto_ops perf_proto_ops = { @@ -811,7 +815,7 @@ static const struct scmi_perf_proto_ops perf_proto_ops = { .freq_get = scmi_dvfs_freq_get, .est_power_get = scmi_dvfs_est_power_get, .fast_switch_possible = scmi_fast_switch_possible, - .power_scale_mw_get = scmi_power_scale_mw_get, + .power_scale_get = scmi_power_scale_get, }; static int scmi_perf_set_notify_enabled(const struct scmi_protocol_handle *ph, diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 704111f63993..a0a246310ba1 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -60,6 +60,12 @@ struct scmi_clock_info { }; }; +enum scmi_power_scale { + SCMI_POWER_BOGOWATTS, + SCMI_POWER_MILLIWATTS, + SCMI_POWER_MICROWATTS +}; + struct scmi_handle; struct scmi_device; struct scmi_protocol_handle; @@ -135,7 +141,7 @@ struct scmi_perf_proto_ops { unsigned long *rate, unsigned long *power); bool (*fast_switch_possible)(const struct scmi_protocol_handle *ph, struct device *dev); - bool (*power_scale_mw_get)(const struct scmi_protocol_handle *ph); + enum scmi_power_scale (*power_scale_get)(const struct scmi_protocol_handle *ph); }; /** -- cgit v1.2.3 From fcfe0ac2fcfae7d5fcad3d0375cb8ff38caf8aba Mon Sep 17 00:00:00 2001 From: Micah Morton Date: Wed, 8 Jun 2022 20:57:11 +0000 Subject: security: Add LSM hook to setgroups() syscall Give the LSM framework the ability to filter setgroups() syscalls. There are already analagous hooks for the set*uid() and set*gid() syscalls. The SafeSetID LSM will use this new hook to ensure setgroups() calls are allowed by the installed security policy. Tested by putting print statement in security_task_fix_setgroups() hook and confirming that it gets hit when userspace does a setgroups() syscall. Acked-by: Casey Schaufler Reviewed-by: Serge Hallyn Signed-off-by: Micah Morton --- include/linux/lsm_hook_defs.h | 1 + include/linux/lsm_hooks.h | 7 +++++++ include/linux/security.h | 7 +++++++ kernel/groups.c | 13 +++++++++++++ security/security.c | 5 +++++ 5 files changed, 33 insertions(+) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index eafa1d2489fd..806448173033 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -201,6 +201,7 @@ LSM_HOOK(int, 0, task_fix_setuid, struct cred *new, const struct cred *old, int flags) LSM_HOOK(int, 0, task_fix_setgid, struct cred *new, const struct cred * old, int flags) +LSM_HOOK(int, 0, task_fix_setgroups, struct cred *new, const struct cred * old) LSM_HOOK(int, 0, task_setpgid, struct task_struct *p, pid_t pgid) LSM_HOOK(int, 0, task_getpgid, struct task_struct *p) LSM_HOOK(int, 0, task_getsid, struct task_struct *p) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 91c8146649f5..84a0d7e02176 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -702,6 +702,13 @@ * @old is the set of credentials that are being replaced. * @flags contains one of the LSM_SETID_* values. * Return 0 on success. + * @task_fix_setgroups: + * Update the module's state after setting the supplementary group + * identity attributes of the current process. + * @new is the set of credentials that will be installed. Modifications + * should be made to this rather than to @current->cred. + * @old is the set of credentials that are being replaced. + * Return 0 on success. * @task_setpgid: * Check permission before setting the process group identifier of the * process @p to @pgid. diff --git a/include/linux/security.h b/include/linux/security.h index 7fc4e9f49f54..1dfd32c49fa3 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -415,6 +415,7 @@ int security_task_fix_setuid(struct cred *new, const struct cred *old, int flags); int security_task_fix_setgid(struct cred *new, const struct cred *old, int flags); +int security_task_fix_setgroups(struct cred *new, const struct cred *old); int security_task_setpgid(struct task_struct *p, pid_t pgid); int security_task_getpgid(struct task_struct *p); int security_task_getsid(struct task_struct *p); @@ -1098,6 +1099,12 @@ static inline int security_task_fix_setgid(struct cred *new, return 0; } +static inline int security_task_fix_setgroups(struct cred *new, + const struct cred *old) +{ + return 0; +} + static inline int security_task_setpgid(struct task_struct *p, pid_t pgid) { return 0; diff --git a/kernel/groups.c b/kernel/groups.c index 787b381c7c00..9aaed2a31073 100644 --- a/kernel/groups.c +++ b/kernel/groups.c @@ -134,13 +134,26 @@ EXPORT_SYMBOL(set_groups); int set_current_groups(struct group_info *group_info) { struct cred *new; + const struct cred *old; + int retval; new = prepare_creds(); if (!new) return -ENOMEM; + old = current_cred(); + set_groups(new, group_info); + + retval = security_task_fix_setgroups(new, old); + if (retval < 0) + goto error; + return commit_creds(new); + +error: + abort_creds(new); + return retval; } EXPORT_SYMBOL(set_current_groups); diff --git a/security/security.c b/security/security.c index 188b8f782220..15c686145ad6 100644 --- a/security/security.c +++ b/security/security.c @@ -1803,6 +1803,11 @@ int security_task_fix_setgid(struct cred *new, const struct cred *old, return call_int_hook(task_fix_setgid, 0, new, old, flags); } +int security_task_fix_setgroups(struct cred *new, const struct cred *old) +{ + return call_int_hook(task_fix_setgroups, 0, new, old); +} + int security_task_setpgid(struct task_struct *p, pid_t pgid) { return call_int_hook(task_setpgid, 0, p, pgid); -- cgit v1.2.3 From 0563231f93c6d1f582b168a47753b345c1e20d81 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 5 Jul 2022 18:44:54 -0400 Subject: tracing/events: Add __vstring() and __assign_vstr() helper macros There's several places that open code the following logic: TP_STRUCT__entry(__dynamic_array(char, msg, MSG_MAX)), TP_fast_assign(vsnprintf(__get_str(msg), MSG_MAX, vaf->fmt, *vaf->va);) To load a string created by variable array va_list. The main issue with this approach is that "MSG_MAX" usage in the __dynamic_array() portion. That actually just reserves the MSG_MAX in the event, and even wastes space because there's dynamic meta data also saved in the event to denote the offset and size of the dynamic array. It would have been better to just use a static __array() field. Instead, create __vstring() and __assign_vstr() that work like __string and __assign_str() but instead of taking a destination string to copy, take a format string and a va_list pointer and fill in the values. It uses the helper: #define __trace_event_vstr_len(fmt, va) \ ({ \ va_list __ap; \ int __ret; \ \ va_copy(__ap, *(va)); \ __ret = vsnprintf(NULL, 0, fmt, __ap) + 1; \ va_end(__ap); \ \ min(__ret, TRACE_EVENT_STR_MAX); \ }) To figure out the length to store the string. It may be slightly slower as it needs to run the vsnprintf() twice, but it now saves space on the ring buffer. Link: https://lkml.kernel.org/r/20220705224749.053570613@goodmis.org Cc: Dennis Dalessandro Cc: Ingo Molnar Cc: Andrew Morton Cc: Jason Gunthorpe Cc: Leon Romanovsky Cc: Kalle Valo Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Arend van Spriel Cc: Franky Lin Cc: Hante Meuleman Cc: Gregory Greenman Cc: Peter Chen Cc: Greg Kroah-Hartman Cc: Mathias Nyman Cc: Chunfeng Yun Cc: Bin Liu Cc: Marek Lindner Cc: Simon Wunderlich Cc: Antonio Quartulli Cc: Sven Eckelmann Cc: Johannes Berg Cc: Jim Cromie Signed-off-by: Steven Rostedt (Google) --- include/linux/trace_events.h | 18 ++++++++++++++++++ include/trace/stages/stage1_struct_define.h | 3 +++ include/trace/stages/stage2_data_offsets.h | 3 +++ include/trace/stages/stage4_event_fields.h | 3 +++ include/trace/stages/stage5_get_offsets.h | 4 ++++ include/trace/stages/stage6_event_callback.h | 7 +++++++ 6 files changed, 38 insertions(+) (limited to 'include/linux') diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index e6e95a9f07a5..b18759a673c6 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -916,6 +916,24 @@ perf_trace_buf_submit(void *raw_data, int size, int rctx, u16 type, #endif +#define TRACE_EVENT_STR_MAX 512 + +/* + * gcc warns that you can not use a va_list in an inlined + * function. But lets me make it into a macro :-/ + */ +#define __trace_event_vstr_len(fmt, va) \ +({ \ + va_list __ap; \ + int __ret; \ + \ + va_copy(__ap, *(va)); \ + __ret = vsnprintf(NULL, 0, fmt, __ap) + 1; \ + va_end(__ap); \ + \ + min(__ret, TRACE_EVENT_STR_MAX); \ +}) + #endif /* _LINUX_TRACE_EVENT_H */ /* diff --git a/include/trace/stages/stage1_struct_define.h b/include/trace/stages/stage1_struct_define.h index a16783419687..1b7bab60434c 100644 --- a/include/trace/stages/stage1_struct_define.h +++ b/include/trace/stages/stage1_struct_define.h @@ -26,6 +26,9 @@ #undef __string_len #define __string_len(item, src, len) __dynamic_array(char, item, -1) +#undef __vstring +#define __vstring(item, fmt, ap) __dynamic_array(char, item, -1) + #undef __bitmask #define __bitmask(item, nr_bits) __dynamic_array(char, item, -1) diff --git a/include/trace/stages/stage2_data_offsets.h b/include/trace/stages/stage2_data_offsets.h index 42fd1e8813ec..1b7a8f764fdd 100644 --- a/include/trace/stages/stage2_data_offsets.h +++ b/include/trace/stages/stage2_data_offsets.h @@ -32,6 +32,9 @@ #undef __string_len #define __string_len(item, src, len) __dynamic_array(char, item, -1) +#undef __vstring +#define __vstring(item, fmt, ap) __dynamic_array(char, item, -1) + #undef __bitmask #define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1) diff --git a/include/trace/stages/stage4_event_fields.h b/include/trace/stages/stage4_event_fields.h index e80cdc397a43..c3790ec7a453 100644 --- a/include/trace/stages/stage4_event_fields.h +++ b/include/trace/stages/stage4_event_fields.h @@ -38,6 +38,9 @@ #undef __string_len #define __string_len(item, src, len) __dynamic_array(char, item, -1) +#undef __vstring +#define __vstring(item, fmt, ap) __dynamic_array(char, item, -1) + #undef __bitmask #define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1) diff --git a/include/trace/stages/stage5_get_offsets.h b/include/trace/stages/stage5_get_offsets.h index 7ee5931300e6..fba4c24ed9e6 100644 --- a/include/trace/stages/stage5_get_offsets.h +++ b/include/trace/stages/stage5_get_offsets.h @@ -39,6 +39,10 @@ #undef __string_len #define __string_len(item, src, len) __dynamic_array(char, item, (len) + 1) +#undef __vstring +#define __vstring(item, fmt, ap) __dynamic_array(char, item, \ + __trace_event_vstr_len(fmt, ap)) + #undef __rel_dynamic_array #define __rel_dynamic_array(type, item, len) \ __item_length = (len) * sizeof(type); \ diff --git a/include/trace/stages/stage6_event_callback.h b/include/trace/stages/stage6_event_callback.h index e1724f73594b..0f51f6b3ab70 100644 --- a/include/trace/stages/stage6_event_callback.h +++ b/include/trace/stages/stage6_event_callback.h @@ -24,6 +24,9 @@ #undef __string_len #define __string_len(item, src, len) __dynamic_array(char, item, -1) +#undef __vstring +#define __vstring(item, fmt, ap) __dynamic_array(char, item, -1) + #undef __assign_str #define __assign_str(dst, src) \ strcpy(__get_str(dst), (src) ? (const char *)(src) : "(null)"); @@ -35,6 +38,10 @@ __get_str(dst)[len] = '\0'; \ } while(0) +#undef __assign_vstr +#define __assign_vstr(dst, fmt, va) \ + vsnprintf(__get_str(dst), TRACE_EVENT_STR_MAX, fmt, *(va)) + #undef __bitmask #define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1) -- cgit v1.2.3 From e68c5dcf0aacc48a23cedcb3ce81b8c60837f48c Mon Sep 17 00:00:00 2001 From: Jaehee Park Date: Wed, 13 Jul 2022 16:40:47 -0700 Subject: net: ipv4: new arp_accept option to accept garp only if in-network In many deployments, we want the option to not learn a neighbor from garp if the src ip is not in the same subnet as an address configured on the interface that received the garp message. net.ipv4.arp_accept sysctl is currently used to control creation of a neigh from a received garp packet. This patch adds a new option '2' to net.ipv4.arp_accept which extends option '1' by including the subnet check. Signed-off-by: Jaehee Park Suggested-by: Roopa Prabhu Signed-off-by: Jakub Kicinski --- Documentation/networking/ip-sysctl.rst | 9 ++++++--- include/linux/inetdevice.h | 2 +- net/ipv4/arp.c | 24 ++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index 2b329042b38c..b31601405c54 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -1633,12 +1633,15 @@ arp_notify - BOOLEAN or hardware address changes. == ========================================================== -arp_accept - BOOLEAN - Define behavior for gratuitous ARP frames who's IP is not - already present in the ARP table: +arp_accept - INTEGER + Define behavior for accepting gratuitous ARP (garp) frames from devices + that are not already present in the ARP table: - 0 - don't create new entries in the ARP table - 1 - create new entries in the ARP table + - 2 - create new entries only if the source IP address is in the same + subnet as an address configured on the interface that received the + garp message. Both replies and requests type gratuitous arp will trigger the ARP table to be updated, if this setting is on. diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index ead323243e7b..ddb27fc0ee8c 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -131,7 +131,7 @@ static inline void ipv4_devconf_setall(struct in_device *in_dev) IN_DEV_ORCONF((in_dev), IGNORE_ROUTES_WITH_LINKDOWN) #define IN_DEV_ARPFILTER(in_dev) IN_DEV_ORCONF((in_dev), ARPFILTER) -#define IN_DEV_ARP_ACCEPT(in_dev) IN_DEV_ORCONF((in_dev), ARP_ACCEPT) +#define IN_DEV_ARP_ACCEPT(in_dev) IN_DEV_MAXCONF((in_dev), ARP_ACCEPT) #define IN_DEV_ARP_ANNOUNCE(in_dev) IN_DEV_MAXCONF((in_dev), ARP_ANNOUNCE) #define IN_DEV_ARP_IGNORE(in_dev) IN_DEV_MAXCONF((in_dev), ARP_IGNORE) #define IN_DEV_ARP_NOTIFY(in_dev) IN_DEV_MAXCONF((in_dev), ARP_NOTIFY) diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index af2f12ffc9ca..87c7e3fc5197 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -429,6 +429,26 @@ static int arp_ignore(struct in_device *in_dev, __be32 sip, __be32 tip) return !inet_confirm_addr(net, in_dev, sip, tip, scope); } +static int arp_accept(struct in_device *in_dev, __be32 sip) +{ + struct net *net = dev_net(in_dev->dev); + int scope = RT_SCOPE_LINK; + + switch (IN_DEV_ARP_ACCEPT(in_dev)) { + case 0: /* Don't create new entries from garp */ + return 0; + case 1: /* Create new entries from garp */ + return 1; + case 2: /* Create a neighbor in the arp table only if sip + * is in the same subnet as an address configured + * on the interface that received the garp message + */ + return !!inet_confirm_addr(net, in_dev, sip, 0, scope); + default: + return 0; + } +} + static int arp_filter(__be32 sip, __be32 tip, struct net_device *dev) { struct rtable *rt; @@ -868,12 +888,12 @@ static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb) n = __neigh_lookup(&arp_tbl, &sip, dev, 0); addr_type = -1; - if (n || IN_DEV_ARP_ACCEPT(in_dev)) { + if (n || arp_accept(in_dev, sip)) { is_garp = arp_is_garp(net, dev, &addr_type, arp->ar_op, sip, tip, sha, tha); } - if (IN_DEV_ARP_ACCEPT(in_dev)) { + if (arp_accept(in_dev, sip)) { /* Unsolicited ARP is not accepted by default. It is possible, that this option should be enabled for some devices (strip is candidate) -- cgit v1.2.3 From 868941b14441282ba08761b770fc6cad69d5bdb7 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 29 Jun 2022 15:07:00 +0200 Subject: fs: remove no_llseek Now that all callers of ->llseek are going through vfs_llseek(), we don't gain anything by keeping no_llseek around. Nothing actually calls it and setting ->llseek to no_lseek is completely equivalent to leaving it NULL. Longer term (== by the end of merge window) we want to remove all such intializations. To simplify the merge window this commit does *not* touch initializers - it only defines no_llseek as NULL (and simplifies the tests on file opening). At -rc1 we'll need do a mechanical removal of no_llseek - git grep -l -w no_llseek | grep -v porting.rst | while read i; do sed -i '/\/d' $i done would do it. Signed-off-by: Jason A. Donenfeld Signed-off-by: Al Viro --- Documentation/filesystems/porting.rst | 8 ++++++++ drivers/gpu/drm/drm_file.c | 3 +-- fs/file_table.c | 2 +- fs/open.c | 2 -- fs/read_write.c | 6 ------ include/linux/fs.h | 2 +- kernel/bpf/bpf_iter.c | 3 +-- 7 files changed, 12 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index 2e0e4f0e0c6f..aee9aaf9f3df 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -914,3 +914,11 @@ Calling conventions for file_open_root() changed; now it takes struct path * instead of passing mount and dentry separately. For callers that used to pass mnt_root> pair (i.e. the root of given mount), a new helper is provided - file_open_root_mnt(). In-tree users adjusted. + +--- + +**mandatory** + +no_llseek is gone; don't set .llseek to that - just leave it NULL instead. +Checks for "does that file have llseek(2), or should it fail with ESPIPE" +should be done by looking at FMODE_LSEEK in file->f_mode. diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index ed25168619fc..dc7d2e5b16c8 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -552,8 +552,7 @@ EXPORT_SYMBOL(drm_release_noglobal); * Since events are used by the KMS API for vblank and page flip completion this * means all modern display drivers must use it. * - * @offset is ignored, DRM events are read like a pipe. Therefore drivers also - * must set the &file_operation.llseek to no_llseek(). Polling support is + * @offset is ignored, DRM events are read like a pipe. Polling support is * provided by drm_poll(). * * This function will only ever read a full event. Therefore userspace must diff --git a/fs/file_table.c b/fs/file_table.c index 0658b822beeb..5727a63a7b67 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -235,7 +235,7 @@ static struct file *alloc_file(const struct path *path, int flags, file->f_mapping = path->dentry->d_inode->i_mapping; file->f_wb_err = filemap_sample_wb_err(file->f_mapping); file->f_sb_err = file_sample_sb_err(file); - if (fop->llseek && fop->llseek != no_llseek) + if (fop->llseek) file->f_mode |= FMODE_LSEEK; if ((file->f_mode & FMODE_READ) && likely(fop->read || fop->read_iter)) diff --git a/fs/open.c b/fs/open.c index 4488bd77c390..07c332753a36 100644 --- a/fs/open.c +++ b/fs/open.c @@ -860,8 +860,6 @@ static int do_dentry_open(struct file *f, f->f_mode |= FMODE_CAN_WRITE; if ((f->f_mode & FMODE_LSEEK) && !f->f_op->llseek) f->f_mode &= ~FMODE_LSEEK; - if ((f->f_mode & FMODE_LSEEK) && f->f_op->llseek == no_llseek) - f->f_mode &= ~FMODE_LSEEK; if (f->f_mapping->a_ops && f->f_mapping->a_ops->direct_IO) f->f_mode |= FMODE_CAN_ODIRECT; diff --git a/fs/read_write.c b/fs/read_write.c index d94b6dbba6f9..6b2849b34781 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -227,12 +227,6 @@ loff_t noop_llseek(struct file *file, loff_t offset, int whence) } EXPORT_SYMBOL(noop_llseek); -loff_t no_llseek(struct file *file, loff_t offset, int whence) -{ - return -ESPIPE; -} -EXPORT_SYMBOL(no_llseek); - loff_t default_llseek(struct file *file, loff_t offset, int whence) { struct inode *inode = file_inode(file); diff --git a/include/linux/fs.h b/include/linux/fs.h index 9ad5e3520fae..294932167335 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3022,7 +3022,7 @@ extern long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, extern void file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping); extern loff_t noop_llseek(struct file *file, loff_t offset, int whence); -extern loff_t no_llseek(struct file *file, loff_t offset, int whence); +#define no_llseek NULL extern loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize); extern loff_t generic_file_llseek(struct file *file, loff_t offset, int whence); extern loff_t generic_file_llseek_size(struct file *file, loff_t offset, diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index d5d96ceca105..8af0cbf9c0cd 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -81,10 +81,9 @@ static bool bpf_iter_support_resched(struct seq_file *seq) #define MAX_ITER_OBJECTS 1000000 /* bpf_seq_read, a customized and simpler version for bpf iterator. - * no_llseek is assumed for this file. * The following are differences from seq_read(): * . fixed buffer size (PAGE_SIZE) - * . assuming no_llseek + * . assuming NULL ->llseek() * . stop() may call bpf program, handling potential overflow there */ static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, -- cgit v1.2.3 From bc72d938c149197688ae3b3ecaa25d4aee8653cb Mon Sep 17 00:00:00 2001 From: Dmitry Rokosov Date: Wed, 1 Jun 2022 17:48:32 +0000 Subject: iio: trigger: move trig->owner init to trigger allocate() stage To provide a new IIO trigger to the IIO core, usually driver executes the following pipeline: allocate()/register()/get(). Before, IIO core assigned trig->owner as a pointer to the module which registered this trigger at the register() stage. But actually the trigger object is owned by the module earlier, on the allocate() stage, when trigger object is successfully allocated for the driver. This patch moves trig->owner initialization from register() stage of trigger initialization pipeline to allocate() stage to eliminate all misunderstandings and time gaps between trigger object creation and owner acquiring. Signed-off-by: Dmitry Rokosov Link: https://lore.kernel.org/r/20220601174837.20292-1-ddrokosov@sberdevices.ru Signed-off-by: Jonathan Cameron --- drivers/iio/industrialio-trigger.c | 46 +++++++++++++++++++++----------------- include/linux/iio/iio.h | 9 +++++--- include/linux/iio/trigger.h | 21 +++++++++-------- 3 files changed, 41 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index 26d610d4cbb8..efc01584b3f9 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -63,13 +63,10 @@ ATTRIBUTE_GROUPS(iio_trig_dev); static struct iio_trigger *__iio_trigger_find_by_name(const char *name); -int __iio_trigger_register(struct iio_trigger *trig_info, - struct module *this_mod) +int iio_trigger_register(struct iio_trigger *trig_info) { int ret; - trig_info->owner = this_mod; - trig_info->id = ida_alloc(&iio_trigger_ida, GFP_KERNEL); if (trig_info->id < 0) return trig_info->id; @@ -100,7 +97,7 @@ error_unregister_id: ida_free(&iio_trigger_ida, trig_info->id); return ret; } -EXPORT_SYMBOL(__iio_trigger_register); +EXPORT_SYMBOL(iio_trigger_register); void iio_trigger_unregister(struct iio_trigger *trig_info) { @@ -547,8 +544,9 @@ static void iio_trig_subirqunmask(struct irq_data *d) trig->subirqs[d->irq - trig->subirq_base].enabled = true; } -static __printf(2, 0) +static __printf(3, 0) struct iio_trigger *viio_trigger_alloc(struct device *parent, + struct module *this_mod, const char *fmt, va_list vargs) { @@ -578,6 +576,8 @@ struct iio_trigger *viio_trigger_alloc(struct device *parent, INIT_LIST_HEAD(&trig->list); + trig->owner = this_mod; + trig->subirq_chip.name = trig->name; trig->subirq_chip.irq_mask = &iio_trig_subirqmask; trig->subirq_chip.irq_unmask = &iio_trig_subirqunmask; @@ -598,8 +598,9 @@ free_trig: } /** - * iio_trigger_alloc - Allocate a trigger + * __iio_trigger_alloc - Allocate a trigger * @parent: Device to allocate iio_trigger for + * @this_mod: module allocating the trigger * @fmt: trigger name format. If it includes format * specifiers, the additional arguments following * format are formatted and inserted in the resulting @@ -607,18 +608,20 @@ free_trig: * RETURNS: * Pointer to allocated iio_trigger on success, NULL on failure. */ -struct iio_trigger *iio_trigger_alloc(struct device *parent, const char *fmt, ...) +struct iio_trigger *__iio_trigger_alloc(struct device *parent, + struct module *this_mod, + const char *fmt, ...) { struct iio_trigger *trig; va_list vargs; va_start(vargs, fmt); - trig = viio_trigger_alloc(parent, fmt, vargs); + trig = viio_trigger_alloc(parent, this_mod, fmt, vargs); va_end(vargs); return trig; } -EXPORT_SYMBOL(iio_trigger_alloc); +EXPORT_SYMBOL(__iio_trigger_alloc); void iio_trigger_free(struct iio_trigger *trig) { @@ -633,10 +636,11 @@ static void devm_iio_trigger_release(struct device *dev, void *res) } /** - * devm_iio_trigger_alloc - Resource-managed iio_trigger_alloc() + * __devm_iio_trigger_alloc - Resource-managed iio_trigger_alloc() * Managed iio_trigger_alloc. iio_trigger allocated with this function is * automatically freed on driver detach. * @parent: Device to allocate iio_trigger for + * @this_mod: module allocating the trigger * @fmt: trigger name format. If it includes format * specifiers, the additional arguments following * format are formatted and inserted in the resulting @@ -646,7 +650,9 @@ static void devm_iio_trigger_release(struct device *dev, void *res) * RETURNS: * Pointer to allocated iio_trigger on success, NULL on failure. */ -struct iio_trigger *devm_iio_trigger_alloc(struct device *parent, const char *fmt, ...) +struct iio_trigger *__devm_iio_trigger_alloc(struct device *parent, + struct module *this_mod, + const char *fmt, ...) { struct iio_trigger **ptr, *trig; va_list vargs; @@ -658,7 +664,7 @@ struct iio_trigger *devm_iio_trigger_alloc(struct device *parent, const char *fm /* use raw alloc_dr for kmalloc caller tracing */ va_start(vargs, fmt); - trig = viio_trigger_alloc(parent, fmt, vargs); + trig = viio_trigger_alloc(parent, this_mod, fmt, vargs); va_end(vargs); if (trig) { *ptr = trig; @@ -669,7 +675,7 @@ struct iio_trigger *devm_iio_trigger_alloc(struct device *parent, const char *fm return trig; } -EXPORT_SYMBOL_GPL(devm_iio_trigger_alloc); +EXPORT_SYMBOL_GPL(__devm_iio_trigger_alloc); static void devm_iio_trigger_unreg(void *trigger_info) { @@ -677,10 +683,9 @@ static void devm_iio_trigger_unreg(void *trigger_info) } /** - * __devm_iio_trigger_register - Resource-managed iio_trigger_register() + * devm_iio_trigger_register - Resource-managed iio_trigger_register() * @dev: device this trigger was allocated for * @trig_info: trigger to register - * @this_mod: module registering the trigger * * Managed iio_trigger_register(). The IIO trigger registered with this * function is automatically unregistered on driver detach. This function @@ -690,19 +695,18 @@ static void devm_iio_trigger_unreg(void *trigger_info) * RETURNS: * 0 on success, negative error number on failure. */ -int __devm_iio_trigger_register(struct device *dev, - struct iio_trigger *trig_info, - struct module *this_mod) +int devm_iio_trigger_register(struct device *dev, + struct iio_trigger *trig_info) { int ret; - ret = __iio_trigger_register(trig_info, this_mod); + ret = iio_trigger_register(trig_info); if (ret) return ret; return devm_add_action_or_reset(dev, devm_iio_trigger_unreg, trig_info); } -EXPORT_SYMBOL_GPL(__devm_iio_trigger_register); +EXPORT_SYMBOL_GPL(devm_iio_trigger_register); bool iio_trigger_using_own(struct iio_dev *indio_dev) { diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index d9b4a9ca9a0f..5dfbfc991c69 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -727,10 +727,13 @@ static inline void *iio_priv(const struct iio_dev *indio_dev) void iio_device_free(struct iio_dev *indio_dev); struct iio_dev *devm_iio_device_alloc(struct device *parent, int sizeof_priv); -__printf(2, 3) -struct iio_trigger *devm_iio_trigger_alloc(struct device *parent, - const char *fmt, ...); +#define devm_iio_trigger_alloc(parent, fmt, ...) \ + __devm_iio_trigger_alloc((parent), THIS_MODULE, (fmt), ##__VA_ARGS__) +__printf(3, 4) +struct iio_trigger *__devm_iio_trigger_alloc(struct device *parent, + struct module *this_mod, + const char *fmt, ...); /** * iio_get_debugfs_dentry() - helper function to get the debugfs_dentry * @indio_dev: IIO device structure for device diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h index 03b1d6863436..f6360d9a492d 100644 --- a/include/linux/iio/trigger.h +++ b/include/linux/iio/trigger.h @@ -131,16 +131,10 @@ static inline void *iio_trigger_get_drvdata(struct iio_trigger *trig) * iio_trigger_register() - register a trigger with the IIO core * @trig_info: trigger to be registered **/ -#define iio_trigger_register(trig_info) \ - __iio_trigger_register((trig_info), THIS_MODULE) -int __iio_trigger_register(struct iio_trigger *trig_info, - struct module *this_mod); +int iio_trigger_register(struct iio_trigger *trig_info); -#define devm_iio_trigger_register(dev, trig_info) \ - __devm_iio_trigger_register((dev), (trig_info), THIS_MODULE) -int __devm_iio_trigger_register(struct device *dev, - struct iio_trigger *trig_info, - struct module *this_mod); +int devm_iio_trigger_register(struct device *dev, + struct iio_trigger *trig_info); /** * iio_trigger_unregister() - unregister a trigger from the core @@ -168,8 +162,13 @@ void iio_trigger_poll_chained(struct iio_trigger *trig); irqreturn_t iio_trigger_generic_data_rdy_poll(int irq, void *private); -__printf(2, 3) -struct iio_trigger *iio_trigger_alloc(struct device *parent, const char *fmt, ...); +#define iio_trigger_alloc(parent, fmt, ...) \ + __iio_trigger_alloc((parent), THIS_MODULE, (fmt), ##__VA_ARGS__) + +__printf(3, 4) +struct iio_trigger *__iio_trigger_alloc(struct device *parent, + struct module *this_mod, + const char *fmt, ...); void iio_trigger_free(struct iio_trigger *trig); /** -- cgit v1.2.3 From f484da847a01936e1d087a80cc96e8254492527c Mon Sep 17 00:00:00 2001 From: Mark Bloch Date: Sun, 3 Jul 2022 13:54:03 -0700 Subject: net/mlx5: Expose the ability to point to any UID from shared UID Expose shared_object_to_user_object_allowed, this capability means an object created with shared UID can point to any UID. Signed-off-by: Mark Bloch Reviewed-by: Maor Gottlieb Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 8e87eb47f9dc..9321d774e2d8 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1371,7 +1371,9 @@ enum { }; struct mlx5_ifc_cmd_hca_cap_bits { - u8 reserved_at_0[0x1f]; + u8 reserved_at_0[0x10]; + u8 shared_object_to_user_object_allowed[0x1]; + u8 reserved_at_13[0xe]; u8 vhca_resource_manager[0x1]; u8 hca_cap_2[0x1]; @@ -8507,7 +8509,7 @@ struct mlx5_ifc_create_flow_table_out_bits { struct mlx5_ifc_create_flow_table_in_bits { u8 opcode[0x10]; - u8 reserved_at_10[0x10]; + u8 uid[0x10]; u8 reserved_at_20[0x10]; u8 op_mod[0x10]; -- cgit v1.2.3 From 6c27c56cdc69608211617eea217e1a6cecccc675 Mon Sep 17 00:00:00 2001 From: Mark Bloch Date: Sun, 3 Jul 2022 13:54:04 -0700 Subject: net/mlx5: fs, expose flow table ID to users Expose the flow table ID to users. This will be used by downstream patches to allow creating steering rules that point to a flow table ID. Signed-off-by: Mark Bloch Reviewed-by: Maor Gottlieb Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 6 ++++++ include/linux/mlx5/fs.h | 1 + 2 files changed, 7 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 14187e50e2f9..1da3dc7c95fa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -1195,6 +1195,12 @@ struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, } EXPORT_SYMBOL(mlx5_create_flow_table); +u32 mlx5_flow_table_id(struct mlx5_flow_table *ft) +{ + return ft->id; +} +EXPORT_SYMBOL(mlx5_flow_table_id); + struct mlx5_flow_table * mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, struct mlx5_flow_table_attr *ft_attr, u16 vport) diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index ece3e35622d7..eee07d416b56 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -315,4 +315,5 @@ struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev, void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev, struct mlx5_pkt_reformat *reformat); +u32 mlx5_flow_table_id(struct mlx5_flow_table *ft); #endif -- cgit v1.2.3 From b0bb369ee451323968b31392a86398f15a2ba183 Mon Sep 17 00:00:00 2001 From: Mark Bloch Date: Sun, 3 Jul 2022 13:54:05 -0700 Subject: net/mlx5: fs, allow flow table creation with a UID Add UID field to flow table attributes to allow creating flow tables with a non default (zero) uid. Signed-off-by: Mark Bloch Reviewed-by: Alex Vesker Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c | 16 ++++++++++------ drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h | 2 +- drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 2 +- .../net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c | 1 + .../net/ethernet/mellanox/mlx5/core/steering/dr_table.c | 8 +++++--- .../net/ethernet/mellanox/mlx5/core/steering/dr_types.h | 1 + drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c | 7 ++++--- .../net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h | 3 ++- include/linux/mlx5/fs.h | 1 + 9 files changed, 26 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 735dc805dad7..e735e19461ba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -50,10 +50,12 @@ static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns, static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int size, + struct mlx5_flow_table_attr *ft_attr, struct mlx5_flow_table *next_ft) { - ft->max_fte = size ? roundup_pow_of_two(size) : 1; + int max_fte = ft_attr->max_fte; + + ft->max_fte = max_fte ? roundup_pow_of_two(max_fte) : 1; return 0; } @@ -258,7 +260,7 @@ static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns, static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int size, + struct mlx5_flow_table_attr *ft_attr, struct mlx5_flow_table *next_ft) { int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT); @@ -267,17 +269,19 @@ static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {}; u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {}; struct mlx5_core_dev *dev = ns->dev; + unsigned int size; int err; - if (size != POOL_NEXT_SIZE) - size = roundup_pow_of_two(size); - size = mlx5_ft_pool_get_avail_sz(dev, ft->type, size); + if (ft_attr->max_fte != POOL_NEXT_SIZE) + size = roundup_pow_of_two(ft_attr->max_fte); + size = mlx5_ft_pool_get_avail_sz(dev, ft->type, ft_attr->max_fte); if (!size) return -ENOSPC; MLX5_SET(create_flow_table_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_TABLE); + MLX5_SET(create_flow_table_in, in, uid, ft_attr->uid); MLX5_SET(create_flow_table_in, in, table_type, ft->type); MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level); MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, size ? ilog2(size) : 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h index 274004e80f03..8ef4254b9ea1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h @@ -38,7 +38,7 @@ struct mlx5_flow_cmds { int (*create_flow_table)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int size, + struct mlx5_flow_table_attr *ft_attr, struct mlx5_flow_table *next_ft); int (*destroy_flow_table)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 1da3dc7c95fa..35d89edb1bcd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -1155,7 +1155,7 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa find_next_chained_ft(fs_prio); ft->def_miss_action = ns->def_miss_action; ft->ns = ns; - err = root->cmds->create_flow_table(root, ft, ft_attr->max_fte, next_ft); + err = root->cmds->create_flow_table(root, ft, ft_attr, next_ft); if (err) goto free_ft; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c index 223c8741b7ae..16d65fe4f654 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c @@ -439,6 +439,7 @@ int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev, MLX5_SET(create_flow_table_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_TABLE); MLX5_SET(create_flow_table_in, in, table_type, attr->table_type); + MLX5_SET(create_flow_table_in, in, uid, attr->uid); ft_mdev = MLX5_ADDR_OF(create_flow_table_in, in, flow_table_context); MLX5_SET(flow_table_context, ft_mdev, termination_table, attr->term_tbl); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c index e5f6412baea9..31d443dd8386 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c @@ -214,7 +214,7 @@ static int dr_table_destroy_sw_owned_tbl(struct mlx5dr_table *tbl) tbl->table_type); } -static int dr_table_create_sw_owned_tbl(struct mlx5dr_table *tbl) +static int dr_table_create_sw_owned_tbl(struct mlx5dr_table *tbl, u16 uid) { bool en_encap = !!(tbl->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT); bool en_decap = !!(tbl->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); @@ -236,6 +236,7 @@ static int dr_table_create_sw_owned_tbl(struct mlx5dr_table *tbl) ft_attr.sw_owner = true; ft_attr.decap_en = en_decap; ft_attr.reformat_en = en_encap; + ft_attr.uid = uid; ret = mlx5dr_cmd_create_flow_table(tbl->dmn->mdev, &ft_attr, NULL, &tbl->table_id); @@ -243,7 +244,8 @@ static int dr_table_create_sw_owned_tbl(struct mlx5dr_table *tbl) return ret; } -struct mlx5dr_table *mlx5dr_table_create(struct mlx5dr_domain *dmn, u32 level, u32 flags) +struct mlx5dr_table *mlx5dr_table_create(struct mlx5dr_domain *dmn, u32 level, + u32 flags, u16 uid) { struct mlx5dr_table *tbl; int ret; @@ -263,7 +265,7 @@ struct mlx5dr_table *mlx5dr_table_create(struct mlx5dr_domain *dmn, u32 level, u if (ret) goto free_tbl; - ret = dr_table_create_sw_owned_tbl(tbl); + ret = dr_table_create_sw_owned_tbl(tbl, uid); if (ret) goto uninit_tbl; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h index 98320e3945ad..50b0dd4fb4a9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h @@ -1200,6 +1200,7 @@ struct mlx5dr_cmd_query_flow_table_details { struct mlx5dr_cmd_create_flow_table_attr { u32 table_type; + u16 uid; u64 icm_addr_rx; u64 icm_addr_tx; u8 level; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c index 6a9abba92df6..c30ed8e18458 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c @@ -62,7 +62,7 @@ static int set_miss_action(struct mlx5_flow_root_namespace *ns, static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int size, + struct mlx5_flow_table_attr *ft_attr, struct mlx5_flow_table *next_ft) { struct mlx5dr_table *tbl; @@ -71,7 +71,7 @@ static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns, if (mlx5_dr_is_fw_table(ft->flags)) return mlx5_fs_cmd_get_fw_cmds()->create_flow_table(ns, ft, - size, + ft_attr, next_ft); flags = ft->flags; /* turn off encap/decap if not supported for sw-str by fw */ @@ -79,7 +79,8 @@ static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns, flags = ft->flags & ~(MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); - tbl = mlx5dr_table_create(ns->fs_dr_domain.dr_domain, ft->level, flags); + tbl = mlx5dr_table_create(ns->fs_dr_domain.dr_domain, ft->level, flags, + ft_attr->uid); if (!tbl) { mlx5_core_err(ns->dev, "Failed creating dr flow_table\n"); return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h index 7626c85643b1..3bb14860b36d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h @@ -51,7 +51,8 @@ void mlx5dr_domain_set_peer(struct mlx5dr_domain *dmn, struct mlx5dr_domain *peer_dmn); struct mlx5dr_table * -mlx5dr_table_create(struct mlx5dr_domain *domain, u32 level, u32 flags); +mlx5dr_table_create(struct mlx5dr_domain *domain, u32 level, u32 flags, + u16 uid); struct mlx5dr_table * mlx5dr_table_get_from_fs_ft(struct mlx5_flow_table *ft); diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index eee07d416b56..8e73c377da2c 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -178,6 +178,7 @@ struct mlx5_flow_table_attr { int max_fte; u32 level; u32 flags; + u16 uid; struct mlx5_flow_table *next_ft; struct { -- cgit v1.2.3 From 335e52c28cf9954d65b819cb68912fd32de3c844 Mon Sep 17 00:00:00 2001 From: David Gow Date: Fri, 1 Jul 2022 17:16:19 +0800 Subject: mm: Add PAGE_ALIGN_DOWN macro This is just the same as PAGE_ALIGN(), but rounds the address down, not up. Suggested-by: Dmitry Vyukov Signed-off-by: David Gow Acked-by: Andrew Morton Signed-off-by: Richard Weinberger --- include/linux/mm.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index cf3d0d673f6b..bd58ee018555 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -221,6 +221,9 @@ int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, /* to align the pointer to the (next) page boundary */ #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) +/* to align the pointer to the (prev) page boundary */ +#define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE) + /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE) -- cgit v1.2.3 From 6077c943beee407168f72ece745b0aeaef6b896f Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Fri, 15 Jul 2022 10:05:08 -0500 Subject: mm: rename is_pinnable_page() to is_longterm_pinnable_page() Patch series "Add MEMORY_DEVICE_COHERENT for coherent device memory mapping", v9. This patch series introduces MEMORY_DEVICE_COHERENT, a type of memory owned by a device that can be mapped into CPU page tables like MEMORY_DEVICE_GENERIC and can also be migrated like MEMORY_DEVICE_PRIVATE. This patch series is mostly self-contained except for a few places where it needs to update other subsystems to handle the new memory type. System stability and performance are not affected according to our ongoing testing, including xfstests. How it works: The system BIOS advertises the GPU device memory (aka VRAM) as SPM (special purpose memory) in the UEFI system address map. The amdgpu driver registers the memory with devmap as MEMORY_DEVICE_COHERENT using devm_memremap_pages. The initial user for this hardware page migration capability is the Frontier supercomputer project. This functionality is not AMD-specific. We expect other GPU vendors to find this functionality useful, and possibly other hardware types in the future. Our test nodes in the lab are similar to the Frontier configuration, with .5 TB of system memory plus 256 GB of device memory split across 4 GPUs, all in a single coherent address space. Page migration is expected to improve application efficiency significantly. We will report empirical results as they become available. Coherent device type pages at gup are now migrated back to system memory if they are being pinned long-term (FOLL_LONGTERM). The reason is, that long-term pinning would interfere with the device memory manager owning the device-coherent pages (e.g. evictions in TTM). These series incorporate Alistair Popple patches to do this migration from pin_user_pages() calls. hmm_gup_test has been added to hmm-test to test different get user pages calls. This series includes handling of device-managed anonymous pages returned by vm_normal_pages. Although they behave like normal pages for purposes of mapping in CPU page tables and for COW, they do not support LRU lists, NUMA migration or THP. We also introduced a FOLL_LRU flag that adds the same behaviour to follow_page and related APIs, to allow callers to specify that they expect to put pages on an LRU list. This patch (of 14): is_pinnable_page() and folio_is_pinnable() are renamed to is_longterm_pinnable_page() and folio_is_longterm_pinnable() respectively. These functions are used in the FOLL_LONGTERM flag context. Link: https://lkml.kernel.org/r/20220715150521.18165-1-alex.sierra@amd.com Link: https://lkml.kernel.org/r/20220715150521.18165-2-alex.sierra@amd.com Signed-off-by: Alex Sierra Reviewed-by: David Hildenbrand Cc: Jason Gunthorpe Cc: Felix Kuehling Cc: Ralph Campbell Cc: Christoph Hellwig Cc: Jerome Glisse Cc: Alistair Popple Cc: Matthew Wilcox Signed-off-by: Andrew Morton --- include/linux/mm.h | 8 ++++---- mm/gup.c | 4 ++-- mm/gup_test.c | 2 +- mm/hugetlb.c | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 9cc02a7e503b..3c044e38958c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1607,7 +1607,7 @@ static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma, /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */ #ifdef CONFIG_MIGRATION -static inline bool is_pinnable_page(struct page *page) +static inline bool is_longterm_pinnable_page(struct page *page) { #ifdef CONFIG_CMA int mt = get_pageblock_migratetype(page); @@ -1618,15 +1618,15 @@ static inline bool is_pinnable_page(struct page *page) return !is_zone_movable_page(page) || is_zero_pfn(page_to_pfn(page)); } #else -static inline bool is_pinnable_page(struct page *page) +static inline bool is_longterm_pinnable_page(struct page *page) { return true; } #endif -static inline bool folio_is_pinnable(struct folio *folio) +static inline bool folio_is_longterm_pinnable(struct folio *folio) { - return is_pinnable_page(&folio->page); + return is_longterm_pinnable_page(&folio->page); } static inline void set_page_zone(struct page *page, enum zone_type zone) diff --git a/mm/gup.c b/mm/gup.c index 3129b754ade3..a9940e3b3181 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -133,7 +133,7 @@ struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags) * path. */ if (unlikely((flags & FOLL_LONGTERM) && - !is_pinnable_page(page))) + !is_longterm_pinnable_page(page))) return NULL; /* @@ -1923,7 +1923,7 @@ static long check_and_migrate_movable_pages(unsigned long nr_pages, continue; prev_folio = folio; - if (folio_is_pinnable(folio)) + if (folio_is_longterm_pinnable(folio)) continue; /* diff --git a/mm/gup_test.c b/mm/gup_test.c index d974dec19e1c..12b0a91767d3 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -53,7 +53,7 @@ static void verify_dma_pinned(unsigned int cmd, struct page **pages, dump_page(page, "gup_test failure"); break; } else if (cmd == PIN_LONGTERM_BENCHMARK && - WARN(!is_pinnable_page(page), + WARN(!is_longterm_pinnable_page(page), "pages[%lu] is NOT pinnable but pinned\n", i)) { dump_page(page, "gup_test failure"); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 064da8ffbac6..ffdf3fc4a83f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1129,7 +1129,7 @@ static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) lockdep_assert_held(&hugetlb_lock); list_for_each_entry(page, &h->hugepage_freelists[nid], lru) { - if (pin && !is_pinnable_page(page)) + if (pin && !is_longterm_pinnable_page(page)) continue; if (PageHWPoison(page)) -- cgit v1.2.3 From 5bb88dc571b1cbf0284100a317fb21ab7d03e40c Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Fri, 15 Jul 2022 10:05:09 -0500 Subject: mm: move page zone helpers from mm.h to mmzone.h It makes more sense to have these helpers in zone specific header file, rather than the generic mm.h Link: https://lkml.kernel.org/r/20220715150521.18165-3-alex.sierra@amd.com Signed-off-by: Alex Sierra Cc: Alistair Popple Cc: Christoph Hellwig Cc: David Hildenbrand Cc: Felix Kuehling Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: Matthew Wilcox Cc: Ralph Campbell Signed-off-by: Andrew Morton --- include/linux/memremap.h | 2 +- include/linux/mm.h | 78 ---------------------------------------------- include/linux/mmzone.h | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 79 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 9f5ee49482de..732dde5988fb 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -2,7 +2,7 @@ #ifndef _LINUX_MEMREMAP_H_ #define _LINUX_MEMREMAP_H_ -#include +#include #include #include #include diff --git a/include/linux/mm.h b/include/linux/mm.h index 3c044e38958c..a2d01e49253b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1045,84 +1045,6 @@ vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf); * back into memory. */ -/* - * The zone field is never updated after free_area_init_core() - * sets it, so none of the operations on it need to be atomic. - */ - -/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */ -#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) -#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) -#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) -#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) -#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) - -/* - * Define the bit shifts to access each section. For non-existent - * sections we define the shift as 0; that plus a 0 mask ensures - * the compiler will optimise away reference to them. - */ -#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) -#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) -#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) -#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0)) -#define KASAN_TAG_PGSHIFT (KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0)) - -/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ -#ifdef NODE_NOT_IN_PAGE_FLAGS -#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT) -#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \ - SECTIONS_PGOFF : ZONES_PGOFF) -#else -#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT) -#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \ - NODES_PGOFF : ZONES_PGOFF) -#endif - -#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0)) - -#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) -#define NODES_MASK ((1UL << NODES_WIDTH) - 1) -#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) -#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1) -#define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) -#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) - -static inline enum zone_type page_zonenum(const struct page *page) -{ - ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT); - return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; -} - -static inline enum zone_type folio_zonenum(const struct folio *folio) -{ - return page_zonenum(&folio->page); -} - -#ifdef CONFIG_ZONE_DEVICE -static inline bool is_zone_device_page(const struct page *page) -{ - return page_zonenum(page) == ZONE_DEVICE; -} -extern void memmap_init_zone_device(struct zone *, unsigned long, - unsigned long, struct dev_pagemap *); -#else -static inline bool is_zone_device_page(const struct page *page) -{ - return false; -} -#endif - -static inline bool folio_is_zone_device(const struct folio *folio) -{ - return is_zone_device_page(&folio->page); -} - -static inline bool is_zone_movable_page(const struct page *page) -{ - return page_zonenum(page) == ZONE_MOVABLE; -} - #if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX) DECLARE_STATIC_KEY_FALSE(devmap_managed_key); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 735bf5b37949..5da1135e6755 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -730,6 +730,86 @@ static inline bool zone_is_empty(struct zone *zone) return zone->spanned_pages == 0; } +#ifndef BUILD_VDSO32_64 +/* + * The zone field is never updated after free_area_init_core() + * sets it, so none of the operations on it need to be atomic. + */ + +/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */ +#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) +#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) +#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) +#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) +#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) + +/* + * Define the bit shifts to access each section. For non-existent + * sections we define the shift as 0; that plus a 0 mask ensures + * the compiler will optimise away reference to them. + */ +#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) +#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) +#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) +#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0)) +#define KASAN_TAG_PGSHIFT (KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0)) + +/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ +#ifdef NODE_NOT_IN_PAGE_FLAGS +#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT) +#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF) ? \ + SECTIONS_PGOFF : ZONES_PGOFF) +#else +#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT) +#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF) ? \ + NODES_PGOFF : ZONES_PGOFF) +#endif + +#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0)) + +#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) +#define NODES_MASK ((1UL << NODES_WIDTH) - 1) +#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) +#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1) +#define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) +#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) + +static inline enum zone_type page_zonenum(const struct page *page) +{ + ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT); + return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; +} + +static inline enum zone_type folio_zonenum(const struct folio *folio) +{ + return page_zonenum(&folio->page); +} + +#ifdef CONFIG_ZONE_DEVICE +static inline bool is_zone_device_page(const struct page *page) +{ + return page_zonenum(page) == ZONE_DEVICE; +} +extern void memmap_init_zone_device(struct zone *, unsigned long, + unsigned long, struct dev_pagemap *); +#else +static inline bool is_zone_device_page(const struct page *page) +{ + return false; +} +#endif + +static inline bool folio_is_zone_device(const struct folio *folio) +{ + return is_zone_device_page(&folio->page); +} + +static inline bool is_zone_movable_page(const struct page *page) +{ + return page_zonenum(page) == ZONE_MOVABLE; +} +#endif + /* * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty * intersection with the given zone -- cgit v1.2.3 From f25cbb7a95a24ff9a2a3bebd308e303942ae6b2c Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Fri, 15 Jul 2022 10:05:10 -0500 Subject: mm: add zone device coherent type memory support Device memory that is cache coherent from device and CPU point of view. This is used on platforms that have an advanced system bus (like CAPI or CXL). Any page of a process can be migrated to such memory. However, no one should be allowed to pin such memory so that it can always be evicted. [hch@lst.de: rebased ontop of the refcount changes, remove is_dev_private_or_coherent_page] Link: https://lkml.kernel.org/r/20220715150521.18165-4-alex.sierra@amd.com Signed-off-by: Alex Sierra Signed-off-by: Christoph Hellwig Acked-by: Felix Kuehling Reviewed-by: Alistair Popple Acked-by: David Hildenbrand Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: Matthew Wilcox Cc: Ralph Campbell Signed-off-by: Andrew Morton --- include/linux/memremap.h | 19 +++++++++++++++++++ include/linux/mm.h | 5 ++++- mm/memcontrol.c | 7 ++++--- mm/memory-failure.c | 8 ++++++-- mm/memremap.c | 10 ++++++++++ mm/migrate_device.c | 16 +++++++--------- mm/rmap.c | 5 +++-- 7 files changed, 53 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 732dde5988fb..09320b7f706c 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -41,6 +41,13 @@ struct vmem_altmap { * A more complete discussion of unaddressable memory may be found in * include/linux/hmm.h and Documentation/mm/hmm.rst. * + * MEMORY_DEVICE_COHERENT: + * Device memory that is cache coherent from device and CPU point of view. This + * is used on platforms that have an advanced system bus (like CAPI or CXL). A + * driver can hotplug the device memory using ZONE_DEVICE and with that memory + * type. Any page of a process can be migrated to such memory. However no one + * should be allowed to pin such memory so that it can always be evicted. + * * MEMORY_DEVICE_FS_DAX: * Host memory that has similar access semantics as System RAM i.e. DMA * coherent and supports page pinning. In support of coordinating page @@ -61,6 +68,7 @@ struct vmem_altmap { enum memory_type { /* 0 is reserved to catch uninitialized type fields */ MEMORY_DEVICE_PRIVATE = 1, + MEMORY_DEVICE_COHERENT, MEMORY_DEVICE_FS_DAX, MEMORY_DEVICE_GENERIC, MEMORY_DEVICE_PCI_P2PDMA, @@ -150,6 +158,17 @@ static inline bool is_pci_p2pdma_page(const struct page *page) page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; } +static inline bool is_device_coherent_page(const struct page *page) +{ + return is_zone_device_page(page) && + page->pgmap->type == MEMORY_DEVICE_COHERENT; +} + +static inline bool folio_is_device_coherent(const struct folio *folio) +{ + return is_device_coherent_page(&folio->page); +} + #ifdef CONFIG_ZONE_DEVICE void *memremap_pages(struct dev_pagemap *pgmap, int nid); void memunmap_pages(struct dev_pagemap *pgmap); diff --git a/include/linux/mm.h b/include/linux/mm.h index a2d01e49253b..64393ed3330a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -28,6 +28,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -1537,7 +1538,9 @@ static inline bool is_longterm_pinnable_page(struct page *page) if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) return false; #endif - return !is_zone_movable_page(page) || is_zero_pfn(page_to_pfn(page)); + return !(is_device_coherent_page(page) || + is_zone_movable_page(page) || + is_zero_pfn(page_to_pfn(page))); } #else static inline bool is_longterm_pinnable_page(struct page *page) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1497affe08c4..b1868784f895 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5716,8 +5716,8 @@ out: * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a * target for charge migration. if @target is not NULL, the entry is stored * in target->ent. - * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE - * (so ZONE_DEVICE page and thus not on the lru). + * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is device memory and + * thus not on the lru. * For now we such page is charge like a regular page would be as for all * intent and purposes it is just special memory taking the place of a * regular page. @@ -5755,7 +5755,8 @@ static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, */ if (page_memcg(page) == mc.from) { ret = MC_TARGET_PAGE; - if (is_device_private_page(page)) + if (is_device_private_page(page) || + is_device_coherent_page(page)) ret = MC_TARGET_DEVICE; if (target) target->page = page; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index f7612ccdb299..b7ca5db7e60e 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1686,12 +1686,16 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, goto unlock; } - if (pgmap->type == MEMORY_DEVICE_PRIVATE) { + switch (pgmap->type) { + case MEMORY_DEVICE_PRIVATE: + case MEMORY_DEVICE_COHERENT: /* - * TODO: Handle HMM pages which may need coordination + * TODO: Handle device pages which may need coordination * with device-side memory. */ goto unlock; + default: + break; } /* diff --git a/mm/memremap.c b/mm/memremap.c index 8b5c8fd4ea8e..f0955785150f 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -315,6 +315,16 @@ void *memremap_pages(struct dev_pagemap *pgmap, int nid) return ERR_PTR(-EINVAL); } break; + case MEMORY_DEVICE_COHERENT: + if (!pgmap->ops->page_free) { + WARN(1, "Missing page_free method\n"); + return ERR_PTR(-EINVAL); + } + if (!pgmap->owner) { + WARN(1, "Missing owner\n"); + return ERR_PTR(-EINVAL); + } + break; case MEMORY_DEVICE_FS_DAX: if (IS_ENABLED(CONFIG_FS_DAX_LIMITED)) { WARN(1, "File system DAX not supported\n"); diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 5052093d0262..a4847ad65da3 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -518,7 +518,7 @@ EXPORT_SYMBOL(migrate_vma_setup); * handle_pte_fault() * do_anonymous_page() * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE - * private page. + * private or coherent page. */ static void migrate_vma_insert_page(struct migrate_vma *migrate, unsigned long addr, @@ -594,11 +594,8 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, page_to_pfn(page)); entry = swp_entry_to_pte(swp_entry); } else { - /* - * For now we only support migrating to un-addressable device - * memory. - */ - if (is_zone_device_page(page)) { + if (is_zone_device_page(page) && + !is_device_coherent_page(page)) { pr_warn_once("Unsupported ZONE_DEVICE page type.\n"); goto abort; } @@ -701,10 +698,11 @@ void migrate_vma_pages(struct migrate_vma *migrate) mapping = page_mapping(page); - if (is_device_private_page(newpage)) { + if (is_device_private_page(newpage) || + is_device_coherent_page(newpage)) { /* - * For now only support private anonymous when migrating - * to un-addressable device memory. + * For now only support anonymous memory migrating to + * device private or coherent memory. */ if (mapping) { migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; diff --git a/mm/rmap.c b/mm/rmap.c index 83172ee0ea35..0532fd92ecb3 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1953,7 +1953,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Update high watermark before we lower rss */ update_hiwater_rss(mm); - if (folio_is_zone_device(folio)) { + if (folio_is_device_private(folio)) { unsigned long pfn = folio_pfn(folio); swp_entry_t entry; pte_t swp_pte; @@ -2124,7 +2124,8 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags) TTU_SYNC))) return; - if (folio_is_zone_device(folio) && !folio_is_device_private(folio)) + if (folio_is_zone_device(folio) && + (!folio_is_device_private(folio) && !folio_is_device_coherent(folio))) return; /* -- cgit v1.2.3 From dd19e6d8ffaa1289d75d7833de97faf1b6b2c8e4 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Fri, 15 Jul 2022 10:05:12 -0500 Subject: mm: add device coherent vma selection for memory migration This case is used to migrate pages from device memory, back to system memory. Device coherent type memory is cache coherent from device and CPU point of view. Link: https://lkml.kernel.org/r/20220715150521.18165-6-alex.sierra@amd.com Signed-off-by: Alex Sierra Signed-off-by: Christoph Hellwig Acked-by: Felix Kuehling Reviewed-by: Alistair Poppple Reviewed-by: David Hildenbrand Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: Matthew Wilcox Cc: Ralph Campbell Signed-off-by: Andrew Morton --- include/linux/migrate.h | 1 + mm/migrate_device.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 069a89e847f3..b84908debe5c 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -148,6 +148,7 @@ static inline unsigned long migrate_pfn(unsigned long pfn) enum migrate_vma_direction { MIGRATE_VMA_SELECT_SYSTEM = 1 << 0, MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1, + MIGRATE_VMA_SELECT_DEVICE_COHERENT = 1 << 2, }; struct migrate_vma { diff --git a/mm/migrate_device.c b/mm/migrate_device.c index a4847ad65da3..18bc6483f63a 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -148,15 +148,21 @@ again: if (is_writable_device_private_entry(entry)) mpfn |= MIGRATE_PFN_WRITE; } else { - if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) - goto next; pfn = pte_pfn(pte); - if (is_zero_pfn(pfn)) { + if (is_zero_pfn(pfn) && + (migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { mpfn = MIGRATE_PFN_MIGRATE; migrate->cpages++; goto next; } page = vm_normal_page(migrate->vma, addr, pte); + if (page && !is_zone_device_page(page) && + !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) + goto next; + else if (page && is_device_coherent_page(page) && + (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_COHERENT) || + page->pgmap->owner != migrate->pgmap_owner)) + goto next; mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE; mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0; } -- cgit v1.2.3 From 8012b866085523758780850087102421dbcce522 Mon Sep 17 00:00:00 2001 From: Shiyang Ruan Date: Fri, 3 Jun 2022 13:37:25 +0800 Subject: dax: introduce holder for dax_device Patch series "v14 fsdax-rmap + v11 fsdax-reflink", v2. The patchset fsdax-rmap is aimed to support shared pages tracking for fsdax. It moves owner tracking from dax_assocaite_entry() to pmem device driver, by introducing an interface ->memory_failure() for struct pagemap. This interface is called by memory_failure() in mm, and implemented by pmem device. Then call holder operations to find the filesystem which the corrupted data located in, and call filesystem handler to track files or metadata associated with this page. Finally we are able to try to fix the corrupted data in filesystem and do other necessary processing, such as killing processes who are using the files affected. The call trace is like this: memory_failure() |* fsdax case |------------ |pgmap->ops->memory_failure() => pmem_pgmap_memory_failure() | dax_holder_notify_failure() => | dax_device->holder_ops->notify_failure() => | - xfs_dax_notify_failure() | |* xfs_dax_notify_failure() | |-------------------------- | | xfs_rmap_query_range() | | xfs_dax_failure_fn() | | * corrupted on metadata | | try to recover data, call xfs_force_shutdown() | | * corrupted on file data | | try to recover data, call mf_dax_kill_procs() |* normal case |------------- |mf_generic_kill_procs() The patchset fsdax-reflink attempts to add CoW support for fsdax, and takes XFS, which has both reflink and fsdax features, as an example. One of the key mechanisms needed to be implemented in fsdax is CoW. Copy the data from srcmap before we actually write data to the destination iomap. And we just copy range in which data won't be changed. Another mechanism is range comparison. In page cache case, readpage() is used to load data on disk to page cache in order to be able to compare data. In fsdax case, readpage() does not work. So, we need another compare data with direct access support. With the two mechanisms implemented in fsdax, we are able to make reflink and fsdax work together in XFS. This patch (of 14): To easily track filesystem from a pmem device, we introduce a holder for dax_device structure, and also its operation. This holder is used to remember who is using this dax_device: - When it is the backend of a filesystem, the holder will be the instance of this filesystem. - When this pmem device is one of the targets in a mapped device, the holder will be this mapped device. In this case, the mapped device has its own dax_device and it will follow the first rule. So that we can finally track to the filesystem we needed. The holder and holder_ops will be set when filesystem is being mounted, or an target device is being activated. Link: https://lkml.kernel.org/r/20220603053738.1218681-1-ruansy.fnst@fujitsu.com Link: https://lkml.kernel.org/r/20220603053738.1218681-2-ruansy.fnst@fujitsu.com Signed-off-by: Shiyang Ruan Reviewed-by: Christoph Hellwig Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Cc: Dave Chinner Cc: Jane Chu Cc: Goldwyn Rodrigues Cc: Al Viro Cc: Matthew Wilcox Cc: Naoya Horiguchi Cc: Miaohe Lin Cc: Dan Williams Cc: Goldwyn Rodrigues Cc: Ritesh Harjani Signed-off-by: Andrew Morton --- drivers/dax/super.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++- drivers/md/dm.c | 2 +- fs/erofs/super.c | 10 ++++---- fs/ext2/super.c | 7 +++--- fs/ext4/super.c | 9 +++---- fs/xfs/xfs_buf.c | 5 ++-- include/linux/dax.h | 33 +++++++++++++++++++------- 7 files changed, 110 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 50a08b2ec247..9b5e2a5eb0ae 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -22,6 +22,8 @@ * @private: dax driver private data * @flags: state and boolean properties * @ops: operations for this device + * @holder_data: holder of a dax_device: could be filesystem or mapped device + * @holder_ops: operations for the inner holder */ struct dax_device { struct inode inode; @@ -29,6 +31,8 @@ struct dax_device { void *private; unsigned long flags; const struct dax_operations *ops; + void *holder_data; + const struct dax_holder_operations *holder_ops; }; static dev_t dax_devt; @@ -71,8 +75,11 @@ EXPORT_SYMBOL_GPL(dax_remove_host); * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax * @bdev: block device to find a dax_device for * @start_off: returns the byte offset into the dax_device that @bdev starts + * @holder: filesystem or mapped device inside the dax_device + * @ops: operations for the inner holder */ -struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off) +struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off, + void *holder, const struct dax_holder_operations *ops) { struct dax_device *dax_dev; u64 part_size; @@ -92,11 +99,26 @@ struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off) dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk); if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode)) dax_dev = NULL; + else if (holder) { + if (!cmpxchg(&dax_dev->holder_data, NULL, holder)) + dax_dev->holder_ops = ops; + else + dax_dev = NULL; + } dax_read_unlock(id); return dax_dev; } EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); + +void fs_put_dax(struct dax_device *dax_dev, void *holder) +{ + if (dax_dev && holder && + cmpxchg(&dax_dev->holder_data, holder, NULL) == holder) + dax_dev->holder_ops = NULL; + put_dax(dax_dev); +} +EXPORT_SYMBOL_GPL(fs_put_dax); #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ enum dax_device_flags { @@ -204,6 +226,29 @@ size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, } EXPORT_SYMBOL_GPL(dax_recovery_write); +int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, + u64 len, int mf_flags) +{ + int rc, id; + + id = dax_read_lock(); + if (!dax_alive(dax_dev)) { + rc = -ENXIO; + goto out; + } + + if (!dax_dev->holder_ops) { + rc = -EOPNOTSUPP; + goto out; + } + + rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags); +out: + dax_read_unlock(id); + return rc; +} +EXPORT_SYMBOL_GPL(dax_holder_notify_failure); + #ifdef CONFIG_ARCH_HAS_PMEM_API void arch_wb_cache_pmem(void *addr, size_t size); void dax_flush(struct dax_device *dax_dev, void *addr, size_t size) @@ -277,8 +322,15 @@ void kill_dax(struct dax_device *dax_dev) if (!dax_dev) return; + if (dax_dev->holder_data != NULL) + dax_holder_notify_failure(dax_dev, 0, U64_MAX, 0); + clear_bit(DAXDEV_ALIVE, &dax_dev->flags); synchronize_srcu(&dax_srcu); + + /* clear holder data */ + dax_dev->holder_ops = NULL; + dax_dev->holder_data = NULL; } EXPORT_SYMBOL_GPL(kill_dax); @@ -420,6 +472,19 @@ void put_dax(struct dax_device *dax_dev) } EXPORT_SYMBOL_GPL(put_dax); +/** + * dax_holder() - obtain the holder of a dax device + * @dax_dev: a dax_device instance + + * Return: the holder's data which represents the holder if registered, + * otherwize NULL. + */ +void *dax_holder(struct dax_device *dax_dev) +{ + return dax_dev->holder_data; +} +EXPORT_SYMBOL_GPL(dax_holder); + /** * inode_dax: convert a public inode into its dax_dev * @inode: An inode with i_cdev pointing to a dax_dev diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 2b75f1ef7386..0177a4ce9a18 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -758,7 +758,7 @@ static int open_table_device(struct table_device *td, dev_t dev, } td->dm_dev.bdev = bdev; - td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off); + td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off, NULL, NULL); return 0; } diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 95addc5c9d34..3173debeaa5a 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -255,7 +255,8 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, if (IS_ERR(bdev)) return PTR_ERR(bdev); dif->bdev = bdev; - dif->dax_dev = fs_dax_get_by_bdev(bdev, &dif->dax_part_off); + dif->dax_dev = fs_dax_get_by_bdev(bdev, &dif->dax_part_off, + NULL, NULL); } dif->blocks = le32_to_cpu(dis->blocks); @@ -720,7 +721,8 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) } sbi->dax_dev = fs_dax_get_by_bdev(sb->s_bdev, - &sbi->dax_part_off); + &sbi->dax_part_off, + NULL, NULL); } err = erofs_read_superblock(sb); @@ -812,7 +814,7 @@ static int erofs_release_device_info(int id, void *ptr, void *data) { struct erofs_device_info *dif = ptr; - fs_put_dax(dif->dax_dev); + fs_put_dax(dif->dax_dev, NULL); if (dif->bdev) blkdev_put(dif->bdev, FMODE_READ | FMODE_EXCL); erofs_fscache_unregister_cookie(&dif->fscache); @@ -886,7 +888,7 @@ static void erofs_kill_sb(struct super_block *sb) return; erofs_free_dev_context(sbi->devs); - fs_put_dax(sbi->dax_dev); + fs_put_dax(sbi->dax_dev, NULL); erofs_fscache_unregister_cookie(&sbi->s_fscache); erofs_fscache_unregister_fs(sb); kfree(sbi->opt.fsid); diff --git a/fs/ext2/super.c b/fs/ext2/super.c index f6a19f6d9f6d..4638946251b9 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -171,7 +171,7 @@ static void ext2_put_super (struct super_block * sb) brelse (sbi->s_sbh); sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); - fs_put_dax(sbi->s_daxdev); + fs_put_dax(sbi->s_daxdev, NULL); kfree(sbi); } @@ -835,7 +835,8 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent) } sb->s_fs_info = sbi; sbi->s_sb_block = sb_block; - sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off); + sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off, + NULL, NULL); spin_lock_init(&sbi->s_lock); ret = -EINVAL; @@ -1204,7 +1205,7 @@ failed_mount_group_desc: failed_mount: brelse(bh); failed_sbi: - fs_put_dax(sbi->s_daxdev); + fs_put_dax(sbi->s_daxdev, NULL); sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); kfree(sbi); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 845f2f8aee5f..1f8bf507ba5a 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1307,7 +1307,7 @@ static void ext4_put_super(struct super_block *sb) if (sbi->s_chksum_driver) crypto_free_shash(sbi->s_chksum_driver); kfree(sbi->s_blockgroup_lock); - fs_put_dax(sbi->s_daxdev); + fs_put_dax(sbi->s_daxdev, NULL); fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy); #if IS_ENABLED(CONFIG_UNICODE) utf8_unload(sb->s_encoding); @@ -4272,7 +4272,7 @@ static void ext4_free_sbi(struct ext4_sb_info *sbi) return; kfree(sbi->s_blockgroup_lock); - fs_put_dax(sbi->s_daxdev); + fs_put_dax(sbi->s_daxdev, NULL); kfree(sbi); } @@ -4284,7 +4284,8 @@ static struct ext4_sb_info *ext4_alloc_sbi(struct super_block *sb) if (!sbi) return NULL; - sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off); + sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off, + NULL, NULL); sbi->s_blockgroup_lock = kzalloc(sizeof(struct blockgroup_lock), GFP_KERNEL); @@ -4296,7 +4297,7 @@ static struct ext4_sb_info *ext4_alloc_sbi(struct super_block *sb) sbi->s_sb = sb; return sbi; err_out: - fs_put_dax(sbi->s_daxdev); + fs_put_dax(sbi->s_daxdev, NULL); kfree(sbi); return NULL; } diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 4aa9c9cf5b6e..1ec2a7b6d44e 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1911,7 +1911,7 @@ xfs_free_buftarg( list_lru_destroy(&btp->bt_lru); blkdev_issue_flush(btp->bt_bdev); - fs_put_dax(btp->bt_daxdev); + fs_put_dax(btp->bt_daxdev, NULL); kmem_free(btp); } @@ -1964,7 +1964,8 @@ xfs_alloc_buftarg( btp->bt_mount = mp; btp->bt_dev = bdev->bd_dev; btp->bt_bdev = bdev; - btp->bt_daxdev = fs_dax_get_by_bdev(bdev, &btp->bt_dax_part_off); + btp->bt_daxdev = fs_dax_get_by_bdev(bdev, &btp->bt_dax_part_off, NULL, + NULL); /* * Buffer IO error rate limiting. Limit it to no more than 10 messages diff --git a/include/linux/dax.h b/include/linux/dax.h index e7b81634c52a..cf85fc36da5f 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -43,8 +43,21 @@ struct dax_operations { void *addr, size_t bytes, struct iov_iter *iter); }; +struct dax_holder_operations { + /* + * notify_failure - notify memory failure into inner holder device + * @dax_dev: the dax device which contains the holder + * @offset: offset on this dax device where memory failure occurs + * @len: length of this memory failure event + * @flags: action flags for memory failure handler + */ + int (*notify_failure)(struct dax_device *dax_dev, u64 offset, + u64 len, int mf_flags); +}; + #if IS_ENABLED(CONFIG_DAX) struct dax_device *alloc_dax(void *private, const struct dax_operations *ops); +void *dax_holder(struct dax_device *dax_dev); void put_dax(struct dax_device *dax_dev); void kill_dax(struct dax_device *dax_dev); void dax_write_cache(struct dax_device *dax_dev, bool wc); @@ -66,6 +79,10 @@ static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, return dax_synchronous(dax_dev); } #else +static inline void *dax_holder(struct dax_device *dax_dev) +{ + return NULL; +} static inline struct dax_device *alloc_dax(void *private, const struct dax_operations *ops) { @@ -114,12 +131,9 @@ struct writeback_control; #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); void dax_remove_host(struct gendisk *disk); -struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, - u64 *start_off); -static inline void fs_put_dax(struct dax_device *dax_dev) -{ - put_dax(dax_dev); -} +struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off, + void *holder, const struct dax_holder_operations *ops); +void fs_put_dax(struct dax_device *dax_dev, void *holder); #else static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) { @@ -129,11 +143,12 @@ static inline void dax_remove_host(struct gendisk *disk) { } static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, - u64 *start_off) + u64 *start_off, void *holder, + const struct dax_holder_operations *ops) { return NULL; } -static inline void fs_put_dax(struct dax_device *dax_dev) +static inline void fs_put_dax(struct dax_device *dax_dev, void *holder) { } #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ @@ -203,6 +218,8 @@ size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, size_t nr_pages); +int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len, + int mf_flags); void dax_flush(struct dax_device *dax_dev, void *addr, size_t size); ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, -- cgit v1.2.3 From 33a8f7f2b3a3437d016d1b4047a4fd37eb6951b3 Mon Sep 17 00:00:00 2001 From: Shiyang Ruan Date: Fri, 3 Jun 2022 13:37:27 +0800 Subject: pagemap,pmem: introduce ->memory_failure() When memory-failure occurs, we call this function which is implemented by each kind of devices. For the fsdax case, pmem device driver implements it. Pmem device driver will find out the filesystem in which the corrupted page located in. With dax_holder notify support, we are able to notify the memory failure from pmem driver to upper layers. If there is something not support in the notify routine, memory_failure will fall back to the generic hanlder. Link: https://lkml.kernel.org/r/20220603053738.1218681-4-ruansy.fnst@fujitsu.com Signed-off-by: Shiyang Ruan Reviewed-by: Christoph Hellwig Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Reviewed-by: Naoya Horiguchi Cc: Al Viro Cc: Dan Williams Cc: Dave Chinner Cc: Goldwyn Rodrigues Cc: Goldwyn Rodrigues Cc: Jane Chu Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Ritesh Harjani Signed-off-by: Andrew Morton --- drivers/nvdimm/pmem.c | 17 +++++++++++++++++ include/linux/memremap.h | 12 ++++++++++++ mm/memory-failure.c | 14 ++++++++++++++ 3 files changed, 43 insertions(+) (limited to 'include/linux') diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 629d10fcf53b..107c9cb3d57d 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -453,6 +453,21 @@ static void pmem_release_disk(void *__pmem) blk_cleanup_disk(pmem->disk); } +static int pmem_pagemap_memory_failure(struct dev_pagemap *pgmap, + unsigned long pfn, unsigned long nr_pages, int mf_flags) +{ + struct pmem_device *pmem = + container_of(pgmap, struct pmem_device, pgmap); + u64 offset = PFN_PHYS(pfn) - pmem->phys_addr - pmem->data_offset; + u64 len = nr_pages << PAGE_SHIFT; + + return dax_holder_notify_failure(pmem->dax_dev, offset, len, mf_flags); +} + +static const struct dev_pagemap_ops fsdax_pagemap_ops = { + .memory_failure = pmem_pagemap_memory_failure, +}; + static int pmem_attach_disk(struct device *dev, struct nd_namespace_common *ndns) { @@ -514,6 +529,7 @@ static int pmem_attach_disk(struct device *dev, pmem->pfn_flags = PFN_DEV; if (is_nd_pfn(dev)) { pmem->pgmap.type = MEMORY_DEVICE_FS_DAX; + pmem->pgmap.ops = &fsdax_pagemap_ops; addr = devm_memremap_pages(dev, &pmem->pgmap); pfn_sb = nd_pfn->pfn_sb; pmem->data_offset = le64_to_cpu(pfn_sb->dataoff); @@ -527,6 +543,7 @@ static int pmem_attach_disk(struct device *dev, pmem->pgmap.range.end = res->end; pmem->pgmap.nr_range = 1; pmem->pgmap.type = MEMORY_DEVICE_FS_DAX; + pmem->pgmap.ops = &fsdax_pagemap_ops; addr = devm_memremap_pages(dev, &pmem->pgmap); pmem->pfn_flags |= PFN_MAP; bb_range = pmem->pgmap.range; diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 09320b7f706c..19010491a603 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -87,6 +87,18 @@ struct dev_pagemap_ops { * the page back to a CPU accessible page. */ vm_fault_t (*migrate_to_ram)(struct vm_fault *vmf); + + /* + * Handle the memory failure happens on a range of pfns. Notify the + * processes who are using these pfns, and try to recover the data on + * them if necessary. The mf_flags is finally passed to the recover + * function through the whole notify routine. + * + * When this is not implemented, or it returns -EOPNOTSUPP, the caller + * will fall back to a common handler called mf_generic_kill_procs(). + */ + int (*memory_failure)(struct dev_pagemap *pgmap, unsigned long pfn, + unsigned long nr_pages, int mf_flags); }; #define PGMAP_ALTMAP_VALID (1 << 0) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index f8a8a5d45eba..46c77151f726 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1748,6 +1748,20 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, if (!pgmap_pfn_valid(pgmap, pfn)) goto out; + /* + * Call driver's implementation to handle the memory failure, otherwise + * fall back to generic handler. + */ + if (pgmap->ops->memory_failure) { + rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); + /* + * Fall back to generic handler too if operation is not + * supported inside the driver/device/filesystem. + */ + if (rc != -EOPNOTSUPP) + goto out; + } + rc = mf_generic_kill_procs(pfn, flags, pgmap); out: /* drop pgmap ref acquired in caller */ -- cgit v1.2.3 From 2f437effc689ef913fbe5e31110580b4e7cf04be Mon Sep 17 00:00:00 2001 From: Shiyang Ruan Date: Fri, 3 Jun 2022 13:37:28 +0800 Subject: fsdax: introduce dax_lock_mapping_entry() The current dax_lock_page() locks dax entry by obtaining mapping and index in page. To support 1-to-N RMAP in NVDIMM, we need a new function to lock a specific dax entry corresponding to this file's mapping,index. And output the page corresponding to the specific dax entry for caller use. Link: https://lkml.kernel.org/r/20220603053738.1218681-5-ruansy.fnst@fujitsu.com Signed-off-by: Shiyang Ruan Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Cc: Al Viro Cc: Dan Williams Cc: Dan Williams Cc: Dave Chinner Cc: Goldwyn Rodrigues Cc: Goldwyn Rodrigues Cc: Jane Chu Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Naoya Horiguchi Cc: Ritesh Harjani Signed-off-by: Andrew Morton --- fs/dax.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/dax.h | 15 +++++++++++++ 2 files changed, 78 insertions(+) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 4155a6107fa1..65e44d78b3bb 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -455,6 +455,69 @@ void dax_unlock_page(struct page *page, dax_entry_t cookie) dax_unlock_entry(&xas, (void *)cookie); } +/* + * dax_lock_mapping_entry - Lock the DAX entry corresponding to a mapping + * @mapping: the file's mapping whose entry we want to lock + * @index: the offset within this file + * @page: output the dax page corresponding to this dax entry + * + * Return: A cookie to pass to dax_unlock_mapping_entry() or 0 if the entry + * could not be locked. + */ +dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, pgoff_t index, + struct page **page) +{ + XA_STATE(xas, NULL, 0); + void *entry; + + rcu_read_lock(); + for (;;) { + entry = NULL; + if (!dax_mapping(mapping)) + break; + + xas.xa = &mapping->i_pages; + xas_lock_irq(&xas); + xas_set(&xas, index); + entry = xas_load(&xas); + if (dax_is_locked(entry)) { + rcu_read_unlock(); + wait_entry_unlocked(&xas, entry); + rcu_read_lock(); + continue; + } + if (!entry || + dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) { + /* + * Because we are looking for entry from file's mapping + * and index, so the entry may not be inserted for now, + * or even a zero/empty entry. We don't think this is + * an error case. So, return a special value and do + * not output @page. + */ + entry = (void *)~0UL; + } else { + *page = pfn_to_page(dax_to_pfn(entry)); + dax_lock_entry(&xas, entry); + } + xas_unlock_irq(&xas); + break; + } + rcu_read_unlock(); + return (dax_entry_t)entry; +} + +void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index, + dax_entry_t cookie) +{ + XA_STATE(xas, &mapping->i_pages, index); + + if (cookie == ~0UL) + return; + + dax_unlock_entry(&xas, (void *)cookie); +} + /* * Find page cache entry at given index. If it is a DAX entry, return it * with the entry locked. If the page cache doesn't contain an entry at diff --git a/include/linux/dax.h b/include/linux/dax.h index cf85fc36da5f..7116681b48c0 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -161,6 +161,10 @@ struct page *dax_layout_busy_page(struct address_space *mapping); struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end); dax_entry_t dax_lock_page(struct page *page); void dax_unlock_page(struct page *page, dax_entry_t cookie); +dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, + unsigned long index, struct page **page); +void dax_unlock_mapping_entry(struct address_space *mapping, + unsigned long index, dax_entry_t cookie); #else static inline struct page *dax_layout_busy_page(struct address_space *mapping) { @@ -188,6 +192,17 @@ static inline dax_entry_t dax_lock_page(struct page *page) static inline void dax_unlock_page(struct page *page, dax_entry_t cookie) { } + +static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, + unsigned long index, struct page **page) +{ + return 0; +} + +static inline void dax_unlock_mapping_entry(struct address_space *mapping, + unsigned long index, dax_entry_t cookie) +{ +} #endif int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, -- cgit v1.2.3 From c36e2024957120566efd99395b5c8cc95b5175c1 Mon Sep 17 00:00:00 2001 From: Shiyang Ruan Date: Fri, 3 Jun 2022 13:37:29 +0800 Subject: mm: introduce mf_dax_kill_procs() for fsdax case This new function is a variant of mf_generic_kill_procs that accepts a file, offset pair instead of a struct to support multiple files sharing a DAX mapping. It is intended to be called by the file systems as part of the memory_failure handler after the file system performed a reverse mapping from the storage address to the file and file offset. Link: https://lkml.kernel.org/r/20220603053738.1218681-6-ruansy.fnst@fujitsu.com Signed-off-by: Shiyang Ruan Reviewed-by: Dan Williams Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Reviewed-by: Miaohe Lin Cc: Al Viro Cc: Dan Williams Cc: Dave Chinner Cc: Goldwyn Rodrigues Cc: Goldwyn Rodrigues Cc: Jane Chu Cc: Matthew Wilcox Cc: Naoya Horiguchi Cc: Ritesh Harjani Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 ++ mm/memory-failure.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 64393ed3330a..d4ebfc206e2b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3178,6 +3178,8 @@ enum mf_flags { MF_UNPOISON = 1 << 4, MF_SW_SIMULATED = 1 << 5, }; +int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, + unsigned long count, int mf_flags); extern int memory_failure(unsigned long pfn, int flags); extern void memory_failure_queue(unsigned long pfn, int flags); extern void memory_failure_queue_kick(int cpu); diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 46c77151f726..c9931c676335 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -297,10 +297,9 @@ void shake_page(struct page *p) } EXPORT_SYMBOL_GPL(shake_page); -static unsigned long dev_pagemap_mapping_shift(struct page *page, - struct vm_area_struct *vma) +static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, + unsigned long address) { - unsigned long address = vma_address(page, vma); unsigned long ret = 0; pgd_t *pgd; p4d_t *p4d; @@ -340,10 +339,14 @@ static unsigned long dev_pagemap_mapping_shift(struct page *page, /* * Schedule a process for later kill. * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. + * + * Notice: @fsdax_pgoff is used only when @p is a fsdax page. + * In other cases, such as anonymous and file-backend page, the address to be + * killed can be caculated by @p itself. */ static void add_to_kill(struct task_struct *tsk, struct page *p, - struct vm_area_struct *vma, - struct list_head *to_kill) + pgoff_t fsdax_pgoff, struct vm_area_struct *vma, + struct list_head *to_kill) { struct to_kill *tk; @@ -354,9 +357,15 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, } tk->addr = page_address_in_vma(p, vma); - if (is_zone_device_page(p)) - tk->size_shift = dev_pagemap_mapping_shift(p, vma); - else + if (is_zone_device_page(p)) { + /* + * Since page->mapping is not used for fsdax, we need + * calculate the address based on the vma. + */ + if (p->pgmap->type == MEMORY_DEVICE_FS_DAX) + tk->addr = vma_pgoff_address(fsdax_pgoff, 1, vma); + tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); + } else tk->size_shift = page_shift(compound_head(p)); /* @@ -505,7 +514,7 @@ static void collect_procs_anon(struct page *page, struct list_head *to_kill, if (!page_mapped_in_vma(page, vma)) continue; if (vma->vm_mm == t->mm) - add_to_kill(t, page, vma, to_kill); + add_to_kill(t, page, 0, vma, to_kill); } } read_unlock(&tasklist_lock); @@ -541,13 +550,41 @@ static void collect_procs_file(struct page *page, struct list_head *to_kill, * to be informed of all such data corruptions. */ if (vma->vm_mm == t->mm) - add_to_kill(t, page, vma, to_kill); + add_to_kill(t, page, 0, vma, to_kill); } } read_unlock(&tasklist_lock); i_mmap_unlock_read(mapping); } +#ifdef CONFIG_FS_DAX +/* + * Collect processes when the error hit a fsdax page. + */ +static void collect_procs_fsdax(struct page *page, + struct address_space *mapping, pgoff_t pgoff, + struct list_head *to_kill) +{ + struct vm_area_struct *vma; + struct task_struct *tsk; + + i_mmap_lock_read(mapping); + read_lock(&tasklist_lock); + for_each_process(tsk) { + struct task_struct *t = task_early_kill(tsk, true); + + if (!t) + continue; + vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + if (vma->vm_mm == t->mm) + add_to_kill(t, page, pgoff, vma, to_kill); + } + } + read_unlock(&tasklist_lock); + i_mmap_unlock_read(mapping); +} +#endif /* CONFIG_FS_DAX */ + /* * Collect the processes who have the corrupted page mapped to kill. */ @@ -1588,6 +1625,45 @@ unlock: return rc; } +#ifdef CONFIG_FS_DAX +/** + * mf_dax_kill_procs - Collect and kill processes who are using this file range + * @mapping: address_space of the file in use + * @index: start pgoff of the range within the file + * @count: length of the range, in unit of PAGE_SIZE + * @mf_flags: memory failure flags + */ +int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, + unsigned long count, int mf_flags) +{ + LIST_HEAD(to_kill); + dax_entry_t cookie; + struct page *page; + size_t end = index + count; + + mf_flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; + + for (; index < end; index++) { + page = NULL; + cookie = dax_lock_mapping_entry(mapping, index, &page); + if (!cookie) + return -EBUSY; + if (!page) + goto unlock; + + SetPageHWPoison(page); + + collect_procs_fsdax(page, mapping, index, &to_kill); + unmap_and_kill(&to_kill, page_to_pfn(page), mapping, + index, mf_flags); +unlock: + dax_unlock_mapping_entry(mapping, index, cookie); + } + return 0; +} +EXPORT_SYMBOL_GPL(mf_dax_kill_procs); +#endif /* CONFIG_FS_DAX */ + /* * Called from hugetlb code with hugetlb_lock held. * -- cgit v1.2.3 From 6061b69b9a550a2ab84e805d0d2315ba6215f112 Mon Sep 17 00:00:00 2001 From: Shiyang Ruan Date: Fri, 3 Jun 2022 13:37:31 +0800 Subject: fsdax: set a CoW flag when associate reflink mappings Introduce a PAGE_MAPPING_DAX_COW flag to support association with CoW file mappings. In this case, since the dax-rmap has already took the responsibility to look up for shared files by given dax page, the page->mapping is no longer to used for rmap but for marking that this dax page is shared. And to make sure disassociation works fine, we use page->index as refcount, and clear page->mapping to the initial state when page->index is decreased to 0. With the help of this new flag, it is able to distinguish normal case and CoW case, and keep the warning in normal case. Link: https://lkml.kernel.org/r/20220603053738.1218681-8-ruansy.fnst@fujitsu.com Signed-off-by: Shiyang Ruan Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Cc: Al Viro Cc: Dan Williams Cc: Dan Williams Cc: Dave Chinner Cc: Goldwyn Rodrigues Cc: Goldwyn Rodrigues Cc: Jane Chu Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Naoya Horiguchi Cc: Ritesh Harjani Signed-off-by: Andrew Morton --- fs/dax.c | 50 +++++++++++++++++++++++++++++++++++++--------- include/linux/page-flags.h | 6 ++++++ 2 files changed, 47 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 65e44d78b3bb..b59b864017ad 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -334,13 +334,35 @@ static unsigned long dax_end_pfn(void *entry) for (pfn = dax_to_pfn(entry); \ pfn < dax_end_pfn(entry); pfn++) +static inline bool dax_mapping_is_cow(struct address_space *mapping) +{ + return (unsigned long)mapping == PAGE_MAPPING_DAX_COW; +} + /* - * TODO: for reflink+dax we need a way to associate a single page with - * multiple address_space instances at different linear_page_index() - * offsets. + * Set the page->mapping with FS_DAX_MAPPING_COW flag, increase the refcount. + */ +static inline void dax_mapping_set_cow(struct page *page) +{ + if ((uintptr_t)page->mapping != PAGE_MAPPING_DAX_COW) { + /* + * Reset the index if the page was already mapped + * regularly before. + */ + if (page->mapping) + page->index = 1; + page->mapping = (void *)PAGE_MAPPING_DAX_COW; + } + page->index++; +} + +/* + * When it is called in dax_insert_entry(), the cow flag will indicate that + * whether this entry is shared by multiple files. If so, set the page->mapping + * FS_DAX_MAPPING_COW, and use page->index as refcount. */ static void dax_associate_entry(void *entry, struct address_space *mapping, - struct vm_area_struct *vma, unsigned long address) + struct vm_area_struct *vma, unsigned long address, bool cow) { unsigned long size = dax_entry_size(entry), pfn, index; int i = 0; @@ -352,9 +374,13 @@ static void dax_associate_entry(void *entry, struct address_space *mapping, for_each_mapped_pfn(entry, pfn) { struct page *page = pfn_to_page(pfn); - WARN_ON_ONCE(page->mapping); - page->mapping = mapping; - page->index = index + i++; + if (cow) { + dax_mapping_set_cow(page); + } else { + WARN_ON_ONCE(page->mapping); + page->mapping = mapping; + page->index = index + i++; + } } } @@ -370,7 +396,12 @@ static void dax_disassociate_entry(void *entry, struct address_space *mapping, struct page *page = pfn_to_page(pfn); WARN_ON_ONCE(trunc && page_ref_count(page) > 1); - WARN_ON_ONCE(page->mapping && page->mapping != mapping); + if (dax_mapping_is_cow(page->mapping)) { + /* keep the CoW flag if this page is still shared */ + if (page->index-- > 0) + continue; + } else + WARN_ON_ONCE(page->mapping && page->mapping != mapping); page->mapping = NULL; page->index = 0; } @@ -830,7 +861,8 @@ static void *dax_insert_entry(struct xa_state *xas, void *old; dax_disassociate_entry(entry, mapping, false); - dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address); + dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address, + false); /* * Only swap our new entry into the page cache if the current * entry is a zero page or an empty entry. If a normal PTE or diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 82719d33c0f1..f2ff65f1bf83 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -661,6 +661,12 @@ PAGEFLAG_FALSE(VmemmapSelfHosted, vmemmap_self_hosted) #define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE) #define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE) +/* + * Different with flags above, this flag is used only for fsdax mode. It + * indicates that this page->mapping is now under reflink case. + */ +#define PAGE_MAPPING_DAX_COW 0x1 + static __always_inline bool folio_mapping_flags(struct folio *folio) { return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0; -- cgit v1.2.3 From 6f7db3894ae23eb5d40af4efb404aa0c072a68d2 Mon Sep 17 00:00:00 2001 From: Shiyang Ruan Date: Fri, 3 Jun 2022 13:37:36 +0800 Subject: fsdax: dedup file range to use a compare function With dax we cannot deal with readpage() etc. So, we create a dax comparison function which is similar with vfs_dedupe_file_range_compare(). And introduce dax_remap_file_range_prep() for filesystem use. Link: https://lkml.kernel.org/r/20220603053738.1218681-13-ruansy.fnst@fujitsu.com Signed-off-by: Goldwyn Rodrigues Signed-off-by: Shiyang Ruan Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Cc: Al Viro Cc: Dan Williams Cc: Dan Williams Cc: Dave Chinner Cc: Goldwyn Rodrigues Cc: Jane Chu Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Naoya Horiguchi Cc: Ritesh Harjani Signed-off-by: Andrew Morton --- fs/dax.c | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ fs/remap_range.c | 31 ++++++++++++++++---- fs/xfs/xfs_reflink.c | 8 +++-- include/linux/dax.h | 8 +++++ include/linux/fs.h | 12 +++++--- 5 files changed, 130 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 0aab32300531..e0f9c4a0a0c1 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1873,3 +1873,85 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, return dax_insert_pfn_mkwrite(vmf, pfn, order); } EXPORT_SYMBOL_GPL(dax_finish_sync_fault); + +static loff_t dax_range_compare_iter(struct iomap_iter *it_src, + struct iomap_iter *it_dest, u64 len, bool *same) +{ + const struct iomap *smap = &it_src->iomap; + const struct iomap *dmap = &it_dest->iomap; + loff_t pos1 = it_src->pos, pos2 = it_dest->pos; + void *saddr, *daddr; + int id, ret; + + len = min(len, min(smap->length, dmap->length)); + + if (smap->type == IOMAP_HOLE && dmap->type == IOMAP_HOLE) { + *same = true; + return len; + } + + if (smap->type == IOMAP_HOLE || dmap->type == IOMAP_HOLE) { + *same = false; + return 0; + } + + id = dax_read_lock(); + ret = dax_iomap_direct_access(smap, pos1, ALIGN(pos1 + len, PAGE_SIZE), + &saddr, NULL); + if (ret < 0) + goto out_unlock; + + ret = dax_iomap_direct_access(dmap, pos2, ALIGN(pos2 + len, PAGE_SIZE), + &daddr, NULL); + if (ret < 0) + goto out_unlock; + + *same = !memcmp(saddr, daddr, len); + if (!*same) + len = 0; + dax_read_unlock(id); + return len; + +out_unlock: + dax_read_unlock(id); + return -EIO; +} + +int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, + struct inode *dst, loff_t dstoff, loff_t len, bool *same, + const struct iomap_ops *ops) +{ + struct iomap_iter src_iter = { + .inode = src, + .pos = srcoff, + .len = len, + .flags = IOMAP_DAX, + }; + struct iomap_iter dst_iter = { + .inode = dst, + .pos = dstoff, + .len = len, + .flags = IOMAP_DAX, + }; + int ret; + + while ((ret = iomap_iter(&src_iter, ops)) > 0) { + while ((ret = iomap_iter(&dst_iter, ops)) > 0) { + dst_iter.processed = dax_range_compare_iter(&src_iter, + &dst_iter, len, same); + } + if (ret <= 0) + src_iter.processed = ret; + } + return ret; +} + +int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags, + const struct iomap_ops *ops) +{ + return __generic_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags, ops); +} +EXPORT_SYMBOL_GPL(dax_remap_file_range_prep); diff --git a/fs/remap_range.c b/fs/remap_range.c index e112b5424cdb..231de627c1b9 100644 --- a/fs/remap_range.c +++ b/fs/remap_range.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "internal.h" #include @@ -271,9 +272,11 @@ out_error: * If there's an error, then the usual negative error code is returned. * Otherwise returns 0 with *len set to the request length. */ -int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, - struct file *file_out, loff_t pos_out, - loff_t *len, unsigned int remap_flags) +int +__generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags, + const struct iomap_ops *dax_read_ops) { struct inode *inode_in = file_inode(file_in); struct inode *inode_out = file_inode(file_out); @@ -333,8 +336,18 @@ int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, if (remap_flags & REMAP_FILE_DEDUP) { bool is_same = false; - ret = vfs_dedupe_file_range_compare(file_in, pos_in, - file_out, pos_out, *len, &is_same); + if (*len == 0) + return 0; + + if (!IS_DAX(inode_in)) + ret = vfs_dedupe_file_range_compare(file_in, pos_in, + file_out, pos_out, *len, &is_same); + else if (dax_read_ops) + ret = dax_dedupe_file_range_compare(inode_in, pos_in, + inode_out, pos_out, *len, &is_same, + dax_read_ops); + else + return -EINVAL; if (ret) return ret; if (!is_same) @@ -352,6 +365,14 @@ int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, return ret; } + +int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags) +{ + return __generic_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags, NULL); +} EXPORT_SYMBOL(generic_remap_file_range_prep); loff_t do_clone_file_range(struct file *file_in, loff_t pos_in, diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index e7a7c00d93be..cbaf36d21020 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -1367,8 +1367,12 @@ xfs_reflink_remap_prep( if (IS_DAX(inode_in) || IS_DAX(inode_out)) goto out_unlock; - ret = generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out, - len, remap_flags); + if (!IS_DAX(inode_in)) + ret = generic_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags); + else + ret = dax_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags, &xfs_read_iomap_ops); if (ret || *len == 0) goto out_unlock; diff --git a/include/linux/dax.h b/include/linux/dax.h index 7116681b48c0..ba985333e26b 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -246,6 +246,14 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); int dax_invalidate_mapping_entry_sync(struct address_space *mapping, pgoff_t index); +int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, + struct inode *dest, loff_t destoff, + loff_t len, bool *is_same, + const struct iomap_ops *ops); +int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags, + const struct iomap_ops *ops); static inline bool dax_mapping(struct address_space *mapping) { return mapping->host && IS_DAX(mapping->host); diff --git a/include/linux/fs.h b/include/linux/fs.h index 9ad5e3520fae..134e9d7ad5d6 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -74,6 +74,7 @@ struct fsverity_operations; struct fs_context; struct fs_parameter_spec; struct fileattr; +struct iomap_ops; extern void __init inode_init(void); extern void __init inode_init_early(void); @@ -2070,10 +2071,13 @@ extern ssize_t vfs_copy_file_range(struct file *, loff_t , struct file *, extern ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, size_t len, unsigned int flags); -extern int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, - struct file *file_out, loff_t pos_out, - loff_t *count, - unsigned int remap_flags); +int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags, + const struct iomap_ops *dax_read_ops); +int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *count, unsigned int remap_flags); extern loff_t do_clone_file_range(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t len, unsigned int remap_flags); -- cgit v1.2.3 From 4fa6893faeaaea4fe4440512d2a708527ef47051 Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Thu, 16 Jun 2022 10:48:35 -0700 Subject: mm: thp: consolidate vma size check to transhuge_vma_suitable There are couple of places that check whether the vma size is ok for THP or whether address fits, they are open coded and duplicate, use transhuge_vma_suitable() to do the job by passing in (vma->end - HPAGE_PMD_SIZE). Move vma size check into hugepage_vma_check(). This will make khugepaged_enter() is as same as khugepaged_enter_vma(). There is just one caller for khugepaged_enter(), replace it to khugepaged_enter_vma() and remove khugepaged_enter(). Link: https://lkml.kernel.org/r/20220616174840.1202070-3-shy828301@gmail.com Signed-off-by: Yang Shi Reviewed-by: Zach O'Keefe Cc: Kirill A. Shutemov Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 11 +++++++++++ include/linux/khugepaged.h | 14 -------------- mm/huge_memory.c | 2 +- mm/khugepaged.c | 19 ++++++------------- 4 files changed, 18 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 648cb3ce7099..8a5a8bfce0f5 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -116,6 +116,17 @@ extern struct kobj_attribute shmem_enabled_attr; extern unsigned long transparent_hugepage_flags; +/* + * Do the below checks: + * - For file vma, check if the linear page offset of vma is + * HPAGE_PMD_NR aligned within the file. The hugepage is + * guaranteed to be hugepage-aligned within the file, but we must + * check that the PMD-aligned addresses in the VMA map to + * PMD-aligned offsets within the file, else the hugepage will + * not be PMD-mappable. + * - For all vmas, check if the haddr is in an aligned HPAGE_PMD_SIZE + * area. + */ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, unsigned long addr) { diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index 392d34c3c59a..31ca8a7f78f4 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -51,16 +51,6 @@ static inline void khugepaged_exit(struct mm_struct *mm) if (test_bit(MMF_VM_HUGEPAGE, &mm->flags)) __khugepaged_exit(mm); } - -static inline void khugepaged_enter(struct vm_area_struct *vma, - unsigned long vm_flags) -{ - if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && - khugepaged_enabled()) { - if (hugepage_vma_check(vma, vm_flags)) - __khugepaged_enter(vma->vm_mm); - } -} #else /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm) { @@ -68,10 +58,6 @@ static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm static inline void khugepaged_exit(struct mm_struct *mm) { } -static inline void khugepaged_enter(struct vm_area_struct *vma, - unsigned long vm_flags) -{ -} static inline void khugepaged_enter_vma(struct vm_area_struct *vma, unsigned long vm_flags) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index a563de8234c1..2751649aaf33 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -726,7 +726,7 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) return VM_FAULT_FALLBACK; if (unlikely(anon_vma_prepare(vma))) return VM_FAULT_OOM; - khugepaged_enter(vma, vma->vm_flags); + khugepaged_enter_vma(vma, vma->vm_flags); if (!(vmf->flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(vma->vm_mm) && diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 3eec970a884d..c7e22135f1b5 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -443,8 +443,8 @@ bool hugepage_vma_check(struct vm_area_struct *vma, if (vma_is_dax(vma)) return false; - if (vma->vm_file && !IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - - vma->vm_pgoff, HPAGE_PMD_NR)) + /* Check alignment for file vma and size for both file and anon vma */ + if (!transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE))) return false; /* Enabled via shmem mount options or sysfs settings. */ @@ -505,9 +505,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, unsigned long vm_flags) { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && - khugepaged_enabled() && - (((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < - (vma->vm_end & HPAGE_PMD_MASK))) { + khugepaged_enabled()) { if (hugepage_vma_check(vma, vm_flags)) __khugepaged_enter(vma->vm_mm); } @@ -948,7 +946,6 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, struct vm_area_struct **vmap) { struct vm_area_struct *vma; - unsigned long hstart, hend; if (unlikely(khugepaged_test_exit(mm))) return SCAN_ANY_PROCESS; @@ -957,9 +954,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, if (!vma) return SCAN_VMA_NULL; - hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; - hend = vma->vm_end & HPAGE_PMD_MASK; - if (address < hstart || address + HPAGE_PMD_SIZE > hend) + if (!transhuge_vma_suitable(vma, address)) return SCAN_ADDRESS_RANGE; if (!hugepage_vma_check(vma, vma->vm_flags)) return SCAN_VMA_CHECK; @@ -2135,10 +2130,8 @@ skip: progress++; continue; } - hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; - hend = vma->vm_end & HPAGE_PMD_MASK; - if (hstart >= hend) - goto skip; + hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE); + hend = round_down(vma->vm_end, HPAGE_PMD_SIZE); if (khugepaged_scan.address > hend) goto skip; if (khugepaged_scan.address < hstart) -- cgit v1.2.3 From 9fec51689ff60d9766b38051a0b1692f93d95364 Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Thu, 16 Jun 2022 10:48:37 -0700 Subject: mm: thp: kill transparent_hugepage_active() The transparent_hugepage_active() was introduced to show THP eligibility bit in smaps in proc, smaps is the only user. But it actually does the similar check as hugepage_vma_check() which is used by khugepaged. We definitely don't have to maintain two similar checks, so kill transparent_hugepage_active(). This patch also fixed the wrong behavior for VM_NO_KHUGEPAGED vmas. Also move hugepage_vma_check() to huge_memory.c and huge_mm.h since it is not only for khugepaged anymore. [akpm@linux-foundation.org: check vma->vm_mm, per Zach] [akpm@linux-foundation.org: add comment to vdso check] Link: https://lkml.kernel.org/r/20220616174840.1202070-5-shy828301@gmail.com Signed-off-by: Yang Shi Reviewed-by: Zach O'Keefe Cc: Kirill A. Shutemov Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 2 +- include/linux/huge_mm.h | 16 ++++++++------ include/linux/khugepaged.h | 2 -- mm/huge_memory.c | 53 ++++++++++++++++++++++++++++++++++++++-------- mm/khugepaged.c | 48 ++++------------------------------------- 5 files changed, 59 insertions(+), 62 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 1d7fd832123b..072cf770b5d0 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -863,7 +863,7 @@ static int show_smap(struct seq_file *m, void *v) __show_smap(m, &mss, false); seq_printf(m, "THPeligible: %d\n", - transparent_hugepage_active(vma)); + hugepage_vma_check(vma, vma->vm_flags, true)); if (arch_pkeys_enabled()) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 8a5a8bfce0f5..64487bcd0c7b 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -202,7 +202,9 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); } -bool transparent_hugepage_active(struct vm_area_struct *vma); +bool hugepage_vma_check(struct vm_area_struct *vma, + unsigned long vm_flags, + bool smaps); #define transparent_hugepage_use_zero_page() \ (transparent_hugepage_flags & \ @@ -351,11 +353,6 @@ static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) return false; } -static inline bool transparent_hugepage_active(struct vm_area_struct *vma) -{ - return false; -} - static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, unsigned long addr) { @@ -368,6 +365,13 @@ static inline bool transhuge_vma_enabled(struct vm_area_struct *vma, return false; } +static inline bool hugepage_vma_check(struct vm_area_struct *vma, + unsigned long vm_flags, + bool smaps) +{ + return false; +} + static inline void prep_transhuge_page(struct page *page) {} #define transparent_hugepage_flags 0UL diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index 31ca8a7f78f4..ea5fd4c398f7 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -10,8 +10,6 @@ extern struct attribute_group khugepaged_attr_group; extern int khugepaged_init(void); extern void khugepaged_destroy(void); extern int start_stop_khugepaged(void); -extern bool hugepage_vma_check(struct vm_area_struct *vma, - unsigned long vm_flags); extern void __khugepaged_enter(struct mm_struct *mm); extern void __khugepaged_exit(struct mm_struct *mm); extern void khugepaged_enter_vma(struct vm_area_struct *vma, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2751649aaf33..8cbd21aaf03e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -69,21 +69,56 @@ static atomic_t huge_zero_refcount; struct page *huge_zero_page __read_mostly; unsigned long huge_zero_pfn __read_mostly = ~0UL; -bool transparent_hugepage_active(struct vm_area_struct *vma) +bool hugepage_vma_check(struct vm_area_struct *vma, + unsigned long vm_flags, + bool smaps) { - /* The addr is used to check if the vma size fits */ - unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE; + if (!vma->vm_mm) /* vdso */ + return false; + + if (!transhuge_vma_enabled(vma, vm_flags)) + return false; - if (!transhuge_vma_suitable(vma, addr)) + if (vm_flags & VM_NO_KHUGEPAGED) return false; - if (vma_is_anonymous(vma)) - return __transparent_hugepage_enabled(vma); - if (vma_is_shmem(vma)) + + /* Don't run khugepaged against DAX vma */ + if (vma_is_dax(vma)) + return false; + + /* Check alignment for file vma and size for both file and anon vma */ + if (!transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE))) + return false; + + /* Enabled via shmem mount options or sysfs settings. */ + if (shmem_file(vma->vm_file)) return shmem_huge_enabled(vma); - if (transhuge_vma_enabled(vma, vma->vm_flags) && file_thp_enabled(vma)) + + if (!khugepaged_enabled()) + return false; + + /* THP settings require madvise. */ + if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always()) + return false; + + /* Only regular file is valid */ + if (file_thp_enabled(vma)) return true; - return false; + if (!vma_is_anonymous(vma)) + return false; + + if (vma_is_temporary_stack(vma)) + return false; + + /* + * THPeligible bit of smaps should show 1 for proper VMAs even + * though anon_vma is not initialized yet. + */ + if (!vma->anon_vma) + return smaps; + + return true; } static bool get_huge_zero_page(void) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 67e144e64b7f..6bbf3adac534 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -430,46 +430,6 @@ static inline int khugepaged_test_exit(struct mm_struct *mm) return atomic_read(&mm->mm_users) == 0; } -bool hugepage_vma_check(struct vm_area_struct *vma, - unsigned long vm_flags) -{ - if (!transhuge_vma_enabled(vma, vm_flags)) - return false; - - if (vm_flags & VM_NO_KHUGEPAGED) - return false; - - /* Don't run khugepaged against DAX vma */ - if (vma_is_dax(vma)) - return false; - - /* Check alignment for file vma and size for both file and anon vma */ - if (!transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE))) - return false; - - /* Enabled via shmem mount options or sysfs settings. */ - if (shmem_file(vma->vm_file)) - return shmem_huge_enabled(vma); - - if (!khugepaged_enabled()) - return false; - - /* THP settings require madvise. */ - if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always()) - return false; - - /* Only regular file is valid */ - if (file_thp_enabled(vma)) - return true; - - if (!vma->anon_vma || !vma_is_anonymous(vma)) - return false; - if (vma_is_temporary_stack(vma)) - return false; - - return true; -} - void __khugepaged_enter(struct mm_struct *mm) { struct mm_slot *mm_slot; @@ -506,7 +466,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && khugepaged_enabled()) { - if (hugepage_vma_check(vma, vm_flags)) + if (hugepage_vma_check(vma, vm_flags, false)) __khugepaged_enter(vma->vm_mm); } } @@ -956,7 +916,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, if (!transhuge_vma_suitable(vma, address)) return SCAN_ADDRESS_RANGE; - if (!hugepage_vma_check(vma, vma->vm_flags)) + if (!hugepage_vma_check(vma, vma->vm_flags, false)) return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then @@ -1441,7 +1401,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() * will not fail the vma for missing VM_HUGEPAGE */ - if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE)) + if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false)) return; /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ @@ -2131,7 +2091,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, progress++; break; } - if (!hugepage_vma_check(vma, vma->vm_flags)) { + if (!hugepage_vma_check(vma, vma->vm_flags, false)) { skip: progress++; continue; -- cgit v1.2.3 From 7da4e2cb8b1ff8221759bfc7512d651ee69516dc Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Thu, 16 Jun 2022 10:48:38 -0700 Subject: mm: thp: kill __transhuge_page_enabled() The page fault path checks THP eligibility with __transhuge_page_enabled() which does the similar thing as hugepage_vma_check(), so use hugepage_vma_check() instead. However page fault allows DAX and !anon_vma cases, so added a new flag, in_pf, to hugepage_vma_check() to make page fault work correctly. The in_pf flag is also used to skip shmem and file THP for page fault since shmem handles THP in its own shmem_fault() and file THP allocation on fault is not supported yet. Also remove hugepage_vma_enabled() since hugepage_vma_check() is the only caller now, it is not necessary to have a helper function. Link: https://lkml.kernel.org/r/20220616174840.1202070-6-shy828301@gmail.com Signed-off-by: Yang Shi Reviewed-by: Zach O'Keefe Cc: Kirill A. Shutemov Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 2 +- include/linux/huge_mm.h | 57 ++----------------------------------------------- mm/huge_memory.c | 51 +++++++++++++++++++++++++++++++++---------- mm/khugepaged.c | 8 +++---- mm/memory.c | 7 ++++-- 5 files changed, 52 insertions(+), 73 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 072cf770b5d0..a3398d0f1927 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -863,7 +863,7 @@ static int show_smap(struct seq_file *m, void *v) __show_smap(m, &mss, false); seq_printf(m, "THPeligible: %d\n", - hugepage_vma_check(vma, vma->vm_flags, true)); + hugepage_vma_check(vma, vma->vm_flags, true, false)); if (arch_pkeys_enabled()) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 64487bcd0c7b..cd8a6c5d9fe5 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -146,48 +146,6 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, return true; } -static inline bool transhuge_vma_enabled(struct vm_area_struct *vma, - unsigned long vm_flags) -{ - /* Explicitly disabled through madvise. */ - if ((vm_flags & VM_NOHUGEPAGE) || - test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) - return false; - return true; -} - -/* - * to be used on vmas which are known to support THP. - * Use transparent_hugepage_active otherwise - */ -static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) -{ - - /* - * If the hardware/firmware marked hugepage support disabled. - */ - if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_NEVER_DAX)) - return false; - - if (!transhuge_vma_enabled(vma, vma->vm_flags)) - return false; - - if (vma_is_temporary_stack(vma)) - return false; - - if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG)) - return true; - - if (vma_is_dax(vma)) - return true; - - if (transparent_hugepage_flags & - (1 << TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)) - return !!(vma->vm_flags & VM_HUGEPAGE); - - return false; -} - static inline bool file_thp_enabled(struct vm_area_struct *vma) { struct inode *inode; @@ -204,7 +162,7 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, - bool smaps); + bool smaps, bool in_pf); #define transparent_hugepage_use_zero_page() \ (transparent_hugepage_flags & \ @@ -348,26 +306,15 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) return false; } -static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) -{ - return false; -} - static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, unsigned long addr) { return false; } -static inline bool transhuge_vma_enabled(struct vm_area_struct *vma, - unsigned long vm_flags) -{ - return false; -} - static inline bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, - bool smaps) + bool smaps, bool in_pf) { return false; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8cbd21aaf03e..4b90c7021e52 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -71,27 +71,53 @@ unsigned long huge_zero_pfn __read_mostly = ~0UL; bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, - bool smaps) + bool smaps, bool in_pf) { if (!vma->vm_mm) /* vdso */ return false; - if (!transhuge_vma_enabled(vma, vm_flags)) + /* + * Explicitly disabled through madvise or prctl, or some + * architectures may disable THP for some mappings, for + * example, s390 kvm. + * */ + if ((vm_flags & VM_NOHUGEPAGE) || + test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) return false; - - if (vm_flags & VM_NO_KHUGEPAGED) + /* + * If the hardware/firmware marked hugepage support disabled. + */ + if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_NEVER_DAX)) return false; - /* Don't run khugepaged against DAX vma */ + /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ if (vma_is_dax(vma)) + return in_pf; + + /* + * Special VMA and hugetlb VMA. + * Must be checked after dax since some dax mappings may have + * VM_MIXEDMAP set. + */ + if (vm_flags & VM_NO_KHUGEPAGED) return false; - /* Check alignment for file vma and size for both file and anon vma */ - if (!transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE))) + /* + * Check alignment for file vma and size for both file and anon vma. + * + * Skip the check for page fault. Huge fault does the check in fault + * handlers. And this check is not suitable for huge PUD fault. + */ + if (!in_pf && + !transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE))) return false; - /* Enabled via shmem mount options or sysfs settings. */ - if (shmem_file(vma->vm_file)) + /* + * Enabled via shmem mount options or sysfs settings. + * Must be done before hugepage flags check since shmem has its + * own flags. + */ + if (!in_pf && shmem_file(vma->vm_file)) return shmem_huge_enabled(vma); if (!khugepaged_enabled()) @@ -102,7 +128,7 @@ bool hugepage_vma_check(struct vm_area_struct *vma, return false; /* Only regular file is valid */ - if (file_thp_enabled(vma)) + if (!in_pf && file_thp_enabled(vma)) return true; if (!vma_is_anonymous(vma)) @@ -114,9 +140,12 @@ bool hugepage_vma_check(struct vm_area_struct *vma, /* * THPeligible bit of smaps should show 1 for proper VMAs even * though anon_vma is not initialized yet. + * + * Allow page fault since anon_vma may be not initialized until + * the first page fault. */ if (!vma->anon_vma) - return smaps; + return (smaps || in_pf); return true; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 6bbf3adac534..d683ef1edeb5 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -466,7 +466,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && khugepaged_enabled()) { - if (hugepage_vma_check(vma, vm_flags, false)) + if (hugepage_vma_check(vma, vm_flags, false, false)) __khugepaged_enter(vma->vm_mm); } } @@ -916,7 +916,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, if (!transhuge_vma_suitable(vma, address)) return SCAN_ADDRESS_RANGE; - if (!hugepage_vma_check(vma, vma->vm_flags, false)) + if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then @@ -1401,7 +1401,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() * will not fail the vma for missing VM_HUGEPAGE */ - if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false)) + if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false, false)) return; /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ @@ -2091,7 +2091,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, progress++; break; } - if (!hugepage_vma_check(vma, vma->vm_flags, false)) { + if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) { skip: progress++; continue; diff --git a/mm/memory.c b/mm/memory.c index dce0b2e686eb..2392d5db473a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4970,6 +4970,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, .gfp_mask = __get_fault_gfp_mask(vma), }; struct mm_struct *mm = vma->vm_mm; + unsigned long vm_flags = vma->vm_flags; pgd_t *pgd; p4d_t *p4d; vm_fault_t ret; @@ -4983,7 +4984,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, if (!vmf.pud) return VM_FAULT_OOM; retry_pud: - if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) { + if (pud_none(*vmf.pud) && + hugepage_vma_check(vma, vm_flags, false, true)) { ret = create_huge_pud(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; @@ -5016,7 +5018,8 @@ retry_pud: if (pud_trans_unstable(vmf.pud)) goto retry_pud; - if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) { + if (pmd_none(*vmf.pmd) && + hugepage_vma_check(vma, vm_flags, false, true)) { ret = create_huge_pmd(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; -- cgit v1.2.3 From 1064026bab9f011bdea1251d44d66bbbcee04f6e Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Thu, 16 Jun 2022 10:48:39 -0700 Subject: mm: khugepaged: reorg some khugepaged helpers The khugepaged_{enabled|always|req_madv} are not khugepaged only anymore, move them to huge_mm.h and rename to hugepage_flags_xxx, and remove khugepaged_req_madv due to no users. Also move khugepaged_defrag to khugepaged.c since its only caller is in that file, it doesn't have to be in a header file. Link: https://lkml.kernel.org/r/20220616174840.1202070-7-shy828301@gmail.com Signed-off-by: Yang Shi Reviewed-by: Zach O'Keefe Cc: Kirill A. Shutemov Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 8 ++++++++ include/linux/khugepaged.h | 14 -------------- mm/huge_memory.c | 4 ++-- mm/khugepaged.c | 18 +++++++++++------- 4 files changed, 21 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index cd8a6c5d9fe5..ae3d8e2fd9e2 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -116,6 +116,14 @@ extern struct kobj_attribute shmem_enabled_attr; extern unsigned long transparent_hugepage_flags; +#define hugepage_flags_enabled() \ + (transparent_hugepage_flags & \ + ((1<flags)) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4b90c7021e52..8e1b3d9f7ebf 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -120,11 +120,11 @@ bool hugepage_vma_check(struct vm_area_struct *vma, if (!in_pf && shmem_file(vma->vm_file)) return shmem_huge_enabled(vma); - if (!khugepaged_enabled()) + if (!hugepage_flags_enabled()) return false; /* THP settings require madvise. */ - if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always()) + if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always()) return false; /* Only regular file is valid */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index d683ef1edeb5..01f71786d530 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -465,7 +465,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, unsigned long vm_flags) { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && - khugepaged_enabled()) { + hugepage_flags_enabled()) { if (hugepage_vma_check(vma, vm_flags, false, false)) __khugepaged_enter(vma->vm_mm); } @@ -761,6 +761,10 @@ static bool khugepaged_scan_abort(int nid) return false; } +#define khugepaged_defrag() \ + (transparent_hugepage_flags & \ + (1< Date: Tue, 21 Jun 2022 16:56:17 -0700 Subject: hugetlb: skip to end of PT page mapping when pte not present Patch series "hugetlb: speed up linear address scanning", v2. At unmap, fork and remap time hugetlb address ranges are linearly scanned. We can optimize these scans if the ranges are sparsely populated. Also, enable page table "Lazy copy" for hugetlb at fork. NOTE: Architectures not defining CONFIG_ARCH_WANT_GENERAL_HUGETLB need to add an arch specific version hugetlb_mask_last_page() to take advantage of sparse address scanning improvements. Baolin Wang added the routine for arm64. Other architectures which could be optimized are: ia64, mips, parisc, powerpc, s390, sh and sparc. This patch (of 4): HugeTLB address ranges are linearly scanned during fork, unmap and remap operations. If a non-present entry is encountered, the code currently continues to the next huge page aligned address. However, a non-present entry implies that the page table page for that entry is not present. Therefore, the linear scan can skip to the end of range mapped by the page table page. This can speed operations on large sparsely populated hugetlb mappings. Create a new routine hugetlb_mask_last_page() that will return an address mask. When the mask is ORed with an address, the result will be the address of the last huge page mapped by the associated page table page. Use this mask to update addresses in routines which linearly scan hugetlb address ranges when a non-present pte is encountered. hugetlb_mask_last_page is related to the implementation of huge_pte_offset as hugetlb_mask_last_page is called when huge_pte_offset returns NULL. This patch only provides a complete hugetlb_mask_last_page implementation when CONFIG_ARCH_WANT_GENERAL_HUGETLB is defined. Architectures which provide their own versions of huge_pte_offset can also provide their own version of hugetlb_mask_last_page. Link: https://lkml.kernel.org/r/20220621235620.291305-1-mike.kravetz@oracle.com Link: https://lkml.kernel.org/r/20220621235620.291305-2-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Tested-by: Baolin Wang Reviewed-by: Baolin Wang Acked-by: Muchun Song Reported-by: kernel test robot Cc: Michal Hocko Cc: Peter Xu Cc: Naoya Horiguchi Cc: James Houghton Cc: Mina Almasry Cc: "Aneesh Kumar K.V" Cc: Anshuman Khandual Cc: Paul Walmsley Cc: Christian Borntraeger Cc: Catalin Marinas Cc: Will Deacon Cc: Rolf Eike Beer Cc: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 1 + mm/hugetlb.c | 56 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index c6cccfaf8708..ce30fad5fd13 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -194,6 +194,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, unsigned long sz); pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz); +unsigned long hugetlb_mask_last_page(struct hstate *h); int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long *addr, pte_t *ptep); void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index ffdf3fc4a83f..95fd1c36c17f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4727,6 +4727,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, unsigned long npages = pages_per_huge_page(h); struct address_space *mapping = src_vma->vm_file->f_mapping; struct mmu_notifier_range range; + unsigned long last_addr_mask; int ret = 0; if (cow) { @@ -4746,11 +4747,14 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, i_mmap_lock_read(mapping); } + last_addr_mask = hugetlb_mask_last_page(h); for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) { spinlock_t *src_ptl, *dst_ptl; src_pte = huge_pte_offset(src, addr, sz); - if (!src_pte) + if (!src_pte) { + addr |= last_addr_mask; continue; + } dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz); if (!dst_pte) { ret = -ENOMEM; @@ -4767,8 +4771,10 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, * after taking the lock below. */ dst_entry = huge_ptep_get(dst_pte); - if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) + if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) { + addr |= last_addr_mask; continue; + } dst_ptl = huge_pte_lock(h, dst, dst_pte); src_ptl = huge_pte_lockptr(h, src, src_pte); @@ -4928,6 +4934,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, unsigned long sz = huge_page_size(h); struct mm_struct *mm = vma->vm_mm; unsigned long old_end = old_addr + len; + unsigned long last_addr_mask; unsigned long old_addr_copy; pte_t *src_pte, *dst_pte; struct mmu_notifier_range range; @@ -4943,12 +4950,16 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, flush_cache_range(vma, range.start, range.end); mmu_notifier_invalidate_range_start(&range); + last_addr_mask = hugetlb_mask_last_page(h); /* Prevent race with file truncation */ i_mmap_lock_write(mapping); for (; old_addr < old_end; old_addr += sz, new_addr += sz) { src_pte = huge_pte_offset(mm, old_addr, sz); - if (!src_pte) + if (!src_pte) { + old_addr |= last_addr_mask; + new_addr |= last_addr_mask; continue; + } if (huge_pte_none(huge_ptep_get(src_pte))) continue; @@ -4993,6 +5004,7 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct struct hstate *h = hstate_vma(vma); unsigned long sz = huge_page_size(h); struct mmu_notifier_range range; + unsigned long last_addr_mask; bool force_flush = false; WARN_ON(!is_vm_hugetlb_page(vma)); @@ -5013,11 +5025,14 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct end); adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); mmu_notifier_invalidate_range_start(&range); + last_addr_mask = hugetlb_mask_last_page(h); address = start; for (; address < end; address += sz) { ptep = huge_pte_offset(mm, address, sz); - if (!ptep) + if (!ptep) { + address |= last_addr_mask; continue; + } ptl = huge_pte_lock(h, mm, ptep); if (huge_pmd_unshare(mm, vma, &address, ptep)) { @@ -6285,6 +6300,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, unsigned long pages = 0, psize = huge_page_size(h); bool shared_pmd = false; struct mmu_notifier_range range; + unsigned long last_addr_mask; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; @@ -6301,12 +6317,15 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, flush_cache_range(vma, range.start, range.end); mmu_notifier_invalidate_range_start(&range); + last_addr_mask = hugetlb_mask_last_page(h); i_mmap_lock_write(vma->vm_file->f_mapping); for (; address < end; address += psize) { spinlock_t *ptl; ptep = huge_pte_offset(mm, address, psize); - if (!ptep) + if (!ptep) { + address |= last_addr_mask; continue; + } ptl = huge_pte_lock(h, mm, ptep); if (huge_pmd_unshare(mm, vma, &address, ptep)) { /* @@ -6856,6 +6875,33 @@ pte_t *huge_pte_offset(struct mm_struct *mm, return (pte_t *)pmd; } +/* + * Return a mask that can be used to update an address to the last huge + * page in a page table page mapping size. Used to skip non-present + * page table entries when linearly scanning address ranges. Architectures + * with unique huge page to page table relationships can define their own + * version of this routine. + */ +unsigned long hugetlb_mask_last_page(struct hstate *h) +{ + unsigned long hp_size = huge_page_size(h); + + if (hp_size == PUD_SIZE) + return P4D_SIZE - PUD_SIZE; + else if (hp_size == PMD_SIZE) + return PUD_SIZE - PMD_SIZE; + else + return 0UL; +} + +#else + +/* See description above. Architectures can provide their own version. */ +__weak unsigned long hugetlb_mask_last_page(struct hstate *h) +{ + return 0UL; +} + #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ /* -- cgit v1.2.3 From 4ddb4d91b82f4b64458fe35bc8e395c7c082ea2b Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Tue, 21 Jun 2022 16:56:19 -0700 Subject: hugetlb: do not update address in huge_pmd_unshare As an optimization for loops sequentially processing hugetlb address ranges, huge_pmd_unshare would update a passed address if it unshared a pmd. Updating a loop control variable outside the loop like this is generally a bad idea. These loops are now using hugetlb_mask_last_page to optimize scanning when non-present ptes are discovered. The same can be done when huge_pmd_unshare returns 1 indicating a pmd was unshared. Remove address update from huge_pmd_unshare. Change the passed argument type and update all callers. In loops sequentially processing addresses use hugetlb_mask_last_page to update address if pmd is unshared. [sfr@canb.auug.org.au: fix an unused variable warning/error] Link: https://lkml.kernel.org/r/20220622171117.70850960@canb.auug.org.au Link: https://lkml.kernel.org/r/20220621235620.291305-4-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Signed-off-by: Stephen Rothwell Acked-by: Muchun Song Reviewed-by: Baolin Wang Cc: "Aneesh Kumar K.V" Cc: Anshuman Khandual Cc: Catalin Marinas Cc: Christian Borntraeger Cc: David Hildenbrand Cc: James Houghton Cc: kernel test robot Cc: Michal Hocko Cc: Mina Almasry Cc: Naoya Horiguchi Cc: Paul Walmsley Cc: Peter Xu Cc: Rolf Eike Beer Cc: Will Deacon Cc: Stephen Rothwell Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 4 ++-- mm/hugetlb.c | 44 +++++++++++++++++--------------------------- mm/rmap.c | 4 ++-- 3 files changed, 21 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ce30fad5fd13..75ee739d815b 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -196,7 +196,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz); unsigned long hugetlb_mask_last_page(struct hstate *h); int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long *addr, pte_t *ptep); + unsigned long addr, pte_t *ptep); void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end); struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, @@ -243,7 +243,7 @@ static inline struct address_space *hugetlb_page_mapping_lock_write( static inline int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long *addr, pte_t *ptep) + unsigned long addr, pte_t *ptep) { return 0; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 95fd1c36c17f..96635a2874e3 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4935,7 +4935,6 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, struct mm_struct *mm = vma->vm_mm; unsigned long old_end = old_addr + len; unsigned long last_addr_mask; - unsigned long old_addr_copy; pte_t *src_pte, *dst_pte; struct mmu_notifier_range range; bool shared_pmd = false; @@ -4963,14 +4962,10 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, if (huge_pte_none(huge_ptep_get(src_pte))) continue; - /* old_addr arg to huge_pmd_unshare() is a pointer and so the - * arg may be modified. Pass a copy instead to preserve the - * value in old_addr. - */ - old_addr_copy = old_addr; - - if (huge_pmd_unshare(mm, vma, &old_addr_copy, src_pte)) { + if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) { shared_pmd = true; + old_addr |= last_addr_mask; + new_addr |= last_addr_mask; continue; } @@ -5035,10 +5030,11 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct } ptl = huge_pte_lock(h, mm, ptep); - if (huge_pmd_unshare(mm, vma, &address, ptep)) { + if (huge_pmd_unshare(mm, vma, address, ptep)) { spin_unlock(ptl); tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE); force_flush = true; + address |= last_addr_mask; continue; } @@ -6327,7 +6323,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, continue; } ptl = huge_pte_lock(h, mm, ptep); - if (huge_pmd_unshare(mm, vma, &address, ptep)) { + if (huge_pmd_unshare(mm, vma, address, ptep)) { /* * When uffd-wp is enabled on the vma, unshare * shouldn't happen at all. Warn about it if it @@ -6337,6 +6333,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, pages++; spin_unlock(ptl); shared_pmd = true; + address |= last_addr_mask; continue; } pte = huge_ptep_get(ptep); @@ -6759,11 +6756,11 @@ out: * 0 the underlying pte page is not shared, or it is the last user */ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long *addr, pte_t *ptep) + unsigned long addr, pte_t *ptep) { - pgd_t *pgd = pgd_offset(mm, *addr); - p4d_t *p4d = p4d_offset(pgd, *addr); - pud_t *pud = pud_offset(p4d, *addr); + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); i_mmap_assert_write_locked(vma->vm_file->f_mapping); BUG_ON(page_count(virt_to_page(ptep)) == 0); @@ -6773,14 +6770,6 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, pud_clear(pud); put_page(virt_to_page(ptep)); mm_dec_nr_pmds(mm); - /* - * This update of passed address optimizes loops sequentially - * processing addresses in increments of huge page size (PMD_SIZE - * in this case). By clearing the pud, a PUD_SIZE area is unmapped. - * Update address to the 'last page' in the cleared area so that - * calling loop can move to first page past this area. - */ - *addr |= PUD_SIZE - PMD_SIZE; return 1; } @@ -6792,7 +6781,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, } int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long *addr, pte_t *ptep) + unsigned long addr, pte_t *ptep) { return 0; } @@ -6899,6 +6888,10 @@ unsigned long hugetlb_mask_last_page(struct hstate *h) /* See description above. Architectures can provide their own version. */ __weak unsigned long hugetlb_mask_last_page(struct hstate *h) { +#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE + if (huge_page_size(h) == PMD_SIZE) + return PUD_SIZE - PMD_SIZE; +#endif return 0UL; } @@ -7125,14 +7118,11 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) mmu_notifier_invalidate_range_start(&range); i_mmap_lock_write(vma->vm_file->f_mapping); for (address = start; address < end; address += PUD_SIZE) { - unsigned long tmp = address; - ptep = huge_pte_offset(mm, address, sz); if (!ptep) continue; ptl = huge_pte_lock(h, mm, ptep); - /* We don't want 'address' to be changed */ - huge_pmd_unshare(mm, vma, &tmp, ptep); + huge_pmd_unshare(mm, vma, address, ptep); spin_unlock(ptl); } flush_hugetlb_tlb_range(vma, start, end); diff --git a/mm/rmap.c b/mm/rmap.c index 0532fd92ecb3..fb6b3b47f3e4 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1559,7 +1559,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * do this outside rmap routines. */ VM_BUG_ON(!anon && !(flags & TTU_RMAP_LOCKED)); - if (!anon && huge_pmd_unshare(mm, vma, &address, pvmw.pte)) { + if (!anon && huge_pmd_unshare(mm, vma, address, pvmw.pte)) { flush_tlb_range(vma, range.start, range.end); mmu_notifier_invalidate_range(mm, range.start, range.end); @@ -1920,7 +1920,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * do this outside rmap routines. */ VM_BUG_ON(!anon && !(flags & TTU_RMAP_LOCKED)); - if (!anon && huge_pmd_unshare(mm, vma, &address, pvmw.pte)) { + if (!anon && huge_pmd_unshare(mm, vma, address, pvmw.pte)) { flush_tlb_range(vma, range.start, range.end); mmu_notifier_invalidate_range(mm, range.start, range.end); -- cgit v1.2.3 From bf75f200569dd05ac2112797f44548beb6b4be26 Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Fri, 24 Jun 2022 13:54:17 +0100 Subject: mm/page_alloc: add page->buddy_list and page->pcp_list Patch series "Drain remote per-cpu directly", v5. Some setups, notably NOHZ_FULL CPUs, may be running realtime or latency-sensitive applications that cannot tolerate interference due to per-cpu drain work queued by __drain_all_pages(). Introduce a new mechanism to remotely drain the per-cpu lists. It is made possible by remotely locking 'struct per_cpu_pages' new per-cpu spinlocks. This has two advantages, the time to drain is more predictable and other unrelated tasks are not interrupted. This series has the same intent as Nicolas' series "mm/page_alloc: Remote per-cpu lists drain support" -- avoid interference of a high priority task due to a workqueue item draining per-cpu page lists. While many workloads can tolerate a brief interruption, it may cause a real-time task running on a NOHZ_FULL CPU to miss a deadline and at minimum, the draining is non-deterministic. Currently an IRQ-safe local_lock protects the page allocator per-cpu lists. The local_lock on its own prevents migration and the IRQ disabling protects from corruption due to an interrupt arriving while a page allocation is in progress. This series adjusts the locking. A spinlock is added to struct per_cpu_pages to protect the list contents while local_lock_irq is ultimately replaced by just the spinlock in the final patch. This allows a remote CPU to safely. Follow-on work should allow the spin_lock_irqsave to be converted to spin_lock to avoid IRQs being disabled/enabled in most cases. The follow-on patch will be one kernel release later as it is relatively high risk and it'll make bisections more clear if there are any problems. Patch 1 is a cosmetic patch to clarify when page->lru is storing buddy pages and when it is storing per-cpu pages. Patch 2 shrinks per_cpu_pages to make room for a spin lock. Strictly speaking this is not necessary but it avoids per_cpu_pages consuming another cache line. Patch 3 is a preparation patch to avoid code duplication. Patch 4 is a minor correction. Patch 5 uses a spin_lock to protect the per_cpu_pages contents while still relying on local_lock to prevent migration, stabilise the pcp lookup and prevent IRQ reentrancy. Patch 6 remote drains per-cpu pages directly instead of using a workqueue. Patch 7 uses a normal spinlock instead of local_lock for remote draining This patch (of 7): The page allocator uses page->lru for storing pages on either buddy or PCP lists. Create page->buddy_list and page->pcp_list as a union with page->lru. This is simply to clarify what type of list a page is on in the page allocator. No functional change intended. [minchan@kernel.org: fix page lru fields in macros] Link: https://lkml.kernel.org/r/20220624125423.6126-2-mgorman@techsingularity.net Signed-off-by: Mel Gorman Tested-by: Minchan Kim Acked-by: Minchan Kim Reviewed-by: Nicolas Saenz Julienne Acked-by: Vlastimil Babka Tested-by: Yu Zhao Cc: Marcelo Tosatti Cc: Michal Hocko Cc: Hugh Dickins Cc: Marek Szyprowski Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 5 +++++ mm/page_alloc.c | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 6b961a29bf26..cf97f3884fda 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -87,6 +87,7 @@ struct page { */ union { struct list_head lru; + /* Or, for the Unevictable "LRU list" slot */ struct { /* Always even, to negate PageTail */ @@ -94,6 +95,10 @@ struct page { /* Count page's or folio's mlocks */ unsigned int mlock_count; }; + + /* Or, free page */ + struct list_head buddy_list; + struct list_head pcp_list; }; /* See page-flags.h for PAGE_MAPPING_FLAGS */ struct address_space *mapping; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c9c02b23f02f..78ba5ba66586 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -793,7 +793,7 @@ static inline bool set_page_guard(struct zone *zone, struct page *page, return false; __SetPageGuard(page); - INIT_LIST_HEAD(&page->lru); + INIT_LIST_HEAD(&page->buddy_list); set_page_private(page, order); /* Guard pages are not available for any usage */ __mod_zone_freepage_state(zone, -(1 << order), migratetype); @@ -936,7 +936,7 @@ static inline void add_to_free_list(struct page *page, struct zone *zone, { struct free_area *area = &zone->free_area[order]; - list_add(&page->lru, &area->free_list[migratetype]); + list_add(&page->buddy_list, &area->free_list[migratetype]); area->nr_free++; } @@ -946,7 +946,7 @@ static inline void add_to_free_list_tail(struct page *page, struct zone *zone, { struct free_area *area = &zone->free_area[order]; - list_add_tail(&page->lru, &area->free_list[migratetype]); + list_add_tail(&page->buddy_list, &area->free_list[migratetype]); area->nr_free++; } @@ -960,7 +960,7 @@ static inline void move_to_free_list(struct page *page, struct zone *zone, { struct free_area *area = &zone->free_area[order]; - list_move_tail(&page->lru, &area->free_list[migratetype]); + list_move_tail(&page->buddy_list, &area->free_list[migratetype]); } static inline void del_page_from_free_list(struct page *page, struct zone *zone, @@ -970,7 +970,7 @@ static inline void del_page_from_free_list(struct page *page, struct zone *zone, if (page_reported(page)) __ClearPageReported(page); - list_del(&page->lru); + list_del(&page->buddy_list); __ClearPageBuddy(page); set_page_private(page, 0); zone->free_area[order].nr_free--; @@ -1508,11 +1508,11 @@ static void free_pcppages_bulk(struct zone *zone, int count, do { int mt; - page = list_last_entry(list, struct page, lru); + page = list_last_entry(list, struct page, pcp_list); mt = get_pcppage_migratetype(page); /* must delete to avoid corrupting pcp list */ - list_del(&page->lru); + list_del(&page->pcp_list); count -= nr_pages; pcp->count -= nr_pages; @@ -3072,7 +3072,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, * for IO devices that can merge IO requests if the physical * pages are ordered properly. */ - list_add_tail(&page->lru, list); + list_add_tail(&page->pcp_list, list); allocated++; if (is_migrate_cma(get_pcppage_migratetype(page))) __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, @@ -3322,7 +3322,7 @@ void mark_free_pages(struct zone *zone) for_each_migratetype_order(order, t) { list_for_each_entry(page, - &zone->free_area[order].free_list[t], lru) { + &zone->free_area[order].free_list[t], buddy_list) { unsigned long i; pfn = page_to_pfn(page); @@ -3411,7 +3411,7 @@ static void free_unref_page_commit(struct page *page, int migratetype, __count_vm_event(PGFREE); pcp = this_cpu_ptr(zone->per_cpu_pageset); pindex = order_to_pindex(migratetype, order); - list_add(&page->lru, &pcp->lists[pindex]); + list_add(&page->pcp_list, &pcp->lists[pindex]); pcp->count += 1 << order; /* @@ -3674,8 +3674,8 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, return NULL; } - page = list_first_entry(list, struct page, lru); - list_del(&page->lru); + page = list_first_entry(list, struct page, pcp_list); + list_del(&page->pcp_list); pcp->count -= 1 << order; } while (check_new_pcp(page, order)); -- cgit v1.2.3 From 5d0a661d808fc8ddc26940b1a12b82ae356f3ae2 Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Fri, 24 Jun 2022 13:54:18 +0100 Subject: mm/page_alloc: use only one PCP list for THP-sized allocations The per_cpu_pages is cache-aligned on a standard x86-64 distribution configuration but a later patch will add a new field which would push the structure into the next cache line. Use only one list to store THP-sized pages on the per-cpu list. This assumes that the vast majority of THP-sized allocations are GFP_MOVABLE but even if it was another type, it would not contribute to serious fragmentation that potentially causes a later THP allocation failure. Align per_cpu_pages on the cacheline boundary to ensure there is no false cache sharing. After this patch, the structure sizing is; struct per_cpu_pages { int count; /* 0 4 */ int high; /* 4 4 */ int batch; /* 8 4 */ short int free_factor; /* 12 2 */ short int expire; /* 14 2 */ struct list_head lists[13]; /* 16 208 */ /* size: 256, cachelines: 4, members: 6 */ /* padding: 32 */ } __attribute__((__aligned__(64))); Link: https://lkml.kernel.org/r/20220624125423.6126-3-mgorman@techsingularity.net Signed-off-by: Mel Gorman Tested-by: Minchan Kim Acked-by: Minchan Kim Acked-by: Vlastimil Babka Tested-by: Yu Zhao Cc: Hugh Dickins Cc: Marcelo Tosatti Cc: Marek Szyprowski Cc: Michal Hocko Cc: Nicolas Saenz Julienne Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 11 +++++++---- mm/page_alloc.c | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 5da1135e6755..041136b5628a 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -355,15 +355,18 @@ enum zone_watermarks { }; /* - * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER plus one additional - * for pageblock size for THP if configured. + * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER. One additional list + * for THP which will usually be GFP_MOVABLE. Even if it is another type, + * it should not contribute to serious fragmentation causing THP allocation + * failures. */ #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define NR_PCP_THP 1 #else #define NR_PCP_THP 0 #endif -#define NR_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1 + NR_PCP_THP)) +#define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1)) +#define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP) /* * Shift to encode migratetype and order in the same integer, with order @@ -389,7 +392,7 @@ struct per_cpu_pages { /* Lists of pages, one per migrate type stored on the pcp-lists */ struct list_head lists[NR_PCP_LISTS]; -}; +} ____cacheline_aligned_in_smp; struct per_cpu_zonestat { #ifdef CONFIG_SMP diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 78ba5ba66586..b5c340d2cb43 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -653,7 +653,7 @@ static inline unsigned int order_to_pindex(int migratetype, int order) #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (order > PAGE_ALLOC_COSTLY_ORDER) { VM_BUG_ON(order != pageblock_order); - base = PAGE_ALLOC_COSTLY_ORDER + 1; + return NR_LOWORDER_PCP_LISTS; } #else VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); @@ -667,7 +667,7 @@ static inline int pindex_to_order(unsigned int pindex) int order = pindex / MIGRATE_PCPTYPES; #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (order > PAGE_ALLOC_COSTLY_ORDER) + if (pindex == NR_LOWORDER_PCP_LISTS) order = pageblock_order; #else VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); -- cgit v1.2.3 From 4b23a68f953628eb4e4b7fe1294ebf93d4b8ceee Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Fri, 24 Jun 2022 13:54:21 +0100 Subject: mm/page_alloc: protect PCP lists with a spinlock Currently the PCP lists are protected by using local_lock_irqsave to prevent migration and IRQ reentrancy but this is inconvenient. Remote draining of the lists is impossible and a workqueue is required and every task allocation/free must disable then enable interrupts which is expensive. As preparation for dealing with both of those problems, protect the lists with a spinlock. The IRQ-unsafe version of the lock is used because IRQs are already disabled by local_lock_irqsave. spin_trylock is used in combination with local_lock_irqsave() but later will be replaced with a spin_trylock_irqsave when the local_lock is removed. The per_cpu_pages still fits within the same number of cache lines after this patch relative to before the series. struct per_cpu_pages { spinlock_t lock; /* 0 4 */ int count; /* 4 4 */ int high; /* 8 4 */ int batch; /* 12 4 */ short int free_factor; /* 16 2 */ short int expire; /* 18 2 */ /* XXX 4 bytes hole, try to pack */ struct list_head lists[13]; /* 24 208 */ /* size: 256, cachelines: 4, members: 7 */ /* sum members: 228, holes: 1, sum holes: 4 */ /* padding: 24 */ } __attribute__((__aligned__(64))); There is overhead in the fast path due to acquiring the spinlock even though the spinlock is per-cpu and uncontended in the common case. Page Fault Test (PFT) running on a 1-socket reported the following results on a 1 socket machine. 5.19.0-rc3 5.19.0-rc3 vanilla mm-pcpspinirq-v5r16 Hmean faults/sec-1 869275.7381 ( 0.00%) 874597.5167 * 0.61%* Hmean faults/sec-3 2370266.6681 ( 0.00%) 2379802.0362 * 0.40%* Hmean faults/sec-5 2701099.7019 ( 0.00%) 2664889.7003 * -1.34%* Hmean faults/sec-7 3517170.9157 ( 0.00%) 3491122.8242 * -0.74%* Hmean faults/sec-8 3965729.6187 ( 0.00%) 3939727.0243 * -0.66%* There is a small hit in the number of faults per second but given that the results are more stable, it's borderline noise. [akpm@linux-foundation.org: add missing local_unlock_irqrestore() on contention path] Link: https://lkml.kernel.org/r/20220624125423.6126-6-mgorman@techsingularity.net Signed-off-by: Mel Gorman Tested-by: Yu Zhao Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne Acked-by: Vlastimil Babka Cc: Hugh Dickins Cc: Marcelo Tosatti Cc: Marek Szyprowski Cc: Michal Hocko Cc: Minchan Kim Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 1 + mm/page_alloc.c | 119 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 99 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 041136b5628a..578247a341b2 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -382,6 +382,7 @@ enum zone_watermarks { /* Fields and list protected by pagesets local_lock in page_alloc.c */ struct per_cpu_pages { + spinlock_t lock; /* Protects lists field */ int count; /* number of pages in the list */ int high; /* high watermark, emptying needed */ int batch; /* chunk size for buddy add/remove */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 026c9437456c..a08ec4ac7ef2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -133,6 +133,20 @@ static DEFINE_PER_CPU(struct pagesets, pagesets) = { .lock = INIT_LOCAL_LOCK(lock), }; +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) +/* + * On SMP, spin_trylock is sufficient protection. + * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP. + */ +#define pcp_trylock_prepare(flags) do { } while (0) +#define pcp_trylock_finish(flag) do { } while (0) +#else + +/* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */ +#define pcp_trylock_prepare(flags) local_irq_save(flags) +#define pcp_trylock_finish(flags) local_irq_restore(flags) +#endif + #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); @@ -3101,15 +3115,22 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, */ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) { - unsigned long flags; int to_drain, batch; - local_lock_irqsave(&pagesets.lock, flags); batch = READ_ONCE(pcp->batch); to_drain = min(pcp->count, batch); - if (to_drain > 0) + if (to_drain > 0) { + unsigned long flags; + + /* + * free_pcppages_bulk expects IRQs disabled for zone->lock + * so even though pcp->lock is not intended to be IRQ-safe, + * it's needed in this context. + */ + spin_lock_irqsave(&pcp->lock, flags); free_pcppages_bulk(zone, to_drain, pcp, 0); - local_unlock_irqrestore(&pagesets.lock, flags); + spin_unlock_irqrestore(&pcp->lock, flags); + } } #endif @@ -3122,16 +3143,17 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) */ static void drain_pages_zone(unsigned int cpu, struct zone *zone) { - unsigned long flags; struct per_cpu_pages *pcp; - local_lock_irqsave(&pagesets.lock, flags); - pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); - if (pcp->count) - free_pcppages_bulk(zone, pcp->count, pcp, 0); + if (pcp->count) { + unsigned long flags; - local_unlock_irqrestore(&pagesets.lock, flags); + /* See drain_zone_pages on why this is disabling IRQs */ + spin_lock_irqsave(&pcp->lock, flags); + free_pcppages_bulk(zone, pcp->count, pcp, 0); + spin_unlock_irqrestore(&pcp->lock, flags); + } } /* @@ -3399,17 +3421,15 @@ static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, return min(READ_ONCE(pcp->batch) << 2, high); } -static void free_unref_page_commit(struct page *page, int migratetype, +static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, + struct page *page, int migratetype, unsigned int order) { - struct zone *zone = page_zone(page); - struct per_cpu_pages *pcp; int high; int pindex; bool free_high; __count_vm_event(PGFREE); - pcp = this_cpu_ptr(zone->per_cpu_pageset); pindex = order_to_pindex(migratetype, order); list_add(&page->pcp_list, &pcp->lists[pindex]); pcp->count += 1 << order; @@ -3436,6 +3456,9 @@ static void free_unref_page_commit(struct page *page, int migratetype, void free_unref_page(struct page *page, unsigned int order) { unsigned long flags; + unsigned long __maybe_unused UP_flags; + struct per_cpu_pages *pcp; + struct zone *zone; unsigned long pfn = page_to_pfn(page); int migratetype; @@ -3459,7 +3482,16 @@ void free_unref_page(struct page *page, unsigned int order) } local_lock_irqsave(&pagesets.lock, flags); - free_unref_page_commit(page, migratetype, order); + zone = page_zone(page); + pcp_trylock_prepare(UP_flags); + pcp = this_cpu_ptr(zone->per_cpu_pageset); + if (spin_trylock(&pcp->lock)) { + free_unref_page_commit(zone, pcp, page, migratetype, order); + spin_unlock(&pcp->lock); + } else { + free_one_page(zone, page, pfn, order, migratetype, FPI_NONE); + } + pcp_trylock_finish(UP_flags); local_unlock_irqrestore(&pagesets.lock, flags); } @@ -3469,6 +3501,8 @@ void free_unref_page(struct page *page, unsigned int order) void free_unref_page_list(struct list_head *list) { struct page *page, *next; + struct per_cpu_pages *pcp = NULL; + struct zone *locked_zone = NULL; unsigned long flags; int batch_count = 0; int migratetype; @@ -3495,6 +3529,17 @@ void free_unref_page_list(struct list_head *list) local_lock_irqsave(&pagesets.lock, flags); list_for_each_entry_safe(page, next, list, lru) { + struct zone *zone = page_zone(page); + + /* Different zone, different pcp lock. */ + if (zone != locked_zone) { + if (pcp) + spin_unlock(&pcp->lock); + locked_zone = zone; + pcp = this_cpu_ptr(zone->per_cpu_pageset); + spin_lock(&pcp->lock); + } + /* * Non-isolated types over MIGRATE_PCPTYPES get added * to the MIGRATE_MOVABLE pcp list. @@ -3504,18 +3549,24 @@ void free_unref_page_list(struct list_head *list) migratetype = MIGRATE_MOVABLE; trace_mm_page_free_batched(page); - free_unref_page_commit(page, migratetype, 0); + free_unref_page_commit(zone, pcp, page, migratetype, 0); /* * Guard against excessive IRQ disabled times when we get * a large list of pages to free. */ if (++batch_count == SWAP_CLUSTER_MAX) { + spin_unlock(&pcp->lock); local_unlock_irqrestore(&pagesets.lock, flags); batch_count = 0; local_lock_irqsave(&pagesets.lock, flags); + pcp = this_cpu_ptr(locked_zone->per_cpu_pageset); + spin_lock(&pcp->lock); } } + + if (pcp) + spin_unlock(&pcp->lock); local_unlock_irqrestore(&pagesets.lock, flags); } @@ -3729,18 +3780,32 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, struct list_head *list; struct page *page; unsigned long flags; + unsigned long __maybe_unused UP_flags; local_lock_irqsave(&pagesets.lock, flags); + /* + * spin_trylock may fail due to a parallel drain. In the future, the + * trylock will also protect against IRQ reentrancy. + */ + pcp = this_cpu_ptr(zone->per_cpu_pageset); + pcp_trylock_prepare(UP_flags); + if (!spin_trylock(&pcp->lock)) { + pcp_trylock_finish(UP_flags); + local_unlock_irqrestore(&pagesets.lock, flags); + return NULL; + } + /* * On allocation, reduce the number of pages that are batch freed. * See nr_pcp_free() where free_factor is increased for subsequent * frees. */ - pcp = this_cpu_ptr(zone->per_cpu_pageset); pcp->free_factor >>= 1; list = &pcp->lists[order_to_pindex(migratetype, order)]; page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); + spin_unlock(&pcp->lock); + pcp_trylock_finish(UP_flags); local_unlock_irqrestore(&pagesets.lock, flags); if (page) { __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); @@ -3775,7 +3840,8 @@ struct page *rmqueue(struct zone *preferred_zone, migratetype != MIGRATE_MOVABLE) { page = rmqueue_pcplist(preferred_zone, zone, order, gfp_flags, migratetype, alloc_flags); - goto out; + if (likely(page)) + goto out; } } @@ -5260,6 +5326,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, { struct page *page; unsigned long flags; + unsigned long __maybe_unused UP_flags; struct zone *zone; struct zoneref *z; struct per_cpu_pages *pcp; @@ -5340,11 +5407,15 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, if (unlikely(!zone)) goto failed; - /* Attempt the batch allocation */ + /* Is a parallel drain in progress? */ local_lock_irqsave(&pagesets.lock, flags); + pcp_trylock_prepare(UP_flags); pcp = this_cpu_ptr(zone->per_cpu_pageset); - pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; + if (!spin_trylock(&pcp->lock)) + goto failed_irq; + /* Attempt the batch allocation */ + pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; while (nr_populated < nr_pages) { /* Skip existing pages */ @@ -5357,8 +5428,10 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, pcp, pcp_list); if (unlikely(!page)) { /* Try and allocate at least one page */ - if (!nr_account) + if (!nr_account) { + spin_unlock(&pcp->lock); goto failed_irq; + } break; } nr_account++; @@ -5371,6 +5444,8 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, nr_populated++; } + spin_unlock(&pcp->lock); + pcp_trylock_finish(UP_flags); local_unlock_irqrestore(&pagesets.lock, flags); __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); @@ -5380,6 +5455,7 @@ out: return nr_populated; failed_irq: + pcp_trylock_finish(UP_flags); local_unlock_irqrestore(&pagesets.lock, flags); failed: @@ -7020,6 +7096,7 @@ static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonesta memset(pcp, 0, sizeof(*pcp)); memset(pzstats, 0, sizeof(*pzstats)); + spin_lock_init(&pcp->lock); for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) INIT_LIST_HEAD(&pcp->lists[pindex]); -- cgit v1.2.3 From 840532711d7299d7e937952482ec899d4622c452 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 11 Jul 2022 12:35:35 +0530 Subject: mm/mmap: build protect protection_map[] with __P000 Patch series "mm/mmap: Drop __SXXX/__PXXX macros from across platforms", v7. __SXXX/__PXXX macros are unnecessary abstraction layer in creating the generic protection_map[] array which is used for vm_get_page_prot(). This abstraction layer can be avoided, if the platforms just define the array protection_map[] for all possible vm_flags access permission combinations and also export vm_get_page_prot() implementation. This series drops __SXXX/__PXXX macros from across platforms in the tree. First it build protects generic protection_map[] array with '#ifdef __P000' and moves it inside platforms which enable ARCH_HAS_VM_GET_PAGE_PROT. Later this build protects same array with '#ifdef ARCH_HAS_VM_GET_PAGE_PROT' and moves inside remaining platforms while enabling ARCH_HAS_VM_GET_PAGE_PROT. This adds a new macro DECLARE_VM_GET_PAGE_PROT defining the current generic vm_get_page_prot(), in order for it to be reused on platforms that do not require custom implementation. Finally, ARCH_HAS_VM_GET_PAGE_PROT can just be dropped, as all platforms now define and export vm_get_page_prot(), via looking up a private and static protection_map[] array. protection_map[] data type has been changed as 'static const' on all platforms that do not change it during boot. This patch (of 26): Build protect generic protection_map[] array with __P000, so that it can be moved inside all the platforms one after the other. Otherwise there will be build failures during this process. CONFIG_ARCH_HAS_VM_GET_PAGE_PROT cannot be used for this purpose as only certain platforms enable this config now. Link: https://lkml.kernel.org/r/20220711070600.2378316-1-anshuman.khandual@arm.com Link: https://lkml.kernel.org/r/20220711070600.2378316-2-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Reviewed-by: Christoph Hellwig Reviewed-by: Christophe Leroy Suggested-by: Christophe Leroy Cc: Arnd Bergmann Cc: Brian Cain Cc: Catalin Marinas Cc: Christoph Hellwig Cc: Chris Zankel Cc: "David S. Miller" Cc: Dinh Nguyen Cc: Geert Uytterhoeven Cc: Guo Ren Cc: Heiko Carstens Cc: Huacai Chen Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Jeff Dike Cc: Jonas Bonn Cc: Michael Ellerman Cc: Michal Simek Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Mackerras Cc: Paul Walmsley Cc: Richard Henderson Cc: Rich Felker Cc: Russell King Cc: Sam Ravnborg Cc: Stafford Horne Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Vasily Gorbik Cc: Vineet Gupta Cc: WANG Xuerui Cc: Will Deacon Cc: Yoshinori Sato Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 ++ mm/mmap.c | 2 ++ 2 files changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index d4ebfc206e2b..1a435ce146a2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -425,7 +425,9 @@ extern unsigned int kobjsize(const void *objp); * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask.. */ +#ifdef __P000 extern pgprot_t protection_map[16]; +#endif /* * The default fault flags that should be used by most of the diff --git a/mm/mmap.c b/mm/mmap.c index c14d7286a379..def0e03cf25c 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -101,6 +101,7 @@ static void unmap_region(struct mm_struct *mm, * w: (no) no * x: (yes) yes */ +#ifdef __P000 pgprot_t protection_map[16] __ro_after_init = { [VM_NONE] = __P000, [VM_READ] = __P001, @@ -119,6 +120,7 @@ pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE] = __S110, [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __S111 }; +#endif #ifndef CONFIG_ARCH_HAS_VM_GET_PAGE_PROT pgprot_t vm_get_page_prot(unsigned long vm_flags) -- cgit v1.2.3 From 43957b5d11037a651d162f65c682ec3c76777fc8 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 11 Jul 2022 12:35:36 +0530 Subject: mm/mmap: define DECLARE_VM_GET_PAGE_PROT This just converts the generic vm_get_page_prot() implementation into a new macro i.e DECLARE_VM_GET_PAGE_PROT which later can be used across platforms when enabling them with ARCH_HAS_VM_GET_PAGE_PROT. This does not create any functional change. Link: https://lkml.kernel.org/r/20220711070600.2378316-3-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Reviewed-by: Christophe Leroy Suggested-by: Christoph Hellwig Cc: Arnd Bergmann Cc: Brian Cain Cc: Catalin Marinas Cc: Christoph Hellwig Cc: Chris Zankel Cc: "David S. Miller" Cc: Dinh Nguyen Cc: Geert Uytterhoeven Cc: Guo Ren Cc: Heiko Carstens Cc: Huacai Chen Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Jeff Dike Cc: Jonas Bonn Cc: Michael Ellerman Cc: Michal Simek Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Mackerras Cc: Paul Walmsley Cc: Richard Henderson Cc: Rich Felker Cc: Russell King Cc: Sam Ravnborg Cc: Stafford Horne Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Vasily Gorbik Cc: Vineet Gupta Cc: WANG Xuerui Cc: Will Deacon Cc: Yoshinori Sato Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 28 ++++++++++++++++++++++++++++ mm/mmap.c | 26 +------------------------- 2 files changed, 29 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 3cdc16cfd867..014ee8f0fbaa 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1689,4 +1689,32 @@ typedef unsigned int pgtbl_mod_mask; #define MAX_PTRS_PER_P4D PTRS_PER_P4D #endif +/* description of effects of mapping type and prot in current implementation. + * this is due to the limited x86 page protection hardware. The expected + * behavior is in parens: + * + * map_type prot + * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC + * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes + * w: (no) no w: (no) no w: (yes) yes w: (no) no + * x: (no) no x: (no) yes x: (no) yes x: (yes) yes + * + * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes + * w: (no) no w: (no) no w: (copy) copy w: (no) no + * x: (no) no x: (no) yes x: (no) yes x: (yes) yes + * + * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and + * MAP_PRIVATE (with Enhanced PAN supported): + * r: (no) no + * w: (no) no + * x: (yes) yes + */ +#define DECLARE_VM_GET_PAGE_PROT \ +pgprot_t vm_get_page_prot(unsigned long vm_flags) \ +{ \ + return protection_map[vm_flags & \ + (VM_READ | VM_WRITE | VM_EXEC | VM_SHARED)]; \ +} \ +EXPORT_SYMBOL(vm_get_page_prot); + #endif /* _LINUX_PGTABLE_H */ diff --git a/mm/mmap.c b/mm/mmap.c index def0e03cf25c..3c0d65743bc4 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -81,26 +81,6 @@ static void unmap_region(struct mm_struct *mm, struct vm_area_struct *vma, struct vm_area_struct *prev, unsigned long start, unsigned long end); -/* description of effects of mapping type and prot in current implementation. - * this is due to the limited x86 page protection hardware. The expected - * behavior is in parens: - * - * map_type prot - * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC - * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes - * w: (no) no w: (no) no w: (yes) yes w: (no) no - * x: (no) no x: (no) yes x: (no) yes x: (yes) yes - * - * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes - * w: (no) no w: (no) no w: (copy) copy w: (no) no - * x: (no) no x: (no) yes x: (no) yes x: (yes) yes - * - * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and - * MAP_PRIVATE (with Enhanced PAN supported): - * r: (no) no - * w: (no) no - * x: (yes) yes - */ #ifdef __P000 pgprot_t protection_map[16] __ro_after_init = { [VM_NONE] = __P000, @@ -123,11 +103,7 @@ pgprot_t protection_map[16] __ro_after_init = { #endif #ifndef CONFIG_ARCH_HAS_VM_GET_PAGE_PROT -pgprot_t vm_get_page_prot(unsigned long vm_flags) -{ - return protection_map[vm_flags & (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]; -} -EXPORT_SYMBOL(vm_get_page_prot); +DECLARE_VM_GET_PAGE_PROT #endif /* CONFIG_ARCH_HAS_VM_GET_PAGE_PROT */ static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) -- cgit v1.2.3 From 09095f74130dfb2110ef2bcdd9ad0d42addaa1d5 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 11 Jul 2022 12:35:41 +0530 Subject: mm/mmap: build protect protection_map[] with ARCH_HAS_VM_GET_PAGE_PROT Now that protection_map[] has been moved inside those platforms that enable ARCH_HAS_VM_GET_PAGE_PROT. Hence generic protection_map[] array now can be protected with CONFIG_ARCH_HAS_VM_GET_PAGE_PROT intead of __P000. Link: https://lkml.kernel.org/r/20220711070600.2378316-8-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Reviewed-by: Christophe Leroy Cc: Arnd Bergmann Cc: Brian Cain Cc: Catalin Marinas Cc: Christoph Hellwig Cc: Christoph Hellwig Cc: Chris Zankel Cc: "David S. Miller" Cc: Dinh Nguyen Cc: Geert Uytterhoeven Cc: Guo Ren Cc: Heiko Carstens Cc: Huacai Chen Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Jeff Dike Cc: Jonas Bonn Cc: Michael Ellerman Cc: Michal Simek Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Mackerras Cc: Paul Walmsley Cc: Richard Henderson Cc: Rich Felker Cc: Russell King Cc: Sam Ravnborg Cc: Stafford Horne Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Vasily Gorbik Cc: Vineet Gupta Cc: WANG Xuerui Cc: Will Deacon Cc: Yoshinori Sato Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 +- mm/mmap.c | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 1a435ce146a2..4b4dc93f9bc3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -425,7 +425,7 @@ extern unsigned int kobjsize(const void *objp); * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask.. */ -#ifdef __P000 +#ifndef CONFIG_ARCH_HAS_VM_GET_PAGE_PROT extern pgprot_t protection_map[16]; #endif diff --git a/mm/mmap.c b/mm/mmap.c index 3c0d65743bc4..2a58a9cd0752 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -81,7 +81,7 @@ static void unmap_region(struct mm_struct *mm, struct vm_area_struct *vma, struct vm_area_struct *prev, unsigned long start, unsigned long end); -#ifdef __P000 +#ifndef CONFIG_ARCH_HAS_VM_GET_PAGE_PROT pgprot_t protection_map[16] __ro_after_init = { [VM_NONE] = __P000, [VM_READ] = __P001, @@ -100,9 +100,6 @@ pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE] = __S110, [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __S111 }; -#endif - -#ifndef CONFIG_ARCH_HAS_VM_GET_PAGE_PROT DECLARE_VM_GET_PAGE_PROT #endif /* CONFIG_ARCH_HAS_VM_GET_PAGE_PROT */ -- cgit v1.2.3 From 3d923c5f1e21ad491acd4c0d62bf2481ce94016c Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 11 Jul 2022 12:36:00 +0530 Subject: mm/mmap: drop ARCH_HAS_VM_GET_PAGE_PROT Now all the platforms enable ARCH_HAS_GET_PAGE_PROT. They define and export own vm_get_page_prot() whether custom or standard DECLARE_VM_GET_PAGE_PROT. Hence there is no need for default generic fallback for vm_get_page_prot(). Just drop this fallback and also ARCH_HAS_GET_PAGE_PROT mechanism. Link: https://lkml.kernel.org/r/20220711070600.2378316-27-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Reviewed-by: Geert Uytterhoeven Reviewed-by: Christoph Hellwig Reviewed-by: Christophe Leroy Acked-by: Geert Uytterhoeven Cc: Arnd Bergmann Cc: Brian Cain Cc: Catalin Marinas Cc: Christoph Hellwig Cc: Chris Zankel Cc: "David S. Miller" Cc: Dinh Nguyen Cc: Guo Ren Cc: Heiko Carstens Cc: Huacai Chen Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Jeff Dike Cc: Jonas Bonn Cc: Michael Ellerman Cc: Michal Simek Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Mackerras Cc: Paul Walmsley Cc: Richard Henderson Cc: Rich Felker Cc: Russell King Cc: Sam Ravnborg Cc: Stafford Horne Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Vasily Gorbik Cc: Vineet Gupta Cc: WANG Xuerui Cc: Will Deacon Cc: Yoshinori Sato Signed-off-by: Andrew Morton --- arch/alpha/Kconfig | 1 - arch/arc/Kconfig | 1 - arch/arm/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/csky/Kconfig | 1 - arch/hexagon/Kconfig | 1 - arch/ia64/Kconfig | 1 - arch/loongarch/Kconfig | 1 - arch/m68k/Kconfig | 1 - arch/microblaze/Kconfig | 1 - arch/mips/Kconfig | 1 - arch/nios2/Kconfig | 1 - arch/openrisc/Kconfig | 1 - arch/parisc/Kconfig | 1 - arch/powerpc/Kconfig | 1 - arch/riscv/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/sh/Kconfig | 1 - arch/sparc/Kconfig | 1 - arch/um/Kconfig | 1 - arch/x86/Kconfig | 1 - arch/xtensa/Kconfig | 1 - include/linux/mm.h | 3 --- mm/Kconfig | 3 --- mm/mmap.c | 22 ---------------------- 25 files changed, 50 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index db1c8b329461..7d0d26b5b3f5 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -2,7 +2,6 @@ config ALPHA bool default y - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_32BIT_USTAT_F_TINODE select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 8be56a5d8a9b..9e3653253ef2 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -13,7 +13,6 @@ config ARC select ARCH_HAS_SETUP_DMA_OPS select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_SUPPORTS_ATOMIC_RMW if ARC_HAS_LLSC select ARCH_32BIT_OFF_T select BUILDTIME_TABLE_SORT diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e153b6d4fc5b..7630ba9cb6cc 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -24,7 +24,6 @@ config ARM select ARCH_HAS_SYNC_DMA_FOR_CPU if SWIOTLB || !MMU select ARCH_HAS_TEARDOWN_DMA_OPS if MMU select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAVE_CUSTOM_GPIO_H select ARCH_HAVE_NMI_SAFE_CMPXCHG if CPU_V7 || CPU_V7M || CPU_V6K select ARCH_HAS_GCOV_PROFILE_ALL diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1652a9800ebe..7030bf3f8d6f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -45,7 +45,6 @@ config ARM64 select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_TEARDOWN_DMA_OPS if IOMMU_SUPPORT select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAS_ZONE_DMA_SET if EXPERT select ARCH_HAVE_ELF_PROT select ARCH_HAVE_NMI_SAFE_CMPXCHG diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 588b8a9c68ed..21d72b078eef 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -6,7 +6,6 @@ config CSKY select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_QUEUED_RWLOCKS select ARCH_WANT_FRAME_POINTERS if !CPU_CK610 && $(cc-option,-mbacktrace) diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index bc4ceecd0588..54eadf265178 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -6,7 +6,6 @@ config HEXAGON def_bool y select ARCH_32BIT_OFF_T select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_NO_PREEMPT select DMA_GLOBAL_POOL # Other pending projects/to-do items. diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 0510a5737711..cb93769a9f2a 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -12,7 +12,6 @@ config IA64 select ARCH_HAS_DMA_MARK_CLEAN select ARCH_HAS_STRNCPY_FROM_USER select ARCH_HAS_STRNLEN_USER - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO select ACPI diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index adf8cf6ec5d5..db2838cf8c02 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -9,7 +9,6 @@ config LOONGARCH select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_PTE_SPECIAL - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_INLINE_READ_LOCK if !PREEMPTION select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 49aa0cf13e96..936cce42ae9a 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -7,7 +7,6 @@ config M68K select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DMA_PREP_COHERENT if HAS_DMA && MMU && !COLDFIRE select ARCH_HAS_SYNC_DMA_FOR_DEVICE if HAS_DMA - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAVE_NMI_SAFE_CMPXCHG if RMW_INSNS select ARCH_MIGHT_HAVE_PC_PARPORT if ISA select ARCH_NO_PREEMPT if !COLDFIRE diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 15f91ba8a0c4..8cf429ad1c84 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -7,7 +7,6 @@ config MICROBLAZE select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_WANT_IPC_PARSE_VERSION select BUILDTIME_TABLE_SORT diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index d0b7eb11ec81..db09d45d59ec 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -14,7 +14,6 @@ config MIPS select ARCH_HAS_STRNLEN_USER select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_KEEP_MEMBLOCK select ARCH_SUPPORTS_UPROBES diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index e0459dffd218..4167f1eb4cd8 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -6,7 +6,6 @@ config NIOS2 select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_DMA_SET_UNCACHED - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_NO_SWAP select COMMON_CLK select TIMER_OF diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index fe0dfb50eb86..e814df4c483c 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -10,7 +10,6 @@ config OPENRISC select ARCH_HAS_DMA_SET_UNCACHED select ARCH_HAS_DMA_CLEAR_UNCACHED select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_HAS_VM_GET_PAGE_PROT select COMMON_CLK select OF select OF_EARLY_FLATTREE diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 891d82393957..fa400055b2d5 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -12,7 +12,6 @@ config PARISC select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_STRICT_MODULE_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAS_PTE_SPECIAL select ARCH_NO_SG_CHAIN select ARCH_SUPPORTS_HUGETLBFS if PA20 diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 1035d172c7dd..250b8658b2d4 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -140,7 +140,6 @@ config PPC select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UACCESS_FLUSHCACHE select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 583389d4e43a..32ffef9f6e5b 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -32,7 +32,6 @@ config RISCV select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT select ARCH_STACKWALK diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index c4481377ca83..91c0b80a8bf0 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -81,7 +81,6 @@ config S390 select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_HAS_VDSO_DATA - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_INLINE_READ_LOCK select ARCH_INLINE_READ_LOCK_BH diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 91f3ea325388..5f220e903e5a 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -12,7 +12,6 @@ config SUPERH select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HIBERNATION_POSSIBLE if MMU select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_WANT_IPC_PARSE_VERSION diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 09f868613a4d..9c1cce74953a 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -13,7 +13,6 @@ config 64BIT config SPARC bool default y - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI select ARCH_MIGHT_HAVE_PC_SERIO select DMA_OPS diff --git a/arch/um/Kconfig b/arch/um/Kconfig index 7fb43654e5b5..4ec22e156a2e 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -10,7 +10,6 @@ config UML select ARCH_HAS_KCOV select ARCH_HAS_STRNCPY_FROM_USER select ARCH_HAS_STRNLEN_USER - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_NO_PREEMPT select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_SECCOMP_FILTER diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index be0b95e51df6..841e4843d0c4 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -94,7 +94,6 @@ config X86 select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_HAS_DEBUG_WX select ARCH_HAS_ZONE_DMA_SET if EXPERT select ARCH_HAVE_NMI_SAFE_CMPXCHG diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 4c0d83520ff1..0b0f0172cced 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -11,7 +11,6 @@ config XTENSA select ARCH_HAS_DMA_SET_UNCACHED if MMU select ARCH_HAS_STRNCPY_FROM_USER if !KASAN select ARCH_HAS_STRNLEN_USER - select ARCH_HAS_VM_GET_PAGE_PROT select ARCH_USE_MEMTEST select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS diff --git a/include/linux/mm.h b/include/linux/mm.h index 4b4dc93f9bc3..61e3101c44ea 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -425,9 +425,6 @@ extern unsigned int kobjsize(const void *objp); * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask.. */ -#ifndef CONFIG_ARCH_HAS_VM_GET_PAGE_PROT -extern pgprot_t protection_map[16]; -#endif /* * The default fault flags that should be used by most of the diff --git a/mm/Kconfig b/mm/Kconfig index c1fa4993a56f..56ca0e7c6f9a 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -951,9 +951,6 @@ config ARCH_HAS_CURRENT_STACK_POINTER register alias named "current_stack_pointer", this config can be selected. -config ARCH_HAS_VM_GET_PAGE_PROT - bool - config ARCH_HAS_PTE_DEVMAP bool diff --git a/mm/mmap.c b/mm/mmap.c index 2a58a9cd0752..edf27a2789a2 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -81,28 +81,6 @@ static void unmap_region(struct mm_struct *mm, struct vm_area_struct *vma, struct vm_area_struct *prev, unsigned long start, unsigned long end); -#ifndef CONFIG_ARCH_HAS_VM_GET_PAGE_PROT -pgprot_t protection_map[16] __ro_after_init = { - [VM_NONE] = __P000, - [VM_READ] = __P001, - [VM_WRITE] = __P010, - [VM_WRITE | VM_READ] = __P011, - [VM_EXEC] = __P100, - [VM_EXEC | VM_READ] = __P101, - [VM_EXEC | VM_WRITE] = __P110, - [VM_EXEC | VM_WRITE | VM_READ] = __P111, - [VM_SHARED] = __S000, - [VM_SHARED | VM_READ] = __S001, - [VM_SHARED | VM_WRITE] = __S010, - [VM_SHARED | VM_WRITE | VM_READ] = __S011, - [VM_SHARED | VM_EXEC] = __S100, - [VM_SHARED | VM_EXEC | VM_READ] = __S101, - [VM_SHARED | VM_EXEC | VM_WRITE] = __S110, - [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __S111 -}; -DECLARE_VM_GET_PAGE_PROT -#endif /* CONFIG_ARCH_HAS_VM_GET_PAGE_PROT */ - static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) { return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); -- cgit v1.2.3 From 3ce4fee4401206cf5a2c476ec0ee6c90191dfade Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Mon, 4 Jul 2022 21:21:55 +0800 Subject: mm/huge_memory: check pmd_present first in is_huge_zero_pmd When pmd is non-present, pmd_pfn returns an insane value. So we should check pmd_present first to avoid acquiring such insane value and also avoid touching possible cold huge_zero_pfn cache line when pmd isn't present. Link: https://lkml.kernel.org/r/20220704132201.14611-11-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Muchun Song Cc: Matthew Wilcox Cc: Yang Shi Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ae3d8e2fd9e2..12b297f9951d 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -273,7 +273,7 @@ static inline bool is_huge_zero_page(struct page *page) static inline bool is_huge_zero_pmd(pmd_t pmd) { - return READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd) && pmd_present(pmd); + return pmd_present(pmd) && READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd); } static inline bool is_huge_zero_pud(pud_t pud) -- cgit v1.2.3 From 121c1781aeb00475d163246d9ae7d8746e377040 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Mon, 4 Jul 2022 21:21:58 +0800 Subject: mm/huge_memory: fix comment of page_deferred_list The current comment is confusing because if global or memcg deferred list in the second tail page is occupied by compound_head, why we still use page[2].deferred_list here? I think it wants to say that Global or memcg deferred list in the first tail page is occupied by compound_mapcount and compound_pincount so we use the second tail page's deferred_list instead. Link: https://lkml.kernel.org/r/20220704132201.14611-14-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Muchun Song Cc: Matthew Wilcox Cc: Yang Shi Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 12b297f9951d..37f2f11a6d7e 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -294,8 +294,8 @@ static inline bool thp_migration_supported(void) static inline struct list_head *page_deferred_list(struct page *page) { /* - * Global or memcg deferred list in the second tail pages is - * occupied by compound_head. + * See organization of tail pages of compound page in + * "struct page" definition. */ return &page[2].deferred_list; } -- cgit v1.2.3 From dcadcf1c30619ead2f3280bfb7f74de8304be2bb Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 6 Jul 2022 11:46:54 +0800 Subject: mm, hugetlb: skip irrelevant nodes in show_free_areas() show_free_areas() allows to filter out node specific data which is irrelevant to the allocation request. But hugetlb_show_meminfo() still shows hugetlb on all nodes, which is redundant and unnecessary. Use show_mem_node_skip() to skip irrelevant nodes. And replace hugetlb_show_meminfo() with hugetlb_show_meminfo_node(nid). before-and-after sample output of OOM: before: ``` [ 214.362453] Node 1 active_anon:148kB inactive_anon:4050920kB active_file:112kB inactive_file:100kB [ 214.375429] Node 1 Normal free:45100kB boost:0kB min:45576kB low:56968kB high:68360kB reserved_hig [ 214.388334] lowmem_reserve[]: 0 0 0 0 0 [ 214.390251] Node 1 Normal: 423*4kB (UE) 320*8kB (UME) 187*16kB (UE) 117*32kB (UE) 57*64kB (UME) 20 [ 214.397626] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [ 214.401518] Node 1 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB ``` after: ``` [ 145.069705] Node 1 active_anon:128kB inactive_anon:4049412kB active_file:56kB inactive_file:84kB u [ 145.110319] Node 1 Normal free:45424kB boost:0kB min:45576kB low:56968kB high:68360kB reserved_hig [ 145.152315] lowmem_reserve[]: 0 0 0 0 0 [ 145.155244] Node 1 Normal: 470*4kB (UME) 373*8kB (UME) 247*16kB (UME) 168*32kB (UE) 86*64kB (UME) [ 145.164119] Node 1 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB ``` Link: https://lkml.kernel.org/r/20220706034655.1834-1-ligang.bdlg@bytedance.com Signed-off-by: Gang Li Reviewed-by: Mike Kravetz Cc: Muchun Song Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 4 ++-- mm/hugetlb.c | 18 ++++++++---------- mm/page_alloc.c | 8 ++++++-- 3 files changed, 16 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 75ee739d815b..4cdfce976644 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -152,7 +152,7 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb, struct page *ref_page, zap_flags_t zap_flags); void hugetlb_report_meminfo(struct seq_file *); int hugetlb_report_node_meminfo(char *buf, int len, int nid); -void hugetlb_show_meminfo(void); +void hugetlb_show_meminfo_node(int nid); unsigned long hugetlb_total_pages(void); vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags); @@ -298,7 +298,7 @@ static inline int hugetlb_report_node_meminfo(char *buf, int len, int nid) return 0; } -static inline void hugetlb_show_meminfo(void) +static inline void hugetlb_show_meminfo_node(int nid) { } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 96635a2874e3..bb763f5d30b9 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4477,22 +4477,20 @@ int hugetlb_report_node_meminfo(char *buf, int len, int nid) nid, h->surplus_huge_pages_node[nid]); } -void hugetlb_show_meminfo(void) +void hugetlb_show_meminfo_node(int nid) { struct hstate *h; - int nid; if (!hugepages_supported()) return; - for_each_node_state(nid, N_MEMORY) - for_each_hstate(h) - pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", - nid, - h->nr_huge_pages_node[nid], - h->free_huge_pages_node[nid], - h->surplus_huge_pages_node[nid], - huge_page_size(h) / SZ_1K); + for_each_hstate(h) + printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", + nid, + h->nr_huge_pages_node[nid], + h->free_huge_pages_node[nid], + h->surplus_huge_pages_node[nid], + huge_page_size(h) / SZ_1K); } void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 215b26664ad7..4fa96d3510fe 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6022,7 +6022,7 @@ static void show_migration_types(unsigned char type) void show_free_areas(unsigned int filter, nodemask_t *nodemask) { unsigned long free_pcp = 0; - int cpu; + int cpu, nid; struct zone *zone; pg_data_t *pgdat; @@ -6210,7 +6210,11 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) printk(KERN_CONT "= %lukB\n", K(total)); } - hugetlb_show_meminfo(); + for_each_online_node(nid) { + if (show_mem_node_skip(filter, nid, nodemask)) + continue; + hugetlb_show_meminfo_node(nid); + } printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); -- cgit v1.2.3 From 0d8bc0b10aeab543bdccb86180f58db1f79f7cee Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Wed, 13 Jul 2022 20:53:14 +0800 Subject: writeback: cleanup bdi_sched_wait() bdi_sched_wait() is no longer used since commit 839a8e8660b6 ("writeback: replace custom worker pool implementation with unbound workqueue"), so remove it. Link: https://lkml.kernel.org/r/20220713125314.171345-1-xiujianfeng@huawei.com Signed-off-by: Xiu Jianfeng Reviewed-by: Jan Kara Reviewed-by: Johannes Thumshirn Acked-by: Jens Axboe Signed-off-by: Andrew Morton --- include/linux/backing-dev.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index d452071db572..e84b745a6811 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -140,12 +140,6 @@ static inline bool mapping_can_writeback(struct address_space *mapping) return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK; } -static inline int bdi_sched_wait(void *word) -{ - schedule(); - return 0; -} - #ifdef CONFIG_CGROUP_WRITEBACK struct bdi_writeback *wb_get_lookup(struct backing_dev_info *bdi, -- cgit v1.2.3 From 62df90b53e6f332bb69b73621998826c49a17323 Mon Sep 17 00:00:00 2001 From: wuchi Date: Sun, 19 Jun 2022 15:46:41 +0800 Subject: net, lib/once: remove {net_}get_random_once_wait macro DO_ONCE(func, ...) will call func with spinlock which acquired by spin_lock_irqsave in __do_once_start. But the get_random_once_wait will sleep in get_random_bytes_wait -> wait_for_random_bytes. Fortunately, there is no place to use {net_}get_random_once_wait, so we could remove them simply. Link: https://lkml.kernel.org/r/20220619074641.40916-1-wuchi.zero@gmail.com Signed-off-by: wuchi Acked-by: Jakub Kicinski Cc: David S. Miller Cc: Eric Dumazet Cc: Paolo Abeni Signed-off-by: Andrew Morton --- include/linux/net.h | 2 -- include/linux/once.h | 2 -- 2 files changed, 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/net.h b/include/linux/net.h index 12093f4db50c..8613772a1f58 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -303,8 +303,6 @@ do { \ #define net_get_random_once(buf, nbytes) \ get_random_once((buf), (nbytes)) -#define net_get_random_once_wait(buf, nbytes) \ - get_random_once_wait((buf), (nbytes)) /* * E.g. XFS meta- & log-data is in slab pages, or bcache meta diff --git a/include/linux/once.h b/include/linux/once.h index f54523052bbc..b14d8b309d52 100644 --- a/include/linux/once.h +++ b/include/linux/once.h @@ -54,7 +54,5 @@ void __do_once_done(bool *done, struct static_key_true *once_key, #define get_random_once(buf, nbytes) \ DO_ONCE(get_random_bytes, (buf), (nbytes)) -#define get_random_once_wait(buf, nbytes) \ - DO_ONCE(get_random_bytes_wait, (buf), (nbytes)) \ #endif /* _LINUX_ONCE_H */ -- cgit v1.2.3 From 43c249ea0b1e10baac4a1264a25d69723ce5d2c2 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 24 Jun 2022 16:14:12 +0200 Subject: compiler-gcc.h: remove ancient workaround for gcc PR 58670 The workaround for 'asm goto' miscompilation introduces a compiler barrier quirk that inhibits many useful compiler optimizations. For example, __try_cmpxchg_user compiles to: 11375: 41 8b 4d 00 mov 0x0(%r13),%ecx 11379: 41 8b 02 mov (%r10),%eax 1137c: f0 0f b1 0a lock cmpxchg %ecx,(%rdx) 11380: 0f 94 c2 sete %dl 11383: 84 d2 test %dl,%dl 11385: 75 c4 jne 1134b <...> 11387: 41 89 02 mov %eax,(%r10) where the barrier inhibits flags propagation from asm when compiled with gcc-12. When the mentioned quirk is removed, the following code is generated: 11553: 41 8b 4d 00 mov 0x0(%r13),%ecx 11557: 41 8b 02 mov (%r10),%eax 1155a: f0 0f b1 0a lock cmpxchg %ecx,(%rdx) 1155e: 74 c9 je 11529 <...> 11560: 41 89 02 mov %eax,(%r10) The refered compiler bug: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 was fixed for gcc-4.8.2. Current minimum required version of GCC is version 5.1 which has the above 'asm goto' miscompilation fixed, so remove the workaround. Link: https://lkml.kernel.org/r/20220624141412.72274-1-ubizjak@gmail.com Signed-off-by: Uros Bizjak Cc: Ingo Molnar Cc: "H. Peter Anvin" Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/compiler-gcc.h | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index a0c55eeaeaf1..9b157b71036f 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -66,17 +66,6 @@ __builtin_unreachable(); \ } while (0) -/* - * GCC 'asm goto' miscompiles certain code sequences: - * - * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 - * - * Work it around via a compiler barrier quirk suggested by Jakub Jelinek. - * - * (asm goto is automatically volatile - the naming reflects this.) - */ -#define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0) - #if defined(CONFIG_ARCH_USE_BUILTIN_BSWAP) #define __HAVE_BUILTIN_BSWAP32__ #define __HAVE_BUILTIN_BSWAP64__ -- cgit v1.2.3 From 045ed31e23aea840648c290dbde04797064960db Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 24 Jun 2022 08:30:04 +0300 Subject: kfifo: fix kfifo_to_user() return type The kfifo_to_user() macro is supposed to return zero for success or negative error codes. Unfortunately, there is a signedness bug so it returns unsigned int. This only affects callers which try to save the result in ssize_t and as far as I can see the only place which does that is line6_hwdep_read(). TL;DR: s/_uint/_int/. Link: https://lkml.kernel.org/r/YrVL3OJVLlNhIMFs@kili Fixes: 144ecf310eb5 ("kfifo: fix kfifo_alloc() to return a signed int value") Signed-off-by: Dan Carpenter Cc: Stefani Seibold Cc: Randy Dunlap Signed-off-by: Andrew Morton --- include/linux/kfifo.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/kfifo.h b/include/linux/kfifo.h index 86249476b57f..0b35a41440ff 100644 --- a/include/linux/kfifo.h +++ b/include/linux/kfifo.h @@ -688,7 +688,7 @@ __kfifo_uint_must_check_helper( \ * writer, you don't need extra locking to use these macro. */ #define kfifo_to_user(fifo, to, len, copied) \ -__kfifo_uint_must_check_helper( \ +__kfifo_int_must_check_helper( \ ({ \ typeof((fifo) + 1) __tmp = (fifo); \ void __user *__to = (to); \ -- cgit v1.2.3 From 4f09903078eeb9138cddce8db06100b82f8620e8 Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 2 Jul 2022 18:08:27 +0200 Subject: cpumask: add UP optimised for_each_*_cpu versions On uniprocessor builds, the following loops will always run over a mask that contains one enabled CPU (cpu0): - for_each_possible_cpu - for_each_online_cpu - for_each_present_cpu Provide uniprocessor-specific macros for these loops, that always run exactly once. Link: https://lkml.kernel.org/r/3a92869b902a075b97be5d1452c9c6badbbff0df.1656777646.git.sander@svanheule.net Signed-off-by: Sander Vanheule Acked-by: Yury Norov Cc: Andy Shevchenko Cc: Borislav Petkov Cc: Dave Hansen Cc: Greg Kroah-Hartman Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Marco Elver Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Valentin Schneider Signed-off-by: Andrew Morton --- include/linux/cpumask.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index fe29ac7cc469..533612770bc0 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -811,9 +811,16 @@ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); /* First bits of cpu_bit_bitmap are in fact unset. */ #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0]) +#if NR_CPUS == 1 +/* Uniprocessor: the possible/online/present masks are always "1" */ +#define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) +#define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) +#define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) +#else #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) +#endif /* Wrappers for arch boot code to manipulate normally-constant masks */ void init_cpu_present(const struct cpumask *src); -- cgit v1.2.3 From b81dce77cedcea6f00292f02d4b1ebbfc2c5988d Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 2 Jul 2022 18:08:25 +0200 Subject: cpumask: Fix invalid uniprocessor mask assumption On uniprocessor builds, any CPU mask is assumed to contain exactly one CPU (cpu0). This assumption ignores the existence of empty masks, resulting in incorrect behaviour. cpumask_first_zero(), cpumask_next_zero(), and for_each_cpu_not() don't provide behaviour matching the assumption that a UP mask is always "1", and instead provide behaviour matching the empty mask. Drop the incorrectly optimised code and use the generic implementations in all cases. Link: https://lkml.kernel.org/r/86bf3f005abba2d92120ddd0809235cab4f759a6.1656777646.git.sander@svanheule.net Signed-off-by: Sander Vanheule Suggested-by: Yury Norov Cc: Andy Shevchenko Cc: Borislav Petkov Cc: Dave Hansen Cc: Greg Kroah-Hartman Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Marco Elver Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Valentin Schneider Signed-off-by: Andrew Morton --- include/linux/cpumask.h | 99 ++++++++++--------------------------------------- lib/Makefile | 3 +- lib/cpumask.c | 2 + 3 files changed, 22 insertions(+), 82 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 533612770bc0..6c5b4ee000f2 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -116,85 +116,6 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu) return cpu; } -#if NR_CPUS == 1 -/* Uniprocessor. Assume all masks are "1". */ -static inline unsigned int cpumask_first(const struct cpumask *srcp) -{ - return 0; -} - -static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) -{ - return 0; -} - -static inline unsigned int cpumask_first_and(const struct cpumask *srcp1, - const struct cpumask *srcp2) -{ - return 0; -} - -static inline unsigned int cpumask_last(const struct cpumask *srcp) -{ - return 0; -} - -/* Valid inputs for n are -1 and 0. */ -static inline unsigned int cpumask_next(int n, const struct cpumask *srcp) -{ - return n+1; -} - -static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) -{ - return n+1; -} - -static inline unsigned int cpumask_next_and(int n, - const struct cpumask *srcp, - const struct cpumask *andp) -{ - return n+1; -} - -static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, - int start, bool wrap) -{ - /* cpu0 unless stop condition, wrap and at cpu0, then nr_cpumask_bits */ - return (wrap && n == 0); -} - -/* cpu must be a valid cpu, ie 0, so there's no other choice. */ -static inline unsigned int cpumask_any_but(const struct cpumask *mask, - unsigned int cpu) -{ - return 1; -} - -static inline unsigned int cpumask_local_spread(unsigned int i, int node) -{ - return 0; -} - -static inline int cpumask_any_and_distribute(const struct cpumask *src1p, - const struct cpumask *src2p) { - return cpumask_first_and(src1p, src2p); -} - -static inline int cpumask_any_distribute(const struct cpumask *srcp) -{ - return cpumask_first(srcp); -} - -#define for_each_cpu(cpu, mask) \ - for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) -#define for_each_cpu_not(cpu, mask) \ - for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) -#define for_each_cpu_wrap(cpu, mask, start) \ - for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start)) -#define for_each_cpu_and(cpu, mask1, mask2) \ - for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask1, (void)mask2) -#else /** * cpumask_first - get the first cpu in a cpumask * @srcp: the cpumask pointer @@ -260,10 +181,29 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) int __pure cpumask_next_and(int n, const struct cpumask *, const struct cpumask *); int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu); + +#if NR_CPUS == 1 +/* Uniprocessor: there is only one valid CPU */ +static inline unsigned int cpumask_local_spread(unsigned int i, int node) +{ + return 0; +} + +static inline int cpumask_any_and_distribute(const struct cpumask *src1p, + const struct cpumask *src2p) { + return cpumask_first_and(src1p, src2p); +} + +static inline int cpumask_any_distribute(const struct cpumask *srcp) +{ + return cpumask_first(srcp); +} +#else unsigned int cpumask_local_spread(unsigned int i, int node); int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p); int cpumask_any_distribute(const struct cpumask *srcp); +#endif /* NR_CPUS */ /** * for_each_cpu - iterate over every cpu in a mask @@ -324,7 +264,6 @@ extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool for ((cpu) = -1; \ (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ (cpu) < nr_cpu_ids;) -#endif /* SMP */ #define CPU_BITS_NONE \ { \ diff --git a/lib/Makefile b/lib/Makefile index f99bf61f8bbc..bcc7e8ea0cde 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -34,10 +34,9 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ nmi_backtrace.o nodemask.o win_minmax.o memcat_p.o \ - buildid.o + buildid.o cpumask.o lib-$(CONFIG_PRINTK) += dump_stack.o -lib-$(CONFIG_SMP) += cpumask.o lib-y += kobject.o klist.o obj-y += lockref.o diff --git a/lib/cpumask.c b/lib/cpumask.c index a971a82d2f43..b9728513a4d4 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -192,6 +192,7 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask) } #endif +#if NR_CPUS > 1 /** * cpumask_local_spread - select the i'th cpu with local numa cpu's first * @i: index number @@ -279,3 +280,4 @@ int cpumask_any_distribute(const struct cpumask *srcp) return next; } EXPORT_SYMBOL(cpumask_any_distribute); +#endif /* NR_CPUS */ -- cgit v1.2.3 From 953257a9252a9b3c58ca68fc5bf26fc65e5b1cb8 Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 2 Jul 2022 18:08:28 +0200 Subject: cpumask: update cpumask_next_wrap() signature The extern specifier is not needed for this declaration, so drop it. The function also depends only on the input parameters, and has no side effects, so it can be marked __pure like other functions in cpumask.h. Link: https://lkml.kernel.org/r/72ab755695b74bb5fbaa756ae4c0edd708d172f1.1656777646.git.sander@svanheule.net Signed-off-by: Sander Vanheule Reviewed-by: Andy Shevchenko Cc: Borislav Petkov Cc: Dave Hansen Cc: Greg Kroah-Hartman Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Marco Elver Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Valentin Schneider Cc: Yury Norov Signed-off-by: Andrew Morton --- include/linux/cpumask.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 6c5b4ee000f2..523857884ae4 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -229,7 +229,7 @@ int cpumask_any_distribute(const struct cpumask *srcp); (cpu) = cpumask_next_zero((cpu), (mask)), \ (cpu) < nr_cpu_ids;) -extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); +int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); /** * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location -- cgit v1.2.3 From 91561d4ecb755f056f8ff04f9dcaec210140e55c Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Fri, 15 Jul 2022 18:45:33 +0800 Subject: swiotlb: remove unused fields in io_tlb_mem Commit 20347fca71a3 ("swiotlb: split up the global swiotlb lock") splits io_tlb_mem into multiple areas. Each area has its own lock and index. The global ones are not used so remove them. Signed-off-by: Chao Gao Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 5 ----- kernel/dma/swiotlb.c | 2 -- 2 files changed, 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index f65ff1930120..d3ae03edbbd2 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -79,11 +79,8 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, * @used: The number of used IO TLB block. * @list: The free list describing the number of free entries available * from each index. - * @index: The index to start searching in the next round. * @orig_addr: The original address corresponding to a mapped entry. * @alloc_size: Size of the allocated buffer. - * @lock: The lock to protect the above data structures in the map and - * unmap calls. * @debugfs: The dentry to debugfs. * @late_alloc: %true if allocated using the page allocator * @force_bounce: %true if swiotlb bouncing is forced @@ -97,8 +94,6 @@ struct io_tlb_mem { void *vaddr; unsigned long nslabs; unsigned long used; - unsigned int index; - spinlock_t lock; struct dentry *debugfs; bool late_alloc; bool force_bounce; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index c50e6fe20f37..cbffa0b1ace5 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -253,14 +253,12 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, mem->nslabs = nslabs; mem->start = start; mem->end = mem->start + bytes; - mem->index = 0; mem->late_alloc = late_alloc; mem->nareas = nareas; mem->area_nslabs = nslabs / mem->nareas; mem->force_bounce = swiotlb_force_bounce || (flags & SWIOTLB_FORCE); - spin_lock_init(&mem->lock); for (i = 0; i < mem->nareas; i++) { spin_lock_init(&mem->areas[i].lock); mem->areas[i].index = 0; -- cgit v1.2.3 From 942a8186eb4451700dadd1d60a2e43ce67605991 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 12 Jul 2022 08:43:07 +0200 Subject: swiotlb: move struct io_tlb_slot to swiotlb.c No need to expose this structure definition in the header. Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 6 +----- kernel/dma/swiotlb.c | 6 ++++++ 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index d3ae03edbbd2..35bc4e281c21 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -101,11 +101,7 @@ struct io_tlb_mem { unsigned int nareas; unsigned int area_nslabs; struct io_tlb_area *areas; - struct io_tlb_slot { - phys_addr_t orig_addr; - size_t alloc_size; - unsigned int list; - } *slots; + struct io_tlb_slot *slots; }; extern struct io_tlb_mem io_tlb_default_mem; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 608923e8dab1..39dee4004439 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -62,6 +62,12 @@ #define INVALID_PHYS_ADDR (~(phys_addr_t)0) +struct io_tlb_slot { + phys_addr_t orig_addr; + size_t alloc_size; + unsigned int list; +}; + static bool swiotlb_force_bounce; static bool swiotlb_force_disable; -- cgit v1.2.3 From 9592eef7c16ec5fb9f36c4d9abe8eeffc2e1d2f3 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 5 Jul 2022 20:48:41 +0200 Subject: random: remove CONFIG_ARCH_RANDOM When RDRAND was introduced, there was much discussion on whether it should be trusted and how the kernel should handle that. Initially, two mechanisms cropped up, CONFIG_ARCH_RANDOM, a compile time switch, and "nordrand", a boot-time switch. Later the thinking evolved. With a properly designed RNG, using RDRAND values alone won't harm anything, even if the outputs are malicious. Rather, the issue is whether those values are being *trusted* to be good or not. And so a new set of options were introduced as the real ones that people use -- CONFIG_RANDOM_TRUST_CPU and "random.trust_cpu". With these options, RDRAND is used, but it's not always credited. So in the worst case, it does nothing, and in the best case, maybe it helps. Along the way, CONFIG_ARCH_RANDOM's meaning got sort of pulled into the center and became something certain platforms force-select. The old options don't really help with much, and it's a bit odd to have special handling for these instructions when the kernel can deal fine with the existence or untrusted existence or broken existence or non-existence of that CPU capability. Simplify the situation by removing CONFIG_ARCH_RANDOM and using the ordinary asm-generic fallback pattern instead, keeping the two options that are actually used. For now it leaves "nordrand" for now, as the removal of that will take a different route. Acked-by: Michael Ellerman Acked-by: Catalin Marinas Acked-by: Borislav Petkov Acked-by: Heiko Carstens Acked-by: Greg Kroah-Hartman Signed-off-by: Jason A. Donenfeld --- arch/arm/include/asm/archrandom.h | 2 ++ arch/arm64/Kconfig | 8 ------- arch/arm64/include/asm/archrandom.h | 10 --------- arch/arm64/kernel/cpufeature.c | 2 -- arch/powerpc/Kconfig | 3 --- arch/powerpc/include/asm/archrandom.h | 3 --- arch/powerpc/include/asm/machdep.h | 2 -- arch/powerpc/platforms/microwatt/Kconfig | 1 - arch/powerpc/platforms/powernv/Kconfig | 1 - arch/powerpc/platforms/pseries/Kconfig | 1 - arch/s390/Kconfig | 15 ------------- arch/s390/configs/zfcpdump_defconfig | 1 - arch/s390/crypto/Makefile | 2 +- arch/s390/include/asm/archrandom.h | 3 --- arch/s390/kernel/setup.c | 2 -- arch/x86/Kconfig | 9 -------- arch/x86/include/asm/archrandom.h | 14 ++++-------- arch/x86/kernel/cpu/rdrand.c | 2 -- drivers/char/Kconfig | 1 - drivers/char/hw_random/s390-trng.c | 9 -------- include/asm-generic/Kbuild | 1 + include/asm-generic/archrandom.h | 25 ++++++++++++++++++++++ include/linux/random.h | 9 +------- .../testing/selftests/wireguard/qemu/kernel.config | 1 - 24 files changed, 34 insertions(+), 93 deletions(-) create mode 100644 include/asm-generic/archrandom.h (limited to 'include/linux') diff --git a/arch/arm/include/asm/archrandom.h b/arch/arm/include/asm/archrandom.h index a8e84ca5c2ee..cc4714eb1a75 100644 --- a/arch/arm/include/asm/archrandom.h +++ b/arch/arm/include/asm/archrandom.h @@ -7,4 +7,6 @@ static inline bool __init smccc_probe_trng(void) return false; } +#include + #endif /* _ASM_ARCHRANDOM_H */ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1652a9800ebe..1880f71c2547 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1858,14 +1858,6 @@ config ARM64_E0PD This option enables E0PD for TTBR1 where available. -config ARCH_RANDOM - bool "Enable support for random number generation" - default y - help - Random number generation (part of the ARMv8.5 Extensions) - provides a high bandwidth, cryptographically secure - hardware random number generator. - config ARM64_AS_HAS_MTE # Initial support for MTE went in binutils 2.32.0, checked with # ".arch armv8.5-a+memtag" below. However, this was incomplete diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index 3a6b6d38c5b8..c3b9fa56af67 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -2,8 +2,6 @@ #ifndef _ASM_ARCHRANDOM_H #define _ASM_ARCHRANDOM_H -#ifdef CONFIG_ARCH_RANDOM - #include #include #include @@ -167,12 +165,4 @@ arch_get_random_seed_long_early(unsigned long *v) } #define arch_get_random_seed_long_early arch_get_random_seed_long_early -#else /* !CONFIG_ARCH_RANDOM */ - -static inline bool __init smccc_probe_trng(void) -{ - return false; -} - -#endif /* CONFIG_ARCH_RANDOM */ #endif /* _ASM_ARCHRANDOM_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 8d88433de81d..0e9462abeb77 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2416,7 +2416,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_e0pd, }, #endif -#ifdef CONFIG_ARCH_RANDOM { .desc = "Random Number Generator", .capability = ARM64_HAS_RNG, @@ -2428,7 +2427,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sign = FTR_UNSIGNED, .min_field_value = 1, }, -#endif #ifdef CONFIG_ARM64_BTI { .desc = "Branch Target Identification", diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 7aa12e88c580..623deb5bedf6 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -1252,9 +1252,6 @@ config PHYSICAL_START default "0x00000000" endif -config ARCH_RANDOM - def_bool n - config PPC_LIB_RHEAP bool diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h index 9a53e29680f4..25ba65df6b1a 100644 --- a/arch/powerpc/include/asm/archrandom.h +++ b/arch/powerpc/include/asm/archrandom.h @@ -2,8 +2,6 @@ #ifndef _ASM_POWERPC_ARCHRANDOM_H #define _ASM_POWERPC_ARCHRANDOM_H -#ifdef CONFIG_ARCH_RANDOM - #include static inline bool __must_check arch_get_random_long(unsigned long *v) @@ -35,7 +33,6 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) return rc; } -#endif /* CONFIG_ARCH_RANDOM */ #ifdef CONFIG_PPC_POWERNV int powernv_hwrng_present(void); diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index 358d171ae8e0..6c1002043367 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -200,9 +200,7 @@ struct machdep_calls { ssize_t (*cpu_release)(const char *, size_t); #endif -#ifdef CONFIG_ARCH_RANDOM int (*get_random_seed)(unsigned long *v); -#endif }; extern void e500_idle(void); diff --git a/arch/powerpc/platforms/microwatt/Kconfig b/arch/powerpc/platforms/microwatt/Kconfig index 5e320f49583a..6af443a1db99 100644 --- a/arch/powerpc/platforms/microwatt/Kconfig +++ b/arch/powerpc/platforms/microwatt/Kconfig @@ -6,7 +6,6 @@ config PPC_MICROWATT select PPC_ICS_NATIVE select PPC_ICP_NATIVE select PPC_UDBG_16550 - select ARCH_RANDOM help This option enables support for FPGA-based Microwatt implementations. diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig index 161dfe024085..e1a05c5a9004 100644 --- a/arch/powerpc/platforms/powernv/Kconfig +++ b/arch/powerpc/platforms/powernv/Kconfig @@ -12,7 +12,6 @@ config PPC_POWERNV select EPAPR_BOOT select PPC_INDIRECT_PIO select PPC_UDBG_16550 - select ARCH_RANDOM select CPU_FREQ select PPC_DOORBELL select MMU_NOTIFIER diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index f7fd91d153a4..f4a647c1f0b2 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -19,7 +19,6 @@ config PPC_PSERIES select PPC_UDBG_16550 select PPC_DOORBELL select HOTPLUG_CPU - select ARCH_RANDOM select FORCE_SMP select SWIOTLB default y diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 8cd9e56c629b..9b6e4e7cb17b 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -507,21 +507,6 @@ config KEXEC_SIG verification for the corresponding kernel image type being loaded in order for this to work. -config ARCH_RANDOM - def_bool y - prompt "s390 architectural random number generation API" - help - Enable the s390 architectural random number generation API - to provide random data for all consumers within the Linux - kernel. - - When enabled the arch_random_* functions declared in linux/random.h - are implemented. The implementation is based on the s390 CPACF - instruction subfunction TRNG which provides a real true random - number generator. - - If unsure, say Y. - config KERNEL_NOBP def_bool n prompt "Enable modified branch prediction for the kernel by default" diff --git a/arch/s390/configs/zfcpdump_defconfig b/arch/s390/configs/zfcpdump_defconfig index a87fcc45e307..f4976f611b94 100644 --- a/arch/s390/configs/zfcpdump_defconfig +++ b/arch/s390/configs/zfcpdump_defconfig @@ -15,7 +15,6 @@ CONFIG_TUNE_ZEC12=y # CONFIG_COMPAT is not set CONFIG_NR_CPUS=2 CONFIG_HZ_100=y -# CONFIG_ARCH_RANDOM is not set # CONFIG_RELOCATABLE is not set # CONFIG_CHSC_SCH is not set # CONFIG_SCM_BUS is not set diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index c63abfeb6d17..1b1cc478fa94 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -15,7 +15,7 @@ obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o obj-$(CONFIG_S390_PRNG) += prng.o obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o -obj-$(CONFIG_ARCH_RANDOM) += arch_random.o +obj-y += arch_random.o crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o chacha_s390-y := chacha-glue.o chacha-s390.o diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h index 2c6e1c6ecbe7..0a1c2e66c709 100644 --- a/arch/s390/include/asm/archrandom.h +++ b/arch/s390/include/asm/archrandom.h @@ -11,8 +11,6 @@ #ifndef _ASM_S390_ARCHRANDOM_H #define _ASM_S390_ARCHRANDOM_H -#ifdef CONFIG_ARCH_RANDOM - #include #include #include @@ -50,5 +48,4 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) return false; } -#endif /* CONFIG_ARCH_RANDOM */ #endif /* _ASM_S390_ARCHRANDOM_H */ diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 0a37f5de2863..ebad41afe355 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -876,10 +876,8 @@ static void __init setup_randomness(void) add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count); memblock_free(vmms, PAGE_SIZE); -#ifdef CONFIG_ARCH_RANDOM if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG)) static_branch_enable(&s390_arch_random_available); -#endif } /* diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index e58798f636d4..ba13749c09c8 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1810,15 +1810,6 @@ config ARCH_USES_PG_UNCACHED def_bool y depends on X86_PAT -config ARCH_RANDOM - def_bool y - prompt "x86 architectural random number generator" if EXPERT - help - Enable the x86 architectural RDRAND instruction - (Intel Bull Mountain technology) to generate random numbers. - If supported, this is a high bandwidth, cryptographically - secure hardware random number generator. - config X86_UMIP def_bool y prompt "User Mode Instruction Prevention" if EXPERT diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h index ebc248e49549..fb235b696175 100644 --- a/arch/x86/include/asm/archrandom.h +++ b/arch/x86/include/asm/archrandom.h @@ -65,10 +65,8 @@ static inline bool __must_check rdseed_int(unsigned int *v) /* * These are the generic interfaces; they must not be declared if the - * stubs in are to be invoked, - * i.e. CONFIG_ARCH_RANDOM is not defined. + * stubs in are to be invoked. */ -#ifdef CONFIG_ARCH_RANDOM static inline bool __must_check arch_get_random_long(unsigned long *v) { @@ -90,12 +88,8 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_int(v) : false; } -extern void x86_init_rdrand(struct cpuinfo_x86 *c); - -#else /* !CONFIG_ARCH_RANDOM */ - -static inline void x86_init_rdrand(struct cpuinfo_x86 *c) { } - -#endif /* !CONFIG_ARCH_RANDOM */ +#ifndef CONFIG_UML +void x86_init_rdrand(struct cpuinfo_x86 *c); +#endif #endif /* ASM_X86_ARCHRANDOM_H */ diff --git a/arch/x86/kernel/cpu/rdrand.c b/arch/x86/kernel/cpu/rdrand.c index c4be62058dd9..8f216669ecb8 100644 --- a/arch/x86/kernel/cpu/rdrand.c +++ b/arch/x86/kernel/cpu/rdrand.c @@ -26,7 +26,6 @@ __setup("nordrand", x86_rdrand_setup); */ #define SANITY_CHECK_LOOPS 8 -#ifdef CONFIG_ARCH_RANDOM void x86_init_rdrand(struct cpuinfo_x86 *c) { unsigned int changed = 0; @@ -63,4 +62,3 @@ void x86_init_rdrand(struct cpuinfo_x86 *c) "RDRAND gives funky smelling output, might consider not using it by booting with \"nordrand\""); } -#endif diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 0b6c03643ddc..30192e123e5f 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -431,7 +431,6 @@ config ADI config RANDOM_TRUST_CPU bool "Initialize RNG using CPU RNG instructions" default y - depends on ARCH_RANDOM help Initialize the RNG using random numbers supplied by the CPU's RNG instructions (e.g. RDRAND), if supported and available. These diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index 2beaa35c0d74..488808dc17a2 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -108,7 +108,6 @@ static ssize_t trng_counter_show(struct device *dev, { u64 dev_counter = atomic64_read(&trng_dev_counter); u64 hwrng_counter = atomic64_read(&trng_hwrng_counter); -#if IS_ENABLED(CONFIG_ARCH_RANDOM) u64 arch_counter = atomic64_read(&s390_arch_random_counter); return sysfs_emit(buf, @@ -118,14 +117,6 @@ static ssize_t trng_counter_show(struct device *dev, "total: %llu\n", dev_counter, hwrng_counter, arch_counter, dev_counter + hwrng_counter + arch_counter); -#else - return sysfs_emit(buf, - "trng: %llu\n" - "hwrng: %llu\n" - "total: %llu\n", - dev_counter, hwrng_counter, - dev_counter + hwrng_counter); -#endif } static DEVICE_ATTR(byte_counter, 0444, trng_counter_show, NULL); diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 8e47d483b524..36db8b9eb68a 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -5,6 +5,7 @@ # asm headers from the host architecutre.) mandatory-y += atomic.h +mandatory-y += archrandom.h mandatory-y += barrier.h mandatory-y += bitops.h mandatory-y += bug.h diff --git a/include/asm-generic/archrandom.h b/include/asm-generic/archrandom.h new file mode 100644 index 000000000000..3a5ee202dd86 --- /dev/null +++ b/include/asm-generic/archrandom.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_ARCHRANDOM_H__ +#define __ASM_GENERIC_ARCHRANDOM_H__ + +static inline bool __must_check arch_get_random_long(unsigned long *v) +{ + return false; +} + +static inline bool __must_check arch_get_random_int(unsigned int *v) +{ + return false; +} + +static inline bool __must_check arch_get_random_seed_long(unsigned long *v) +{ + return false; +} + +static inline bool __must_check arch_get_random_seed_int(unsigned int *v) +{ + return false; +} + +#endif diff --git a/include/linux/random.h b/include/linux/random.h index 20e389a14e5c..865770e29f3e 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -106,14 +106,7 @@ declare_get_random_var_wait(long, unsigned long) */ #include -#ifdef CONFIG_ARCH_RANDOM -# include -#else -static inline bool __must_check arch_get_random_long(unsigned long *v) { return false; } -static inline bool __must_check arch_get_random_int(unsigned int *v) { return false; } -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) { return false; } -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) { return false; } -#endif +#include /* * Called from the boot CPU during startup; not valid to call once diff --git a/tools/testing/selftests/wireguard/qemu/kernel.config b/tools/testing/selftests/wireguard/qemu/kernel.config index bad88f4b0a03..e1858ce7003f 100644 --- a/tools/testing/selftests/wireguard/qemu/kernel.config +++ b/tools/testing/selftests/wireguard/qemu/kernel.config @@ -58,7 +58,6 @@ CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n CONFIG_HZ_PERIODIC=n CONFIG_HIGH_RES_TIMERS=y -CONFIG_ARCH_RANDOM=y CONFIG_FILE_LOCKING=y CONFIG_POSIX_TIMERS=y CONFIG_DEVTMPFS=y -- cgit v1.2.3 From 0b4ae3f6d1210c11f9baf159009c7227eacf90f2 Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Mon, 11 Jul 2022 07:47:16 -0700 Subject: iio: cros: Register FIFO callback after sensor is registered Instead of registering callback to process sensor events right at initialization time, wait for the sensor to be register in the iio subsystem. Events can come at probe time (in case the kernel rebooted abruptly without switching the sensor off for instance), and be sent to IIO core before the sensor is fully registered. Fixes: aa984f1ba4a4 ("iio: cros_ec: Register to cros_ec_sensorhub when EC supports FIFO") Reported-by: Douglas Anderson Signed-off-by: Gwendal Grignou Reviewed-by: Douglas Anderson Link: https://lore.kernel.org/r/20220711144716.642617-1-gwendal@chromium.org Signed-off-by: Jonathan Cameron --- drivers/iio/accel/cros_ec_accel_legacy.c | 4 +- .../iio/common/cros_ec_sensors/cros_ec_lid_angle.c | 4 +- .../iio/common/cros_ec_sensors/cros_ec_sensors.c | 6 +-- .../common/cros_ec_sensors/cros_ec_sensors_core.c | 58 ++++++++++++++++------ drivers/iio/light/cros_ec_light_prox.c | 6 +-- drivers/iio/pressure/cros_ec_baro.c | 6 +-- include/linux/iio/common/cros_ec_sensors_core.h | 7 ++- 7 files changed, 60 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/accel/cros_ec_accel_legacy.c b/drivers/iio/accel/cros_ec_accel_legacy.c index 1c0171f26e99..0f403342b1fc 100644 --- a/drivers/iio/accel/cros_ec_accel_legacy.c +++ b/drivers/iio/accel/cros_ec_accel_legacy.c @@ -215,7 +215,7 @@ static int cros_ec_accel_legacy_probe(struct platform_device *pdev) return -ENOMEM; ret = cros_ec_sensors_core_init(pdev, indio_dev, true, - cros_ec_sensors_capture, NULL); + cros_ec_sensors_capture); if (ret) return ret; @@ -235,7 +235,7 @@ static int cros_ec_accel_legacy_probe(struct platform_device *pdev) state->sign[CROS_EC_SENSOR_Z] = -1; } - return devm_iio_device_register(dev, indio_dev); + return cros_ec_sensors_core_register(dev, indio_dev, NULL); } static struct platform_driver cros_ec_accel_platform_driver = { diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c b/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c index 9f780fafaed9..119acb078af3 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c @@ -98,7 +98,7 @@ static int cros_ec_lid_angle_probe(struct platform_device *pdev) if (!indio_dev) return -ENOMEM; - ret = cros_ec_sensors_core_init(pdev, indio_dev, false, NULL, NULL); + ret = cros_ec_sensors_core_init(pdev, indio_dev, false, NULL); if (ret) return ret; @@ -114,7 +114,7 @@ static int cros_ec_lid_angle_probe(struct platform_device *pdev) if (ret) return ret; - return devm_iio_device_register(dev, indio_dev); + return cros_ec_sensors_core_register(dev, indio_dev, NULL); } static const struct platform_device_id cros_ec_lid_angle_ids[] = { diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c index 61e07a7bb199..66153b1850f1 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c @@ -236,8 +236,7 @@ static int cros_ec_sensors_probe(struct platform_device *pdev) return -ENOMEM; ret = cros_ec_sensors_core_init(pdev, indio_dev, true, - cros_ec_sensors_capture, - cros_ec_sensors_push_data); + cros_ec_sensors_capture); if (ret) return ret; @@ -298,7 +297,8 @@ static int cros_ec_sensors_probe(struct platform_device *pdev) else state->core.read_ec_sensors_data = cros_ec_sensors_read_cmd; - return devm_iio_device_register(dev, indio_dev); + return cros_ec_sensors_core_register(dev, indio_dev, + cros_ec_sensors_push_data); } static const struct platform_device_id cros_ec_sensors_ids[] = { diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index e5ccedef13a8..05a28d353e34 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -228,21 +228,18 @@ static void cros_ec_sensors_core_clean(void *arg) /** * cros_ec_sensors_core_init() - basic initialization of the core structure - * @pdev: platform device created for the sensors + * @pdev: platform device created for the sensor * @indio_dev: iio device structure of the device * @physical_device: true if the device refers to a physical device * @trigger_capture: function pointer to call buffer is triggered, * for backward compatibility. - * @push_data: function to call when cros_ec_sensorhub receives - * a sample for that sensor. * * Return: 0 on success, -errno on failure. */ int cros_ec_sensors_core_init(struct platform_device *pdev, struct iio_dev *indio_dev, bool physical_device, - cros_ec_sensors_capture_t trigger_capture, - cros_ec_sensorhub_push_data_cb_t push_data) + cros_ec_sensors_capture_t trigger_capture) { struct device *dev = &pdev->dev; struct cros_ec_sensors_core_state *state = iio_priv(indio_dev); @@ -340,17 +337,6 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, if (ret) return ret; - ret = cros_ec_sensorhub_register_push_data( - sensor_hub, sensor_platform->sensor_num, - indio_dev, push_data); - if (ret) - return ret; - - ret = devm_add_action_or_reset( - dev, cros_ec_sensors_core_clean, pdev); - if (ret) - return ret; - /* Timestamp coming from FIFO are in ns since boot. */ ret = iio_device_set_clock(indio_dev, CLOCK_BOOTTIME); if (ret) @@ -372,6 +358,46 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, } EXPORT_SYMBOL_GPL(cros_ec_sensors_core_init); +/** + * cros_ec_sensors_core_register() - Register callback to FIFO and IIO when + * sensor is ready. + * It must be called at the end of the sensor probe routine. + * @dev: device created for the sensor + * @indio_dev: iio device structure of the device + * @push_data: function to call when cros_ec_sensorhub receives + * a sample for that sensor. + * + * Return: 0 on success, -errno on failure. + */ +int cros_ec_sensors_core_register(struct device *dev, + struct iio_dev *indio_dev, + cros_ec_sensorhub_push_data_cb_t push_data) +{ + struct cros_ec_sensor_platform *sensor_platform = dev_get_platdata(dev); + struct cros_ec_sensorhub *sensor_hub = dev_get_drvdata(dev->parent); + struct platform_device *pdev = to_platform_device(dev); + struct cros_ec_dev *ec = sensor_hub->ec; + int ret; + + ret = devm_iio_device_register(dev, indio_dev); + if (ret) + return ret; + + if (!push_data || + !cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) + return 0; + + ret = cros_ec_sensorhub_register_push_data( + sensor_hub, sensor_platform->sensor_num, + indio_dev, push_data); + if (ret) + return ret; + + return devm_add_action_or_reset( + dev, cros_ec_sensors_core_clean, pdev); +} +EXPORT_SYMBOL_GPL(cros_ec_sensors_core_register); + /** * cros_ec_motion_send_host_cmd() - send motion sense host command * @state: pointer to state information for device diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c index e345e0f71b74..19e529c84e95 100644 --- a/drivers/iio/light/cros_ec_light_prox.c +++ b/drivers/iio/light/cros_ec_light_prox.c @@ -182,8 +182,7 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev) return -ENOMEM; ret = cros_ec_sensors_core_init(pdev, indio_dev, true, - cros_ec_sensors_capture, - cros_ec_sensors_push_data); + cros_ec_sensors_capture); if (ret) return ret; @@ -239,7 +238,8 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev) state->core.read_ec_sensors_data = cros_ec_sensors_read_cmd; - return devm_iio_device_register(dev, indio_dev); + return cros_ec_sensors_core_register(dev, indio_dev, + cros_ec_sensors_push_data); } static const struct platform_device_id cros_ec_light_prox_ids[] = { diff --git a/drivers/iio/pressure/cros_ec_baro.c b/drivers/iio/pressure/cros_ec_baro.c index 25217279f350..2649c2f89e89 100644 --- a/drivers/iio/pressure/cros_ec_baro.c +++ b/drivers/iio/pressure/cros_ec_baro.c @@ -139,8 +139,7 @@ static int cros_ec_baro_probe(struct platform_device *pdev) return -ENOMEM; ret = cros_ec_sensors_core_init(pdev, indio_dev, true, - cros_ec_sensors_capture, - cros_ec_sensors_push_data); + cros_ec_sensors_capture); if (ret) return ret; @@ -185,7 +184,8 @@ static int cros_ec_baro_probe(struct platform_device *pdev) state->core.read_ec_sensors_data = cros_ec_sensors_read_cmd; - return devm_iio_device_register(dev, indio_dev); + return cros_ec_sensors_core_register(dev, indio_dev, + cros_ec_sensors_push_data); } static const struct platform_device_id cros_ec_baro_ids[] = { diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index a8259c8822f5..e72167b96d27 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -93,8 +93,11 @@ int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev, unsigned long scan_mask, struct platform_device; int cros_ec_sensors_core_init(struct platform_device *pdev, struct iio_dev *indio_dev, bool physical_device, - cros_ec_sensors_capture_t trigger_capture, - cros_ec_sensorhub_push_data_cb_t push_data); + cros_ec_sensors_capture_t trigger_capture); + +int cros_ec_sensors_core_register(struct device *dev, + struct iio_dev *indio_dev, + cros_ec_sensorhub_push_data_cb_t push_data); irqreturn_t cros_ec_sensors_capture(int irq, void *p); int cros_ec_sensors_push_data(struct iio_dev *indio_dev, -- cgit v1.2.3 From 2e07a521e1e424787af3bfc59615de4220856c35 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Jul 2022 21:52:28 +0100 Subject: skbuff: add SKBFL_DONT_ORPHAN flag We don't want to list every single ubuf_info callback in skb_orphan_frags(), add a flag controlling the behaviour. Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 8 +++++--- net/core/skbuff.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d3d10556f0fa..8e12b3b9ad6c 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -686,10 +686,13 @@ enum { * charged to the kernel memory. */ SKBFL_PURE_ZEROCOPY = BIT(2), + + SKBFL_DONT_ORPHAN = BIT(3), }; #define SKBFL_ZEROCOPY_FRAG (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG) -#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY) +#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \ + SKBFL_DONT_ORPHAN) /* * The callback notifies userspace to release buffers when skb DMA is done in @@ -3182,8 +3185,7 @@ static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask) { if (likely(!skb_zcopy(skb))) return 0; - if (!skb_zcopy_is_nouarg(skb) && - skb_uarg(skb)->callback == msg_zerocopy_callback) + if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN) return 0; return skb_copy_ubufs(skb, gfp_mask); } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 09f56bfa2771..fc22b3d32052 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1193,7 +1193,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) uarg->len = 1; uarg->bytelen = size; uarg->zerocopy = 1; - uarg->flags = SKBFL_ZEROCOPY_FRAG; + uarg->flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; refcount_set(&uarg->refcnt, 1); sock_hold(sk); -- cgit v1.2.3 From a229cc14f3395311b899e5e582b71efa8dd01df0 Mon Sep 17 00:00:00 2001 From: John Garry Date: Thu, 14 Jul 2022 19:15:24 +0800 Subject: dma-mapping: add dma_opt_mapping_size() Streaming DMA mapping involving an IOMMU may be much slower for larger total mapping size. This is because every IOMMU DMA mapping requires an IOVA to be allocated and freed. IOVA sizes above a certain limit are not cached, which can have a big impact on DMA mapping performance. Provide an API for device drivers to know this "optimal" limit, such that they may try to produce mapping which don't exceed it. Signed-off-by: John Garry Reviewed-by: Damien Le Moal Acked-by: Martin K. Petersen Signed-off-by: Christoph Hellwig --- Documentation/core-api/dma-api.rst | 14 ++++++++++++++ include/linux/dma-map-ops.h | 1 + include/linux/dma-mapping.h | 5 +++++ kernel/dma/mapping.c | 12 ++++++++++++ 4 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst index 6d6d0edd2d27..829f20a193ca 100644 --- a/Documentation/core-api/dma-api.rst +++ b/Documentation/core-api/dma-api.rst @@ -204,6 +204,20 @@ Returns the maximum size of a mapping for the device. The size parameter of the mapping functions like dma_map_single(), dma_map_page() and others should not be larger than the returned value. +:: + + size_t + dma_opt_mapping_size(struct device *dev); + +Returns the maximum optimal size of a mapping for the device. + +Mapping larger buffers may take much longer in certain scenarios. In +addition, for high-rate short-lived streaming mappings, the upfront time +spent on the mapping may account for an appreciable part of the total +request lifetime. As such, if splitting larger requests incurs no +significant performance penalty, then device drivers are advised to +limit total DMA streaming mappings length to the returned value. + :: bool diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 0d5b06b3a4a6..98ceba6fa848 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -69,6 +69,7 @@ struct dma_map_ops { int (*dma_supported)(struct device *dev, u64 mask); u64 (*get_required_mask)(struct device *dev); size_t (*max_mapping_size)(struct device *dev); + size_t (*opt_mapping_size)(void); unsigned long (*get_merge_boundary)(struct device *dev); }; diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index dca2b1355bb1..fe3849434b2a 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -144,6 +144,7 @@ int dma_set_mask(struct device *dev, u64 mask); int dma_set_coherent_mask(struct device *dev, u64 mask); u64 dma_get_required_mask(struct device *dev); size_t dma_max_mapping_size(struct device *dev); +size_t dma_opt_mapping_size(struct device *dev); bool dma_need_sync(struct device *dev, dma_addr_t dma_addr); unsigned long dma_get_merge_boundary(struct device *dev); struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, @@ -266,6 +267,10 @@ static inline size_t dma_max_mapping_size(struct device *dev) { return 0; } +static inline size_t dma_opt_mapping_size(struct device *dev) +{ + return 0; +} static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) { return false; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index db7244291b74..1bfe11b1edb6 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -773,6 +773,18 @@ size_t dma_max_mapping_size(struct device *dev) } EXPORT_SYMBOL_GPL(dma_max_mapping_size); +size_t dma_opt_mapping_size(struct device *dev) +{ + const struct dma_map_ops *ops = get_dma_ops(dev); + size_t size = SIZE_MAX; + + if (ops && ops->opt_mapping_size) + size = ops->opt_mapping_size(); + + return min(dma_max_mapping_size(dev), size); +} +EXPORT_SYMBOL_GPL(dma_opt_mapping_size); + bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) { const struct dma_map_ops *ops = get_dma_ops(dev); -- cgit v1.2.3 From 6d9870b7e5def2450e21316515b9efc0529204dd Mon Sep 17 00:00:00 2001 From: John Garry Date: Thu, 14 Jul 2022 19:15:25 +0800 Subject: dma-iommu: add iommu_dma_opt_mapping_size() Add the IOMMU callback for DMA mapping API dma_opt_mapping_size(), which allows the drivers to know the optimal mapping limit and thus limit the requested IOVA lengths. This value is based on the IOVA rcache range limit, as IOVAs allocated above this limit must always be newly allocated, which may be quite slow. Signed-off-by: John Garry Reviewed-by: Damien Le Moal Acked-by: Robin Murphy Acked-by: Martin K. Petersen Signed-off-by: Christoph Hellwig --- drivers/iommu/dma-iommu.c | 6 ++++++ drivers/iommu/iova.c | 5 +++++ include/linux/iova.h | 2 ++ 3 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index f90251572a5d..9e1586447ee8 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1459,6 +1459,11 @@ static unsigned long iommu_dma_get_merge_boundary(struct device *dev) return (1UL << __ffs(domain->pgsize_bitmap)) - 1; } +static size_t iommu_dma_opt_mapping_size(void) +{ + return iova_rcache_range(); +} + static const struct dma_map_ops iommu_dma_ops = { .alloc = iommu_dma_alloc, .free = iommu_dma_free, @@ -1479,6 +1484,7 @@ static const struct dma_map_ops iommu_dma_ops = { .map_resource = iommu_dma_map_resource, .unmap_resource = iommu_dma_unmap_resource, .get_merge_boundary = iommu_dma_get_merge_boundary, + .opt_mapping_size = iommu_dma_opt_mapping_size, }; /* diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index db77aa675145..9f00b58d546e 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -26,6 +26,11 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_iova_rcaches(struct iova_domain *iovad); +unsigned long iova_rcache_range(void) +{ + return PAGE_SIZE << (IOVA_RANGE_CACHE_MAX_SIZE - 1); +} + static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) { struct iova_domain *iovad; diff --git a/include/linux/iova.h b/include/linux/iova.h index 320a70e40233..c6ba6d95d79c 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -79,6 +79,8 @@ static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova) int iova_cache_get(void); void iova_cache_put(void); +unsigned long iova_rcache_range(void); + void free_iova(struct iova_domain *iovad, unsigned long pfn); void __free_iova(struct iova_domain *iovad, struct iova *iova); struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, -- cgit v1.2.3 From 2b038e786f8338a3bc22d791000753e0ec113e00 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 22 Jun 2022 20:28:42 +0300 Subject: gpiolib: devres: Get rid of unused devm_gpio_free() The last user, which in fact was a dead code, has gone a year ago, previous one 3 years ago. On top of that we want to drop away the legacy GPIO APIs in the kernel, so take a chance to get rid of unused devm_gpio_free() and accompanying stuff. Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- Documentation/driver-api/driver-model/devres.rst | 1 - drivers/gpio/gpiolib-devres.c | 32 ------------------------ include/linux/gpio.h | 6 ----- 3 files changed, 39 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index 2d39967bafcc..55272942e721 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -277,7 +277,6 @@ GPIO devm_gpiochip_add_data() devm_gpio_request() devm_gpio_request_one() - devm_gpio_free() I2C devm_i2c_new_dummy_device() diff --git a/drivers/gpio/gpiolib-devres.c b/drivers/gpio/gpiolib-devres.c index 79da85d17b71..16a696249229 100644 --- a/drivers/gpio/gpiolib-devres.c +++ b/drivers/gpio/gpiolib-devres.c @@ -375,9 +375,6 @@ void devm_gpiod_put_array(struct device *dev, struct gpio_descs *descs) } EXPORT_SYMBOL_GPL(devm_gpiod_put_array); - - - static void devm_gpio_release(struct device *dev, void *res) { unsigned *gpio = res; @@ -385,13 +382,6 @@ static void devm_gpio_release(struct device *dev, void *res) gpio_free(*gpio); } -static int devm_gpio_match(struct device *dev, void *res, void *data) -{ - unsigned *this = res, *gpio = data; - - return *this == *gpio; -} - /** * devm_gpio_request - request a GPIO for a managed device * @dev: device to request the GPIO for @@ -402,11 +392,7 @@ static int devm_gpio_match(struct device *dev, void *res, void *data) * same arguments and performs the same function as * gpio_request(). GPIOs requested with this function will be * automatically freed on driver detach. - * - * If an GPIO allocated with this function needs to be freed - * separately, devm_gpio_free() must be used. */ - int devm_gpio_request(struct device *dev, unsigned gpio, const char *label) { unsigned *dr; @@ -459,24 +445,6 @@ int devm_gpio_request_one(struct device *dev, unsigned gpio, } EXPORT_SYMBOL_GPL(devm_gpio_request_one); -/** - * devm_gpio_free - free a GPIO - * @dev: device to free GPIO for - * @gpio: GPIO to free - * - * Except for the extra @dev argument, this function takes the - * same arguments and performs the same function as gpio_free(). - * This function instead of gpio_free() should be used to manually - * free GPIOs allocated with devm_gpio_request(). - */ -void devm_gpio_free(struct device *dev, unsigned int gpio) -{ - - WARN_ON(devres_release(dev, devm_gpio_release, devm_gpio_match, - &gpio)); -} -EXPORT_SYMBOL_GPL(devm_gpio_free); - static void devm_gpio_chip_release(void *data) { struct gpio_chip *gc = data; diff --git a/include/linux/gpio.h b/include/linux/gpio.h index 008ad3ee56b7..a370387fa406 100644 --- a/include/linux/gpio.h +++ b/include/linux/gpio.h @@ -95,7 +95,6 @@ struct device; int devm_gpio_request(struct device *dev, unsigned gpio, const char *label); int devm_gpio_request_one(struct device *dev, unsigned gpio, unsigned long flags, const char *label); -void devm_gpio_free(struct device *dev, unsigned int gpio); #else /* ! CONFIG_GPIOLIB */ @@ -240,11 +239,6 @@ static inline int devm_gpio_request_one(struct device *dev, unsigned gpio, return -EINVAL; } -static inline void devm_gpio_free(struct device *dev, unsigned int gpio) -{ - WARN_ON(1); -} - #endif /* ! CONFIG_GPIOLIB */ #endif /* __LINUX_GPIO_H */ -- cgit v1.2.3 From 2a1192ff0835cb4b0ccd6f6e85c93aa0dc6f66b7 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Tue, 14 Jun 2022 17:23:38 +0200 Subject: gpio: twl4030: Drop platform teardown callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no machine providing a teardown callback, so drop the unused code. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-twl4030.c | 11 ----------- include/linux/mfd/twl.h | 2 -- 2 files changed, 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpio-twl4030.c b/drivers/gpio/gpio-twl4030.c index de249726230e..e2cb7cb90c8c 100644 --- a/drivers/gpio/gpio-twl4030.c +++ b/drivers/gpio/gpio-twl4030.c @@ -593,18 +593,7 @@ out: /* Cannot use as gpio_twl4030_probe() calls us */ static int gpio_twl4030_remove(struct platform_device *pdev) { - struct twl4030_gpio_platform_data *pdata = dev_get_platdata(&pdev->dev); struct gpio_twl4030_priv *priv = platform_get_drvdata(pdev); - int status; - - if (pdata && pdata->teardown) { - status = pdata->teardown(&pdev->dev, priv->gpio_chip.base, - TWL4030_GPIO_MAX); - if (status) { - dev_dbg(&pdev->dev, "teardown --> %d\n", status); - return status; - } - } gpiochip_remove(&priv->gpio_chip); diff --git a/include/linux/mfd/twl.h b/include/linux/mfd/twl.h index 8871cc5188a0..c8cd31756037 100644 --- a/include/linux/mfd/twl.h +++ b/include/linux/mfd/twl.h @@ -594,8 +594,6 @@ struct twl4030_gpio_platform_data { int (*setup)(struct device *dev, unsigned gpio, unsigned ngpio); - int (*teardown)(struct device *dev, - unsigned gpio, unsigned ngpio); }; struct twl4030_madc_platform_data { -- cgit v1.2.3 From 7e55b33d3f18fde5c7a57b6c52d80499485c737f Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Tue, 14 Jun 2022 21:48:02 +0200 Subject: gpio: ucb1400: Remove platform setup and teardown support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no user of these callbacks. The motivation for this change is to stop returning an error code from the remove callback. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-ucb1400.c | 20 -------------------- drivers/mfd/ucb1400_core.c | 6 ++---- include/linux/ucb1400.h | 2 -- 3 files changed, 2 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpio-ucb1400.c b/drivers/gpio/gpio-ucb1400.c index d2a8644864c3..386e69300332 100644 --- a/drivers/gpio/gpio-ucb1400.c +++ b/drivers/gpio/gpio-ucb1400.c @@ -64,34 +64,14 @@ static int ucb1400_gpio_probe(struct platform_device *dev) ucb->gc.can_sleep = true; err = devm_gpiochip_add_data(&dev->dev, &ucb->gc, ucb); - if (err) - goto err; - - if (ucb->gpio_setup) - err = ucb->gpio_setup(&dev->dev, ucb->gc.ngpio); err: return err; } -static int ucb1400_gpio_remove(struct platform_device *dev) -{ - int err = 0; - struct ucb1400_gpio *ucb = platform_get_drvdata(dev); - - if (ucb && ucb->gpio_teardown) { - err = ucb->gpio_teardown(&dev->dev, ucb->gc.ngpio); - if (err) - return err; - } - - return err; -} - static struct platform_driver ucb1400_gpio_driver = { .probe = ucb1400_gpio_probe, - .remove = ucb1400_gpio_remove, .driver = { .name = "ucb1400_gpio" }, diff --git a/drivers/mfd/ucb1400_core.c b/drivers/mfd/ucb1400_core.c index 8c3832a58ef6..ac1d18039568 100644 --- a/drivers/mfd/ucb1400_core.c +++ b/drivers/mfd/ucb1400_core.c @@ -72,11 +72,9 @@ static int ucb1400_core_probe(struct device *dev) /* GPIO */ ucb_gpio.ac97 = ac97; - if (pdata) { - ucb_gpio.gpio_setup = pdata->gpio_setup; - ucb_gpio.gpio_teardown = pdata->gpio_teardown; + if (pdata) ucb_gpio.gpio_offset = pdata->gpio_offset; - } + ucb->ucb1400_gpio = platform_device_alloc("ucb1400_gpio", -1); if (!ucb->ucb1400_gpio) { err = -ENOMEM; diff --git a/include/linux/ucb1400.h b/include/linux/ucb1400.h index 0968ef458447..22345391350b 100644 --- a/include/linux/ucb1400.h +++ b/include/linux/ucb1400.h @@ -84,8 +84,6 @@ struct ucb1400_gpio { struct gpio_chip gc; struct snd_ac97 *ac97; int gpio_offset; - int (*gpio_setup)(struct device *dev, int ngpio); - int (*gpio_teardown)(struct device *dev, int ngpio); }; struct ucb1400_ts { -- cgit v1.2.3 From c269df8c5ad316eac47a9078e7a2339e1956637a Mon Sep 17 00:00:00 2001 From: Nuno Sá Date: Wed, 13 Jul 2022 15:14:18 +0200 Subject: gpiolib: add support for bias pull disable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change prepares the gpio core to look at firmware flags and set 'FLAG_BIAS_DISABLE' if necessary. It works in similar way to 'GPIO_PULL_DOWN' and 'GPIO_PULL_UP'. Signed-off-by: Nuno Sá Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 8 ++++++-- include/linux/gpio/machine.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 9535f48e18d1..0692ec84d3b0 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -3945,9 +3945,11 @@ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id, if (lflags & GPIO_OPEN_SOURCE) set_bit(FLAG_OPEN_SOURCE, &desc->flags); - if ((lflags & GPIO_PULL_UP) && (lflags & GPIO_PULL_DOWN)) { + if (((lflags & GPIO_PULL_UP) && (lflags & GPIO_PULL_DOWN)) || + ((lflags & GPIO_PULL_UP) && (lflags & GPIO_PULL_DISABLE)) || + ((lflags & GPIO_PULL_DOWN) && (lflags & GPIO_PULL_DISABLE))) { gpiod_err(desc, - "both pull-up and pull-down enabled, invalid configuration\n"); + "multiple pull-up, pull-down or pull-disable enabled, invalid configuration\n"); return -EINVAL; } @@ -3955,6 +3957,8 @@ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id, set_bit(FLAG_PULL_UP, &desc->flags); else if (lflags & GPIO_PULL_DOWN) set_bit(FLAG_PULL_DOWN, &desc->flags); + else if (lflags & GPIO_PULL_DISABLE) + set_bit(FLAG_BIAS_DISABLE, &desc->flags); ret = gpiod_set_transitory(desc, (lflags & GPIO_TRANSITORY)); if (ret < 0) diff --git a/include/linux/gpio/machine.h b/include/linux/gpio/machine.h index 4d55da28e664..0b619eb7ae83 100644 --- a/include/linux/gpio/machine.h +++ b/include/linux/gpio/machine.h @@ -14,6 +14,7 @@ enum gpio_lookup_flags { GPIO_TRANSITORY = (1 << 3), GPIO_PULL_UP = (1 << 4), GPIO_PULL_DOWN = (1 << 5), + GPIO_PULL_DISABLE = (1 << 6), GPIO_LOOKUP_FLAGS_DEFAULT = GPIO_ACTIVE_HIGH | GPIO_PERSISTENT, }; -- cgit v1.2.3 From 31bea23119cda87088c6bd4085a1e442c6c5974c Mon Sep 17 00:00:00 2001 From: Nuno Sá Date: Wed, 13 Jul 2022 15:14:19 +0200 Subject: gpiolib: of: support bias pull disable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On top of looking at PULL_UP and PULL_DOWN flags, also look at PULL_DISABLE and set the appropriate GPIO flag. The GPIO core will then pass down this to controllers that support it. Signed-off-by: Nuno Sá Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-of.c | 7 +++++++ include/linux/of_gpio.h | 1 + 2 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index f80307be37d5..a037b50bef33 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -354,6 +354,9 @@ struct gpio_desc *gpiod_get_from_of_node(const struct device_node *node, if (flags & OF_GPIO_PULL_DOWN) lflags |= GPIO_PULL_DOWN; + if (flags & OF_GPIO_PULL_DISABLE) + lflags |= GPIO_PULL_DISABLE; + ret = gpiod_configure_flags(desc, propname, lflags, dflags); if (ret < 0) { gpiod_put(desc); @@ -556,6 +559,8 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, *flags |= GPIO_PULL_UP; if (of_flags & OF_GPIO_PULL_DOWN) *flags |= GPIO_PULL_DOWN; + if (of_flags & OF_GPIO_PULL_DISABLE) + *flags |= GPIO_PULL_DISABLE; return desc; } @@ -621,6 +626,8 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, *lflags |= GPIO_PULL_UP; if (xlate_flags & OF_GPIO_PULL_DOWN) *lflags |= GPIO_PULL_DOWN; + if (xlate_flags & OF_GPIO_PULL_DISABLE) + *lflags |= GPIO_PULL_DISABLE; if (of_property_read_bool(np, "input")) *dflags |= GPIOD_IN; diff --git a/include/linux/of_gpio.h b/include/linux/of_gpio.h index 8bf2ea859653..a5166eb93437 100644 --- a/include/linux/of_gpio.h +++ b/include/linux/of_gpio.h @@ -29,6 +29,7 @@ enum of_gpio_flags { OF_GPIO_TRANSITORY = 0x8, OF_GPIO_PULL_UP = 0x10, OF_GPIO_PULL_DOWN = 0x20, + OF_GPIO_PULL_DISABLE = 0x40, }; #ifdef CONFIG_OF_GPIO -- cgit v1.2.3 From 62fa5c9800a0b9aecf9ce0743f12a16057c9400d Mon Sep 17 00:00:00 2001 From: Luca Ceresoli Date: Fri, 3 Jun 2022 17:57:25 +0200 Subject: mfd: max77714: Update Luca Ceresoli's e-mail address My Bootlin address is preferred from now on. Signed-off-by: Luca Ceresoli Signed-off-by: Luca Ceresoli Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220603155727.1232061-4-luca@lucaceresoli.net --- drivers/mfd/max77714.c | 4 ++-- include/linux/mfd/max77714.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/mfd/max77714.c b/drivers/mfd/max77714.c index d1e4247800d2..143a432ea343 100644 --- a/drivers/mfd/max77714.c +++ b/drivers/mfd/max77714.c @@ -3,7 +3,7 @@ * Maxim MAX77714 Core Driver * * Copyright (C) 2022 Luca Ceresoli - * Author: Luca Ceresoli + * Author: Luca Ceresoli */ #include @@ -148,5 +148,5 @@ static struct i2c_driver max77714_driver = { module_i2c_driver(max77714_driver); MODULE_DESCRIPTION("Maxim MAX77714 MFD core driver"); -MODULE_AUTHOR("Luca Ceresoli "); +MODULE_AUTHOR("Luca Ceresoli "); MODULE_LICENSE("GPL"); diff --git a/include/linux/mfd/max77714.h b/include/linux/mfd/max77714.h index a970dc455426..7947e0d697a5 100644 --- a/include/linux/mfd/max77714.h +++ b/include/linux/mfd/max77714.h @@ -3,7 +3,7 @@ * Maxim MAX77714 Register and data structures definition. * * Copyright (C) 2022 Luca Ceresoli - * Author: Luca Ceresoli + * Author: Luca Ceresoli */ #ifndef __LINUX_MFD_MAX77714_H_ -- cgit v1.2.3 From 128ac294e1b437cb8a7f2ff8ede1cde9082bddbe Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Mon, 30 May 2022 21:24:28 +0200 Subject: mfd: t7l66xb: Drop platform disable callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit None of the in-tree instantiations of struct t7l66xb_platform_data provides a disable callback. So better don't dereference this function pointer unconditionally. As there is no user, drop it completely instead of calling it conditional. This is a preparation for making platform remove callbacks return void. Fixes: 1f192015ca5b ("mfd: driver for the T7L66XB TMIO SoC") Signed-off-by: Uwe Kleine-König Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220530192430.2108217-3-u.kleine-koenig@pengutronix.de --- drivers/mfd/t7l66xb.c | 6 +----- include/linux/mfd/t7l66xb.h | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/mfd/t7l66xb.c b/drivers/mfd/t7l66xb.c index 5369c67e3280..663ffd4b8570 100644 --- a/drivers/mfd/t7l66xb.c +++ b/drivers/mfd/t7l66xb.c @@ -397,11 +397,8 @@ err_noirq: static int t7l66xb_remove(struct platform_device *dev) { - struct t7l66xb_platform_data *pdata = dev_get_platdata(&dev->dev); struct t7l66xb *t7l66xb = platform_get_drvdata(dev); - int ret; - ret = pdata->disable(dev); clk_disable_unprepare(t7l66xb->clk48m); clk_put(t7l66xb->clk48m); clk_disable_unprepare(t7l66xb->clk32k); @@ -412,8 +409,7 @@ static int t7l66xb_remove(struct platform_device *dev) mfd_remove_devices(&dev->dev); kfree(t7l66xb); - return ret; - + return 0; } static struct platform_driver t7l66xb_platform_driver = { diff --git a/include/linux/mfd/t7l66xb.h b/include/linux/mfd/t7l66xb.h index 69632c1b07bd..ae3e7a5c5219 100644 --- a/include/linux/mfd/t7l66xb.h +++ b/include/linux/mfd/t7l66xb.h @@ -12,7 +12,6 @@ struct t7l66xb_platform_data { int (*enable)(struct platform_device *dev); - int (*disable)(struct platform_device *dev); int (*suspend)(struct platform_device *dev); int (*resume)(struct platform_device *dev); -- cgit v1.2.3 From 6e1f1b1c93ceec1d9bfa9213775abc19161de5f1 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Mon, 30 May 2022 21:24:29 +0200 Subject: mfd: tc6387xb: Drop disable callback that is never called MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver never calls the disable callback, so drop the member from the platform struct and all callbacks from the actual platform datas. Signed-off-by: Uwe Kleine-König Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220530192430.2108217-4-u.kleine-koenig@pengutronix.de --- arch/arm/mach-pxa/eseries.c | 1 - include/linux/mfd/tc6387xb.h | 1 - 2 files changed, 2 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/mach-pxa/eseries.c b/arch/arm/mach-pxa/eseries.c index 08f8737aa8fd..99781eec065e 100644 --- a/arch/arm/mach-pxa/eseries.c +++ b/arch/arm/mach-pxa/eseries.c @@ -139,7 +139,6 @@ static void __init __maybe_unused eseries_register_clks(void) static struct tc6387xb_platform_data e330_tc6387xb_info = { .enable = &eseries_tmio_enable, - .disable = &eseries_tmio_disable, .suspend = &eseries_tmio_suspend, .resume = &eseries_tmio_resume, }; diff --git a/include/linux/mfd/tc6387xb.h b/include/linux/mfd/tc6387xb.h index b4888209494a..aacf1dcc86b9 100644 --- a/include/linux/mfd/tc6387xb.h +++ b/include/linux/mfd/tc6387xb.h @@ -12,7 +12,6 @@ struct tc6387xb_platform_data { int (*enable)(struct platform_device *dev); - int (*disable)(struct platform_device *dev); int (*suspend)(struct platform_device *dev); int (*resume)(struct platform_device *dev); }; -- cgit v1.2.3 From de58cee8c6b803dda3304eace346919fe880a40a Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Tue, 31 May 2022 14:49:56 +0200 Subject: mfd: mt6397-core: Add MT6357 PMIC support Adds support for PMIC keys, Regulator, and RTC for the MT6357 PMIC. Signed-off-by: Fabien Parent Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220531124959.202787-5-fparent@baylibre.com --- drivers/mfd/mt6397-core.c | 44 + include/linux/mfd/mt6357/core.h | 119 +++ include/linux/mfd/mt6357/registers.h | 1574 ++++++++++++++++++++++++++++++++++ 3 files changed, 1737 insertions(+) create mode 100644 include/linux/mfd/mt6357/core.h create mode 100644 include/linux/mfd/mt6357/registers.h (limited to 'include/linux') diff --git a/drivers/mfd/mt6397-core.c b/drivers/mfd/mt6397-core.c index 1a368ad08f58..3cb8836bd08d 100644 --- a/drivers/mfd/mt6397-core.c +++ b/drivers/mfd/mt6397-core.c @@ -12,10 +12,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -23,6 +25,9 @@ #define MT6323_RTC_BASE 0x8000 #define MT6323_RTC_SIZE 0x40 +#define MT6357_RTC_BASE 0x0588 +#define MT6357_RTC_SIZE 0x3c + #define MT6358_RTC_BASE 0x0588 #define MT6358_RTC_SIZE 0x3c @@ -37,6 +42,11 @@ static const struct resource mt6323_rtc_resources[] = { DEFINE_RES_IRQ(MT6323_IRQ_STATUS_RTC), }; +static const struct resource mt6357_rtc_resources[] = { + DEFINE_RES_MEM(MT6357_RTC_BASE, MT6357_RTC_SIZE), + DEFINE_RES_IRQ(MT6357_IRQ_RTC), +}; + static const struct resource mt6358_rtc_resources[] = { DEFINE_RES_MEM(MT6358_RTC_BASE, MT6358_RTC_SIZE), DEFINE_RES_IRQ(MT6358_IRQ_RTC), @@ -66,6 +76,13 @@ static const struct resource mt6323_keys_resources[] = { DEFINE_RES_IRQ_NAMED(MT6323_IRQ_STATUS_FCHRKEY, "homekey"), }; +static const struct resource mt6357_keys_resources[] = { + DEFINE_RES_IRQ_NAMED(MT6357_IRQ_PWRKEY, "powerkey"), + DEFINE_RES_IRQ_NAMED(MT6357_IRQ_HOMEKEY, "homekey"), + DEFINE_RES_IRQ_NAMED(MT6357_IRQ_PWRKEY_R, "powerkey_r"), + DEFINE_RES_IRQ_NAMED(MT6357_IRQ_HOMEKEY_R, "homekey_r"), +}; + static const struct resource mt6397_keys_resources[] = { DEFINE_RES_IRQ_NAMED(MT6397_IRQ_PWRKEY, "powerkey"), DEFINE_RES_IRQ_NAMED(MT6397_IRQ_HOMEKEY, "homekey"), @@ -100,6 +117,22 @@ static const struct mfd_cell mt6323_devs[] = { }, }; +static const struct mfd_cell mt6357_devs[] = { + { + .name = "mt6357-regulator", + }, { + .name = "mt6357-rtc", + .num_resources = ARRAY_SIZE(mt6357_rtc_resources), + .resources = mt6357_rtc_resources, + .of_compatible = "mediatek,mt6357-rtc", + }, { + .name = "mtk-pmic-keys", + .num_resources = ARRAY_SIZE(mt6357_keys_resources), + .resources = mt6357_keys_resources, + .of_compatible = "mediatek,mt6357-keys" + }, +}; + static const struct mfd_cell mt6358_devs[] = { { .name = "mt6358-regulator", @@ -179,6 +212,14 @@ static const struct chip_data mt6323_core = { .irq_init = mt6397_irq_init, }; +static const struct chip_data mt6357_core = { + .cid_addr = MT6357_SWCID, + .cid_shift = 8, + .cells = mt6357_devs, + .cell_size = ARRAY_SIZE(mt6357_devs), + .irq_init = mt6358_irq_init, +}; + static const struct chip_data mt6358_core = { .cid_addr = MT6358_SWCID, .cid_shift = 8, @@ -261,6 +302,9 @@ static const struct of_device_id mt6397_of_match[] = { { .compatible = "mediatek,mt6323", .data = &mt6323_core, + }, { + .compatible = "mediatek,mt6357", + .data = &mt6357_core, }, { .compatible = "mediatek,mt6358", .data = &mt6358_core, diff --git a/include/linux/mfd/mt6357/core.h b/include/linux/mfd/mt6357/core.h new file mode 100644 index 000000000000..2441611264fd --- /dev/null +++ b/include/linux/mfd/mt6357/core.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 BayLibre, SAS + * Author: Fabien Parent + */ + +#ifndef __MFD_MT6357_CORE_H__ +#define __MFD_MT6357_CORE_H__ + +enum mt6357_irq_top_status_shift { + MT6357_BUCK_TOP = 0, + MT6357_LDO_TOP, + MT6357_PSC_TOP, + MT6357_SCK_TOP, + MT6357_BM_TOP, + MT6357_HK_TOP, + MT6357_XPP_TOP, + MT6357_AUD_TOP, + MT6357_MISC_TOP, +}; + +enum mt6357_irq_numbers { + MT6357_IRQ_VPROC_OC = 0, + MT6357_IRQ_VCORE_OC, + MT6357_IRQ_VMODEM_OC, + MT6357_IRQ_VS1_OC, + MT6357_IRQ_VPA_OC, + MT6357_IRQ_VCORE_PREOC, + MT6357_IRQ_VFE28_OC = 16, + MT6357_IRQ_VXO22_OC, + MT6357_IRQ_VRF18_OC, + MT6357_IRQ_VRF12_OC, + MT6357_IRQ_VEFUSE_OC, + MT6357_IRQ_VCN33_OC, + MT6357_IRQ_VCN28_OC, + MT6357_IRQ_VCN18_OC, + MT6357_IRQ_VCAMA_OC, + MT6357_IRQ_VCAMD_OC, + MT6357_IRQ_VCAMIO_OC, + MT6357_IRQ_VLDO28_OC, + MT6357_IRQ_VUSB33_OC, + MT6357_IRQ_VAUX18_OC, + MT6357_IRQ_VAUD28_OC, + MT6357_IRQ_VIO28_OC, + MT6357_IRQ_VIO18_OC, + MT6357_IRQ_VSRAM_PROC_OC, + MT6357_IRQ_VSRAM_OTHERS_OC, + MT6357_IRQ_VIBR_OC, + MT6357_IRQ_VDRAM_OC, + MT6357_IRQ_VMC_OC, + MT6357_IRQ_VMCH_OC, + MT6357_IRQ_VEMC_OC, + MT6357_IRQ_VSIM1_OC, + MT6357_IRQ_VSIM2_OC, + MT6357_IRQ_PWRKEY = 48, + MT6357_IRQ_HOMEKEY, + MT6357_IRQ_PWRKEY_R, + MT6357_IRQ_HOMEKEY_R, + MT6357_IRQ_NI_LBAT_INT, + MT6357_IRQ_CHRDET, + MT6357_IRQ_CHRDET_EDGE, + MT6357_IRQ_VCDT_HV_DET, + MT6357_IRQ_WATCHDOG, + MT6357_IRQ_VBATON_UNDET, + MT6357_IRQ_BVALID_DET, + MT6357_IRQ_OV, + MT6357_IRQ_RTC = 64, + MT6357_IRQ_FG_BAT0_H = 80, + MT6357_IRQ_FG_BAT0_L, + MT6357_IRQ_FG_CUR_H, + MT6357_IRQ_FG_CUR_L, + MT6357_IRQ_FG_ZCV, + MT6357_IRQ_BATON_LV = 96, + MT6357_IRQ_BATON_HT, + MT6357_IRQ_BAT_H = 112, + MT6357_IRQ_BAT_L, + MT6357_IRQ_AUXADC_IMP, + MT6357_IRQ_NAG_C_DLTV, + MT6357_IRQ_AUDIO = 128, + MT6357_IRQ_ACCDET = 133, + MT6357_IRQ_ACCDET_EINT0, + MT6357_IRQ_ACCDET_EINT1, + MT6357_IRQ_SPI_CMD_ALERT = 144, + MT6357_IRQ_NR, +}; + +#define MT6357_IRQ_BUCK_BASE MT6357_IRQ_VPROC_OC +#define MT6357_IRQ_LDO_BASE MT6357_IRQ_VFE28_OC +#define MT6357_IRQ_PSC_BASE MT6357_IRQ_PWRKEY +#define MT6357_IRQ_SCK_BASE MT6357_IRQ_RTC +#define MT6357_IRQ_BM_BASE MT6357_IRQ_FG_BAT0_H +#define MT6357_IRQ_HK_BASE MT6357_IRQ_BAT_H +#define MT6357_IRQ_AUD_BASE MT6357_IRQ_AUDIO +#define MT6357_IRQ_MISC_BASE MT6357_IRQ_SPI_CMD_ALERT + +#define MT6357_IRQ_BUCK_BITS (MT6357_IRQ_VCORE_PREOC - MT6357_IRQ_BUCK_BASE + 1) +#define MT6357_IRQ_LDO_BITS (MT6357_IRQ_VSIM2_OC - MT6357_IRQ_LDO_BASE + 1) +#define MT6357_IRQ_PSC_BITS (MT6357_IRQ_VCDT_HV_DET - MT6357_IRQ_PSC_BASE + 1) +#define MT6357_IRQ_SCK_BITS (MT6357_IRQ_RTC - MT6357_IRQ_SCK_BASE + 1) +#define MT6357_IRQ_BM_BITS (MT6357_IRQ_BATON_HT - MT6357_IRQ_BM_BASE + 1) +#define MT6357_IRQ_HK_BITS (MT6357_IRQ_NAG_C_DLTV - MT6357_IRQ_HK_BASE + 1) +#define MT6357_IRQ_AUD_BITS (MT6357_IRQ_ACCDET_EINT1 - MT6357_IRQ_AUD_BASE + 1) +#define MT6357_IRQ_MISC_BITS \ + (MT6357_IRQ_SPI_CMD_ALERT - MT6357_IRQ_MISC_BASE + 1) + +#define MT6357_TOP_GEN(sp) \ +{ \ + .hwirq_base = MT6357_IRQ_##sp##_BASE, \ + .num_int_regs = \ + ((MT6357_IRQ_##sp##_BITS - 1) / \ + MTK_PMIC_REG_WIDTH) + 1, \ + .en_reg = MT6357_##sp##_TOP_INT_CON0, \ + .en_reg_shift = 0x6, \ + .sta_reg = MT6357_##sp##_TOP_INT_STATUS0, \ + .sta_reg_shift = 0x2, \ + .top_offset = MT6357_##sp##_TOP, \ +} + +#endif /* __MFD_MT6357_CORE_H__ */ diff --git a/include/linux/mfd/mt6357/registers.h b/include/linux/mfd/mt6357/registers.h new file mode 100644 index 000000000000..e24af83b618d --- /dev/null +++ b/include/linux/mfd/mt6357/registers.h @@ -0,0 +1,1574 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 MediaTek Inc. + */ + +#ifndef __MFD_MT6357_REGISTERS_H__ +#define __MFD_MT6357_REGISTERS_H__ + +/* PMIC Registers */ +#define MT6357_TOP0_ID 0x0 +#define MT6357_TOP0_REV0 0x2 +#define MT6357_TOP0_DSN_DBI 0x4 +#define MT6357_TOP0_DSN_DXI 0x6 +#define MT6357_HWCID 0x8 +#define MT6357_SWCID 0xa +#define MT6357_PONSTS 0xc +#define MT6357_POFFSTS 0xe +#define MT6357_PSTSCTL 0x10 +#define MT6357_PG_DEB_STS0 0x12 +#define MT6357_PG_SDN_STS0 0x14 +#define MT6357_OC_SDN_STS0 0x16 +#define MT6357_THERMALSTATUS 0x18 +#define MT6357_TOP_CON 0x1a +#define MT6357_TEST_OUT 0x1c +#define MT6357_TEST_CON0 0x1e +#define MT6357_TEST_CON1 0x20 +#define MT6357_TESTMODE_SW 0x22 +#define MT6357_TOPSTATUS 0x24 +#define MT6357_TDSEL_CON 0x26 +#define MT6357_RDSEL_CON 0x28 +#define MT6357_SMT_CON0 0x2a +#define MT6357_SMT_CON1 0x2c +#define MT6357_TOP_RSV0 0x2e +#define MT6357_TOP_RSV1 0x30 +#define MT6357_DRV_CON0 0x32 +#define MT6357_DRV_CON1 0x34 +#define MT6357_DRV_CON2 0x36 +#define MT6357_DRV_CON3 0x38 +#define MT6357_FILTER_CON0 0x3a +#define MT6357_FILTER_CON1 0x3c +#define MT6357_FILTER_CON2 0x3e +#define MT6357_FILTER_CON3 0x40 +#define MT6357_TOP_STATUS 0x42 +#define MT6357_TOP_STATUS_SET 0x44 +#define MT6357_TOP_STATUS_CLR 0x46 +#define MT6357_TOP_TRAP 0x48 +#define MT6357_TOP1_ID 0x80 +#define MT6357_TOP1_REV0 0x82 +#define MT6357_TOP1_DSN_DBI 0x84 +#define MT6357_TOP1_DSN_DXI 0x86 +#define MT6357_GPIO_DIR0 0x88 +#define MT6357_GPIO_DIR0_SET 0x8a +#define MT6357_GPIO_DIR0_CLR 0x8c +#define MT6357_GPIO_PULLEN0 0x8e +#define MT6357_GPIO_PULLEN0_SET 0x90 +#define MT6357_GPIO_PULLEN0_CLR 0x92 +#define MT6357_GPIO_PULLSEL0 0x94 +#define MT6357_GPIO_PULLSEL0_SET 0x96 +#define MT6357_GPIO_PULLSEL0_CLR 0x98 +#define MT6357_GPIO_DINV0 0x9a +#define MT6357_GPIO_DINV0_SET 0x9c +#define MT6357_GPIO_DINV0_CLR 0x9e +#define MT6357_GPIO_DOUT0 0xa0 +#define MT6357_GPIO_DOUT0_SET 0xa2 +#define MT6357_GPIO_DOUT0_CLR 0xa4 +#define MT6357_GPIO_PI0 0xa6 +#define MT6357_GPIO_POE0 0xa8 +#define MT6357_GPIO_MODE0 0xaa +#define MT6357_GPIO_MODE0_SET 0xac +#define MT6357_GPIO_MODE0_CLR 0xae +#define MT6357_GPIO_MODE1 0xb0 +#define MT6357_GPIO_MODE1_SET 0xb2 +#define MT6357_GPIO_MODE1_CLR 0xb4 +#define MT6357_GPIO_MODE2 0xb6 +#define MT6357_GPIO_MODE2_SET 0xb8 +#define MT6357_GPIO_MODE2_CLR 0xba +#define MT6357_GPIO_MODE3 0xbc +#define MT6357_GPIO_MODE3_SET 0xbe +#define MT6357_GPIO_MODE3_CLR 0xc0 +#define MT6357_GPIO_RSV 0xc2 +#define MT6357_TOP2_ID 0x100 +#define MT6357_TOP2_REV0 0x102 +#define MT6357_TOP2_DSN_DBI 0x104 +#define MT6357_TOP2_DSN_DXI 0x106 +#define MT6357_TOP_PAM0 0x108 +#define MT6357_TOP_PAM1 0x10a +#define MT6357_TOP_CKPDN_CON0 0x10c +#define MT6357_TOP_CKPDN_CON0_SET 0x10e +#define MT6357_TOP_CKPDN_CON0_CLR 0x110 +#define MT6357_TOP_CKPDN_CON1 0x112 +#define MT6357_TOP_CKPDN_CON1_SET 0x114 +#define MT6357_TOP_CKPDN_CON1_CLR 0x116 +#define MT6357_TOP_CKSEL_CON0 0x118 +#define MT6357_TOP_CKSEL_CON0_SET 0x11a +#define MT6357_TOP_CKSEL_CON0_CLR 0x11c +#define MT6357_TOP_CKSEL_CON1 0x11e +#define MT6357_TOP_CKSEL_CON1_SET 0x120 +#define MT6357_TOP_CKSEL_CON1_CLR 0x122 +#define MT6357_TOP_CKDIVSEL_CON0 0x124 +#define MT6357_TOP_CKDIVSEL_CON0_SET 0x126 +#define MT6357_TOP_CKDIVSEL_CON0_CLR 0x128 +#define MT6357_TOP_CKHWEN_CON0 0x12a +#define MT6357_TOP_CKHWEN_CON0_SET 0x12c +#define MT6357_TOP_CKHWEN_CON0_CLR 0x12e +#define MT6357_TOP_CKTST_CON0 0x130 +#define MT6357_TOP_CKTST_CON1 0x132 +#define MT6357_TOP_CLK_CON0 0x134 +#define MT6357_TOP_CLK_CON0_SET 0x136 +#define MT6357_TOP_CLK_CON0_CLR 0x138 +#define MT6357_TOP_DCM_CON0 0x13a +#define MT6357_TOP_HANDOVER_DEBUG0 0x13c +#define MT6357_TOP_RST_CON0 0x13e +#define MT6357_TOP_RST_CON0_SET 0x140 +#define MT6357_TOP_RST_CON0_CLR 0x142 +#define MT6357_TOP_RST_CON1 0x144 +#define MT6357_TOP_RST_CON1_SET 0x146 +#define MT6357_TOP_RST_CON1_CLR 0x148 +#define MT6357_TOP_RST_CON2 0x14a +#define MT6357_TOP_RST_MISC 0x14c +#define MT6357_TOP_RST_MISC_SET 0x14e +#define MT6357_TOP_RST_MISC_CLR 0x150 +#define MT6357_TOP_RST_STATUS 0x152 +#define MT6357_TOP_RST_STATUS_SET 0x154 +#define MT6357_TOP_RST_STATUS_CLR 0x156 +#define MT6357_TOP2_ELR_NUM 0x158 +#define MT6357_TOP2_ELR0 0x15a +#define MT6357_TOP2_ELR1 0x15c +#define MT6357_TOP3_ID 0x180 +#define MT6357_TOP3_REV0 0x182 +#define MT6357_TOP3_DSN_DBI 0x184 +#define MT6357_TOP3_DSN_DXI 0x186 +#define MT6357_MISC_TOP_INT_CON0 0x188 +#define MT6357_MISC_TOP_INT_CON0_SET 0x18a +#define MT6357_MISC_TOP_INT_CON0_CLR 0x18c +#define MT6357_MISC_TOP_INT_MASK_CON0 0x18e +#define MT6357_MISC_TOP_INT_MASK_CON0_SET 0x190 +#define MT6357_MISC_TOP_INT_MASK_CON0_CLR 0x192 +#define MT6357_MISC_TOP_INT_STATUS0 0x194 +#define MT6357_MISC_TOP_INT_RAW_STATUS0 0x196 +#define MT6357_TOP_INT_MASK_CON0 0x198 +#define MT6357_TOP_INT_MASK_CON0_SET 0x19a +#define MT6357_TOP_INT_MASK_CON0_CLR 0x19c +#define MT6357_TOP_INT_STATUS0 0x19e +#define MT6357_TOP_INT_RAW_STATUS0 0x1a0 +#define MT6357_TOP_INT_CON0 0x1a2 +#define MT6357_PLT0_ID 0x380 +#define MT6357_PLT0_REV0 0x382 +#define MT6357_PLT0_REV1 0x384 +#define MT6357_PLT0_DSN_DXI 0x386 +#define MT6357_FQMTR_CON0 0x388 +#define MT6357_FQMTR_CON1 0x38a +#define MT6357_FQMTR_CON2 0x38c +#define MT6357_TOP_CLK_TRIM 0x38e +#define MT6357_OTP_CON0 0x390 +#define MT6357_OTP_CON1 0x392 +#define MT6357_OTP_CON2 0x394 +#define MT6357_OTP_CON3 0x396 +#define MT6357_OTP_CON4 0x398 +#define MT6357_OTP_CON5 0x39a +#define MT6357_OTP_CON6 0x39c +#define MT6357_OTP_CON7 0x39e +#define MT6357_OTP_CON8 0x3a0 +#define MT6357_OTP_CON9 0x3a2 +#define MT6357_OTP_CON10 0x3a4 +#define MT6357_OTP_CON11 0x3a6 +#define MT6357_OTP_CON12 0x3a8 +#define MT6357_OTP_CON13 0x3aa +#define MT6357_OTP_CON14 0x3ac +#define MT6357_TOP_TMA_KEY 0x3ae +#define MT6357_TOP_MDB_CONF0 0x3b0 +#define MT6357_TOP_MDB_CONF1 0x3b2 +#define MT6357_TOP_MDB_CONF2 0x3b4 +#define MT6357_PLT0_ELR_NUM 0x3b6 +#define MT6357_PLT0_ELR0 0x3b8 +#define MT6357_PLT0_ELR1 0x3ba +#define MT6357_SPISLV_ID 0x400 +#define MT6357_SPISLV_REV0 0x402 +#define MT6357_SPISLV_REV1 0x404 +#define MT6357_SPISLV_DSN_DXI 0x406 +#define MT6357_RG_SPI_CON0 0x408 +#define MT6357_DEW_DIO_EN 0x40a +#define MT6357_DEW_READ_TEST 0x40c +#define MT6357_DEW_WRITE_TEST 0x40e +#define MT6357_DEW_CRC_SWRST 0x410 +#define MT6357_DEW_CRC_EN 0x412 +#define MT6357_DEW_CRC_VAL 0x414 +#define MT6357_DEW_DBG_MON_SEL 0x416 +#define MT6357_DEW_CIPHER_KEY_SEL 0x418 +#define MT6357_DEW_CIPHER_IV_SEL 0x41a +#define MT6357_DEW_CIPHER_EN 0x41c +#define MT6357_DEW_CIPHER_RDY 0x41e +#define MT6357_DEW_CIPHER_MODE 0x420 +#define MT6357_DEW_CIPHER_SWRST 0x422 +#define MT6357_DEW_RDDMY_NO 0x424 +#define MT6357_INT_TYPE_CON0 0x426 +#define MT6357_INT_TYPE_CON0_SET 0x428 +#define MT6357_INT_TYPE_CON0_CLR 0x42a +#define MT6357_INT_STA 0x42c +#define MT6357_RG_SPI_CON1 0x42e +#define MT6357_RG_SPI_CON2 0x430 +#define MT6357_RG_SPI_CON3 0x432 +#define MT6357_RG_SPI_CON4 0x434 +#define MT6357_RG_SPI_CON5 0x436 +#define MT6357_RG_SPI_CON6 0x438 +#define MT6357_RG_SPI_CON7 0x43a +#define MT6357_RG_SPI_CON8 0x43c +#define MT6357_RG_SPI_CON9 0x43e +#define MT6357_RG_SPI_CON10 0x440 +#define MT6357_RG_SPI_CON11 0x442 +#define MT6357_RG_SPI_CON12 0x444 +#define MT6357_RG_SPI_CON13 0x446 +#define MT6357_TOP_SPI_CON0 0x448 +#define MT6357_TOP_SPI_CON1 0x44a +#define MT6357_SCK_TOP_DSN_ID 0x500 +#define MT6357_SCK_TOP_DSN_REV0 0x502 +#define MT6357_SCK_TOP_DBI 0x504 +#define MT6357_SCK_TOP_DXI 0x506 +#define MT6357_SCK_TOP_TPM0 0x508 +#define MT6357_SCK_TOP_TPM1 0x50a +#define MT6357_SCK_TOP_CON0 0x50c +#define MT6357_SCK_TOP_CON1 0x50e +#define MT6357_SCK_TOP_TEST_OUT 0x510 +#define MT6357_SCK_TOP_TEST_CON0 0x512 +#define MT6357_SCK_TOP_CKPDN_CON0 0x514 +#define MT6357_SCK_TOP_CKPDN_CON0_SET 0x516 +#define MT6357_SCK_TOP_CKPDN_CON0_CLR 0x518 +#define MT6357_SCK_TOP_CKHWEN_CON0 0x51a +#define MT6357_SCK_TOP_CKHWEN_CON0_SET 0x51c +#define MT6357_SCK_TOP_CKHWEN_CON0_CLR 0x51e +#define MT6357_SCK_TOP_CKTST_CON 0x520 +#define MT6357_SCK_TOP_RST_CON0 0x522 +#define MT6357_SCK_TOP_RST_CON0_SET 0x524 +#define MT6357_SCK_TOP_RST_CON0_CLR 0x526 +#define MT6357_SCK_TOP_INT_CON0 0x528 +#define MT6357_SCK_TOP_INT_CON0_SET 0x52a +#define MT6357_SCK_TOP_INT_CON0_CLR 0x52c +#define MT6357_SCK_TOP_INT_MASK_CON0 0x52e +#define MT6357_SCK_TOP_INT_MASK_CON0_SET 0x530 +#define MT6357_SCK_TOP_INT_MASK_CON0_CLR 0x532 +#define MT6357_SCK_TOP_INT_STATUS0 0x534 +#define MT6357_SCK_TOP_INT_RAW_STATUS0 0x536 +#define MT6357_SCK_TOP_INT_MISC_CON 0x538 +#define MT6357_EOSC_CALI_CON0 0x53a +#define MT6357_EOSC_CALI_CON1 0x53c +#define MT6357_RTC_MIX_CON0 0x53e +#define MT6357_RTC_MIX_CON1 0x540 +#define MT6357_RTC_MIX_CON2 0x542 +#define MT6357_RTC_DSN_ID 0x580 +#define MT6357_RTC_DSN_REV0 0x582 +#define MT6357_RTC_DBI 0x584 +#define MT6357_RTC_DXI 0x586 +#define MT6357_RTC_BBPU 0x588 +#define MT6357_RTC_IRQ_STA 0x58a +#define MT6357_RTC_IRQ_EN 0x58c +#define MT6357_RTC_CII_EN 0x58e +#define MT6357_RTC_AL_MASK 0x590 +#define MT6357_RTC_TC_SEC 0x592 +#define MT6357_RTC_TC_MIN 0x594 +#define MT6357_RTC_TC_HOU 0x596 +#define MT6357_RTC_TC_DOM 0x598 +#define MT6357_RTC_TC_DOW 0x59a +#define MT6357_RTC_TC_MTH 0x59c +#define MT6357_RTC_TC_YEA 0x59e +#define MT6357_RTC_AL_SEC 0x5a0 +#define MT6357_RTC_AL_MIN 0x5a2 +#define MT6357_RTC_AL_HOU 0x5a4 +#define MT6357_RTC_AL_DOM 0x5a6 +#define MT6357_RTC_AL_DOW 0x5a8 +#define MT6357_RTC_AL_MTH 0x5aa +#define MT6357_RTC_AL_YEA 0x5ac +#define MT6357_RTC_OSC32CON 0x5ae +#define MT6357_RTC_POWERKEY1 0x5b0 +#define MT6357_RTC_POWERKEY2 0x5b2 +#define MT6357_RTC_PDN1 0x5b4 +#define MT6357_RTC_PDN2 0x5b6 +#define MT6357_RTC_SPAR0 0x5b8 +#define MT6357_RTC_SPAR1 0x5ba +#define MT6357_RTC_PROT 0x5bc +#define MT6357_RTC_DIFF 0x5be +#define MT6357_RTC_CALI 0x5c0 +#define MT6357_RTC_WRTGR 0x5c2 +#define MT6357_RTC_CON 0x5c4 +#define MT6357_RTC_SEC_CTRL 0x5c6 +#define MT6357_RTC_INT_CNT 0x5c8 +#define MT6357_RTC_SEC_DAT0 0x5ca +#define MT6357_RTC_SEC_DAT1 0x5cc +#define MT6357_RTC_SEC_DAT2 0x5ce +#define MT6357_RTC_SEC_DSN_ID 0x600 +#define MT6357_RTC_SEC_DSN_REV0 0x602 +#define MT6357_RTC_SEC_DBI 0x604 +#define MT6357_RTC_SEC_DXI 0x606 +#define MT6357_RTC_TC_SEC_SEC 0x608 +#define MT6357_RTC_TC_MIN_SEC 0x60a +#define MT6357_RTC_TC_HOU_SEC 0x60c +#define MT6357_RTC_TC_DOM_SEC 0x60e +#define MT6357_RTC_TC_DOW_SEC 0x610 +#define MT6357_RTC_TC_MTH_SEC 0x612 +#define MT6357_RTC_TC_YEA_SEC 0x614 +#define MT6357_RTC_SEC_CK_PDN 0x616 +#define MT6357_RTC_SEC_WRTGR 0x618 +#define MT6357_DCXO_DSN_ID 0x780 +#define MT6357_DCXO_DSN_REV0 0x782 +#define MT6357_DCXO_DSN_DBI 0x784 +#define MT6357_DCXO_DSN_DXI 0x786 +#define MT6357_DCXO_CW00 0x788 +#define MT6357_DCXO_CW00_SET 0x78a +#define MT6357_DCXO_CW00_CLR 0x78c +#define MT6357_DCXO_CW01 0x78e +#define MT6357_DCXO_CW02 0x790 +#define MT6357_DCXO_CW03 0x792 +#define MT6357_DCXO_CW04 0x794 +#define MT6357_DCXO_CW05 0x796 +#define MT6357_DCXO_CW06 0x798 +#define MT6357_DCXO_CW07 0x79a +#define MT6357_DCXO_CW08 0x79c +#define MT6357_DCXO_CW09 0x79e +#define MT6357_DCXO_CW10 0x7a0 +#define MT6357_DCXO_CW11 0x7a2 +#define MT6357_DCXO_CW11_SET 0x7a4 +#define MT6357_DCXO_CW11_CLR 0x7a6 +#define MT6357_DCXO_CW12 0x7a8 +#define MT6357_DCXO_CW13 0x7aa +#define MT6357_DCXO_CW14 0x7ac +#define MT6357_DCXO_CW15 0x7ae +#define MT6357_DCXO_CW16 0x7b0 +#define MT6357_DCXO_CW17 0x7b2 +#define MT6357_DCXO_CW18 0x7b4 +#define MT6357_DCXO_CW19 0x7b6 +#define MT6357_DCXO_CW20 0x7b8 +#define MT6357_DCXO_CW21 0x7ba +#define MT6357_DCXO_CW22 0x7bc +#define MT6357_DCXO_ELR_NUM 0x7be +#define MT6357_DCXO_ELR0 0x7c0 +#define MT6357_PSC_TOP_ID 0x900 +#define MT6357_PSC_TOP_REV0 0x902 +#define MT6357_PSC_TOP_DBI 0x904 +#define MT6357_PSC_TOP_DXI 0x906 +#define MT6357_PSC_TPM0 0x908 +#define MT6357_PSC_TPM1 0x90a +#define MT6357_PSC_TOP_RSTCTL_0 0x90c +#define MT6357_PSC_TOP_INT_CON0 0x90e +#define MT6357_PSC_TOP_INT_CON0_SET 0x910 +#define MT6357_PSC_TOP_INT_CON0_CLR 0x912 +#define MT6357_PSC_TOP_INT_MASK_CON0 0x914 +#define MT6357_PSC_TOP_INT_MASK_CON0_SET 0x916 +#define MT6357_PSC_TOP_INT_MASK_CON0_CLR 0x918 +#define MT6357_PSC_TOP_INT_STATUS0 0x91a +#define MT6357_PSC_TOP_INT_RAW_STATUS0 0x91c +#define MT6357_PSC_TOP_INT_MISC_CON 0x91e +#define MT6357_PSC_TOP_INT_MISC_CON_SET 0x920 +#define MT6357_PSC_TOP_INT_MISC_CON_CLR 0x922 +#define MT6357_PSC_TOP_MON_CTL 0x924 +#define MT6357_STRUP_ID 0x980 +#define MT6357_STRUP_REV0 0x982 +#define MT6357_STRUP_DBI 0x984 +#define MT6357_STRUP_DXI 0x986 +#define MT6357_STRUP_ANA_CON0 0x988 +#define MT6357_STRUP_ANA_CON1 0x98a +#define MT6357_STRUP_ANA_CON2 0x98c +#define MT6357_STRUP_ELR_NUM 0x98e +#define MT6357_STRUP_ELR_0 0x990 +#define MT6357_PSEQ_ID 0xa00 +#define MT6357_PSEQ_REV0 0xa02 +#define MT6357_PSEQ_DBI 0xa04 +#define MT6357_PSEQ_DXI 0xa06 +#define MT6357_PPCCTL0 0xa08 +#define MT6357_PPCCTL1 0xa0a +#define MT6357_PPCCTL2 0xa0c +#define MT6357_PPCCFG0 0xa0e +#define MT6357_PPCTST0 0xa10 +#define MT6357_PORFLAG 0xa12 +#define MT6357_STRUP_CON0 0xa14 +#define MT6357_STRUP_CON1 0xa16 +#define MT6357_STRUP_CON2 0xa18 +#define MT6357_STRUP_CON3 0xa1a +#define MT6357_STRUP_CON4 0xa1c +#define MT6357_STRUP_CON5 0xa1e +#define MT6357_STRUP_CON6 0xa20 +#define MT6357_STRUP_CON7 0xa22 +#define MT6357_CPSCFG0 0xa24 +#define MT6357_STRUP_CON9 0xa26 +#define MT6357_STRUP_CON10 0xa28 +#define MT6357_STRUP_CON11 0xa2a +#define MT6357_STRUP_CON12 0xa2c +#define MT6357_STRUP_CON13 0xa2e +#define MT6357_STRUP_CON14 0xa30 +#define MT6357_STRUP_CON15 0xa32 +#define MT6357_STRUP_CON16 0xa34 +#define MT6357_STRUP_CON19 0xa36 +#define MT6357_PSEQ_ELR_NUM 0xa38 +#define MT6357_PSEQ_ELR7 0xa3a +#define MT6357_PSEQ_ELR8 0xa3c +#define MT6357_PCHR_DIG_DSN_ID 0xa80 +#define MT6357_PCHR_DIG_DSN_REV0 0xa82 +#define MT6357_PCHR_DIG_DSN_DBI 0xa84 +#define MT6357_PCHR_DIG_DSN_DXI 0xa86 +#define MT6357_CHR_TOP_CON0 0xa88 +#define MT6357_CHR_TOP_CON1 0xa8a +#define MT6357_CHR_TOP_CON2 0xa8c +#define MT6357_CHR_TOP_CON3 0xa8e +#define MT6357_CHR_TOP_CON4 0xa90 +#define MT6357_CHR_TOP_CON5 0xa92 +#define MT6357_CHR_TOP_CON6 0xa94 +#define MT6357_PCHR_DIG_ELR_NUM 0xa96 +#define MT6357_PCHR_ELR0 0xa98 +#define MT6357_PCHR_ELR1 0xa9a +#define MT6357_PCHR_MACRO_DSN_ID 0xb80 +#define MT6357_PCHR_MACRO_DSN_REV0 0xb82 +#define MT6357_PCHR_MACRO_DSN_DBI 0xb84 +#define MT6357_PCHR_MACRO_DSN_DXI 0xb86 +#define MT6357_CHR_CON0 0xb88 +#define MT6357_CHR_CON1 0xb8a +#define MT6357_CHR_CON2 0xb8c +#define MT6357_CHR_CON3 0xb8e +#define MT6357_CHR_CON4 0xb90 +#define MT6357_CHR_CON5 0xb92 +#define MT6357_CHR_CON6 0xb94 +#define MT6357_CHR_CON7 0xb96 +#define MT6357_CHR_CON8 0xb98 +#define MT6357_CHR_CON9 0xb9a +#define MT6357_BM_TOP_DSN_ID 0xc00 +#define MT6357_BM_TOP_DSN_REV0 0xc02 +#define MT6357_BM_TOP_DBI 0xc04 +#define MT6357_BM_TOP_DXI 0xc06 +#define MT6357_BM_TPM0 0xc08 +#define MT6357_BM_TPM1 0xc0a +#define MT6357_BM_TOP_CKPDN_CON0 0xc0c +#define MT6357_BM_TOP_CKPDN_CON0_SET 0xc0e +#define MT6357_BM_TOP_CKPDN_CON0_CLR 0xc10 +#define MT6357_BM_TOP_CKSEL_CON0 0xc12 +#define MT6357_BM_TOP_CKSEL_CON0_SET 0xc14 +#define MT6357_BM_TOP_CKSEL_CON0_CLR 0xc16 +#define MT6357_BM_TOP_CKTST_CON0 0xc18 +#define MT6357_BM_TOP_RST_CON0 0xc1a +#define MT6357_BM_TOP_RST_CON0_SET 0xc1c +#define MT6357_BM_TOP_RST_CON0_CLR 0xc1e +#define MT6357_BM_TOP_INT_CON0 0xc20 +#define MT6357_BM_TOP_INT_CON0_SET 0xc22 +#define MT6357_BM_TOP_INT_CON0_CLR 0xc24 +#define MT6357_BM_TOP_INT_CON1 0xc26 +#define MT6357_BM_TOP_INT_CON1_SET 0xc28 +#define MT6357_BM_TOP_INT_CON1_CLR 0xc2a +#define MT6357_BM_TOP_INT_MASK_CON0 0xc2c +#define MT6357_BM_TOP_INT_MASK_CON0_SET 0xc2e +#define MT6357_BM_TOP_INT_MASK_CON0_CLR 0xc30 +#define MT6357_BM_TOP_INT_MASK_CON1 0xc32 +#define MT6357_BM_TOP_INT_MASK_CON1_SET 0xc34 +#define MT6357_BM_TOP_INT_MASK_CON1_CLR 0xc36 +#define MT6357_BM_TOP_INT_STATUS0 0xc38 +#define MT6357_BM_TOP_INT_STATUS1 0xc3a +#define MT6357_BM_TOP_INT_RAW_STATUS0 0xc3c +#define MT6357_BM_TOP_INT_RAW_STATUS1 0xc3e +#define MT6357_BM_TOP_INT_MISC_CON 0xc40 +#define MT6357_BM_TOP_DBG_CON 0xc42 +#define MT6357_BM_TOP_RSV0 0xc44 +#define MT6357_FGADC_ANA_DSN_ID 0xc80 +#define MT6357_FGADC_ANA_DSN_REV0 0xc82 +#define MT6357_FGADC_ANA_DSN_DBI 0xc84 +#define MT6357_FGADC_ANA_DSN_DXI 0xc86 +#define MT6357_FGADC_ANA_CON0 0xc88 +#define MT6357_FGADC_ANA_TEST_CON0 0xc8a +#define MT6357_FGADC_ANA_ELR_NUM 0xc8c +#define MT6357_FGADC_ANA_ELR0 0xc8e +#define MT6357_FGADC_ANA_ELR1 0xc90 +#define MT6357_FGADC0_DSN_ID 0xd00 +#define MT6357_FGADC0_DSN_REV0 0xd02 +#define MT6357_FGADC0_DSN_DBI 0xd04 +#define MT6357_FGADC0_DSN_DXI 0xd06 +#define MT6357_FGADC_CON0 0xd08 +#define MT6357_FGADC_CON1 0xd0a +#define MT6357_FGADC_CON2 0xd0c +#define MT6357_FGADC_CON3 0xd0e +#define MT6357_FGADC_CON4 0xd10 +#define MT6357_FGADC_CAR_CON0 0xd12 +#define MT6357_FGADC_CAR_CON1 0xd14 +#define MT6357_FGADC_CAR_CON2 0xd16 +#define MT6357_FGADC_CARTH_CON0 0xd18 +#define MT6357_FGADC_CARTH_CON1 0xd1a +#define MT6357_FGADC_CARTH_CON2 0xd1c +#define MT6357_FGADC_CARTH_CON3 0xd1e +#define MT6357_FGADC_NTER_CON0 0xd20 +#define MT6357_FGADC_NTER_CON1 0xd22 +#define MT6357_FGADC_NTER_CON2 0xd24 +#define MT6357_FGADC_SON_CON0 0xd26 +#define MT6357_FGADC_SON_CON1 0xd28 +#define MT6357_FGADC_SON_CON2 0xd2a +#define MT6357_FGADC_SON_CON3 0xd2c +#define MT6357_FGADC_ZCV_CON0 0xd2e +#define MT6357_FGADC_ZCV_CON1 0xd30 +#define MT6357_FGADC_ZCV_CON2 0xd32 +#define MT6357_FGADC_ZCV_CON3 0xd34 +#define MT6357_FGADC_ZCV_CON4 0xd36 +#define MT6357_FGADC_ZCVTH_CON0 0xd38 +#define MT6357_FGADC_ZCVTH_CON1 0xd3a +#define MT6357_FGADC_ZCVTH_CON2 0xd3c +#define MT6357_FGADC1_DSN_ID 0xd80 +#define MT6357_FGADC1_DSN_REV0 0xd82 +#define MT6357_FGADC1_DSN_DBI 0xd84 +#define MT6357_FGADC1_DSN_DXI 0xd86 +#define MT6357_FGADC_R_CON0 0xd88 +#define MT6357_FGADC_CUR_CON0 0xd8a +#define MT6357_FGADC_CUR_CON1 0xd8c +#define MT6357_FGADC_CUR_CON2 0xd8e +#define MT6357_FGADC_CUR_CON3 0xd90 +#define MT6357_FGADC_OFFSET_CON0 0xd92 +#define MT6357_FGADC_OFFSET_CON1 0xd94 +#define MT6357_FGADC_GAIN_CON0 0xd96 +#define MT6357_FGADC_TEST_CON0 0xd98 +#define MT6357_SYSTEM_INFO_CON0 0xd9a +#define MT6357_SYSTEM_INFO_CON1 0xd9c +#define MT6357_SYSTEM_INFO_CON2 0xd9e +#define MT6357_SYSTEM_INFO_CON3 0xda0 +#define MT6357_SYSTEM_INFO_CON4 0xda2 +#define MT6357_BATON_ANA_DSN_ID 0xe00 +#define MT6357_BATON_ANA_DSN_REV0 0xe02 +#define MT6357_BATON_ANA_DSN_DBI 0xe04 +#define MT6357_BATON_ANA_DSN_DXI 0xe06 +#define MT6357_BATON_ANA_CON0 0xe08 +#define MT6357_BATON_ANA_ELR_NUM 0xe0a +#define MT6357_BATON_ANA_ELR0 0xe0c +#define MT6357_HK_TOP_ID 0xf80 +#define MT6357_HK_TOP_REV0 0xf82 +#define MT6357_HK_TOP_DBI 0xf84 +#define MT6357_HK_TOP_DXI 0xf86 +#define MT6357_HK_TPM0 0xf88 +#define MT6357_HK_TPM1 0xf8a +#define MT6357_HK_TOP_CLK_CON0 0xf8c +#define MT6357_HK_TOP_CLK_CON1 0xf8e +#define MT6357_HK_TOP_RST_CON0 0xf90 +#define MT6357_HK_TOP_INT_CON0 0xf92 +#define MT6357_HK_TOP_INT_CON0_SET 0xf94 +#define MT6357_HK_TOP_INT_CON0_CLR 0xf96 +#define MT6357_HK_TOP_INT_MASK_CON0 0xf98 +#define MT6357_HK_TOP_INT_MASK_CON0_SET 0xf9a +#define MT6357_HK_TOP_INT_MASK_CON0_CLR 0xf9c +#define MT6357_HK_TOP_INT_STATUS0 0xf9e +#define MT6357_HK_TOP_INT_RAW_STATUS0 0xfa0 +#define MT6357_HK_TOP_MON_CON0 0xfa2 +#define MT6357_HK_TOP_MON_CON1 0xfa4 +#define MT6357_HK_TOP_MON_CON2 0xfa6 +#define MT6357_AUXADC_DSN_ID 0x1000 +#define MT6357_AUXADC_DSN_REV0 0x1002 +#define MT6357_AUXADC_DSN_DBI 0x1004 +#define MT6357_AUXADC_DSN_DXI 0x1006 +#define MT6357_AUXADC_ANA_CON0 0x1008 +#define MT6357_AUXADC_DIG_1_DSN_ID 0x1080 +#define MT6357_AUXADC_DIG_1_DSN_REV0 0x1082 +#define MT6357_AUXADC_DIG_1_DSN_DBI 0x1084 +#define MT6357_AUXADC_DIG_1_DSN_DXI 0x1086 +#define MT6357_AUXADC_ADC0 0x1088 +#define MT6357_AUXADC_ADC1 0x108a +#define MT6357_AUXADC_ADC2 0x108c +#define MT6357_AUXADC_ADC3 0x108e +#define MT6357_AUXADC_ADC4 0x1090 +#define MT6357_AUXADC_ADC5 0x1092 +#define MT6357_AUXADC_ADC6 0x1094 +#define MT6357_AUXADC_ADC7 0x1096 +#define MT6357_AUXADC_ADC8 0x1098 +#define MT6357_AUXADC_ADC9 0x109a +#define MT6357_AUXADC_ADC10 0x109c +#define MT6357_AUXADC_ADC11 0x109e +#define MT6357_AUXADC_ADC12 0x10a0 +#define MT6357_AUXADC_ADC14 0x10a2 +#define MT6357_AUXADC_ADC16 0x10a4 +#define MT6357_AUXADC_ADC17 0x10a6 +#define MT6357_AUXADC_ADC18 0x10a8 +#define MT6357_AUXADC_ADC19 0x10aa +#define MT6357_AUXADC_ADC20 0x10ac +#define MT6357_AUXADC_ADC21 0x10ae +#define MT6357_AUXADC_ADC22 0x10b0 +#define MT6357_AUXADC_ADC23 0x10b2 +#define MT6357_AUXADC_ADC24 0x10b4 +#define MT6357_AUXADC_ADC25 0x10b6 +#define MT6357_AUXADC_ADC26 0x10b8 +#define MT6357_AUXADC_ADC27 0x10ba +#define MT6357_AUXADC_ADC29 0x10bc +#define MT6357_AUXADC_ADC30 0x10be +#define MT6357_AUXADC_ADC31 0x10c0 +#define MT6357_AUXADC_ADC32 0x10c2 +#define MT6357_AUXADC_ADC33 0x10c4 +#define MT6357_AUXADC_ADC34 0x10c6 +#define MT6357_AUXADC_ADC35 0x10c8 +#define MT6357_AUXADC_ADC36 0x10ca +#define MT6357_AUXADC_ADC38 0x10cc +#define MT6357_AUXADC_ADC39 0x10ce +#define MT6357_AUXADC_ADC40 0x10d0 +#define MT6357_AUXADC_ADC41 0x10d2 +#define MT6357_AUXADC_ADC42 0x10d4 +#define MT6357_AUXADC_ADC43 0x10d6 +#define MT6357_AUXADC_ADC46 0x10d8 +#define MT6357_AUXADC_ADC47 0x10da +#define MT6357_AUXADC_DIG_1_ELR_NUM 0x10dc +#define MT6357_AUXADC_DIG_1_ELR0 0x10de +#define MT6357_AUXADC_DIG_1_ELR1 0x10e0 +#define MT6357_AUXADC_DIG_2_DSN_ID 0x1100 +#define MT6357_AUXADC_DIG_2_DSN_REV0 0x1102 +#define MT6357_AUXADC_DIG_2_DSN_DBI 0x1104 +#define MT6357_AUXADC_DIG_2_DSN_DXI 0x1106 +#define MT6357_AUXADC_STA0 0x1108 +#define MT6357_AUXADC_STA1 0x110a +#define MT6357_AUXADC_STA2 0x110c +#define MT6357_AUXADC_RQST0 0x110e +#define MT6357_AUXADC_RQST0_SET 0x1110 +#define MT6357_AUXADC_RQST0_CLR 0x1112 +#define MT6357_AUXADC_RQST2 0x1114 +#define MT6357_AUXADC_RQST2_SET 0x1116 +#define MT6357_AUXADC_RQST2_CLR 0x1118 +#define MT6357_AUXADC_RQST1 0x111a +#define MT6357_AUXADC_RQST1_SET 0x111c +#define MT6357_AUXADC_RQST1_CLR 0x111e +#define MT6357_AUXADC_CON0 0x1120 +#define MT6357_AUXADC_CON0_SET 0x1122 +#define MT6357_AUXADC_CON0_CLR 0x1124 +#define MT6357_AUXADC_CON1 0x1126 +#define MT6357_AUXADC_CON2 0x1128 +#define MT6357_AUXADC_CON3 0x112a +#define MT6357_AUXADC_CON4 0x112c +#define MT6357_AUXADC_CON5 0x112e +#define MT6357_AUXADC_CON6 0x1130 +#define MT6357_AUXADC_CON7 0x1132 +#define MT6357_AUXADC_CON8 0x1134 +#define MT6357_AUXADC_CON9 0x1136 +#define MT6357_AUXADC_CON10 0x1138 +#define MT6357_AUXADC_CON11 0x113a +#define MT6357_AUXADC_CON12 0x113c +#define MT6357_AUXADC_CON13 0x113e +#define MT6357_AUXADC_CON14 0x1140 +#define MT6357_AUXADC_CON15 0x1142 +#define MT6357_AUXADC_CON16 0x1144 +#define MT6357_AUXADC_CON17 0x1146 +#define MT6357_AUXADC_CON18 0x1148 +#define MT6357_AUXADC_CON19 0x114a +#define MT6357_AUXADC_CON20 0x114c +#define MT6357_AUXADC_DIG_3_DSN_ID 0x1180 +#define MT6357_AUXADC_DIG_3_DSN_REV0 0x1182 +#define MT6357_AUXADC_DIG_3_DSN_DBI 0x1184 +#define MT6357_AUXADC_DIG_3_DSN_DXI 0x1186 +#define MT6357_AUXADC_AUTORPT0 0x1188 +#define MT6357_AUXADC_LBAT0 0x118a +#define MT6357_AUXADC_LBAT1 0x118c +#define MT6357_AUXADC_LBAT2 0x118e +#define MT6357_AUXADC_LBAT3 0x1190 +#define MT6357_AUXADC_LBAT4 0x1192 +#define MT6357_AUXADC_LBAT5 0x1194 +#define MT6357_AUXADC_LBAT6 0x1196 +#define MT6357_AUXADC_ACCDET 0x1198 +#define MT6357_AUXADC_DBG0 0x119a +#define MT6357_AUXADC_IMP0 0x119c +#define MT6357_AUXADC_IMP1 0x119e +#define MT6357_AUXADC_DIG_3_ELR_NUM 0x11a0 +#define MT6357_AUXADC_DIG_3_ELR0 0x11a2 +#define MT6357_AUXADC_DIG_3_ELR1 0x11a4 +#define MT6357_AUXADC_DIG_3_ELR2 0x11a6 +#define MT6357_AUXADC_DIG_3_ELR3 0x11a8 +#define MT6357_AUXADC_DIG_3_ELR4 0x11aa +#define MT6357_AUXADC_DIG_3_ELR5 0x11ac +#define MT6357_AUXADC_DIG_3_ELR6 0x11ae +#define MT6357_AUXADC_DIG_3_ELR7 0x11b0 +#define MT6357_AUXADC_DIG_3_ELR8 0x11b2 +#define MT6357_AUXADC_DIG_3_ELR9 0x11b4 +#define MT6357_AUXADC_DIG_3_ELR10 0x11b6 +#define MT6357_AUXADC_DIG_3_ELR11 0x11b8 +#define MT6357_AUXADC_DIG_4_DSN_ID 0x1200 +#define MT6357_AUXADC_DIG_4_DSN_REV0 0x1202 +#define MT6357_AUXADC_DIG_4_DSN_DBI 0x1204 +#define MT6357_AUXADC_DIG_4_DSN_DXI 0x1206 +#define MT6357_AUXADC_MDRT_0 0x1208 +#define MT6357_AUXADC_MDRT_1 0x120a +#define MT6357_AUXADC_MDRT_2 0x120c +#define MT6357_AUXADC_MDRT_3 0x120e +#define MT6357_AUXADC_MDRT_4 0x1210 +#define MT6357_AUXADC_DCXO_MDRT_0 0x1212 +#define MT6357_AUXADC_DCXO_MDRT_1 0x1214 +#define MT6357_AUXADC_DCXO_MDRT_2 0x1216 +#define MT6357_AUXADC_NAG_0 0x1218 +#define MT6357_AUXADC_NAG_1 0x121a +#define MT6357_AUXADC_NAG_2 0x121c +#define MT6357_AUXADC_NAG_3 0x121e +#define MT6357_AUXADC_NAG_4 0x1220 +#define MT6357_AUXADC_NAG_5 0x1222 +#define MT6357_AUXADC_NAG_6 0x1224 +#define MT6357_AUXADC_NAG_7 0x1226 +#define MT6357_AUXADC_NAG_8 0x1228 +#define MT6357_AUXADC_RSV_1 0x122a +#define MT6357_AUXADC_ANA_0 0x122c +#define MT6357_AUXADC_IMP_CG0 0x122e +#define MT6357_AUXADC_LBAT_CG0 0x1230 +#define MT6357_AUXADC_NAG_CG0 0x1232 +#define MT6357_AUXADC_PRI_NEW 0x1234 +#define MT6357_AUXADC_CHR_TOP_CON2 0x1236 +#define MT6357_BUCK_TOP_DSN_ID 0x1400 +#define MT6357_BUCK_TOP_DSN_REV0 0x1402 +#define MT6357_BUCK_TOP_DBI 0x1404 +#define MT6357_BUCK_TOP_DXI 0x1406 +#define MT6357_BUCK_TOP_PAM0 0x1408 +#define MT6357_BUCK_TOP_PAM1 0x140a +#define MT6357_BUCK_TOP_CLK_CON0 0x140c +#define MT6357_BUCK_TOP_CLK_CON0_SET 0x140e +#define MT6357_BUCK_TOP_CLK_CON0_CLR 0x1410 +#define MT6357_BUCK_TOP_CLK_HWEN_CON0 0x1412 +#define MT6357_BUCK_TOP_CLK_HWEN_CON0_SET 0x1414 +#define MT6357_BUCK_TOP_CLK_HWEN_CON0_CLR 0x1416 +#define MT6357_BUCK_TOP_CLK_MISC_CON0 0x1418 +#define MT6357_BUCK_TOP_INT_CON0 0x141a +#define MT6357_BUCK_TOP_INT_CON0_SET 0x141c +#define MT6357_BUCK_TOP_INT_CON0_CLR 0x141e +#define MT6357_BUCK_TOP_INT_MASK_CON0 0x1420 +#define MT6357_BUCK_TOP_INT_MASK_CON0_SET 0x1422 +#define MT6357_BUCK_TOP_INT_MASK_CON0_CLR 0x1424 +#define MT6357_BUCK_TOP_INT_STATUS0 0x1426 +#define MT6357_BUCK_TOP_INT_RAW_STATUS0 0x1428 +#define MT6357_BUCK_TOP_STB_CON 0x142a +#define MT6357_BUCK_TOP_SLP_CON0 0x142c +#define MT6357_BUCK_TOP_SLP_CON1 0x142e +#define MT6357_BUCK_TOP_SLP_CON2 0x1430 +#define MT6357_BUCK_TOP_MINFREQ_CON 0x1432 +#define MT6357_BUCK_TOP_OC_CON0 0x1434 +#define MT6357_BUCK_TOP_K_CON0 0x1436 +#define MT6357_BUCK_TOP_K_CON1 0x1438 +#define MT6357_BUCK_TOP_K_CON2 0x143a +#define MT6357_BUCK_TOP_WDTDBG0 0x143c +#define MT6357_BUCK_TOP_WDTDBG1 0x143e +#define MT6357_BUCK_TOP_WDTDBG2 0x1440 +#define MT6357_BUCK_TOP_ELR_NUM 0x1442 +#define MT6357_BUCK_TOP_ELR0 0x1444 +#define MT6357_BUCK_TOP_ELR1 0x1446 +#define MT6357_BUCK_VPROC_DSN_ID 0x1480 +#define MT6357_BUCK_VPROC_DSN_REV0 0x1482 +#define MT6357_BUCK_VPROC_DSN_DBI 0x1484 +#define MT6357_BUCK_VPROC_DSN_DXI 0x1486 +#define MT6357_BUCK_VPROC_CON0 0x1488 +#define MT6357_BUCK_VPROC_CON1 0x148a +#define MT6357_BUCK_VPROC_CFG0 0x148c +#define MT6357_BUCK_VPROC_CFG1 0x148e +#define MT6357_BUCK_VPROC_OP_EN 0x1490 +#define MT6357_BUCK_VPROC_OP_EN_SET 0x1492 +#define MT6357_BUCK_VPROC_OP_EN_CLR 0x1494 +#define MT6357_BUCK_VPROC_OP_CFG 0x1496 +#define MT6357_BUCK_VPROC_OP_CFG_SET 0x1498 +#define MT6357_BUCK_VPROC_OP_CFG_CLR 0x149a +#define MT6357_BUCK_VPROC_SP_CON 0x149c +#define MT6357_BUCK_VPROC_SP_CFG 0x149e +#define MT6357_BUCK_VPROC_OC_CFG 0x14a0 +#define MT6357_BUCK_VPROC_DBG0 0x14a2 +#define MT6357_BUCK_VPROC_DBG1 0x14a4 +#define MT6357_BUCK_VPROC_DBG2 0x14a6 +#define MT6357_BUCK_VPROC_ELR_NUM 0x14a8 +#define MT6357_BUCK_VPROC_ELR0 0x14aa +#define MT6357_BUCK_VCORE_DSN_ID 0x1500 +#define MT6357_BUCK_VCORE_DSN_REV0 0x1502 +#define MT6357_BUCK_VCORE_DSN_DBI 0x1504 +#define MT6357_BUCK_VCORE_DSN_DXI 0x1506 +#define MT6357_BUCK_VCORE_CON0 0x1508 +#define MT6357_BUCK_VCORE_CON1 0x150a +#define MT6357_BUCK_VCORE_CFG0 0x150c +#define MT6357_BUCK_VCORE_CFG1 0x150e +#define MT6357_BUCK_VCORE_OP_EN 0x1510 +#define MT6357_BUCK_VCORE_OP_EN_SET 0x1512 +#define MT6357_BUCK_VCORE_OP_EN_CLR 0x1514 +#define MT6357_BUCK_VCORE_OP_CFG 0x1516 +#define MT6357_BUCK_VCORE_OP_CFG_SET 0x1518 +#define MT6357_BUCK_VCORE_OP_CFG_CLR 0x151a +#define MT6357_BUCK_VCORE_SP_CON 0x151c +#define MT6357_BUCK_VCORE_SP_CFG 0x151e +#define MT6357_BUCK_VCORE_OC_CFG 0x1520 +#define MT6357_BUCK_VCORE_DBG0 0x1522 +#define MT6357_BUCK_VCORE_DBG1 0x1524 +#define MT6357_BUCK_VCORE_DBG2 0x1526 +#define MT6357_BUCK_VCORE_ELR_NUM 0x1528 +#define MT6357_BUCK_VCORE_ELR0 0x152a +#define MT6357_BUCK_VMODEM_DSN_ID 0x1580 +#define MT6357_BUCK_VMODEM_DSN_REV0 0x1582 +#define MT6357_BUCK_VMODEM_DSN_DBI 0x1584 +#define MT6357_BUCK_VMODEM_DSN_DXI 0x1586 +#define MT6357_BUCK_VMODEM_CON0 0x1588 +#define MT6357_BUCK_VMODEM_CON1 0x158a +#define MT6357_BUCK_VMODEM_CFG0 0x158c +#define MT6357_BUCK_VMODEM_CFG1 0x158e +#define MT6357_BUCK_VMODEM_OP_EN 0x1590 +#define MT6357_BUCK_VMODEM_OP_EN_SET 0x1592 +#define MT6357_BUCK_VMODEM_OP_EN_CLR 0x1594 +#define MT6357_BUCK_VMODEM_OP_CFG 0x1596 +#define MT6357_BUCK_VMODEM_OP_CFG_SET 0x1598 +#define MT6357_BUCK_VMODEM_OP_CFG_CLR 0x159a +#define MT6357_BUCK_VMODEM_SP_CON 0x159c +#define MT6357_BUCK_VMODEM_SP_CFG 0x159e +#define MT6357_BUCK_VMODEM_OC_CFG 0x15a0 +#define MT6357_BUCK_VMODEM_DBG0 0x15a2 +#define MT6357_BUCK_VMODEM_DBG1 0x15a4 +#define MT6357_BUCK_VMODEM_DBG2 0x15a6 +#define MT6357_BUCK_VMODEM_ELR_NUM 0x15a8 +#define MT6357_BUCK_VMODEM_ELR0 0x15aa +#define MT6357_BUCK_VS1_DSN_ID 0x1600 +#define MT6357_BUCK_VS1_DSN_REV0 0x1602 +#define MT6357_BUCK_VS1_DSN_DBI 0x1604 +#define MT6357_BUCK_VS1_DSN_DXI 0x1606 +#define MT6357_BUCK_VS1_CON0 0x1608 +#define MT6357_BUCK_VS1_CON1 0x160a +#define MT6357_BUCK_VS1_CFG0 0x160c +#define MT6357_BUCK_VS1_CFG1 0x160e +#define MT6357_BUCK_VS1_OP_EN 0x1610 +#define MT6357_BUCK_VS1_OP_EN_SET 0x1612 +#define MT6357_BUCK_VS1_OP_EN_CLR 0x1614 +#define MT6357_BUCK_VS1_OP_CFG 0x1616 +#define MT6357_BUCK_VS1_OP_CFG_SET 0x1618 +#define MT6357_BUCK_VS1_OP_CFG_CLR 0x161a +#define MT6357_BUCK_VS1_SP_CON 0x161c +#define MT6357_BUCK_VS1_SP_CFG 0x161e +#define MT6357_BUCK_VS1_OC_CFG 0x1620 +#define MT6357_BUCK_VS1_DBG0 0x1622 +#define MT6357_BUCK_VS1_DBG1 0x1624 +#define MT6357_BUCK_VS1_DBG2 0x1626 +#define MT6357_BUCK_VS1_VOTER 0x1628 +#define MT6357_BUCK_VS1_VOTER_SET 0x162a +#define MT6357_BUCK_VS1_VOTER_CLR 0x162c +#define MT6357_BUCK_VS1_VOTER_CFG 0x162e +#define MT6357_BUCK_VS1_ELR_NUM 0x1630 +#define MT6357_BUCK_VS1_ELR0 0x1632 +#define MT6357_BUCK_VPA_DSN_ID 0x1680 +#define MT6357_BUCK_VPA_DSN_REV0 0x1682 +#define MT6357_BUCK_VPA_DSN_DBI 0x1684 +#define MT6357_BUCK_VPA_DSN_DXI 0x1686 +#define MT6357_BUCK_VPA_CON0 0x1688 +#define MT6357_BUCK_VPA_CON1 0x168a +#define MT6357_BUCK_VPA_CFG0 0x168c +#define MT6357_BUCK_VPA_CFG1 0x168e +#define MT6357_BUCK_VPA_OC_CFG 0x1690 +#define MT6357_BUCK_VPA_DBG0 0x1692 +#define MT6357_BUCK_VPA_DBG1 0x1694 +#define MT6357_BUCK_VPA_DBG2 0x1696 +#define MT6357_BUCK_VPA_DLC_CON0 0x1698 +#define MT6357_BUCK_VPA_DLC_CON1 0x169a +#define MT6357_BUCK_VPA_DLC_CON2 0x169c +#define MT6357_BUCK_VPA_MSFG_CON0 0x169e +#define MT6357_BUCK_VPA_MSFG_CON1 0x16a0 +#define MT6357_BUCK_VPA_MSFG_RRATE0 0x16a2 +#define MT6357_BUCK_VPA_MSFG_RRATE1 0x16a4 +#define MT6357_BUCK_VPA_MSFG_RRATE2 0x16a6 +#define MT6357_BUCK_VPA_MSFG_RTHD0 0x16a8 +#define MT6357_BUCK_VPA_MSFG_RTHD1 0x16aa +#define MT6357_BUCK_VPA_MSFG_RTHD2 0x16ac +#define MT6357_BUCK_VPA_MSFG_FRATE0 0x16ae +#define MT6357_BUCK_VPA_MSFG_FRATE1 0x16b0 +#define MT6357_BUCK_VPA_MSFG_FRATE2 0x16b2 +#define MT6357_BUCK_VPA_MSFG_FTHD0 0x16b4 +#define MT6357_BUCK_VPA_MSFG_FTHD1 0x16b6 +#define MT6357_BUCK_VPA_MSFG_FTHD2 0x16b8 +#define MT6357_BUCK_ANA_DSN_ID 0x1700 +#define MT6357_BUCK_ANA_DSN_REV0 0x1702 +#define MT6357_BUCK_ANA_DSN_DBI 0x1704 +#define MT6357_BUCK_ANA_DSN_FPI 0x1706 +#define MT6357_SMPS_ANA_CON0 0x1708 +#define MT6357_SMPS_ANA_CON1 0x170a +#define MT6357_SMPS_ANA_CON2 0x170c +#define MT6357_VCORE_VPROC_ANA_CON0 0x170e +#define MT6357_VCORE_VPROC_ANA_CON1 0x1710 +#define MT6357_VCORE_VPROC_ANA_CON2 0x1712 +#define MT6357_VCORE_VPROC_ANA_CON3 0x1714 +#define MT6357_VCORE_VPROC_ANA_CON4 0x1716 +#define MT6357_VCORE_VPROC_ANA_CON5 0x1718 +#define MT6357_VCORE_VPROC_ANA_CON6 0x171a +#define MT6357_VCORE_VPROC_ANA_CON7 0x171c +#define MT6357_VCORE_VPROC_ANA_CON8 0x171e +#define MT6357_VCORE_VPROC_ANA_CON9 0x1720 +#define MT6357_VCORE_VPROC_ANA_CON10 0x1722 +#define MT6357_VCORE_VPROC_ANA_CON11 0x1724 +#define MT6357_VMODEM_ANA_CON0 0x1726 +#define MT6357_VMODEM_ANA_CON1 0x1728 +#define MT6357_VMODEM_ANA_CON2 0x172a +#define MT6357_VMODEM_ANA_CON3 0x172c +#define MT6357_VMODEM_ANA_CON4 0x172e +#define MT6357_VMODEM_ANA_CON5 0x1730 +#define MT6357_VS1_ANA_CON0 0x1732 +#define MT6357_VS1_ANA_CON1 0x1734 +#define MT6357_VS1_ANA_CON2 0x1736 +#define MT6357_VS1_ANA_CON3 0x1738 +#define MT6357_VS1_ANA_CON4 0x173a +#define MT6357_VS1_ANA_CON5 0x173c +#define MT6357_VPA_ANA_CON0 0x173e +#define MT6357_VPA_ANA_CON1 0x1740 +#define MT6357_VPA_ANA_CON2 0x1742 +#define MT6357_VPA_ANA_CON3 0x1744 +#define MT6357_VPA_ANA_CON4 0x1746 +#define MT6357_VPA_ANA_CON5 0x1748 +#define MT6357_BUCK_ANA_ELR_NUM 0x174a +#define MT6357_SMPS_ELR_0 0x174c +#define MT6357_SMPS_ELR_1 0x174e +#define MT6357_SMPS_ELR_2 0x1750 +#define MT6357_SMPS_ELR_3 0x1752 +#define MT6357_SMPS_ELR_4 0x1754 +#define MT6357_SMPS_ELR_5 0x1756 +#define MT6357_VCORE_VPROC_ELR_0 0x1758 +#define MT6357_VCORE_VPROC_ELR_1 0x175a +#define MT6357_VCORE_VPROC_ELR_2 0x175c +#define MT6357_VCORE_VPROC_ELR_3 0x175e +#define MT6357_VCORE_VPROC_ELR_4 0x1760 +#define MT6357_VMODEM_ELR_0 0x1762 +#define MT6357_VMODEM_ELR_1 0x1764 +#define MT6357_VMODEM_ELR_2 0x1766 +#define MT6357_VS1_ELR_0 0x1768 +#define MT6357_VS1_ELR_1 0x176a +#define MT6357_VPA_ELR_0 0x176c +#define MT6357_LDO_TOP_ID 0x1880 +#define MT6357_LDO_TOP_REV0 0x1882 +#define MT6357_LDO_TOP_DBI 0x1884 +#define MT6357_LDO_TOP_DXI 0x1886 +#define MT6357_LDO_TPM0 0x1888 +#define MT6357_LDO_TPM1 0x188a +#define MT6357_LDO_TOP_CLK_DCM_CON0 0x188c +#define MT6357_LDO_TOP_CLK_VIO28_CON0 0x188e +#define MT6357_LDO_TOP_CLK_VIO18_CON0 0x1890 +#define MT6357_LDO_TOP_CLK_VAUD28_CON0 0x1892 +#define MT6357_LDO_TOP_CLK_VDRAM_CON0 0x1894 +#define MT6357_LDO_TOP_CLK_VSRAM_PROC_CON0 0x1896 +#define MT6357_LDO_TOP_CLK_VSRAM_OTHERS_CON0 0x1898 +#define MT6357_LDO_TOP_CLK_VAUX18_CON0 0x189a +#define MT6357_LDO_TOP_CLK_VUSB33_CON0 0x189c +#define MT6357_LDO_TOP_CLK_VEMC_CON0 0x189e +#define MT6357_LDO_TOP_CLK_VXO22_CON0 0x18a0 +#define MT6357_LDO_TOP_CLK_VSIM1_CON0 0x18a2 +#define MT6357_LDO_TOP_CLK_VSIM2_CON0 0x18a4 +#define MT6357_LDO_TOP_CLK_VCAMD_CON0 0x18a6 +#define MT6357_LDO_TOP_CLK_VCAMIO_CON0 0x18a8 +#define MT6357_LDO_TOP_CLK_VEFUSE_CON0 0x18aa +#define MT6357_LDO_TOP_CLK_VCN33_CON0 0x18ac +#define MT6357_LDO_TOP_CLK_VCN18_CON0 0x18ae +#define MT6357_LDO_TOP_CLK_VCN28_CON0 0x18b0 +#define MT6357_LDO_TOP_CLK_VIBR_CON0 0x18b2 +#define MT6357_LDO_TOP_CLK_VFE28_CON0 0x18b4 +#define MT6357_LDO_TOP_CLK_VMCH_CON0 0x18b6 +#define MT6357_LDO_TOP_CLK_VMC_CON0 0x18b8 +#define MT6357_LDO_TOP_CLK_VRF18_CON0 0x18ba +#define MT6357_LDO_TOP_CLK_VLDO28_CON0 0x18bc +#define MT6357_LDO_TOP_CLK_VRF12_CON0 0x18be +#define MT6357_LDO_TOP_CLK_VCAMA_CON0 0x18c0 +#define MT6357_LDO_TOP_CLK_TREF_CON0 0x18c2 +#define MT6357_LDO_TOP_INT_CON0 0x18c4 +#define MT6357_LDO_TOP_INT_CON0_SET 0x18c6 +#define MT6357_LDO_TOP_INT_CON0_CLR 0x18c8 +#define MT6357_LDO_TOP_INT_CON1 0x18ca +#define MT6357_LDO_TOP_INT_CON1_SET 0x18cc +#define MT6357_LDO_TOP_INT_CON1_CLR 0x18ce +#define MT6357_LDO_TOP_INT_MASK_CON0 0x18d0 +#define MT6357_LDO_TOP_INT_MASK_CON0_SET 0x18d2 +#define MT6357_LDO_TOP_INT_MASK_CON0_CLR 0x18d4 +#define MT6357_LDO_TOP_INT_MASK_CON1 0x18d6 +#define MT6357_LDO_TOP_INT_MASK_CON1_SET 0x18d8 +#define MT6357_LDO_TOP_INT_MASK_CON1_CLR 0x18da +#define MT6357_LDO_TOP_INT_STATUS0 0x18dc +#define MT6357_LDO_TOP_INT_STATUS1 0x18de +#define MT6357_LDO_TOP_INT_RAW_STATUS0 0x18e0 +#define MT6357_LDO_TOP_INT_RAW_STATUS1 0x18e2 +#define MT6357_LDO_TEST_CON0 0x18e4 +#define MT6357_LDO_TOP_WDT_CON0 0x18e6 +#define MT6357_LDO_TOP_RSV_CON0 0x18e8 +#define MT6357_LDO_TOP_RSV_CON1 0x18ea +#define MT6357_LDO_OCFB0 0x18ec +#define MT6357_LDO_LP_PROTECTION 0x18ee +#define MT6357_LDO_DUMMY_LOAD_GATED 0x18f0 +#define MT6357_LDO_GON0_DSN_ID 0x1900 +#define MT6357_LDO_GON0_DSN_REV0 0x1902 +#define MT6357_LDO_GON0_DSN_DBI 0x1904 +#define MT6357_LDO_GON0_DSN_DXI 0x1906 +#define MT6357_LDO_VXO22_CON0 0x1908 +#define MT6357_LDO_VXO22_OP_EN 0x190a +#define MT6357_LDO_VXO22_OP_EN_SET 0x190c +#define MT6357_LDO_VXO22_OP_EN_CLR 0x190e +#define MT6357_LDO_VXO22_OP_CFG 0x1910 +#define MT6357_LDO_VXO22_OP_CFG_SET 0x1912 +#define MT6357_LDO_VXO22_OP_CFG_CLR 0x1914 +#define MT6357_LDO_VXO22_CON1 0x1916 +#define MT6357_LDO_VXO22_CON2 0x1918 +#define MT6357_LDO_VXO22_CON3 0x191a +#define MT6357_LDO_VAUX18_CON0 0x191c +#define MT6357_LDO_VAUX18_OP_EN 0x191e +#define MT6357_LDO_VAUX18_OP_EN_SET 0x1920 +#define MT6357_LDO_VAUX18_OP_EN_CLR 0x1922 +#define MT6357_LDO_VAUX18_OP_CFG 0x1924 +#define MT6357_LDO_VAUX18_OP_CFG_SET 0x1926 +#define MT6357_LDO_VAUX18_OP_CFG_CLR 0x1928 +#define MT6357_LDO_VAUX18_CON1 0x192a +#define MT6357_LDO_VAUX18_CON2 0x192c +#define MT6357_LDO_VAUX18_CON3 0x192e +#define MT6357_LDO_VAUD28_CON0 0x1930 +#define MT6357_LDO_VAUD28_OP_EN 0x1932 +#define MT6357_LDO_VAUD28_OP_EN_SET 0x1934 +#define MT6357_LDO_VAUD28_OP_EN_CLR 0x1936 +#define MT6357_LDO_VAUD28_OP_CFG 0x1938 +#define MT6357_LDO_VAUD28_OP_CFG_SET 0x193a +#define MT6357_LDO_VAUD28_OP_CFG_CLR 0x193c +#define MT6357_LDO_VAUD28_CON1 0x193e +#define MT6357_LDO_VAUD28_CON2 0x1940 +#define MT6357_LDO_VAUD28_CON3 0x1942 +#define MT6357_LDO_VIO28_CON0 0x1944 +#define MT6357_LDO_VIO28_OP_EN 0x1946 +#define MT6357_LDO_VIO28_OP_EN_SET 0x1948 +#define MT6357_LDO_VIO28_OP_EN_CLR 0x194a +#define MT6357_LDO_VIO28_OP_CFG 0x194c +#define MT6357_LDO_VIO28_OP_CFG_SET 0x194e +#define MT6357_LDO_VIO28_OP_CFG_CLR 0x1950 +#define MT6357_LDO_VIO28_CON1 0x1952 +#define MT6357_LDO_VIO28_CON2 0x1954 +#define MT6357_LDO_VIO28_CON3 0x1956 +#define MT6357_LDO_VIO18_CON0 0x1958 +#define MT6357_LDO_VIO18_OP_EN 0x195a +#define MT6357_LDO_VIO18_OP_EN_SET 0x195c +#define MT6357_LDO_VIO18_OP_EN_CLR 0x195e +#define MT6357_LDO_VIO18_OP_CFG 0x1960 +#define MT6357_LDO_VIO18_OP_CFG_SET 0x1962 +#define MT6357_LDO_VIO18_OP_CFG_CLR 0x1964 +#define MT6357_LDO_VIO18_CON1 0x1966 +#define MT6357_LDO_VIO18_CON2 0x1968 +#define MT6357_LDO_VIO18_CON3 0x196a +#define MT6357_LDO_VDRAM_CON0 0x196c +#define MT6357_LDO_VDRAM_OP_EN 0x196e +#define MT6357_LDO_VDRAM_OP_EN_SET 0x1970 +#define MT6357_LDO_VDRAM_OP_EN_CLR 0x1972 +#define MT6357_LDO_VDRAM_OP_CFG 0x1974 +#define MT6357_LDO_VDRAM_OP_CFG_SET 0x1976 +#define MT6357_LDO_VDRAM_OP_CFG_CLR 0x1978 +#define MT6357_LDO_VDRAM_CON1 0x197a +#define MT6357_LDO_VDRAM_CON2 0x197c +#define MT6357_LDO_VDRAM_CON3 0x197e +#define MT6357_LDO_GON1_DSN_ID 0x1980 +#define MT6357_LDO_GON1_DSN_REV0 0x1982 +#define MT6357_LDO_GON1_DSN_DBI 0x1984 +#define MT6357_LDO_GON1_DSN_DXI 0x1986 +#define MT6357_LDO_VEMC_CON0 0x1988 +#define MT6357_LDO_VEMC_OP_EN 0x198a +#define MT6357_LDO_VEMC_OP_EN_SET 0x198c +#define MT6357_LDO_VEMC_OP_EN_CLR 0x198e +#define MT6357_LDO_VEMC_OP_CFG 0x1990 +#define MT6357_LDO_VEMC_OP_CFG_SET 0x1992 +#define MT6357_LDO_VEMC_OP_CFG_CLR 0x1994 +#define MT6357_LDO_VEMC_CON1 0x1996 +#define MT6357_LDO_VEMC_CON2 0x1998 +#define MT6357_LDO_VEMC_CON3 0x199a +#define MT6357_LDO_VUSB33_CON0_0 0x199c +#define MT6357_LDO_VUSB33_OP_EN 0x199e +#define MT6357_LDO_VUSB33_OP_EN_SET 0x19a0 +#define MT6357_LDO_VUSB33_OP_EN_CLR 0x19a2 +#define MT6357_LDO_VUSB33_OP_CFG 0x19a4 +#define MT6357_LDO_VUSB33_OP_CFG_SET 0x19a6 +#define MT6357_LDO_VUSB33_OP_CFG_CLR 0x19a8 +#define MT6357_LDO_VUSB33_CON0_1 0x19aa +#define MT6357_LDO_VUSB33_CON1 0x19ac +#define MT6357_LDO_VUSB33_CON2 0x19ae +#define MT6357_LDO_VUSB33_CON3 0x19b0 +#define MT6357_LDO_VSRAM_PROC_CON0 0x19b2 +#define MT6357_LDO_VSRAM_PROC_CON2 0x19b4 +#define MT6357_LDO_VSRAM_PROC_CFG0 0x19b6 +#define MT6357_LDO_VSRAM_PROC_CFG1 0x19b8 +#define MT6357_LDO_VSRAM_PROC_OP_EN 0x19ba +#define MT6357_LDO_VSRAM_PROC_OP_EN_SET 0x19bc +#define MT6357_LDO_VSRAM_PROC_OP_EN_CLR 0x19be +#define MT6357_LDO_VSRAM_PROC_OP_CFG 0x19c0 +#define MT6357_LDO_VSRAM_PROC_OP_CFG_SET 0x19c2 +#define MT6357_LDO_VSRAM_PROC_OP_CFG_CLR 0x19c4 +#define MT6357_LDO_VSRAM_PROC_CON3 0x19c6 +#define MT6357_LDO_VSRAM_PROC_CON4 0x19c8 +#define MT6357_LDO_VSRAM_PROC_CON5 0x19ca +#define MT6357_LDO_VSRAM_PROC_DBG0 0x19cc +#define MT6357_LDO_VSRAM_PROC_DBG1 0x19ce +#define MT6357_LDO_VSRAM_OTHERS_CON0 0x19d0 +#define MT6357_LDO_VSRAM_OTHERS_CON2 0x19d2 +#define MT6357_LDO_VSRAM_OTHERS_CFG0 0x19d4 +#define MT6357_LDO_VSRAM_OTHERS_CFG1 0x19d6 +#define MT6357_LDO_VSRAM_OTHERS_OP_EN 0x19d8 +#define MT6357_LDO_VSRAM_OTHERS_OP_EN_SET 0x19da +#define MT6357_LDO_VSRAM_OTHERS_OP_EN_CLR 0x19dc +#define MT6357_LDO_VSRAM_OTHERS_OP_CFG 0x19de +#define MT6357_LDO_VSRAM_OTHERS_OP_CFG_SET 0x19e0 +#define MT6357_LDO_VSRAM_OTHERS_OP_CFG_CLR 0x19e2 +#define MT6357_LDO_VSRAM_OTHERS_CON3 0x19e4 +#define MT6357_LDO_VSRAM_OTHERS_CON4 0x19e6 +#define MT6357_LDO_VSRAM_OTHERS_CON5 0x19e8 +#define MT6357_LDO_VSRAM_OTHERS_DBG0 0x19ea +#define MT6357_LDO_VSRAM_OTHERS_DBG1 0x19ec +#define MT6357_LDO_VSRAM_PROC_SP 0x19ee +#define MT6357_LDO_VSRAM_OTHERS_SP 0x19f0 +#define MT6357_LDO_VSRAM_PROC_R2R_PDN_DIS 0x19f2 +#define MT6357_LDO_VSRAM_OTHERS_R2R_PDN_DIS 0x19f4 +#define MT6357_LDO_VSRAM_WDT_DBG0 0x19f6 +#define MT6357_LDO_GON1_ELR_NUM 0x19f8 +#define MT6357_LDO_VSRAM_CON0 0x19fa +#define MT6357_LDO_VSRAM_CON1 0x19fc +#define MT6357_LDO_VSRAM_CON2 0x19fe +#define MT6357_LDO_GOFF0_DSN_ID 0x1a00 +#define MT6357_LDO_GOFF0_DSN_REV0 0x1a02 +#define MT6357_LDO_GOFF0_DSN_DBI 0x1a04 +#define MT6357_LDO_GOFF0_DSN_DXI 0x1a06 +#define MT6357_LDO_VFE28_CON0 0x1a08 +#define MT6357_LDO_VFE28_OP_EN 0x1a0a +#define MT6357_LDO_VFE28_OP_EN_SET 0x1a0c +#define MT6357_LDO_VFE28_OP_EN_CLR 0x1a0e +#define MT6357_LDO_VFE28_OP_CFG 0x1a10 +#define MT6357_LDO_VFE28_OP_CFG_SET 0x1a12 +#define MT6357_LDO_VFE28_OP_CFG_CLR 0x1a14 +#define MT6357_LDO_VFE28_CON1 0x1a16 +#define MT6357_LDO_VFE28_CON2 0x1a18 +#define MT6357_LDO_VFE28_CON3 0x1a1a +#define MT6357_LDO_VRF18_CON0 0x1a1c +#define MT6357_LDO_VRF18_OP_EN 0x1a1e +#define MT6357_LDO_VRF18_OP_EN_SET 0x1a20 +#define MT6357_LDO_VRF18_OP_EN_CLR 0x1a22 +#define MT6357_LDO_VRF18_OP_CFG 0x1a24 +#define MT6357_LDO_VRF18_OP_CFG_SET 0x1a26 +#define MT6357_LDO_VRF18_OP_CFG_CLR 0x1a28 +#define MT6357_LDO_VRF18_CON1 0x1a2a +#define MT6357_LDO_VRF18_CON2 0x1a2c +#define MT6357_LDO_VRF18_CON3 0x1a2e +#define MT6357_LDO_VRF12_CON0 0x1a30 +#define MT6357_LDO_VRF12_OP_EN 0x1a32 +#define MT6357_LDO_VRF12_OP_EN_SET 0x1a34 +#define MT6357_LDO_VRF12_OP_EN_CLR 0x1a36 +#define MT6357_LDO_VRF12_OP_CFG 0x1a38 +#define MT6357_LDO_VRF12_OP_CFG_SET 0x1a3a +#define MT6357_LDO_VRF12_OP_CFG_CLR 0x1a3c +#define MT6357_LDO_VRF12_CON1 0x1a3e +#define MT6357_LDO_VRF12_CON2 0x1a40 +#define MT6357_LDO_VRF12_CON3 0x1a42 +#define MT6357_LDO_VEFUSE_CON0 0x1a44 +#define MT6357_LDO_VEFUSE_OP_EN 0x1a46 +#define MT6357_LDO_VEFUSE_OP_EN_SET 0x1a48 +#define MT6357_LDO_VEFUSE_OP_EN_CLR 0x1a4a +#define MT6357_LDO_VEFUSE_OP_CFG 0x1a4c +#define MT6357_LDO_VEFUSE_OP_CFG_SET 0x1a4e +#define MT6357_LDO_VEFUSE_OP_CFG_CLR 0x1a50 +#define MT6357_LDO_VEFUSE_CON1 0x1a52 +#define MT6357_LDO_VEFUSE_CON2 0x1a54 +#define MT6357_LDO_VEFUSE_CON3 0x1a56 +#define MT6357_LDO_VCN18_CON0 0x1a58 +#define MT6357_LDO_VCN18_OP_EN 0x1a5a +#define MT6357_LDO_VCN18_OP_EN_SET 0x1a5c +#define MT6357_LDO_VCN18_OP_EN_CLR 0x1a5e +#define MT6357_LDO_VCN18_OP_CFG 0x1a60 +#define MT6357_LDO_VCN18_OP_CFG_SET 0x1a62 +#define MT6357_LDO_VCN18_OP_CFG_CLR 0x1a64 +#define MT6357_LDO_VCN18_CON1 0x1a66 +#define MT6357_LDO_VCN18_CON2 0x1a68 +#define MT6357_LDO_VCN18_CON3 0x1a6a +#define MT6357_LDO_VCAMA_CON0 0x1a6c +#define MT6357_LDO_VCAMA_OP_EN 0x1a6e +#define MT6357_LDO_VCAMA_OP_EN_SET 0x1a70 +#define MT6357_LDO_VCAMA_OP_EN_CLR 0x1a72 +#define MT6357_LDO_VCAMA_OP_CFG 0x1a74 +#define MT6357_LDO_VCAMA_OP_CFG_SET 0x1a76 +#define MT6357_LDO_VCAMA_OP_CFG_CLR 0x1a78 +#define MT6357_LDO_VCAMA_CON1 0x1a7a +#define MT6357_LDO_VCAMA_CON2 0x1a7c +#define MT6357_LDO_VCAMA_CON3 0x1a7e +#define MT6357_LDO_GOFF1_DSN_ID 0x1a80 +#define MT6357_LDO_GOFF1_DSN_REV0 0x1a82 +#define MT6357_LDO_GOFF1_DSN_DBI 0x1a84 +#define MT6357_LDO_GOFF1_DSN_DXI 0x1a86 +#define MT6357_LDO_VCAMD_CON0 0x1a88 +#define MT6357_LDO_VCAMD_OP_EN 0x1a8a +#define MT6357_LDO_VCAMD_OP_EN_SET 0x1a8c +#define MT6357_LDO_VCAMD_OP_EN_CLR 0x1a8e +#define MT6357_LDO_VCAMD_OP_CFG 0x1a90 +#define MT6357_LDO_VCAMD_OP_CFG_SET 0x1a92 +#define MT6357_LDO_VCAMD_OP_CFG_CLR 0x1a94 +#define MT6357_LDO_VCAMD_CON1 0x1a96 +#define MT6357_LDO_VCAMD_CON2 0x1a98 +#define MT6357_LDO_VCAMD_CON3 0x1a9a +#define MT6357_LDO_VCAMIO_CON0 0x1a9c +#define MT6357_LDO_VCAMIO_OP_EN 0x1a9e +#define MT6357_LDO_VCAMIO_OP_EN_SET 0x1aa0 +#define MT6357_LDO_VCAMIO_OP_EN_CLR 0x1aa2 +#define MT6357_LDO_VCAMIO_OP_CFG 0x1aa4 +#define MT6357_LDO_VCAMIO_OP_CFG_SET 0x1aa6 +#define MT6357_LDO_VCAMIO_OP_CFG_CLR 0x1aa8 +#define MT6357_LDO_VCAMIO_CON1 0x1aaa +#define MT6357_LDO_VCAMIO_CON2 0x1aac +#define MT6357_LDO_VCAMIO_CON3 0x1aae +#define MT6357_LDO_VMC_CON0 0x1ab0 +#define MT6357_LDO_VMC_OP_EN 0x1ab2 +#define MT6357_LDO_VMC_OP_EN_SET 0x1ab4 +#define MT6357_LDO_VMC_OP_EN_CLR 0x1ab6 +#define MT6357_LDO_VMC_OP_CFG 0x1ab8 +#define MT6357_LDO_VMC_OP_CFG_SET 0x1aba +#define MT6357_LDO_VMC_OP_CFG_CLR 0x1abc +#define MT6357_LDO_VMC_CON1 0x1abe +#define MT6357_LDO_VMC_CON2 0x1ac0 +#define MT6357_LDO_VMC_CON3 0x1ac2 +#define MT6357_LDO_VMCH_CON0 0x1ac4 +#define MT6357_LDO_VMCH_OP_EN 0x1ac6 +#define MT6357_LDO_VMCH_OP_EN_SET 0x1ac8 +#define MT6357_LDO_VMCH_OP_EN_CLR 0x1aca +#define MT6357_LDO_VMCH_OP_CFG 0x1acc +#define MT6357_LDO_VMCH_OP_CFG_SET 0x1ace +#define MT6357_LDO_VMCH_OP_CFG_CLR 0x1ad0 +#define MT6357_LDO_VMCH_CON1 0x1ad2 +#define MT6357_LDO_VMCH_CON2 0x1ad4 +#define MT6357_LDO_VMCH_CON3 0x1ad6 +#define MT6357_LDO_VSIM1_CON0 0x1ad8 +#define MT6357_LDO_VSIM1_OP_EN 0x1ada +#define MT6357_LDO_VSIM1_OP_EN_SET 0x1adc +#define MT6357_LDO_VSIM1_OP_EN_CLR 0x1ade +#define MT6357_LDO_VSIM1_OP_CFG 0x1ae0 +#define MT6357_LDO_VSIM1_OP_CFG_SET 0x1ae2 +#define MT6357_LDO_VSIM1_OP_CFG_CLR 0x1ae4 +#define MT6357_LDO_VSIM1_CON1 0x1ae6 +#define MT6357_LDO_VSIM1_CON2 0x1ae8 +#define MT6357_LDO_VSIM1_CON3 0x1aea +#define MT6357_LDO_VSIM2_CON0 0x1aec +#define MT6357_LDO_VSIM2_OP_EN 0x1aee +#define MT6357_LDO_VSIM2_OP_EN_SET 0x1af0 +#define MT6357_LDO_VSIM2_OP_EN_CLR 0x1af2 +#define MT6357_LDO_VSIM2_OP_CFG 0x1af4 +#define MT6357_LDO_VSIM2_OP_CFG_SET 0x1af6 +#define MT6357_LDO_VSIM2_OP_CFG_CLR 0x1af8 +#define MT6357_LDO_VSIM2_CON1 0x1afa +#define MT6357_LDO_VSIM2_CON2 0x1afc +#define MT6357_LDO_VSIM2_CON3 0x1afe +#define MT6357_LDO_GOFF2_DSN_ID 0x1b00 +#define MT6357_LDO_GOFF2_DSN_REV0 0x1b02 +#define MT6357_LDO_GOFF2_DSN_DBI 0x1b04 +#define MT6357_LDO_GOFF2_DSN_DXI 0x1b06 +#define MT6357_LDO_VIBR_CON0 0x1b08 +#define MT6357_LDO_VIBR_OP_EN 0x1b0a +#define MT6357_LDO_VIBR_OP_EN_SET 0x1b0c +#define MT6357_LDO_VIBR_OP_EN_CLR 0x1b0e +#define MT6357_LDO_VIBR_OP_CFG 0x1b10 +#define MT6357_LDO_VIBR_OP_CFG_SET 0x1b12 +#define MT6357_LDO_VIBR_OP_CFG_CLR 0x1b14 +#define MT6357_LDO_VIBR_CON1 0x1b16 +#define MT6357_LDO_VIBR_CON2 0x1b18 +#define MT6357_LDO_VIBR_CON3 0x1b1a +#define MT6357_LDO_VCN33_CON0_0 0x1b1c +#define MT6357_LDO_VCN33_OP_EN 0x1b1e +#define MT6357_LDO_VCN33_OP_EN_SET 0x1b20 +#define MT6357_LDO_VCN33_OP_EN_CLR 0x1b22 +#define MT6357_LDO_VCN33_OP_CFG 0x1b24 +#define MT6357_LDO_VCN33_OP_CFG_SET 0x1b26 +#define MT6357_LDO_VCN33_OP_CFG_CLR 0x1b28 +#define MT6357_LDO_VCN33_CON0_1 0x1b2a +#define MT6357_LDO_VCN33_CON1 0x1b2c +#define MT6357_LDO_VCN33_CON2 0x1b2e +#define MT6357_LDO_VCN33_CON3 0x1b30 +#define MT6357_LDO_VLDO28_CON0_0 0x1b32 +#define MT6357_LDO_VLDO28_OP_EN 0x1b34 +#define MT6357_LDO_VLDO28_OP_EN_SET 0x1b36 +#define MT6357_LDO_VLDO28_OP_EN_CLR 0x1b38 +#define MT6357_LDO_VLDO28_OP_CFG 0x1b3a +#define MT6357_LDO_VLDO28_OP_CFG_SET 0x1b3c +#define MT6357_LDO_VLDO28_OP_CFG_CLR 0x1b3e +#define MT6357_LDO_VLDO28_CON0_1 0x1b40 +#define MT6357_LDO_VLDO28_CON1 0x1b42 +#define MT6357_LDO_VLDO28_CON2 0x1b44 +#define MT6357_LDO_VLDO28_CON3 0x1b46 +#define MT6357_LDO_GOFF2_RSV_CON0 0x1b48 +#define MT6357_LDO_GOFF2_RSV_CON1 0x1b4a +#define MT6357_LDO_GOFF3_DSN_ID 0x1b80 +#define MT6357_LDO_GOFF3_DSN_REV0 0x1b82 +#define MT6357_LDO_GOFF3_DSN_DBI 0x1b84 +#define MT6357_LDO_GOFF3_DSN_DXI 0x1b86 +#define MT6357_LDO_VCN28_CON0 0x1b88 +#define MT6357_LDO_VCN28_OP_EN 0x1b8a +#define MT6357_LDO_VCN28_OP_EN_SET 0x1b8c +#define MT6357_LDO_VCN28_OP_EN_CLR 0x1b8e +#define MT6357_LDO_VCN28_OP_CFG 0x1b90 +#define MT6357_LDO_VCN28_OP_CFG_SET 0x1b92 +#define MT6357_LDO_VCN28_OP_CFG_CLR 0x1b94 +#define MT6357_LDO_VCN28_CON1 0x1b96 +#define MT6357_LDO_VCN28_CON2 0x1b98 +#define MT6357_LDO_VCN28_CON3 0x1b9a +#define MT6357_VRTC_CON0 0x1b9c +#define MT6357_LDO_TREF_CON0 0x1b9e +#define MT6357_LDO_TREF_OP_EN 0x1ba0 +#define MT6357_LDO_TREF_OP_EN_SET 0x1ba2 +#define MT6357_LDO_TREF_OP_EN_CLR 0x1ba4 +#define MT6357_LDO_TREF_OP_CFG 0x1ba6 +#define MT6357_LDO_TREF_OP_CFG_SET 0x1ba8 +#define MT6357_LDO_TREF_OP_CFG_CLR 0x1baa +#define MT6357_LDO_TREF_CON1 0x1bac +#define MT6357_LDO_GOFF3_RSV_CON0 0x1bae +#define MT6357_LDO_GOFF3_RSV_CON1 0x1bb0 +#define MT6357_LDO_ANA0_DSN_ID 0x1c00 +#define MT6357_LDO_ANA0_DSN_REV0 0x1c02 +#define MT6357_LDO_ANA0_DSN_DBI 0x1c04 +#define MT6357_LDO_ANA0_DSN_DXI 0x1c06 +#define MT6357_VFE28_ANA_CON0 0x1c08 +#define MT6357_VFE28_ANA_CON1 0x1c0a +#define MT6357_VCN28_ANA_CON0 0x1c0c +#define MT6357_VCN28_ANA_CON1 0x1c0e +#define MT6357_VAUD28_ANA_CON0 0x1c10 +#define MT6357_VAUD28_ANA_CON1 0x1c12 +#define MT6357_VAUX18_ANA_CON0 0x1c14 +#define MT6357_VAUX18_ANA_CON1 0x1c16 +#define MT6357_VXO22_ANA_CON0 0x1c18 +#define MT6357_VXO22_ANA_CON1 0x1c1a +#define MT6357_VCN33_ANA_CON0 0x1c1c +#define MT6357_VCN33_ANA_CON1 0x1c1e +#define MT6357_VEMC_ANA_CON0 0x1c20 +#define MT6357_VEMC_ANA_CON1 0x1c22 +#define MT6357_VLDO28_ANA_CON0 0x1c24 +#define MT6357_VLDO28_ANA_CON1 0x1c26 +#define MT6357_VIO28_ANA_CON0 0x1c28 +#define MT6357_VIO28_ANA_CON1 0x1c2a +#define MT6357_VIBR_ANA_CON0 0x1c2c +#define MT6357_VIBR_ANA_CON1 0x1c2e +#define MT6357_VSIM1_ANA_CON0 0x1c30 +#define MT6357_VSIM1_ANA_CON1 0x1c32 +#define MT6357_VSIM2_ANA_CON0 0x1c34 +#define MT6357_VSIM2_ANA_CON1 0x1c36 +#define MT6357_VMCH_ANA_CON0 0x1c38 +#define MT6357_VMCH_ANA_CON1 0x1c3a +#define MT6357_VMC_ANA_CON0 0x1c3c +#define MT6357_VMC_ANA_CON1 0x1c3e +#define MT6357_VCAMIO_ANA_CON0 0x1c40 +#define MT6357_VCAMIO_ANA_CON1 0x1c42 +#define MT6357_VCN18_ANA_CON0 0x1c44 +#define MT6357_VCN18_ANA_CON1 0x1c46 +#define MT6357_VRF18_ANA_CON0 0x1c48 +#define MT6357_VRF18_ANA_CON1 0x1c4a +#define MT6357_VIO18_ANA_CON0 0x1c4c +#define MT6357_VIO18_ANA_CON1 0x1c4e +#define MT6357_VDRAM_ANA_CON1 0x1c50 +#define MT6357_VRF12_ANA_CON0 0x1c52 +#define MT6357_VRF12_ANA_CON1 0x1c54 +#define MT6357_VSRAM_PROC_ANA_CON0 0x1c56 +#define MT6357_VSRAM_OTHERS_ANA_CON0 0x1c58 +#define MT6357_LDO_ANA0_ELR_NUM 0x1c5a +#define MT6357_VFE28_ELR_0 0x1c5c +#define MT6357_VCN28_ELR_0 0x1c5e +#define MT6357_VAUD28_ELR_0 0x1c60 +#define MT6357_VAUX18_ELR_0 0x1c62 +#define MT6357_VXO22_ELR_0 0x1c64 +#define MT6357_VCN33_ELR_0 0x1c66 +#define MT6357_VEMC_ELR_0 0x1c68 +#define MT6357_VLDO28_ELR_0 0x1c6a +#define MT6357_VIO28_ELR_0 0x1c6c +#define MT6357_VIBR_ELR_0 0x1c6e +#define MT6357_VSIM1_ELR_0 0x1c70 +#define MT6357_VSIM2_ELR_0 0x1c72 +#define MT6357_VMCH_ELR_0 0x1c74 +#define MT6357_VMC_ELR_0 0x1c76 +#define MT6357_VCAMIO_ELR_0 0x1c78 +#define MT6357_VCN18_ELR_0 0x1c7a +#define MT6357_VRF18_ELR_0 0x1c7c +#define MT6357_LDO_ANA1_DSN_ID 0x1c80 +#define MT6357_LDO_ANA1_DSN_REV0 0x1c82 +#define MT6357_LDO_ANA1_DSN_DBI 0x1c84 +#define MT6357_LDO_ANA1_DSN_DXI 0x1c86 +#define MT6357_VUSB33_ANA_CON0 0x1c88 +#define MT6357_VUSB33_ANA_CON1 0x1c8a +#define MT6357_VCAMA_ANA_CON0 0x1c8c +#define MT6357_VCAMA_ANA_CON1 0x1c8e +#define MT6357_VEFUSE_ANA_CON0 0x1c90 +#define MT6357_VEFUSE_ANA_CON1 0x1c92 +#define MT6357_VCAMD_ANA_CON0 0x1c94 +#define MT6357_VCAMD_ANA_CON1 0x1c96 +#define MT6357_LDO_ANA1_ELR_NUM 0x1c98 +#define MT6357_VUSB33_ELR_0 0x1c9a +#define MT6357_VCAMA_ELR_0 0x1c9c +#define MT6357_VEFUSE_ELR_0 0x1c9e +#define MT6357_VCAMD_ELR_0 0x1ca0 +#define MT6357_VIO18_ELR_0 0x1ca2 +#define MT6357_VDRAM_ELR_0 0x1ca4 +#define MT6357_VRF12_ELR_0 0x1ca6 +#define MT6357_VRTC_ELR_0 0x1ca8 +#define MT6357_VDRAM_ELR_1 0x1caa +#define MT6357_VDRAM_ELR_2 0x1cac +#define MT6357_XPP_TOP_ID 0x1e00 +#define MT6357_XPP_TOP_REV0 0x1e02 +#define MT6357_XPP_TOP_DBI 0x1e04 +#define MT6357_XPP_TOP_DXI 0x1e06 +#define MT6357_XPP_TPM0 0x1e08 +#define MT6357_XPP_TPM1 0x1e0a +#define MT6357_XPP_TOP_TEST_OUT 0x1e0c +#define MT6357_XPP_TOP_TEST_CON0 0x1e0e +#define MT6357_XPP_TOP_CKPDN_CON0 0x1e10 +#define MT6357_XPP_TOP_CKPDN_CON0_SET 0x1e12 +#define MT6357_XPP_TOP_CKPDN_CON0_CLR 0x1e14 +#define MT6357_XPP_TOP_CKSEL_CON0 0x1e16 +#define MT6357_XPP_TOP_CKSEL_CON0_SET 0x1e18 +#define MT6357_XPP_TOP_CKSEL_CON0_CLR 0x1e1a +#define MT6357_XPP_TOP_RST_CON0 0x1e1c +#define MT6357_XPP_TOP_RST_CON0_SET 0x1e1e +#define MT6357_XPP_TOP_RST_CON0_CLR 0x1e20 +#define MT6357_XPP_TOP_RST_BANK_CON0 0x1e22 +#define MT6357_XPP_TOP_RST_BANK_CON0_SET 0x1e24 +#define MT6357_XPP_TOP_RST_BANK_CON0_CLR 0x1e26 +#define MT6357_DRIVER_BL_DSN_ID 0x1e80 +#define MT6357_DRIVER_BL_DSN_REV0 0x1e82 +#define MT6357_DRIVER_BL_DSN_DBI 0x1e84 +#define MT6357_DRIVER_BL_DSN_DXI 0x1e86 +#define MT6357_ISINK1_CON0 0x1e88 +#define MT6357_ISINK1_CON1 0x1e8a +#define MT6357_ISINK1_CON2 0x1e8c +#define MT6357_ISINK1_CON3 0x1e8e +#define MT6357_ISINK_ANA1 0x1e90 +#define MT6357_ISINK_PHASE_DLY 0x1e92 +#define MT6357_ISINK_SFSTR 0x1e94 +#define MT6357_ISINK_EN_CTRL 0x1e96 +#define MT6357_ISINK_MODE_CTRL 0x1e98 +#define MT6357_DRIVER_ANA_CON0 0x1e9a +#define MT6357_ISINK_ANA_CON0 0x1e9c +#define MT6357_ISINK_ANA_CON1 0x1e9e +#define MT6357_DRIVER_BL_ELR_NUM 0x1ea0 +#define MT6357_DRIVER_BL_ELR_0 0x1ea2 +#define MT6357_DRIVER_CI_DSN_ID 0x1f00 +#define MT6357_DRIVER_CI_DSN_REV0 0x1f02 +#define MT6357_DRIVER_CI_DSN_DBI 0x1f04 +#define MT6357_DRIVER_CI_DSN_DXI 0x1f06 +#define MT6357_CHRIND_CON0 0x1f08 +#define MT6357_CHRIND_CON1 0x1f0a +#define MT6357_CHRIND_CON2 0x1f0c +#define MT6357_CHRIND_CON3 0x1f0e +#define MT6357_CHRIND_CON4 0x1f10 +#define MT6357_CHRIND_EN_CTRL 0x1f12 +#define MT6357_CHRIND_ANA_CON0 0x1f14 +#define MT6357_DRIVER_DL_DSN_ID 0x1f80 +#define MT6357_DRIVER_DL_DSN_REV0 0x1f82 +#define MT6357_DRIVER_DL_DSN_DBI 0x1f84 +#define MT6357_DRIVER_DL_DSN_DXI 0x1f86 +#define MT6357_ISINK2_CON0 0x1f88 +#define MT6357_ISINK3_CON0 0x1f8a +#define MT6357_ISINK_EN_CTRL_SMPL 0x1f8c +#define MT6357_AUD_TOP_ID 0x2080 +#define MT6357_AUD_TOP_REV0 0x2082 +#define MT6357_AUD_TOP_DBI 0x2084 +#define MT6357_AUD_TOP_DXI 0x2086 +#define MT6357_AUD_TOP_CKPDN_TPM0 0x2088 +#define MT6357_AUD_TOP_CKPDN_TPM1 0x208a +#define MT6357_AUD_TOP_CKPDN_CON0 0x208c +#define MT6357_AUD_TOP_CKPDN_CON0_SET 0x208e +#define MT6357_AUD_TOP_CKPDN_CON0_CLR 0x2090 +#define MT6357_AUD_TOP_CKSEL_CON0 0x2092 +#define MT6357_AUD_TOP_CKSEL_CON0_SET 0x2094 +#define MT6357_AUD_TOP_CKSEL_CON0_CLR 0x2096 +#define MT6357_AUD_TOP_CKTST_CON0 0x2098 +#define MT6357_AUD_TOP_RST_CON0 0x209a +#define MT6357_AUD_TOP_RST_CON0_SET 0x209c +#define MT6357_AUD_TOP_RST_CON0_CLR 0x209e +#define MT6357_AUD_TOP_RST_BANK_CON0 0x20a0 +#define MT6357_AUD_TOP_INT_CON0 0x20a2 +#define MT6357_AUD_TOP_INT_CON0_SET 0x20a4 +#define MT6357_AUD_TOP_INT_CON0_CLR 0x20a6 +#define MT6357_AUD_TOP_INT_MASK_CON0 0x20a8 +#define MT6357_AUD_TOP_INT_MASK_CON0_SET 0x20aa +#define MT6357_AUD_TOP_INT_MASK_CON0_CLR 0x20ac +#define MT6357_AUD_TOP_INT_STATUS0 0x20ae +#define MT6357_AUD_TOP_INT_RAW_STATUS0 0x20b0 +#define MT6357_AUD_TOP_INT_MISC_CON0 0x20b2 +#define MT6357_AUDNCP_CLKDIV_CON0 0x20b4 +#define MT6357_AUDNCP_CLKDIV_CON1 0x20b6 +#define MT6357_AUDNCP_CLKDIV_CON2 0x20b8 +#define MT6357_AUDNCP_CLKDIV_CON3 0x20ba +#define MT6357_AUDNCP_CLKDIV_CON4 0x20bc +#define MT6357_AUD_TOP_MON_CON0 0x20be +#define MT6357_AUDIO_DIG_DSN_ID 0x2100 +#define MT6357_AUDIO_DIG_DSN_REV0 0x2102 +#define MT6357_AUDIO_DIG_DSN_DBI 0x2104 +#define MT6357_AUDIO_DIG_DSN_DXI 0x2106 +#define MT6357_AFE_UL_DL_CON0 0x2108 +#define MT6357_AFE_DL_SRC2_CON0_L 0x210a +#define MT6357_AFE_UL_SRC_CON0_H 0x210c +#define MT6357_AFE_UL_SRC_CON0_L 0x210e +#define MT6357_AFE_TOP_CON0 0x2110 +#define MT6357_AUDIO_TOP_CON0 0x2112 +#define MT6357_AFE_MON_DEBUG0 0x2114 +#define MT6357_AFUNC_AUD_CON0 0x2116 +#define MT6357_AFUNC_AUD_CON1 0x2118 +#define MT6357_AFUNC_AUD_CON2 0x211a +#define MT6357_AFUNC_AUD_CON3 0x211c +#define MT6357_AFUNC_AUD_CON4 0x211e +#define MT6357_AFUNC_AUD_CON5 0x2120 +#define MT6357_AFUNC_AUD_CON6 0x2122 +#define MT6357_AFUNC_AUD_MON0 0x2124 +#define MT6357_AUDRC_TUNE_MON0 0x2126 +#define MT6357_AFE_ADDA_MTKAIF_FIFO_CFG0 0x2128 +#define MT6357_AFE_ADDA_MTKAIF_FIFO_LOG_MON1 0x212a +#define MT6357_AFE_ADDA_MTKAIF_MON0 0x212c +#define MT6357_AFE_ADDA_MTKAIF_MON1 0x212e +#define MT6357_AFE_ADDA_MTKAIF_MON2 0x2130 +#define MT6357_AFE_ADDA_MTKAIF_MON3 0x2132 +#define MT6357_AFE_ADDA_MTKAIF_CFG0 0x2134 +#define MT6357_AFE_ADDA_MTKAIF_RX_CFG0 0x2136 +#define MT6357_AFE_ADDA_MTKAIF_RX_CFG1 0x2138 +#define MT6357_AFE_ADDA_MTKAIF_RX_CFG2 0x213a +#define MT6357_AFE_ADDA_MTKAIF_RX_CFG3 0x213c +#define MT6357_AFE_ADDA_MTKAIF_TX_CFG1 0x213e +#define MT6357_AFE_SGEN_CFG0 0x2140 +#define MT6357_AFE_SGEN_CFG1 0x2142 +#define MT6357_AFE_ADC_ASYNC_FIFO_CFG 0x2144 +#define MT6357_AFE_DCCLK_CFG0 0x2146 +#define MT6357_AFE_DCCLK_CFG1 0x2148 +#define MT6357_AUDIO_DIG_CFG 0x214a +#define MT6357_AFE_AUD_PAD_TOP 0x214c +#define MT6357_AFE_AUD_PAD_TOP_MON 0x214e +#define MT6357_AFE_AUD_PAD_TOP_MON1 0x2150 +#define MT6357_AUDENC_DSN_ID 0x2180 +#define MT6357_AUDENC_DSN_REV0 0x2182 +#define MT6357_AUDENC_DSN_DBI 0x2184 +#define MT6357_AUDENC_DSN_FPI 0x2186 +#define MT6357_AUDENC_ANA_CON0 0x2188 +#define MT6357_AUDENC_ANA_CON1 0x218a +#define MT6357_AUDENC_ANA_CON2 0x218c +#define MT6357_AUDENC_ANA_CON3 0x218e +#define MT6357_AUDENC_ANA_CON4 0x2190 +#define MT6357_AUDENC_ANA_CON5 0x2192 +#define MT6357_AUDENC_ANA_CON6 0x2194 +#define MT6357_AUDENC_ANA_CON7 0x2196 +#define MT6357_AUDENC_ANA_CON8 0x2198 +#define MT6357_AUDENC_ANA_CON9 0x219a +#define MT6357_AUDENC_ANA_CON10 0x219c +#define MT6357_AUDENC_ANA_CON11 0x219e +#define MT6357_AUDDEC_DSN_ID 0x2200 +#define MT6357_AUDDEC_DSN_REV0 0x2202 +#define MT6357_AUDDEC_DSN_DBI 0x2204 +#define MT6357_AUDDEC_DSN_FPI 0x2206 +#define MT6357_AUDDEC_ANA_CON0 0x2208 +#define MT6357_AUDDEC_ANA_CON1 0x220a +#define MT6357_AUDDEC_ANA_CON2 0x220c +#define MT6357_AUDDEC_ANA_CON3 0x220e +#define MT6357_AUDDEC_ANA_CON4 0x2210 +#define MT6357_AUDDEC_ANA_CON5 0x2212 +#define MT6357_AUDDEC_ANA_CON6 0x2214 +#define MT6357_AUDDEC_ANA_CON7 0x2216 +#define MT6357_AUDDEC_ANA_CON8 0x2218 +#define MT6357_AUDDEC_ANA_CON9 0x221a +#define MT6357_AUDDEC_ANA_CON10 0x221c +#define MT6357_AUDDEC_ANA_CON11 0x221e +#define MT6357_AUDDEC_ANA_CON12 0x2220 +#define MT6357_AUDDEC_ANA_CON13 0x2222 +#define MT6357_AUDDEC_ELR_NUM 0x2224 +#define MT6357_AUDDEC_ELR_0 0x2226 +#define MT6357_AUDZCD_DSN_ID 0x2280 +#define MT6357_AUDZCD_DSN_REV0 0x2282 +#define MT6357_AUDZCD_DSN_DBI 0x2284 +#define MT6357_AUDZCD_DSN_FPI 0x2286 +#define MT6357_ZCD_CON0 0x2288 +#define MT6357_ZCD_CON1 0x228a +#define MT6357_ZCD_CON2 0x228c +#define MT6357_ZCD_CON3 0x228e +#define MT6357_ZCD_CON4 0x2290 +#define MT6357_ZCD_CON5 0x2292 +#define MT6357_ACCDET_DSN_DIG_ID 0x2300 +#define MT6357_ACCDET_DSN_DIG_REV0 0x2302 +#define MT6357_ACCDET_DSN_DBI 0x2304 +#define MT6357_ACCDET_DSN_FPI 0x2306 +#define MT6357_ACCDET_CON0 0x2308 +#define MT6357_ACCDET_CON1 0x230a +#define MT6357_ACCDET_CON2 0x230c +#define MT6357_ACCDET_CON3 0x230e +#define MT6357_ACCDET_CON4 0x2310 +#define MT6357_ACCDET_CON5 0x2312 +#define MT6357_ACCDET_CON6 0x2314 +#define MT6357_ACCDET_CON7 0x2316 +#define MT6357_ACCDET_CON8 0x2318 +#define MT6357_ACCDET_CON9 0x231a +#define MT6357_ACCDET_CON10 0x231c +#define MT6357_ACCDET_CON11 0x231e +#define MT6357_ACCDET_CON12 0x2320 +#define MT6357_ACCDET_CON13 0x2322 +#define MT6357_ACCDET_CON14 0x2324 +#define MT6357_ACCDET_CON15 0x2326 +#define MT6357_ACCDET_CON16 0x2328 +#define MT6357_ACCDET_CON17 0x232a +#define MT6357_ACCDET_CON18 0x232c +#define MT6357_ACCDET_CON19 0x232e +#define MT6357_ACCDET_CON20 0x2330 +#define MT6357_ACCDET_CON21 0x2332 +#define MT6357_ACCDET_CON22 0x2334 +#define MT6357_ACCDET_CON23 0x2336 +#define MT6357_ACCDET_CON24 0x2338 +#define MT6357_ACCDET_CON25 0x233a +#define MT6357_ACCDET_CON26 0x233c +#define MT6357_ACCDET_CON27 0x233e +#define MT6357_ACCDET_CON28 0x2340 + +#endif /* __MFD_MT6357_REGISTERS_H__ */ -- cgit v1.2.3 From 738654be3cf7af4a0a131bd92142db045f0660dc Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Tue, 31 May 2022 14:49:57 +0200 Subject: mfd: mt6358-irq: Add MT6357 PMIC support Add MT6357 PMIC IRQ support. Signed-off-by: Fabien Parent Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220531124959.202787-6-fparent@baylibre.com --- drivers/mfd/mt6358-irq.c | 24 ++++++++++++++++++++++++ include/linux/mfd/mt6397/core.h | 1 + 2 files changed, 25 insertions(+) (limited to 'include/linux') diff --git a/drivers/mfd/mt6358-irq.c b/drivers/mfd/mt6358-irq.c index ea5e452510eb..389756436af6 100644 --- a/drivers/mfd/mt6358-irq.c +++ b/drivers/mfd/mt6358-irq.c @@ -3,6 +3,8 @@ // Copyright (c) 2020 MediaTek Inc. #include +#include +#include #include #include #include @@ -17,6 +19,17 @@ #define MTK_PMIC_REG_WIDTH 16 +static const struct irq_top_t mt6357_ints[] = { + MT6357_TOP_GEN(BUCK), + MT6357_TOP_GEN(LDO), + MT6357_TOP_GEN(PSC), + MT6357_TOP_GEN(SCK), + MT6357_TOP_GEN(BM), + MT6357_TOP_GEN(HK), + MT6357_TOP_GEN(AUD), + MT6357_TOP_GEN(MISC), +}; + static const struct irq_top_t mt6358_ints[] = { MT6358_TOP_GEN(BUCK), MT6358_TOP_GEN(LDO), @@ -39,6 +52,13 @@ static const struct irq_top_t mt6359_ints[] = { MT6359_TOP_GEN(MISC), }; +static struct pmic_irq_data mt6357_irqd = { + .num_top = ARRAY_SIZE(mt6357_ints), + .num_pmic_irqs = MT6357_IRQ_NR, + .top_int_status_reg = MT6357_TOP_INT_STATUS0, + .pmic_ints = mt6357_ints, +}; + static struct pmic_irq_data mt6358_irqd = { .num_top = ARRAY_SIZE(mt6358_ints), .num_pmic_irqs = MT6358_IRQ_NR, @@ -211,6 +231,10 @@ int mt6358_irq_init(struct mt6397_chip *chip) struct pmic_irq_data *irqd; switch (chip->chip_id) { + case MT6357_CHIP_ID: + chip->irq_data = &mt6357_irqd; + break; + case MT6358_CHIP_ID: case MT6366_CHIP_ID: chip->irq_data = &mt6358_irqd; diff --git a/include/linux/mfd/mt6397/core.h b/include/linux/mfd/mt6397/core.h index 1cf78726503b..3fecaffe5019 100644 --- a/include/linux/mfd/mt6397/core.h +++ b/include/linux/mfd/mt6397/core.h @@ -12,6 +12,7 @@ enum chip_id { MT6323_CHIP_ID = 0x23, + MT6357_CHIP_ID = 0x57, MT6358_CHIP_ID = 0x58, MT6359_CHIP_ID = 0x59, MT6366_CHIP_ID = 0x66, -- cgit v1.2.3 From 66ee379d743c69c726b61d078119a34d5be96a35 Mon Sep 17 00:00:00 2001 From: Tinghan Shen Date: Wed, 1 Jun 2022 19:22:01 +0800 Subject: mfd: cros_ec: Add SCP Core-1 as a new CrOS EC MCU MT8195 System Companion Processors(SCP) is a dual-core RISC-V MCU. Add a new CrOS feature ID to represent the SCP's 2nd core. The 1st core is referred to as 'core 0', and the 2nd core is referred to as 'core 1'. Signed-off-by: Tinghan Shen Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220601112201.15510-16-tinghan.shen@mediatek.com --- drivers/mfd/cros_ec_dev.c | 5 +++++ include/linux/platform_data/cros_ec_commands.h | 2 ++ include/linux/platform_data/cros_ec_proto.h | 1 + 3 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c index 596731caf407..07cc31d92edc 100644 --- a/drivers/mfd/cros_ec_dev.c +++ b/drivers/mfd/cros_ec_dev.c @@ -64,6 +64,11 @@ static const struct cros_feature_to_name cros_mcu_devices[] = { .name = CROS_EC_DEV_SCP_NAME, .desc = "System Control Processor", }, + { + .id = EC_FEATURE_SCP_C1, + .name = CROS_EC_DEV_SCP_C1_NAME, + .desc = "System Control Processor 2nd Core", + }, { .id = EC_FEATURE_TOUCHPAD, .name = CROS_EC_DEV_TP_NAME, diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 8cfa8cfca77e..9fbf1c5eb8d3 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -1300,6 +1300,8 @@ enum ec_feature_code { * mux. */ EC_FEATURE_TYPEC_MUX_REQUIRE_AP_ACK = 43, + /* The MCU is a System Companion Processor (SCP) 2nd Core. */ + EC_FEATURE_SCP_C1 = 45, }; #define EC_FEATURE_MASK_0(event_code) BIT(event_code % 32) diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index 138fd912c808..da06dc7cf1cb 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -19,6 +19,7 @@ #define CROS_EC_DEV_ISH_NAME "cros_ish" #define CROS_EC_DEV_PD_NAME "cros_pd" #define CROS_EC_DEV_SCP_NAME "cros_scp" +#define CROS_EC_DEV_SCP_C1_NAME "cros_scp_c1" #define CROS_EC_DEV_TP_NAME "cros_tp" /* -- cgit v1.2.3 From 4a346a03a63cb45f7766d9d6559cf3fee783e926 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Tue, 14 Jun 2022 17:21:48 +0200 Subject: mfd: twl: Remove platform data support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no in-tree machine that provides a struct twl4030_platform_data since commit e92fc4f04a34 ("ARM: OMAP2+: Drop legacy board file for LDP"). So assume dev_get_platdata() returns NULL in twl_probe() and simplify accordingly. Signed-off-by: Uwe Kleine-König Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220614152148.252820-1-u.kleine-koenig@pengutronix.de --- drivers/mfd/twl-core.c | 323 +----------------------------------------------- include/linux/mfd/twl.h | 55 --------- 2 files changed, 5 insertions(+), 373 deletions(-) (limited to 'include/linux') diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c index bd6659cf3bc0..2cb9326f3e61 100644 --- a/drivers/mfd/twl-core.c +++ b/drivers/mfd/twl-core.c @@ -656,309 +656,6 @@ static inline struct device *add_child(unsigned mod_no, const char *name, can_wakeup, irq0, irq1); } -static struct device * -add_regulator_linked(int num, struct regulator_init_data *pdata, - struct regulator_consumer_supply *consumers, - unsigned num_consumers, unsigned long features) -{ - struct twl_regulator_driver_data drv_data; - - /* regulator framework demands init_data ... */ - if (!pdata) - return NULL; - - if (consumers) { - pdata->consumer_supplies = consumers; - pdata->num_consumer_supplies = num_consumers; - } - - if (pdata->driver_data) { - /* If we have existing drv_data, just add the flags */ - struct twl_regulator_driver_data *tmp; - tmp = pdata->driver_data; - tmp->features |= features; - } else { - /* add new driver data struct, used only during init */ - drv_data.features = features; - drv_data.set_voltage = NULL; - drv_data.get_voltage = NULL; - drv_data.data = NULL; - pdata->driver_data = &drv_data; - } - - /* NOTE: we currently ignore regulator IRQs, e.g. for short circuits */ - return add_numbered_child(TWL_MODULE_PM_MASTER, "twl_reg", num, - pdata, sizeof(*pdata), false, 0, 0); -} - -static struct device * -add_regulator(int num, struct regulator_init_data *pdata, - unsigned long features) -{ - return add_regulator_linked(num, pdata, NULL, 0, features); -} - -/* - * NOTE: We know the first 8 IRQs after pdata->base_irq are - * for the PIH, and the next are for the PWR_INT SIH, since - * that's how twl_init_irq() sets things up. - */ - -static int -add_children(struct twl4030_platform_data *pdata, unsigned irq_base, - unsigned long features) -{ - struct device *child; - - if (IS_ENABLED(CONFIG_GPIO_TWL4030) && pdata->gpio) { - child = add_child(TWL4030_MODULE_GPIO, "twl4030_gpio", - pdata->gpio, sizeof(*pdata->gpio), - false, irq_base + GPIO_INTR_OFFSET, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_KEYBOARD_TWL4030) && pdata->keypad) { - child = add_child(TWL4030_MODULE_KEYPAD, "twl4030_keypad", - pdata->keypad, sizeof(*pdata->keypad), - true, irq_base + KEYPAD_INTR_OFFSET, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_TWL4030_MADC) && pdata->madc && - twl_class_is_4030()) { - child = add_child(TWL4030_MODULE_MADC, "twl4030_madc", - pdata->madc, sizeof(*pdata->madc), - true, irq_base + MADC_INTR_OFFSET, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_RTC_DRV_TWL4030)) { - /* - * REVISIT platform_data here currently might expose the - * "msecure" line ... but for now we just expect board - * setup to tell the chip "it's always ok to SET_TIME". - * Eventually, Linux might become more aware of such - * HW security concerns, and "least privilege". - */ - child = add_child(TWL_MODULE_RTC, "twl_rtc", NULL, 0, - true, irq_base + RTC_INTR_OFFSET, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_PWM_TWL)) { - child = add_child(TWL_MODULE_PWM, "twl-pwm", NULL, 0, - false, 0, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_PWM_TWL_LED)) { - child = add_child(TWL_MODULE_LED, "twl-pwmled", NULL, 0, - false, 0, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_TWL4030_USB) && pdata->usb && - twl_class_is_4030()) { - - static struct regulator_consumer_supply usb1v5 = { - .supply = "usb1v5", - }; - static struct regulator_consumer_supply usb1v8 = { - .supply = "usb1v8", - }; - static struct regulator_consumer_supply usb3v1 = { - .supply = "usb3v1", - }; - - /* First add the regulators so that they can be used by transceiver */ - if (IS_ENABLED(CONFIG_REGULATOR_TWL4030)) { - /* this is a template that gets copied */ - struct regulator_init_data usb_fixed = { - .constraints.valid_modes_mask = - REGULATOR_MODE_NORMAL - | REGULATOR_MODE_STANDBY, - .constraints.valid_ops_mask = - REGULATOR_CHANGE_MODE - | REGULATOR_CHANGE_STATUS, - }; - - child = add_regulator_linked(TWL4030_REG_VUSB1V5, - &usb_fixed, &usb1v5, 1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator_linked(TWL4030_REG_VUSB1V8, - &usb_fixed, &usb1v8, 1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator_linked(TWL4030_REG_VUSB3V1, - &usb_fixed, &usb3v1, 1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - } - - child = add_child(TWL_MODULE_USB, "twl4030_usb", - pdata->usb, sizeof(*pdata->usb), true, - /* irq0 = USB_PRES, irq1 = USB */ - irq_base + USB_PRES_INTR_OFFSET, - irq_base + USB_INTR_OFFSET); - - if (IS_ERR(child)) - return PTR_ERR(child); - - /* we need to connect regulators to this transceiver */ - if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && child) { - usb1v5.dev_name = dev_name(child); - usb1v8.dev_name = dev_name(child); - usb3v1.dev_name = dev_name(child); - } - } - - if (IS_ENABLED(CONFIG_TWL4030_WATCHDOG) && twl_class_is_4030()) { - child = add_child(TWL_MODULE_PM_RECEIVER, "twl4030_wdt", NULL, - 0, false, 0, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_INPUT_TWL4030_PWRBUTTON) && twl_class_is_4030()) { - child = add_child(TWL_MODULE_PM_MASTER, "twl4030_pwrbutton", - NULL, 0, true, irq_base + 8 + 0, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_MFD_TWL4030_AUDIO) && pdata->audio && - twl_class_is_4030()) { - child = add_child(TWL4030_MODULE_AUDIO_VOICE, "twl4030-audio", - pdata->audio, sizeof(*pdata->audio), - false, 0, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - /* twl4030 regulators */ - if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && twl_class_is_4030()) { - child = add_regulator(TWL4030_REG_VPLL1, pdata->vpll1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VIO, pdata->vio, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VDD1, pdata->vdd1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VDD2, pdata->vdd2, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VMMC1, pdata->vmmc1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VDAC, pdata->vdac, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator((features & TWL4030_VAUX2) - ? TWL4030_REG_VAUX2_4030 - : TWL4030_REG_VAUX2, - pdata->vaux2, features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VINTANA1, pdata->vintana1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VINTANA2, pdata->vintana2, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VINTDIG, pdata->vintdig, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - /* maybe add LDOs that are omitted on cost-reduced parts */ - if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && !(features & TPS_SUBSET) - && twl_class_is_4030()) { - child = add_regulator(TWL4030_REG_VPLL2, pdata->vpll2, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VMMC2, pdata->vmmc2, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VSIM, pdata->vsim, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VAUX1, pdata->vaux1, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VAUX3, pdata->vaux3, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - - child = add_regulator(TWL4030_REG_VAUX4, pdata->vaux4, - features); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_CHARGER_TWL4030) && pdata->bci && - !(features & (TPS_SUBSET | TWL5031))) { - child = add_child(TWL_MODULE_MAIN_CHARGE, "twl4030_bci", - pdata->bci, sizeof(*pdata->bci), false, - /* irq0 = CHG_PRES, irq1 = BCI */ - irq_base + BCI_PRES_INTR_OFFSET, - irq_base + BCI_INTR_OFFSET); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - if (IS_ENABLED(CONFIG_TWL4030_POWER) && pdata->power) { - child = add_child(TWL_MODULE_PM_MASTER, "twl4030_power", - pdata->power, sizeof(*pdata->power), false, - 0, 0); - if (IS_ERR(child)) - return PTR_ERR(child); - } - - return 0; -} - /*----------------------------------------------------------------------*/ /* @@ -987,8 +684,7 @@ static inline int unprotect_pm_master(void) return e; } -static void clocks_init(struct device *dev, - struct twl4030_clock_init_data *clock) +static void clocks_init(struct device *dev) { int e = 0; struct clk *osc; @@ -1018,8 +714,6 @@ static void clocks_init(struct device *dev, } ctrl |= HIGH_PERF_SQ; - if (clock && clock->ck32k_lowpwr_enable) - ctrl |= CK32K_LOWPWR_EN; e |= unprotect_pm_master(); /* effect->MADC+USB ck en */ @@ -1063,7 +757,6 @@ static struct of_dev_auxdata twl_auxdata_lookup[] = { static int twl_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct twl4030_platform_data *pdata = dev_get_platdata(&client->dev); struct device_node *node = client->dev.of_node; struct platform_device *pdev; const struct regmap_config *twl_regmap_config; @@ -1071,7 +764,7 @@ twl_probe(struct i2c_client *client, const struct i2c_device_id *id) int status; unsigned i, num_slaves; - if (!node && !pdata) { + if (!node) { dev_err(&client->dev, "no platform data\n"); return -EINVAL; } @@ -1161,7 +854,7 @@ twl_probe(struct i2c_client *client, const struct i2c_device_id *id) twl_priv->ready = true; /* setup clock framework */ - clocks_init(&client->dev, pdata ? pdata->clock : NULL); + clocks_init(&client->dev); /* read TWL IDCODE Register */ if (twl_class_is_4030()) { @@ -1209,14 +902,8 @@ twl_probe(struct i2c_client *client, const struct i2c_device_id *id) TWL4030_DCDC_GLOBAL_CFG); } - if (node) { - if (pdata) - twl_auxdata_lookup[0].platform_data = pdata->gpio; - status = of_platform_populate(node, NULL, twl_auxdata_lookup, - &client->dev); - } else { - status = add_children(pdata, irq_base, id->driver_data); - } + status = of_platform_populate(node, NULL, twl_auxdata_lookup, + &client->dev); fail: if (status < 0) diff --git a/include/linux/mfd/twl.h b/include/linux/mfd/twl.h index 8871cc5188a0..e426938eafd5 100644 --- a/include/linux/mfd/twl.h +++ b/include/linux/mfd/twl.h @@ -694,61 +694,6 @@ struct twl4030_audio_data { unsigned int irq_base; }; -struct twl4030_platform_data { - struct twl4030_clock_init_data *clock; - struct twl4030_bci_platform_data *bci; - struct twl4030_gpio_platform_data *gpio; - struct twl4030_madc_platform_data *madc; - struct twl4030_keypad_data *keypad; - struct twl4030_usb_data *usb; - struct twl4030_power_data *power; - struct twl4030_audio_data *audio; - - /* Common LDO regulators for TWL4030/TWL6030 */ - struct regulator_init_data *vdac; - struct regulator_init_data *vaux1; - struct regulator_init_data *vaux2; - struct regulator_init_data *vaux3; - struct regulator_init_data *vdd1; - struct regulator_init_data *vdd2; - struct regulator_init_data *vdd3; - /* TWL4030 LDO regulators */ - struct regulator_init_data *vpll1; - struct regulator_init_data *vpll2; - struct regulator_init_data *vmmc1; - struct regulator_init_data *vmmc2; - struct regulator_init_data *vsim; - struct regulator_init_data *vaux4; - struct regulator_init_data *vio; - struct regulator_init_data *vintana1; - struct regulator_init_data *vintana2; - struct regulator_init_data *vintdig; - /* TWL6030 LDO regulators */ - struct regulator_init_data *vmmc; - struct regulator_init_data *vpp; - struct regulator_init_data *vusim; - struct regulator_init_data *vana; - struct regulator_init_data *vcxio; - struct regulator_init_data *vusb; - struct regulator_init_data *clk32kg; - struct regulator_init_data *v1v8; - struct regulator_init_data *v2v1; - /* TWL6032 LDO regulators */ - struct regulator_init_data *ldo1; - struct regulator_init_data *ldo2; - struct regulator_init_data *ldo3; - struct regulator_init_data *ldo4; - struct regulator_init_data *ldo5; - struct regulator_init_data *ldo6; - struct regulator_init_data *ldo7; - struct regulator_init_data *ldoln; - struct regulator_init_data *ldousb; - /* TWL6032 DCDC regulators */ - struct regulator_init_data *smps3; - struct regulator_init_data *smps4; - struct regulator_init_data *vio6025; -}; - struct twl_regulator_driver_data { int (*set_voltage)(void *data, int target_uV); int (*get_voltage)(void *data); -- cgit v1.2.3 From c55333064d6ea9a26c7e8cfbe2ba1fa77c3a8e48 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sun, 19 Jun 2022 10:26:55 +0200 Subject: mfd: tc6393xb: Make disable callback return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All implementations return 0, so simplify accordingly. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Reported-by: kernel test robot Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220619082655.53728-1-u.kleine-koenig@pengutronix.de --- arch/arm/mach-pxa/eseries.c | 3 +-- arch/arm/mach-pxa/tosa.c | 4 +--- drivers/mfd/tc6393xb.c | 5 ++--- include/linux/mfd/tc6393xb.h | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/mach-pxa/eseries.c b/arch/arm/mach-pxa/eseries.c index 99781eec065e..6226aee152d3 100644 --- a/arch/arm/mach-pxa/eseries.c +++ b/arch/arm/mach-pxa/eseries.c @@ -86,11 +86,10 @@ int eseries_tmio_enable(struct platform_device *dev) return 0; } -int eseries_tmio_disable(struct platform_device *dev) +void eseries_tmio_disable(struct platform_device *dev) { gpio_set_value(GPIO_ESERIES_TMIO_SUSPEND, 0); gpio_set_value(GPIO_ESERIES_TMIO_PCLR, 0); - return 0; } int eseries_tmio_suspend(struct platform_device *dev) diff --git a/arch/arm/mach-pxa/tosa.c b/arch/arm/mach-pxa/tosa.c index 6af8bc404825..d41641d6cfcd 100644 --- a/arch/arm/mach-pxa/tosa.c +++ b/arch/arm/mach-pxa/tosa.c @@ -678,13 +678,11 @@ err_req_pclr: return rc; } -static int tosa_tc6393xb_disable(struct platform_device *dev) +static void tosa_tc6393xb_disable(struct platform_device *dev) { gpio_free(TOSA_GPIO_TC6393XB_L3V_ON); gpio_free(TOSA_GPIO_TC6393XB_SUSPEND); gpio_free(TOSA_GPIO_TC6393XB_REST_IN); - - return 0; } static int tosa_tc6393xb_resume(struct platform_device *dev) diff --git a/drivers/mfd/tc6393xb.c b/drivers/mfd/tc6393xb.c index 0be5731685b4..aa903a31dd43 100644 --- a/drivers/mfd/tc6393xb.c +++ b/drivers/mfd/tc6393xb.c @@ -798,20 +798,19 @@ static int tc6393xb_remove(struct platform_device *dev) { struct tc6393xb_platform_data *tcpd = dev_get_platdata(&dev->dev); struct tc6393xb *tc6393xb = platform_get_drvdata(dev); - int ret; mfd_remove_devices(&dev->dev); tc6393xb_detach_irq(dev); - ret = tcpd->disable(dev); + tcpd->disable(dev); clk_disable_unprepare(tc6393xb->clk); iounmap(tc6393xb->scr); release_resource(&tc6393xb->rscr); clk_put(tc6393xb->clk); kfree(tc6393xb); - return ret; + return 0; } #ifdef CONFIG_PM diff --git a/include/linux/mfd/tc6393xb.h b/include/linux/mfd/tc6393xb.h index d336c541b7df..d17807f2d0c9 100644 --- a/include/linux/mfd/tc6393xb.h +++ b/include/linux/mfd/tc6393xb.h @@ -22,7 +22,7 @@ struct tc6393xb_platform_data { u16 scr_gper; /* GP Enable */ int (*enable)(struct platform_device *dev); - int (*disable)(struct platform_device *dev); + void (*disable)(struct platform_device *dev); int (*suspend)(struct platform_device *dev); int (*resume)(struct platform_device *dev); -- cgit v1.2.3 From 79f821b5a3bf46d2d5ee2dc0e2b2428b1062a040 Mon Sep 17 00:00:00 2001 From: Zhang Jiaming Date: Thu, 23 Jun 2022 15:33:33 +0800 Subject: mfd: ipaq-micro: Fix spelling mistake of "receive{d}" Change 'receieved' to 'received' and 'recieve' to 'receive'. Signed-off-by: Zhang Jiaming Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220623073333.5675-1-jiaming@nfschina.com --- include/linux/mfd/ipaq-micro.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/ipaq-micro.h b/include/linux/mfd/ipaq-micro.h index ee48a4321c57..d5caa4c86ecc 100644 --- a/include/linux/mfd/ipaq-micro.h +++ b/include/linux/mfd/ipaq-micro.h @@ -75,8 +75,8 @@ struct ipaq_micro_rxdev { * @id: 4-bit ID of the message * @tx_len: length of TX data * @tx_data: TX data to send - * @rx_len: length of receieved RX data - * @rx_data: RX data to recieve + * @rx_len: length of received RX data + * @rx_data: RX data to receive * @ack: a completion that will be completed when RX is complete * @node: list node if message gets queued */ -- cgit v1.2.3 From d9cd0bc604705eb01497c3e483f8820a9677c682 Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Mon, 27 Jun 2022 14:39:54 +0200 Subject: mfd: mt6397: Add basic support for MT6331+MT6332 PMIC Add support for the MT6331 PMIC with MT6332 Companion PMIC, found in MT6795 Helio X10 smartphone platforms. This combo has support for multiple devices but, for a start, only the following have been implemented: - Regulators (two instances, one in MT6331, one in MT6332) - RTC (MT6331) - Keys (MT6331) - Interrupts (MT6331 also dispatches MT6332's interrupts) There's more to be implemented, especially for MT6332, which will come at a later stage. Signed-off-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20220627123954.64299-1-angelogioacchino.delregno@collabora.com --- drivers/mfd/mt6397-core.c | 47 +++ drivers/mfd/mt6397-irq.c | 9 +- include/linux/mfd/mt6331/core.h | 40 +++ include/linux/mfd/mt6331/registers.h | 584 +++++++++++++++++++++++++++++++ include/linux/mfd/mt6332/core.h | 65 ++++ include/linux/mfd/mt6332/registers.h | 642 +++++++++++++++++++++++++++++++++++ include/linux/mfd/mt6397/core.h | 2 + 7 files changed, 1388 insertions(+), 1 deletion(-) create mode 100644 include/linux/mfd/mt6331/core.h create mode 100644 include/linux/mfd/mt6331/registers.h create mode 100644 include/linux/mfd/mt6332/core.h create mode 100644 include/linux/mfd/mt6332/registers.h (limited to 'include/linux') diff --git a/drivers/mfd/mt6397-core.c b/drivers/mfd/mt6397-core.c index 3cb8836bd08d..f6c1f80f94a4 100644 --- a/drivers/mfd/mt6397-core.c +++ b/drivers/mfd/mt6397-core.c @@ -12,11 +12,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -28,6 +30,9 @@ #define MT6357_RTC_BASE 0x0588 #define MT6357_RTC_SIZE 0x3c +#define MT6331_RTC_BASE 0x4000 +#define MT6331_RTC_SIZE 0x40 + #define MT6358_RTC_BASE 0x0588 #define MT6358_RTC_SIZE 0x3c @@ -47,6 +52,11 @@ static const struct resource mt6357_rtc_resources[] = { DEFINE_RES_IRQ(MT6357_IRQ_RTC), }; +static const struct resource mt6331_rtc_resources[] = { + DEFINE_RES_MEM(MT6331_RTC_BASE, MT6331_RTC_SIZE), + DEFINE_RES_IRQ(MT6331_IRQ_STATUS_RTC), +}; + static const struct resource mt6358_rtc_resources[] = { DEFINE_RES_MEM(MT6358_RTC_BASE, MT6358_RTC_SIZE), DEFINE_RES_IRQ(MT6358_IRQ_RTC), @@ -83,6 +93,11 @@ static const struct resource mt6357_keys_resources[] = { DEFINE_RES_IRQ_NAMED(MT6357_IRQ_HOMEKEY_R, "homekey_r"), }; +static const struct resource mt6331_keys_resources[] = { + DEFINE_RES_IRQ_NAMED(MT6331_IRQ_STATUS_PWRKEY, "powerkey"), + DEFINE_RES_IRQ_NAMED(MT6331_IRQ_STATUS_HOMEKEY, "homekey"), +}; + static const struct resource mt6397_keys_resources[] = { DEFINE_RES_IRQ_NAMED(MT6397_IRQ_PWRKEY, "powerkey"), DEFINE_RES_IRQ_NAMED(MT6397_IRQ_HOMEKEY, "homekey"), @@ -133,6 +148,27 @@ static const struct mfd_cell mt6357_devs[] = { }, }; +/* MT6331 is always used in combination with MT6332 */ +static const struct mfd_cell mt6331_mt6332_devs[] = { + { + .name = "mt6331-rtc", + .num_resources = ARRAY_SIZE(mt6331_rtc_resources), + .resources = mt6331_rtc_resources, + .of_compatible = "mediatek,mt6331-rtc", + }, { + .name = "mt6331-regulator", + .of_compatible = "mediatek,mt6331-regulator" + }, { + .name = "mt6332-regulator", + .of_compatible = "mediatek,mt6332-regulator" + }, { + .name = "mtk-pmic-keys", + .num_resources = ARRAY_SIZE(mt6331_keys_resources), + .resources = mt6331_keys_resources, + .of_compatible = "mediatek,mt6331-keys" + }, +}; + static const struct mfd_cell mt6358_devs[] = { { .name = "mt6358-regulator", @@ -220,6 +256,14 @@ static const struct chip_data mt6357_core = { .irq_init = mt6358_irq_init, }; +static const struct chip_data mt6331_mt6332_core = { + .cid_addr = MT6331_HWCID, + .cid_shift = 0, + .cells = mt6331_mt6332_devs, + .cell_size = ARRAY_SIZE(mt6331_mt6332_devs), + .irq_init = mt6397_irq_init, +}; + static const struct chip_data mt6358_core = { .cid_addr = MT6358_SWCID, .cid_shift = 8, @@ -302,6 +346,9 @@ static const struct of_device_id mt6397_of_match[] = { { .compatible = "mediatek,mt6323", .data = &mt6323_core, + }, { + .compatible = "mediatek,mt6331", + .data = &mt6331_mt6332_core, }, { .compatible = "mediatek,mt6357", .data = &mt6357_core, diff --git a/drivers/mfd/mt6397-irq.c b/drivers/mfd/mt6397-irq.c index 2924919da991..eff53fed8fe7 100644 --- a/drivers/mfd/mt6397-irq.c +++ b/drivers/mfd/mt6397-irq.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -172,7 +174,12 @@ int mt6397_irq_init(struct mt6397_chip *chip) chip->int_status[0] = MT6323_INT_STATUS0; chip->int_status[1] = MT6323_INT_STATUS1; break; - + case MT6331_CHIP_ID: + chip->int_con[0] = MT6331_INT_CON0; + chip->int_con[1] = MT6331_INT_CON1; + chip->int_status[0] = MT6331_INT_STATUS_CON0; + chip->int_status[1] = MT6331_INT_STATUS_CON1; + break; case MT6391_CHIP_ID: case MT6397_CHIP_ID: chip->int_con[0] = MT6397_INT_CON0; diff --git a/include/linux/mfd/mt6331/core.h b/include/linux/mfd/mt6331/core.h new file mode 100644 index 000000000000..df8e6b1e4bc1 --- /dev/null +++ b/include/linux/mfd/mt6331/core.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 AngeloGioacchino Del Regno + */ + +#ifndef __MFD_MT6331_CORE_H__ +#define __MFD_MT6331_CORE_H__ + +enum mt6331_irq_status_numbers { + MT6331_IRQ_STATUS_PWRKEY = 0, + MT6331_IRQ_STATUS_HOMEKEY, + MT6331_IRQ_STATUS_CHRDET, + MT6331_IRQ_STATUS_THR_H, + MT6331_IRQ_STATUS_THR_L, + MT6331_IRQ_STATUS_BAT_H, + MT6331_IRQ_STATUS_BAT_L, + MT6331_IRQ_STATUS_RTC, + MT6331_IRQ_STATUS_AUDIO, + MT6331_IRQ_STATUS_MAD, + MT6331_IRQ_STATUS_ACCDET, + MT6331_IRQ_STATUS_ACCDET_EINT, + MT6331_IRQ_STATUS_ACCDET_NEGV = 12, + MT6331_IRQ_STATUS_VDVFS11_OC = 16, + MT6331_IRQ_STATUS_VDVFS12_OC, + MT6331_IRQ_STATUS_VDVFS13_OC, + MT6331_IRQ_STATUS_VDVFS14_OC, + MT6331_IRQ_STATUS_GPU_OC, + MT6331_IRQ_STATUS_VCORE1_OC, + MT6331_IRQ_STATUS_VCORE2_OC, + MT6331_IRQ_STATUS_VIO18_OC, + MT6331_IRQ_STATUS_LDO_OC, + MT6331_IRQ_STATUS_NR, +}; + +#define MT6331_IRQ_CON0_BASE MT6331_IRQ_STATUS_PWRKEY +#define MT6331_IRQ_CON0_BITS (MT6331_IRQ_STATUS_ACCDET_NEGV + 1) +#define MT6331_IRQ_CON1_BASE MT6331_IRQ_STATUS_VDVFS11_OC +#define MT6331_IRQ_CON1_BITS (MT6331_IRQ_STATUS_LDO_OC - MT6331_IRQ_STATUS_VDFS11_OC + 1) + +#endif /* __MFD_MT6331_CORE_H__ */ diff --git a/include/linux/mfd/mt6331/registers.h b/include/linux/mfd/mt6331/registers.h new file mode 100644 index 000000000000..e2be6bccd1a7 --- /dev/null +++ b/include/linux/mfd/mt6331/registers.h @@ -0,0 +1,584 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 AngeloGioacchino Del Regno + */ + +#ifndef __MFD_MT6331_REGISTERS_H__ +#define __MFD_MT6331_REGISTERS_H__ + +/* PMIC Registers */ +#define MT6331_STRUP_CON0 0x0 +#define MT6331_STRUP_CON2 0x2 +#define MT6331_STRUP_CON3 0x4 +#define MT6331_STRUP_CON4 0x6 +#define MT6331_STRUP_CON5 0x8 +#define MT6331_STRUP_CON6 0xA +#define MT6331_STRUP_CON7 0xC +#define MT6331_STRUP_CON8 0xE +#define MT6331_STRUP_CON9 0x10 +#define MT6331_STRUP_CON10 0x12 +#define MT6331_STRUP_CON11 0x14 +#define MT6331_STRUP_CON12 0x16 +#define MT6331_STRUP_CON13 0x18 +#define MT6331_STRUP_CON14 0x1A +#define MT6331_STRUP_CON15 0x1C +#define MT6331_STRUP_CON16 0x1E +#define MT6331_STRUP_CON17 0x20 +#define MT6331_STRUP_CON18 0x22 +#define MT6331_HWCID 0x100 +#define MT6331_SWCID 0x102 +#define MT6331_EXT_PMIC_STATUS 0x104 +#define MT6331_TOP_CON 0x106 +#define MT6331_TEST_OUT 0x108 +#define MT6331_TEST_CON0 0x10A +#define MT6331_TEST_CON1 0x10C +#define MT6331_TESTMODE_SW 0x10E +#define MT6331_EN_STATUS0 0x110 +#define MT6331_EN_STATUS1 0x112 +#define MT6331_EN_STATUS2 0x114 +#define MT6331_OCSTATUS0 0x116 +#define MT6331_OCSTATUS1 0x118 +#define MT6331_OCSTATUS2 0x11A +#define MT6331_PGSTATUS 0x11C +#define MT6331_TOPSTATUS 0x11E +#define MT6331_TDSEL_CON 0x120 +#define MT6331_RDSEL_CON 0x122 +#define MT6331_SMT_CON0 0x124 +#define MT6331_SMT_CON1 0x126 +#define MT6331_SMT_CON2 0x128 +#define MT6331_DRV_CON0 0x12A +#define MT6331_DRV_CON1 0x12C +#define MT6331_DRV_CON2 0x12E +#define MT6331_DRV_CON3 0x130 +#define MT6331_TOP_STATUS 0x132 +#define MT6331_TOP_STATUS_SET 0x134 +#define MT6331_TOP_STATUS_CLR 0x136 +#define MT6331_TOP_CKPDN_CON0 0x138 +#define MT6331_TOP_CKPDN_CON0_SET 0x13A +#define MT6331_TOP_CKPDN_CON0_CLR 0x13C +#define MT6331_TOP_CKPDN_CON1 0x13E +#define MT6331_TOP_CKPDN_CON1_SET 0x140 +#define MT6331_TOP_CKPDN_CON1_CLR 0x142 +#define MT6331_TOP_CKPDN_CON2 0x144 +#define MT6331_TOP_CKPDN_CON2_SET 0x146 +#define MT6331_TOP_CKPDN_CON2_CLR 0x148 +#define MT6331_TOP_CKSEL_CON 0x14A +#define MT6331_TOP_CKSEL_CON_SET 0x14C +#define MT6331_TOP_CKSEL_CON_CLR 0x14E +#define MT6331_TOP_CKHWEN_CON 0x150 +#define MT6331_TOP_CKHWEN_CON_SET 0x152 +#define MT6331_TOP_CKHWEN_CON_CLR 0x154 +#define MT6331_TOP_CKTST_CON0 0x156 +#define MT6331_TOP_CKTST_CON1 0x158 +#define MT6331_TOP_CLKSQ 0x15A +#define MT6331_TOP_CLKSQ_SET 0x15C +#define MT6331_TOP_CLKSQ_CLR 0x15E +#define MT6331_TOP_RST_CON 0x160 +#define MT6331_TOP_RST_CON_SET 0x162 +#define MT6331_TOP_RST_CON_CLR 0x164 +#define MT6331_TOP_RST_MISC 0x166 +#define MT6331_TOP_RST_MISC_SET 0x168 +#define MT6331_TOP_RST_MISC_CLR 0x16A +#define MT6331_INT_CON0 0x16C +#define MT6331_INT_CON0_SET 0x16E +#define MT6331_INT_CON0_CLR 0x170 +#define MT6331_INT_CON1 0x172 +#define MT6331_INT_CON1_SET 0x174 +#define MT6331_INT_CON1_CLR 0x176 +#define MT6331_INT_MISC_CON 0x178 +#define MT6331_INT_MISC_CON_SET 0x17A +#define MT6331_INT_MISC_CON_CLR 0x17C +#define MT6331_INT_STATUS_CON0 0x17E +#define MT6331_INT_STATUS_CON1 0x180 +#define MT6331_OC_GEAR_0 0x182 +#define MT6331_FQMTR_CON0 0x184 +#define MT6331_FQMTR_CON1 0x186 +#define MT6331_FQMTR_CON2 0x188 +#define MT6331_RG_SPI_CON 0x18A +#define MT6331_DEW_DIO_EN 0x18C +#define MT6331_DEW_READ_TEST 0x18E +#define MT6331_DEW_WRITE_TEST 0x190 +#define MT6331_DEW_CRC_SWRST 0x192 +#define MT6331_DEW_CRC_EN 0x194 +#define MT6331_DEW_CRC_VAL 0x196 +#define MT6331_DEW_DBG_MON_SEL 0x198 +#define MT6331_DEW_CIPHER_KEY_SEL 0x19A +#define MT6331_DEW_CIPHER_IV_SEL 0x19C +#define MT6331_DEW_CIPHER_EN 0x19E +#define MT6331_DEW_CIPHER_RDY 0x1A0 +#define MT6331_DEW_CIPHER_MODE 0x1A2 +#define MT6331_DEW_CIPHER_SWRST 0x1A4 +#define MT6331_DEW_RDDMY_NO 0x1A6 +#define MT6331_INT_TYPE_CON0 0x1A8 +#define MT6331_INT_TYPE_CON0_SET 0x1AA +#define MT6331_INT_TYPE_CON0_CLR 0x1AC +#define MT6331_INT_TYPE_CON1 0x1AE +#define MT6331_INT_TYPE_CON1_SET 0x1B0 +#define MT6331_INT_TYPE_CON1_CLR 0x1B2 +#define MT6331_INT_STA 0x1B4 +#define MT6331_BUCK_ALL_CON0 0x200 +#define MT6331_BUCK_ALL_CON1 0x202 +#define MT6331_BUCK_ALL_CON2 0x204 +#define MT6331_BUCK_ALL_CON3 0x206 +#define MT6331_BUCK_ALL_CON4 0x208 +#define MT6331_BUCK_ALL_CON5 0x20A +#define MT6331_BUCK_ALL_CON6 0x20C +#define MT6331_BUCK_ALL_CON7 0x20E +#define MT6331_BUCK_ALL_CON8 0x210 +#define MT6331_BUCK_ALL_CON9 0x212 +#define MT6331_BUCK_ALL_CON10 0x214 +#define MT6331_BUCK_ALL_CON11 0x216 +#define MT6331_BUCK_ALL_CON12 0x218 +#define MT6331_BUCK_ALL_CON13 0x21A +#define MT6331_BUCK_ALL_CON14 0x21C +#define MT6331_BUCK_ALL_CON15 0x21E +#define MT6331_BUCK_ALL_CON16 0x220 +#define MT6331_BUCK_ALL_CON17 0x222 +#define MT6331_BUCK_ALL_CON18 0x224 +#define MT6331_BUCK_ALL_CON19 0x226 +#define MT6331_BUCK_ALL_CON20 0x228 +#define MT6331_BUCK_ALL_CON21 0x22A +#define MT6331_BUCK_ALL_CON22 0x22C +#define MT6331_BUCK_ALL_CON23 0x22E +#define MT6331_BUCK_ALL_CON24 0x230 +#define MT6331_BUCK_ALL_CON25 0x232 +#define MT6331_BUCK_ALL_CON26 0x234 +#define MT6331_VDVFS11_CON0 0x236 +#define MT6331_VDVFS11_CON1 0x238 +#define MT6331_VDVFS11_CON2 0x23A +#define MT6331_VDVFS11_CON3 0x23C +#define MT6331_VDVFS11_CON4 0x23E +#define MT6331_VDVFS11_CON5 0x240 +#define MT6331_VDVFS11_CON6 0x242 +#define MT6331_VDVFS11_CON7 0x244 +#define MT6331_VDVFS11_CON8 0x246 +#define MT6331_VDVFS11_CON9 0x248 +#define MT6331_VDVFS11_CON10 0x24A +#define MT6331_VDVFS11_CON11 0x24C +#define MT6331_VDVFS11_CON12 0x24E +#define MT6331_VDVFS11_CON13 0x250 +#define MT6331_VDVFS11_CON14 0x252 +#define MT6331_VDVFS11_CON18 0x25A +#define MT6331_VDVFS11_CON19 0x25C +#define MT6331_VDVFS11_CON20 0x25E +#define MT6331_VDVFS11_CON21 0x260 +#define MT6331_VDVFS11_CON22 0x262 +#define MT6331_VDVFS11_CON23 0x264 +#define MT6331_VDVFS11_CON24 0x266 +#define MT6331_VDVFS11_CON25 0x268 +#define MT6331_VDVFS11_CON26 0x26A +#define MT6331_VDVFS11_CON27 0x26C +#define MT6331_VDVFS12_CON0 0x26E +#define MT6331_VDVFS12_CON1 0x270 +#define MT6331_VDVFS12_CON2 0x272 +#define MT6331_VDVFS12_CON3 0x274 +#define MT6331_VDVFS12_CON4 0x276 +#define MT6331_VDVFS12_CON5 0x278 +#define MT6331_VDVFS12_CON6 0x27A +#define MT6331_VDVFS12_CON7 0x27C +#define MT6331_VDVFS12_CON8 0x27E +#define MT6331_VDVFS12_CON9 0x280 +#define MT6331_VDVFS12_CON10 0x282 +#define MT6331_VDVFS12_CON11 0x284 +#define MT6331_VDVFS12_CON12 0x286 +#define MT6331_VDVFS12_CON13 0x288 +#define MT6331_VDVFS12_CON14 0x28A +#define MT6331_VDVFS12_CON18 0x292 +#define MT6331_VDVFS12_CON19 0x294 +#define MT6331_VDVFS12_CON20 0x296 +#define MT6331_VDVFS13_CON0 0x298 +#define MT6331_VDVFS13_CON1 0x29A +#define MT6331_VDVFS13_CON2 0x29C +#define MT6331_VDVFS13_CON3 0x29E +#define MT6331_VDVFS13_CON4 0x2A0 +#define MT6331_VDVFS13_CON5 0x2A2 +#define MT6331_VDVFS13_CON6 0x2A4 +#define MT6331_VDVFS13_CON7 0x2A6 +#define MT6331_VDVFS13_CON8 0x2A8 +#define MT6331_VDVFS13_CON9 0x2AA +#define MT6331_VDVFS13_CON10 0x2AC +#define MT6331_VDVFS13_CON11 0x2AE +#define MT6331_VDVFS13_CON12 0x2B0 +#define MT6331_VDVFS13_CON13 0x2B2 +#define MT6331_VDVFS13_CON14 0x2B4 +#define MT6331_VDVFS13_CON18 0x2BC +#define MT6331_VDVFS13_CON19 0x2BE +#define MT6331_VDVFS13_CON20 0x2C0 +#define MT6331_VDVFS14_CON0 0x2C2 +#define MT6331_VDVFS14_CON1 0x2C4 +#define MT6331_VDVFS14_CON2 0x2C6 +#define MT6331_VDVFS14_CON3 0x2C8 +#define MT6331_VDVFS14_CON4 0x2CA +#define MT6331_VDVFS14_CON5 0x2CC +#define MT6331_VDVFS14_CON6 0x2CE +#define MT6331_VDVFS14_CON7 0x2D0 +#define MT6331_VDVFS14_CON8 0x2D2 +#define MT6331_VDVFS14_CON9 0x2D4 +#define MT6331_VDVFS14_CON10 0x2D6 +#define MT6331_VDVFS14_CON11 0x2D8 +#define MT6331_VDVFS14_CON12 0x2DA +#define MT6331_VDVFS14_CON13 0x2DC +#define MT6331_VDVFS14_CON14 0x2DE +#define MT6331_VDVFS14_CON18 0x2E6 +#define MT6331_VDVFS14_CON19 0x2E8 +#define MT6331_VDVFS14_CON20 0x2EA +#define MT6331_VGPU_CON0 0x300 +#define MT6331_VGPU_CON1 0x302 +#define MT6331_VGPU_CON2 0x304 +#define MT6331_VGPU_CON3 0x306 +#define MT6331_VGPU_CON4 0x308 +#define MT6331_VGPU_CON5 0x30A +#define MT6331_VGPU_CON6 0x30C +#define MT6331_VGPU_CON7 0x30E +#define MT6331_VGPU_CON8 0x310 +#define MT6331_VGPU_CON9 0x312 +#define MT6331_VGPU_CON10 0x314 +#define MT6331_VGPU_CON11 0x316 +#define MT6331_VGPU_CON12 0x318 +#define MT6331_VGPU_CON13 0x31A +#define MT6331_VGPU_CON14 0x31C +#define MT6331_VGPU_CON15 0x31E +#define MT6331_VGPU_CON16 0x320 +#define MT6331_VGPU_CON17 0x322 +#define MT6331_VGPU_CON18 0x324 +#define MT6331_VGPU_CON19 0x326 +#define MT6331_VGPU_CON20 0x328 +#define MT6331_VCORE1_CON0 0x32A +#define MT6331_VCORE1_CON1 0x32C +#define MT6331_VCORE1_CON2 0x32E +#define MT6331_VCORE1_CON3 0x330 +#define MT6331_VCORE1_CON4 0x332 +#define MT6331_VCORE1_CON5 0x334 +#define MT6331_VCORE1_CON6 0x336 +#define MT6331_VCORE1_CON7 0x338 +#define MT6331_VCORE1_CON8 0x33A +#define MT6331_VCORE1_CON9 0x33C +#define MT6331_VCORE1_CON10 0x33E +#define MT6331_VCORE1_CON11 0x340 +#define MT6331_VCORE1_CON12 0x342 +#define MT6331_VCORE1_CON13 0x344 +#define MT6331_VCORE1_CON14 0x346 +#define MT6331_VCORE1_CON15 0x348 +#define MT6331_VCORE1_CON16 0x34A +#define MT6331_VCORE1_CON17 0x34C +#define MT6331_VCORE1_CON18 0x34E +#define MT6331_VCORE1_CON19 0x350 +#define MT6331_VCORE1_CON20 0x352 +#define MT6331_VCORE2_CON0 0x354 +#define MT6331_VCORE2_CON1 0x356 +#define MT6331_VCORE2_CON2 0x358 +#define MT6331_VCORE2_CON3 0x35A +#define MT6331_VCORE2_CON4 0x35C +#define MT6331_VCORE2_CON5 0x35E +#define MT6331_VCORE2_CON6 0x360 +#define MT6331_VCORE2_CON7 0x362 +#define MT6331_VCORE2_CON8 0x364 +#define MT6331_VCORE2_CON9 0x366 +#define MT6331_VCORE2_CON10 0x368 +#define MT6331_VCORE2_CON11 0x36A +#define MT6331_VCORE2_CON12 0x36C +#define MT6331_VCORE2_CON13 0x36E +#define MT6331_VCORE2_CON14 0x370 +#define MT6331_VCORE2_CON15 0x372 +#define MT6331_VCORE2_CON16 0x374 +#define MT6331_VCORE2_CON17 0x376 +#define MT6331_VCORE2_CON18 0x378 +#define MT6331_VCORE2_CON19 0x37A +#define MT6331_VCORE2_CON20 0x37C +#define MT6331_VCORE2_CON21 0x37E +#define MT6331_VIO18_CON0 0x380 +#define MT6331_VIO18_CON1 0x382 +#define MT6331_VIO18_CON2 0x384 +#define MT6331_VIO18_CON3 0x386 +#define MT6331_VIO18_CON4 0x388 +#define MT6331_VIO18_CON5 0x38A +#define MT6331_VIO18_CON6 0x38C +#define MT6331_VIO18_CON7 0x38E +#define MT6331_VIO18_CON8 0x390 +#define MT6331_VIO18_CON9 0x392 +#define MT6331_VIO18_CON10 0x394 +#define MT6331_VIO18_CON11 0x396 +#define MT6331_VIO18_CON12 0x398 +#define MT6331_VIO18_CON13 0x39A +#define MT6331_VIO18_CON14 0x39C +#define MT6331_VIO18_CON15 0x39E +#define MT6331_VIO18_CON16 0x3A0 +#define MT6331_VIO18_CON17 0x3A2 +#define MT6331_VIO18_CON18 0x3A4 +#define MT6331_VIO18_CON19 0x3A6 +#define MT6331_VIO18_CON20 0x3A8 +#define MT6331_BUCK_K_CON0 0x3AA +#define MT6331_BUCK_K_CON1 0x3AC +#define MT6331_BUCK_K_CON2 0x3AE +#define MT6331_BUCK_K_CON3 0x3B0 +#define MT6331_ZCD_CON0 0x400 +#define MT6331_ZCD_CON1 0x402 +#define MT6331_ZCD_CON2 0x404 +#define MT6331_ZCD_CON3 0x406 +#define MT6331_ZCD_CON4 0x408 +#define MT6331_ZCD_CON5 0x40A +#define MT6331_ISINK0_CON0 0x40C +#define MT6331_ISINK0_CON1 0x40E +#define MT6331_ISINK0_CON2 0x410 +#define MT6331_ISINK0_CON3 0x412 +#define MT6331_ISINK0_CON4 0x414 +#define MT6331_ISINK1_CON0 0x416 +#define MT6331_ISINK1_CON1 0x418 +#define MT6331_ISINK1_CON2 0x41A +#define MT6331_ISINK1_CON3 0x41C +#define MT6331_ISINK1_CON4 0x41E +#define MT6331_ISINK2_CON0 0x420 +#define MT6331_ISINK2_CON1 0x422 +#define MT6331_ISINK2_CON2 0x424 +#define MT6331_ISINK2_CON3 0x426 +#define MT6331_ISINK2_CON4 0x428 +#define MT6331_ISINK3_CON0 0x42A +#define MT6331_ISINK3_CON1 0x42C +#define MT6331_ISINK3_CON2 0x42E +#define MT6331_ISINK3_CON3 0x430 +#define MT6331_ISINK3_CON4 0x432 +#define MT6331_ISINK_ANA0 0x434 +#define MT6331_ISINK_ANA1 0x436 +#define MT6331_ISINK_PHASE_DLY 0x438 +#define MT6331_ISINK_EN_CTRL 0x43A +#define MT6331_ANALDO_CON0 0x500 +#define MT6331_ANALDO_CON1 0x502 +#define MT6331_ANALDO_CON2 0x504 +#define MT6331_ANALDO_CON3 0x506 +#define MT6331_ANALDO_CON4 0x508 +#define MT6331_ANALDO_CON5 0x50A +#define MT6331_ANALDO_CON6 0x50C +#define MT6331_ANALDO_CON7 0x50E +#define MT6331_ANALDO_CON8 0x510 +#define MT6331_ANALDO_CON9 0x512 +#define MT6331_ANALDO_CON10 0x514 +#define MT6331_ANALDO_CON11 0x516 +#define MT6331_ANALDO_CON12 0x518 +#define MT6331_ANALDO_CON13 0x51A +#define MT6331_SYSLDO_CON0 0x51C +#define MT6331_SYSLDO_CON1 0x51E +#define MT6331_SYSLDO_CON2 0x520 +#define MT6331_SYSLDO_CON3 0x522 +#define MT6331_SYSLDO_CON4 0x524 +#define MT6331_SYSLDO_CON5 0x526 +#define MT6331_SYSLDO_CON6 0x528 +#define MT6331_SYSLDO_CON7 0x52A +#define MT6331_SYSLDO_CON8 0x52C +#define MT6331_SYSLDO_CON9 0x52E +#define MT6331_SYSLDO_CON10 0x530 +#define MT6331_SYSLDO_CON11 0x532 +#define MT6331_SYSLDO_CON12 0x534 +#define MT6331_SYSLDO_CON13 0x536 +#define MT6331_SYSLDO_CON14 0x538 +#define MT6331_SYSLDO_CON15 0x53A +#define MT6331_SYSLDO_CON16 0x53C +#define MT6331_SYSLDO_CON17 0x53E +#define MT6331_SYSLDO_CON18 0x540 +#define MT6331_SYSLDO_CON19 0x542 +#define MT6331_SYSLDO_CON20 0x544 +#define MT6331_SYSLDO_CON21 0x546 +#define MT6331_DIGLDO_CON0 0x548 +#define MT6331_DIGLDO_CON1 0x54A +#define MT6331_DIGLDO_CON2 0x54C +#define MT6331_DIGLDO_CON3 0x54E +#define MT6331_DIGLDO_CON4 0x550 +#define MT6331_DIGLDO_CON5 0x552 +#define MT6331_DIGLDO_CON6 0x554 +#define MT6331_DIGLDO_CON7 0x556 +#define MT6331_DIGLDO_CON8 0x558 +#define MT6331_DIGLDO_CON9 0x55A +#define MT6331_DIGLDO_CON10 0x55C +#define MT6331_DIGLDO_CON11 0x55E +#define MT6331_DIGLDO_CON12 0x560 +#define MT6331_DIGLDO_CON13 0x562 +#define MT6331_DIGLDO_CON14 0x564 +#define MT6331_DIGLDO_CON15 0x566 +#define MT6331_DIGLDO_CON16 0x568 +#define MT6331_DIGLDO_CON17 0x56A +#define MT6331_DIGLDO_CON18 0x56C +#define MT6331_DIGLDO_CON19 0x56E +#define MT6331_DIGLDO_CON20 0x570 +#define MT6331_DIGLDO_CON21 0x572 +#define MT6331_DIGLDO_CON22 0x574 +#define MT6331_DIGLDO_CON23 0x576 +#define MT6331_DIGLDO_CON24 0x578 +#define MT6331_DIGLDO_CON25 0x57A +#define MT6331_DIGLDO_CON26 0x57C +#define MT6331_DIGLDO_CON27 0x57E +#define MT6331_DIGLDO_CON28 0x580 +#define MT6331_OTP_CON0 0x600 +#define MT6331_OTP_CON1 0x602 +#define MT6331_OTP_CON2 0x604 +#define MT6331_OTP_CON3 0x606 +#define MT6331_OTP_CON4 0x608 +#define MT6331_OTP_CON5 0x60A +#define MT6331_OTP_CON6 0x60C +#define MT6331_OTP_CON7 0x60E +#define MT6331_OTP_CON8 0x610 +#define MT6331_OTP_CON9 0x612 +#define MT6331_OTP_CON10 0x614 +#define MT6331_OTP_CON11 0x616 +#define MT6331_OTP_CON12 0x618 +#define MT6331_OTP_CON13 0x61A +#define MT6331_OTP_CON14 0x61C +#define MT6331_OTP_DOUT_0_15 0x61E +#define MT6331_OTP_DOUT_16_31 0x620 +#define MT6331_OTP_DOUT_32_47 0x622 +#define MT6331_OTP_DOUT_48_63 0x624 +#define MT6331_OTP_DOUT_64_79 0x626 +#define MT6331_OTP_DOUT_80_95 0x628 +#define MT6331_OTP_DOUT_96_111 0x62A +#define MT6331_OTP_DOUT_112_127 0x62C +#define MT6331_OTP_DOUT_128_143 0x62E +#define MT6331_OTP_DOUT_144_159 0x630 +#define MT6331_OTP_DOUT_160_175 0x632 +#define MT6331_OTP_DOUT_176_191 0x634 +#define MT6331_OTP_DOUT_192_207 0x636 +#define MT6331_OTP_DOUT_208_223 0x638 +#define MT6331_OTP_DOUT_224_239 0x63A +#define MT6331_OTP_DOUT_240_255 0x63C +#define MT6331_OTP_VAL_0_15 0x63E +#define MT6331_OTP_VAL_16_31 0x640 +#define MT6331_OTP_VAL_32_47 0x642 +#define MT6331_OTP_VAL_48_63 0x644 +#define MT6331_OTP_VAL_64_79 0x646 +#define MT6331_OTP_VAL_80_95 0x648 +#define MT6331_OTP_VAL_96_111 0x64A +#define MT6331_OTP_VAL_112_127 0x64C +#define MT6331_OTP_VAL_128_143 0x64E +#define MT6331_OTP_VAL_144_159 0x650 +#define MT6331_OTP_VAL_160_175 0x652 +#define MT6331_OTP_VAL_176_191 0x654 +#define MT6331_OTP_VAL_192_207 0x656 +#define MT6331_OTP_VAL_208_223 0x658 +#define MT6331_OTP_VAL_224_239 0x65A +#define MT6331_OTP_VAL_240_255 0x65C +#define MT6331_RTC_MIX_CON0 0x65E +#define MT6331_RTC_MIX_CON1 0x660 +#define MT6331_AUDDAC_CFG0 0x662 +#define MT6331_AUDBUF_CFG0 0x664 +#define MT6331_AUDBUF_CFG1 0x666 +#define MT6331_AUDBUF_CFG2 0x668 +#define MT6331_AUDBUF_CFG3 0x66A +#define MT6331_AUDBUF_CFG4 0x66C +#define MT6331_AUDBUF_CFG5 0x66E +#define MT6331_AUDBUF_CFG6 0x670 +#define MT6331_AUDBUF_CFG7 0x672 +#define MT6331_AUDBUF_CFG8 0x674 +#define MT6331_IBIASDIST_CFG0 0x676 +#define MT6331_AUDCLKGEN_CFG0 0x678 +#define MT6331_AUDLDO_CFG0 0x67A +#define MT6331_AUDDCDC_CFG0 0x67C +#define MT6331_AUDDCDC_CFG1 0x67E +#define MT6331_AUDNVREGGLB_CFG0 0x680 +#define MT6331_AUD_NCP0 0x682 +#define MT6331_AUD_ZCD_CFG0 0x684 +#define MT6331_AUDPREAMP_CFG0 0x686 +#define MT6331_AUDPREAMP_CFG1 0x688 +#define MT6331_AUDPREAMP_CFG2 0x68A +#define MT6331_AUDADC_CFG0 0x68C +#define MT6331_AUDADC_CFG1 0x68E +#define MT6331_AUDADC_CFG2 0x690 +#define MT6331_AUDADC_CFG3 0x692 +#define MT6331_AUDADC_CFG4 0x694 +#define MT6331_AUDADC_CFG5 0x696 +#define MT6331_AUDDIGMI_CFG0 0x698 +#define MT6331_AUDDIGMI_CFG1 0x69A +#define MT6331_AUDMICBIAS_CFG0 0x69C +#define MT6331_AUDMICBIAS_CFG1 0x69E +#define MT6331_AUDENCSPARE_CFG0 0x6A0 +#define MT6331_AUDPREAMPGAIN_CFG0 0x6A2 +#define MT6331_AUDMADPLL_CFG0 0x6A4 +#define MT6331_AUDMADPLL_CFG1 0x6A6 +#define MT6331_AUDMADPLL_CFG2 0x6A8 +#define MT6331_AUDLDO_NVREG_CFG0 0x6AA +#define MT6331_AUDLDO_NVREG_CFG1 0x6AC +#define MT6331_AUDLDO_NVREG_CFG2 0x6AE +#define MT6331_AUXADC_ADC0 0x700 +#define MT6331_AUXADC_ADC1 0x702 +#define MT6331_AUXADC_ADC2 0x704 +#define MT6331_AUXADC_ADC3 0x706 +#define MT6331_AUXADC_ADC4 0x708 +#define MT6331_AUXADC_ADC5 0x70A +#define MT6331_AUXADC_ADC6 0x70C +#define MT6331_AUXADC_ADC7 0x70E +#define MT6331_AUXADC_ADC8 0x710 +#define MT6331_AUXADC_ADC9 0x712 +#define MT6331_AUXADC_ADC10 0x714 +#define MT6331_AUXADC_ADC11 0x716 +#define MT6331_AUXADC_ADC12 0x718 +#define MT6331_AUXADC_ADC13 0x71A +#define MT6331_AUXADC_ADC14 0x71C +#define MT6331_AUXADC_ADC15 0x71E +#define MT6331_AUXADC_ADC16 0x720 +#define MT6331_AUXADC_ADC17 0x722 +#define MT6331_AUXADC_ADC18 0x724 +#define MT6331_AUXADC_ADC19 0x726 +#define MT6331_AUXADC_STA0 0x728 +#define MT6331_AUXADC_STA1 0x72A +#define MT6331_AUXADC_RQST0 0x72C +#define MT6331_AUXADC_RQST0_SET 0x72E +#define MT6331_AUXADC_RQST0_CLR 0x730 +#define MT6331_AUXADC_RQST1 0x732 +#define MT6331_AUXADC_RQST1_SET 0x734 +#define MT6331_AUXADC_RQST1_CLR 0x736 +#define MT6331_AUXADC_CON0 0x738 +#define MT6331_AUXADC_CON1 0x73A +#define MT6331_AUXADC_CON2 0x73C +#define MT6331_AUXADC_CON3 0x73E +#define MT6331_AUXADC_CON4 0x740 +#define MT6331_AUXADC_CON5 0x742 +#define MT6331_AUXADC_CON6 0x744 +#define MT6331_AUXADC_CON7 0x746 +#define MT6331_AUXADC_CON8 0x748 +#define MT6331_AUXADC_CON9 0x74A +#define MT6331_AUXADC_CON10 0x74C +#define MT6331_AUXADC_CON11 0x74E +#define MT6331_AUXADC_CON12 0x750 +#define MT6331_AUXADC_CON13 0x752 +#define MT6331_AUXADC_CON14 0x754 +#define MT6331_AUXADC_CON15 0x756 +#define MT6331_AUXADC_CON16 0x758 +#define MT6331_AUXADC_CON17 0x75A +#define MT6331_AUXADC_CON18 0x75C +#define MT6331_AUXADC_CON19 0x75E +#define MT6331_AUXADC_CON20 0x760 +#define MT6331_AUXADC_CON21 0x762 +#define MT6331_AUXADC_CON22 0x764 +#define MT6331_AUXADC_CON23 0x766 +#define MT6331_AUXADC_CON24 0x768 +#define MT6331_AUXADC_CON25 0x76A +#define MT6331_AUXADC_CON26 0x76C +#define MT6331_AUXADC_CON27 0x76E +#define MT6331_AUXADC_CON28 0x770 +#define MT6331_AUXADC_CON29 0x772 +#define MT6331_AUXADC_CON30 0x774 +#define MT6331_AUXADC_CON31 0x776 +#define MT6331_AUXADC_CON32 0x778 +#define MT6331_ACCDET_CON0 0x77A +#define MT6331_ACCDET_CON1 0x77C +#define MT6331_ACCDET_CON2 0x77E +#define MT6331_ACCDET_CON3 0x780 +#define MT6331_ACCDET_CON4 0x782 +#define MT6331_ACCDET_CON5 0x784 +#define MT6331_ACCDET_CON6 0x786 +#define MT6331_ACCDET_CON7 0x788 +#define MT6331_ACCDET_CON8 0x78A +#define MT6331_ACCDET_CON9 0x78C +#define MT6331_ACCDET_CON10 0x78E +#define MT6331_ACCDET_CON11 0x790 +#define MT6331_ACCDET_CON12 0x792 +#define MT6331_ACCDET_CON13 0x794 +#define MT6331_ACCDET_CON14 0x796 +#define MT6331_ACCDET_CON15 0x798 +#define MT6331_ACCDET_CON16 0x79A +#define MT6331_ACCDET_CON17 0x79C +#define MT6331_ACCDET_CON18 0x79E +#define MT6331_ACCDET_CON19 0x7A0 +#define MT6331_ACCDET_CON20 0x7A2 +#define MT6331_ACCDET_CON21 0x7A4 +#define MT6331_ACCDET_CON22 0x7A6 +#define MT6331_ACCDET_CON23 0x7A8 +#define MT6331_ACCDET_CON24 0x7AA + +#endif /* __MFD_MT6331_REGISTERS_H__ */ diff --git a/include/linux/mfd/mt6332/core.h b/include/linux/mfd/mt6332/core.h new file mode 100644 index 000000000000..cd6013eb82d9 --- /dev/null +++ b/include/linux/mfd/mt6332/core.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 AngeloGioacchino Del Regno + */ + +#ifndef __MFD_MT6332_CORE_H__ +#define __MFD_MT6332_CORE_H__ + +enum mt6332_irq_status_numbers { + MT6332_IRQ_STATUS_CHR_COMPLETE = 0, + MT6332_IRQ_STATUS_THERMAL_SD, + MT6332_IRQ_STATUS_THERMAL_REG_IN, + MT6332_IRQ_STATUS_THERMAL_REG_OUT, + MT6332_IRQ_STATUS_OTG_OC, + MT6332_IRQ_STATUS_CHR_OC, + MT6332_IRQ_STATUS_OTG_THERMAL, + MT6332_IRQ_STATUS_CHRIN_SHORT, + MT6332_IRQ_STATUS_DRVCDT_SHORT, + MT6332_IRQ_STATUS_PLUG_IN_FLASH, + MT6332_IRQ_STATUS_CHRWDT_FLAG, + MT6332_IRQ_STATUS_FLASH_EN_TIMEOUT, + MT6332_IRQ_STATUS_FLASH_VLED1_SHORT, + MT6332_IRQ_STATUS_FLASH_VLED1_OPEN = 13, + MT6332_IRQ_STATUS_OV = 16, + MT6332_IRQ_STATUS_BVALID_DET, + MT6332_IRQ_STATUS_VBATON_UNDET, + MT6332_IRQ_STATUS_CHR_PLUG_IN, + MT6332_IRQ_STATUS_CHR_PLUG_OUT, + MT6332_IRQ_STATUS_BC11_TIMEOUT, + MT6332_IRQ_STATUS_FLASH_VLED2_SHORT, + MT6332_IRQ_STATUS_FLASH_VLED2_OPEN = 23, + MT6332_IRQ_STATUS_THR_H = 32, + MT6332_IRQ_STATUS_THR_L, + MT6332_IRQ_STATUS_BAT_H, + MT6332_IRQ_STATUS_BAT_L, + MT6332_IRQ_STATUS_M3_H, + MT6332_IRQ_STATUS_M3_L, + MT6332_IRQ_STATUS_FG_BAT_H, + MT6332_IRQ_STATUS_FG_BAT_L, + MT6332_IRQ_STATUS_FG_CUR_H, + MT6332_IRQ_STATUS_FG_CUR_L, + MT6332_IRQ_STATUS_SPKL_D, + MT6332_IRQ_STATUS_SPKL_AB, + MT6332_IRQ_STATUS_BIF, + MT6332_IRQ_STATUS_VWLED_OC = 45, + MT6332_IRQ_STATUS_VDRAM_OC = 48, + MT6332_IRQ_STATUS_VDVFS2_OC, + MT6332_IRQ_STATUS_VRF1_OC, + MT6332_IRQ_STATUS_VRF2_OC, + MT6332_IRQ_STATUS_VPA_OC, + MT6332_IRQ_STATUS_VSBST_OC, + MT6332_IRQ_STATUS_LDO_OC, + MT6332_IRQ_STATUS_NR, +}; + +#define MT6332_IRQ_CON0_BASE MT6332_IRQ_STATUS_CHR_COMPLETE +#define MT6332_IRQ_CON0_BITS (MT6332_IRQ_STATUS_FLASH_VLED1_OPEN + 1) +#define MT6332_IRQ_CON1_BASE MT6332_IRQ_STATUS_OV +#define MT6332_IRQ_CON1_BITS (MT6332_IRQ_STATUS_FLASH_VLED2_OPEN - MT6332_IRQ_STATUS_OV + 1) +#define MT6332_IRQ_CON2_BASE MT6332_IRQ_STATUS_THR_H +#define MT6332_IRQ_CON2_BITS (MT6332_IRQ_STATUS_VWLED_OC - MT6332_IRQ_STATUS_THR_H + 1) +#define MT6332_IRQ_CON3_BASE MT6332_IRQ_STATUS_VDRAM_OC +#define MT6332_IRQ_CON3_BITS (MT6332_IRQ_STATUS_LDO_OC - MT6332_IRQ_STATUS_VDRAM_OC + 1) + +#endif /* __MFD_MT6332_CORE_H__ */ diff --git a/include/linux/mfd/mt6332/registers.h b/include/linux/mfd/mt6332/registers.h new file mode 100644 index 000000000000..65e0b86fceac --- /dev/null +++ b/include/linux/mfd/mt6332/registers.h @@ -0,0 +1,642 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 AngeloGioacchino Del Regno + */ + +#ifndef __MFD_MT6332_REGISTERS_H__ +#define __MFD_MT6332_REGISTERS_H__ + +/* PMIC Registers */ +#define MT6332_HWCID 0x8000 +#define MT6332_SWCID 0x8002 +#define MT6332_TOP_CON 0x8004 +#define MT6332_DDR_VREF_AP_CON 0x8006 +#define MT6332_DDR_VREF_DQ_CON 0x8008 +#define MT6332_DDR_VREF_CA_CON 0x800A +#define MT6332_TEST_OUT 0x800C +#define MT6332_TEST_CON0 0x800E +#define MT6332_TEST_CON1 0x8010 +#define MT6332_TESTMODE_SW 0x8012 +#define MT6332_TESTMODE_ANA 0x8014 +#define MT6332_TDSEL_CON 0x8016 +#define MT6332_RDSEL_CON 0x8018 +#define MT6332_SMT_CON0 0x801A +#define MT6332_SMT_CON1 0x801C +#define MT6332_DRV_CON0 0x801E +#define MT6332_DRV_CON1 0x8020 +#define MT6332_DRV_CON2 0x8022 +#define MT6332_EN_STATUS0 0x8024 +#define MT6332_OCSTATUS0 0x8026 +#define MT6332_TOP_STATUS 0x8028 +#define MT6332_TOP_STATUS_SET 0x802A +#define MT6332_TOP_STATUS_CLR 0x802C +#define MT6332_FLASH_CON0 0x802E +#define MT6332_FLASH_CON1 0x8030 +#define MT6332_FLASH_CON2 0x8032 +#define MT6332_CORE_CON0 0x8034 +#define MT6332_CORE_CON1 0x8036 +#define MT6332_CORE_CON2 0x8038 +#define MT6332_CORE_CON3 0x803A +#define MT6332_CORE_CON4 0x803C +#define MT6332_CORE_CON5 0x803E +#define MT6332_CORE_CON6 0x8040 +#define MT6332_CORE_CON7 0x8042 +#define MT6332_CORE_CON8 0x8044 +#define MT6332_CORE_CON9 0x8046 +#define MT6332_CORE_CON10 0x8048 +#define MT6332_CORE_CON11 0x804A +#define MT6332_CORE_CON12 0x804C +#define MT6332_CORE_CON13 0x804E +#define MT6332_CORE_CON14 0x8050 +#define MT6332_CORE_CON15 0x8052 +#define MT6332_STA_CON0 0x8054 +#define MT6332_STA_CON1 0x8056 +#define MT6332_STA_CON2 0x8058 +#define MT6332_STA_CON3 0x805A +#define MT6332_STA_CON4 0x805C +#define MT6332_STA_CON5 0x805E +#define MT6332_STA_CON6 0x8060 +#define MT6332_STA_CON7 0x8062 +#define MT6332_CHR_CON0 0x8064 +#define MT6332_CHR_CON1 0x8066 +#define MT6332_CHR_CON2 0x8068 +#define MT6332_CHR_CON3 0x806A +#define MT6332_CHR_CON4 0x806C +#define MT6332_CHR_CON5 0x806E +#define MT6332_CHR_CON6 0x8070 +#define MT6332_CHR_CON7 0x8072 +#define MT6332_CHR_CON8 0x8074 +#define MT6332_CHR_CON9 0x8076 +#define MT6332_CHR_CON10 0x8078 +#define MT6332_CHR_CON11 0x807A +#define MT6332_CHR_CON12 0x807C +#define MT6332_CHR_CON13 0x807E +#define MT6332_CHR_CON14 0x8080 +#define MT6332_CHR_CON15 0x8082 +#define MT6332_BOOST_CON0 0x8084 +#define MT6332_BOOST_CON1 0x8086 +#define MT6332_BOOST_CON2 0x8088 +#define MT6332_BOOST_CON3 0x808A +#define MT6332_BOOST_CON4 0x808C +#define MT6332_BOOST_CON5 0x808E +#define MT6332_BOOST_CON6 0x8090 +#define MT6332_BOOST_CON7 0x8092 +#define MT6332_TOP_CKPDN_CON0 0x8094 +#define MT6332_TOP_CKPDN_CON0_SET 0x8096 +#define MT6332_TOP_CKPDN_CON0_CLR 0x8098 +#define MT6332_TOP_CKPDN_CON1 0x809A +#define MT6332_TOP_CKPDN_CON1_SET 0x809C +#define MT6332_TOP_CKPDN_CON1_CLR 0x809E +#define MT6332_TOP_CKPDN_CON2 0x80A0 +#define MT6332_TOP_CKPDN_CON2_SET 0x80A2 +#define MT6332_TOP_CKPDN_CON2_CLR 0x80A4 +#define MT6332_TOP_CKSEL_CON0 0x80A6 +#define MT6332_TOP_CKSEL_CON0_SET 0x80A8 +#define MT6332_TOP_CKSEL_CON0_CLR 0x80AA +#define MT6332_TOP_CKSEL_CON1 0x80AC +#define MT6332_TOP_CKSEL_CON1_SET 0x80AE +#define MT6332_TOP_CKSEL_CON1_CLR 0x80B0 +#define MT6332_TOP_CKHWEN_CON 0x80B2 +#define MT6332_TOP_CKHWEN_CON_SET 0x80B4 +#define MT6332_TOP_CKHWEN_CON_CLR 0x80B6 +#define MT6332_TOP_CKTST_CON0 0x80B8 +#define MT6332_TOP_CKTST_CON1 0x80BA +#define MT6332_TOP_RST_CON 0x80BC +#define MT6332_TOP_RST_CON_SET 0x80BE +#define MT6332_TOP_RST_CON_CLR 0x80C0 +#define MT6332_TOP_RST_MISC 0x80C2 +#define MT6332_TOP_RST_MISC_SET 0x80C4 +#define MT6332_TOP_RST_MISC_CLR 0x80C6 +#define MT6332_INT_CON0 0x80C8 +#define MT6332_INT_CON0_SET 0x80CA +#define MT6332_INT_CON0_CLR 0x80CC +#define MT6332_INT_CON1 0x80CE +#define MT6332_INT_CON1_SET 0x80D0 +#define MT6332_INT_CON1_CLR 0x80D2 +#define MT6332_INT_CON2 0x80D4 +#define MT6332_INT_CON2_SET 0x80D6 +#define MT6332_INT_CON2_CLR 0x80D8 +#define MT6332_INT_CON3 0x80DA +#define MT6332_INT_CON3_SET 0x80DC +#define MT6332_INT_CON3_CLR 0x80DE +#define MT6332_CHRWDT_CON0 0x80E0 +#define MT6332_CHRWDT_STATUS0 0x80E2 +#define MT6332_INT_STATUS0 0x80E4 +#define MT6332_INT_STATUS1 0x80E6 +#define MT6332_INT_STATUS2 0x80E8 +#define MT6332_INT_STATUS3 0x80EA +#define MT6332_OC_GEAR_0 0x80EC +#define MT6332_OC_GEAR_1 0x80EE +#define MT6332_OC_GEAR_2 0x80F0 +#define MT6332_INT_MISC_CON 0x80F2 +#define MT6332_RG_SPI_CON 0x80F4 +#define MT6332_DEW_DIO_EN 0x80F6 +#define MT6332_DEW_READ_TEST 0x80F8 +#define MT6332_DEW_WRITE_TEST 0x80FA +#define MT6332_DEW_CRC_SWRST 0x80FC +#define MT6332_DEW_CRC_EN 0x80FE +#define MT6332_DEW_CRC_VAL 0x8100 +#define MT6332_DEW_DBG_MON_SEL 0x8102 +#define MT6332_DEW_CIPHER_KEY_SEL 0x8104 +#define MT6332_DEW_CIPHER_IV_SEL 0x8106 +#define MT6332_DEW_CIPHER_EN 0x8108 +#define MT6332_DEW_CIPHER_RDY 0x810A +#define MT6332_DEW_CIPHER_MODE 0x810C +#define MT6332_DEW_CIPHER_SWRST 0x810E +#define MT6332_DEW_RDDMY_NO 0x8110 +#define MT6332_INT_STA 0x8112 +#define MT6332_BIF_CON0 0x8114 +#define MT6332_BIF_CON1 0x8116 +#define MT6332_BIF_CON2 0x8118 +#define MT6332_BIF_CON3 0x811A +#define MT6332_BIF_CON4 0x811C +#define MT6332_BIF_CON5 0x811E +#define MT6332_BIF_CON6 0x8120 +#define MT6332_BIF_CON7 0x8122 +#define MT6332_BIF_CON8 0x8124 +#define MT6332_BIF_CON9 0x8126 +#define MT6332_BIF_CON10 0x8128 +#define MT6332_BIF_CON11 0x812A +#define MT6332_BIF_CON12 0x812C +#define MT6332_BIF_CON13 0x812E +#define MT6332_BIF_CON14 0x8130 +#define MT6332_BIF_CON15 0x8132 +#define MT6332_BIF_CON16 0x8134 +#define MT6332_BIF_CON17 0x8136 +#define MT6332_BIF_CON18 0x8138 +#define MT6332_BIF_CON19 0x813A +#define MT6332_BIF_CON20 0x813C +#define MT6332_BIF_CON21 0x813E +#define MT6332_BIF_CON22 0x8140 +#define MT6332_BIF_CON23 0x8142 +#define MT6332_BIF_CON24 0x8144 +#define MT6332_BIF_CON25 0x8146 +#define MT6332_BIF_CON26 0x8148 +#define MT6332_BIF_CON27 0x814A +#define MT6332_BIF_CON28 0x814C +#define MT6332_BIF_CON29 0x814E +#define MT6332_BIF_CON30 0x8150 +#define MT6332_BIF_CON31 0x8152 +#define MT6332_BIF_CON32 0x8154 +#define MT6332_BIF_CON33 0x8156 +#define MT6332_BIF_CON34 0x8158 +#define MT6332_BIF_CON35 0x815A +#define MT6332_BIF_CON36 0x815C +#define MT6332_BATON_CON0 0x815E +#define MT6332_BIF_CON37 0x8160 +#define MT6332_BIF_CON38 0x8162 +#define MT6332_CHR_CON16 0x8164 +#define MT6332_CHR_CON17 0x8166 +#define MT6332_CHR_CON18 0x8168 +#define MT6332_CHR_CON19 0x816A +#define MT6332_CHR_CON20 0x816C +#define MT6332_CHR_CON21 0x816E +#define MT6332_CHR_CON22 0x8170 +#define MT6332_CHR_CON23 0x8172 +#define MT6332_CHR_CON24 0x8174 +#define MT6332_CHR_CON25 0x8176 +#define MT6332_STA_CON8 0x8178 +#define MT6332_BUCK_ALL_CON0 0x8400 +#define MT6332_BUCK_ALL_CON1 0x8402 +#define MT6332_BUCK_ALL_CON2 0x8404 +#define MT6332_BUCK_ALL_CON3 0x8406 +#define MT6332_BUCK_ALL_CON4 0x8408 +#define MT6332_BUCK_ALL_CON5 0x840A +#define MT6332_BUCK_ALL_CON6 0x840C +#define MT6332_BUCK_ALL_CON7 0x840E +#define MT6332_BUCK_ALL_CON8 0x8410 +#define MT6332_BUCK_ALL_CON9 0x8412 +#define MT6332_BUCK_ALL_CON10 0x8414 +#define MT6332_BUCK_ALL_CON11 0x8416 +#define MT6332_BUCK_ALL_CON12 0x8418 +#define MT6332_BUCK_ALL_CON13 0x841A +#define MT6332_BUCK_ALL_CON14 0x841C +#define MT6332_BUCK_ALL_CON15 0x841E +#define MT6332_BUCK_ALL_CON16 0x8420 +#define MT6332_BUCK_ALL_CON17 0x8422 +#define MT6332_BUCK_ALL_CON18 0x8424 +#define MT6332_BUCK_ALL_CON19 0x8426 +#define MT6332_BUCK_ALL_CON20 0x8428 +#define MT6332_BUCK_ALL_CON21 0x842A +#define MT6332_BUCK_ALL_CON22 0x842C +#define MT6332_BUCK_ALL_CON23 0x842E +#define MT6332_BUCK_ALL_CON24 0x8430 +#define MT6332_BUCK_ALL_CON25 0x8432 +#define MT6332_BUCK_ALL_CON26 0x8434 +#define MT6332_BUCK_ALL_CON27 0x8436 +#define MT6332_VDRAM_CON0 0x8438 +#define MT6332_VDRAM_CON1 0x843A +#define MT6332_VDRAM_CON2 0x843C +#define MT6332_VDRAM_CON3 0x843E +#define MT6332_VDRAM_CON4 0x8440 +#define MT6332_VDRAM_CON5 0x8442 +#define MT6332_VDRAM_CON6 0x8444 +#define MT6332_VDRAM_CON7 0x8446 +#define MT6332_VDRAM_CON8 0x8448 +#define MT6332_VDRAM_CON9 0x844A +#define MT6332_VDRAM_CON10 0x844C +#define MT6332_VDRAM_CON11 0x844E +#define MT6332_VDRAM_CON12 0x8450 +#define MT6332_VDRAM_CON13 0x8452 +#define MT6332_VDRAM_CON14 0x8454 +#define MT6332_VDRAM_CON15 0x8456 +#define MT6332_VDRAM_CON16 0x8458 +#define MT6332_VDRAM_CON17 0x845A +#define MT6332_VDRAM_CON18 0x845C +#define MT6332_VDRAM_CON19 0x845E +#define MT6332_VDRAM_CON20 0x8460 +#define MT6332_VDRAM_CON21 0x8462 +#define MT6332_VDVFS2_CON0 0x8464 +#define MT6332_VDVFS2_CON1 0x8466 +#define MT6332_VDVFS2_CON2 0x8468 +#define MT6332_VDVFS2_CON3 0x846A +#define MT6332_VDVFS2_CON4 0x846C +#define MT6332_VDVFS2_CON5 0x846E +#define MT6332_VDVFS2_CON6 0x8470 +#define MT6332_VDVFS2_CON7 0x8472 +#define MT6332_VDVFS2_CON8 0x8474 +#define MT6332_VDVFS2_CON9 0x8476 +#define MT6332_VDVFS2_CON10 0x8478 +#define MT6332_VDVFS2_CON11 0x847A +#define MT6332_VDVFS2_CON12 0x847C +#define MT6332_VDVFS2_CON13 0x847E +#define MT6332_VDVFS2_CON14 0x8480 +#define MT6332_VDVFS2_CON15 0x8482 +#define MT6332_VDVFS2_CON16 0x8484 +#define MT6332_VDVFS2_CON17 0x8486 +#define MT6332_VDVFS2_CON18 0x8488 +#define MT6332_VDVFS2_CON19 0x848A +#define MT6332_VDVFS2_CON20 0x848C +#define MT6332_VDVFS2_CON21 0x848E +#define MT6332_VDVFS2_CON22 0x8490 +#define MT6332_VDVFS2_CON23 0x8492 +#define MT6332_VDVFS2_CON24 0x8494 +#define MT6332_VDVFS2_CON25 0x8496 +#define MT6332_VDVFS2_CON26 0x8498 +#define MT6332_VDVFS2_CON27 0x849A +#define MT6332_VRF1_CON0 0x849C +#define MT6332_VRF1_CON1 0x849E +#define MT6332_VRF1_CON2 0x84A0 +#define MT6332_VRF1_CON3 0x84A2 +#define MT6332_VRF1_CON4 0x84A4 +#define MT6332_VRF1_CON5 0x84A6 +#define MT6332_VRF1_CON6 0x84A8 +#define MT6332_VRF1_CON7 0x84AA +#define MT6332_VRF1_CON8 0x84AC +#define MT6332_VRF1_CON9 0x84AE +#define MT6332_VRF1_CON10 0x84B0 +#define MT6332_VRF1_CON11 0x84B2 +#define MT6332_VRF1_CON12 0x84B4 +#define MT6332_VRF1_CON13 0x84B6 +#define MT6332_VRF1_CON14 0x84B8 +#define MT6332_VRF1_CON15 0x84BA +#define MT6332_VRF1_CON16 0x84BC +#define MT6332_VRF1_CON17 0x84BE +#define MT6332_VRF1_CON18 0x84C0 +#define MT6332_VRF1_CON19 0x84C2 +#define MT6332_VRF1_CON20 0x84C4 +#define MT6332_VRF1_CON21 0x84C6 +#define MT6332_VRF2_CON0 0x84C8 +#define MT6332_VRF2_CON1 0x84CA +#define MT6332_VRF2_CON2 0x84CC +#define MT6332_VRF2_CON3 0x84CE +#define MT6332_VRF2_CON4 0x84D0 +#define MT6332_VRF2_CON5 0x84D2 +#define MT6332_VRF2_CON6 0x84D4 +#define MT6332_VRF2_CON7 0x84D6 +#define MT6332_VRF2_CON8 0x84D8 +#define MT6332_VRF2_CON9 0x84DA +#define MT6332_VRF2_CON10 0x84DC +#define MT6332_VRF2_CON11 0x84DE +#define MT6332_VRF2_CON12 0x84E0 +#define MT6332_VRF2_CON13 0x84E2 +#define MT6332_VRF2_CON14 0x84E4 +#define MT6332_VRF2_CON15 0x84E6 +#define MT6332_VRF2_CON16 0x84E8 +#define MT6332_VRF2_CON17 0x84EA +#define MT6332_VRF2_CON18 0x84EC +#define MT6332_VRF2_CON19 0x84EE +#define MT6332_VRF2_CON20 0x84F0 +#define MT6332_VRF2_CON21 0x84F2 +#define MT6332_VPA_CON0 0x84F4 +#define MT6332_VPA_CON1 0x84F6 +#define MT6332_VPA_CON2 0x84F8 +#define MT6332_VPA_CON3 0x84FC +#define MT6332_VPA_CON4 0x84FE +#define MT6332_VPA_CON5 0x8500 +#define MT6332_VPA_CON6 0x8502 +#define MT6332_VPA_CON7 0x8504 +#define MT6332_VPA_CON8 0x8506 +#define MT6332_VPA_CON9 0x8508 +#define MT6332_VPA_CON10 0x850A +#define MT6332_VPA_CON11 0x850C +#define MT6332_VPA_CON12 0x850E +#define MT6332_VPA_CON13 0x8510 +#define MT6332_VPA_CON14 0x8512 +#define MT6332_VPA_CON15 0x8514 +#define MT6332_VPA_CON16 0x8516 +#define MT6332_VPA_CON17 0x8518 +#define MT6332_VPA_CON18 0x851A +#define MT6332_VPA_CON19 0x851C +#define MT6332_VPA_CON20 0x851E +#define MT6332_VPA_CON21 0x8520 +#define MT6332_VPA_CON22 0x8522 +#define MT6332_VPA_CON23 0x8524 +#define MT6332_VPA_CON24 0x8526 +#define MT6332_VPA_CON25 0x8528 +#define MT6332_VSBST_CON0 0x852A +#define MT6332_VSBST_CON1 0x852C +#define MT6332_VSBST_CON2 0x852E +#define MT6332_VSBST_CON3 0x8530 +#define MT6332_VSBST_CON4 0x8532 +#define MT6332_VSBST_CON5 0x8534 +#define MT6332_VSBST_CON6 0x8536 +#define MT6332_VSBST_CON7 0x8538 +#define MT6332_VSBST_CON8 0x853A +#define MT6332_VSBST_CON9 0x853C +#define MT6332_VSBST_CON10 0x853E +#define MT6332_VSBST_CON11 0x8540 +#define MT6332_VSBST_CON12 0x8542 +#define MT6332_VSBST_CON13 0x8544 +#define MT6332_VSBST_CON14 0x8546 +#define MT6332_VSBST_CON15 0x8548 +#define MT6332_VSBST_CON16 0x854A +#define MT6332_VSBST_CON17 0x854C +#define MT6332_VSBST_CON18 0x854E +#define MT6332_VSBST_CON19 0x8550 +#define MT6332_VSBST_CON20 0x8552 +#define MT6332_VSBST_CON21 0x8554 +#define MT6332_BUCK_K_CON0 0x8556 +#define MT6332_BUCK_K_CON1 0x8558 +#define MT6332_BUCK_K_CON2 0x855A +#define MT6332_BUCK_K_CON3 0x855C +#define MT6332_BUCK_K_CON4 0x855E +#define MT6332_BUCK_K_CON5 0x8560 +#define MT6332_AUXADC_ADC0 0x8800 +#define MT6332_AUXADC_ADC1 0x8802 +#define MT6332_AUXADC_ADC2 0x8804 +#define MT6332_AUXADC_ADC3 0x8806 +#define MT6332_AUXADC_ADC4 0x8808 +#define MT6332_AUXADC_ADC5 0x880A +#define MT6332_AUXADC_ADC6 0x880C +#define MT6332_AUXADC_ADC7 0x880E +#define MT6332_AUXADC_ADC8 0x8810 +#define MT6332_AUXADC_ADC9 0x8812 +#define MT6332_AUXADC_ADC10 0x8814 +#define MT6332_AUXADC_ADC11 0x8816 +#define MT6332_AUXADC_ADC12 0x8818 +#define MT6332_AUXADC_ADC13 0x881A +#define MT6332_AUXADC_ADC14 0x881C +#define MT6332_AUXADC_ADC15 0x881E +#define MT6332_AUXADC_ADC16 0x8820 +#define MT6332_AUXADC_ADC17 0x8822 +#define MT6332_AUXADC_ADC18 0x8824 +#define MT6332_AUXADC_ADC19 0x8826 +#define MT6332_AUXADC_ADC20 0x8828 +#define MT6332_AUXADC_ADC21 0x882A +#define MT6332_AUXADC_ADC22 0x882C +#define MT6332_AUXADC_ADC23 0x882E +#define MT6332_AUXADC_ADC24 0x8830 +#define MT6332_AUXADC_ADC25 0x8832 +#define MT6332_AUXADC_ADC26 0x8834 +#define MT6332_AUXADC_ADC27 0x8836 +#define MT6332_AUXADC_ADC28 0x8838 +#define MT6332_AUXADC_ADC29 0x883A +#define MT6332_AUXADC_ADC30 0x883C +#define MT6332_AUXADC_ADC31 0x883E +#define MT6332_AUXADC_ADC32 0x8840 +#define MT6332_AUXADC_ADC33 0x8842 +#define MT6332_AUXADC_ADC34 0x8844 +#define MT6332_AUXADC_ADC35 0x8846 +#define MT6332_AUXADC_ADC36 0x8848 +#define MT6332_AUXADC_ADC37 0x884A +#define MT6332_AUXADC_ADC38 0x884C +#define MT6332_AUXADC_ADC39 0x884E +#define MT6332_AUXADC_ADC40 0x8850 +#define MT6332_AUXADC_ADC41 0x8852 +#define MT6332_AUXADC_ADC42 0x8854 +#define MT6332_AUXADC_ADC43 0x8856 +#define MT6332_AUXADC_STA0 0x8858 +#define MT6332_AUXADC_STA1 0x885A +#define MT6332_AUXADC_RQST0 0x885C +#define MT6332_AUXADC_RQST0_SET 0x885E +#define MT6332_AUXADC_RQST0_CLR 0x8860 +#define MT6332_AUXADC_RQST1 0x8862 +#define MT6332_AUXADC_RQST1_SET 0x8864 +#define MT6332_AUXADC_RQST1_CLR 0x8866 +#define MT6332_AUXADC_CON0 0x8868 +#define MT6332_AUXADC_CON1 0x886A +#define MT6332_AUXADC_CON2 0x886C +#define MT6332_AUXADC_CON3 0x886E +#define MT6332_AUXADC_CON4 0x8870 +#define MT6332_AUXADC_CON5 0x8872 +#define MT6332_AUXADC_CON6 0x8874 +#define MT6332_AUXADC_CON7 0x8876 +#define MT6332_AUXADC_CON8 0x8878 +#define MT6332_AUXADC_CON9 0x887A +#define MT6332_AUXADC_CON10 0x887C +#define MT6332_AUXADC_CON11 0x887E +#define MT6332_AUXADC_CON12 0x8880 +#define MT6332_AUXADC_CON13 0x8882 +#define MT6332_AUXADC_CON14 0x8884 +#define MT6332_AUXADC_CON15 0x8886 +#define MT6332_AUXADC_CON16 0x8888 +#define MT6332_AUXADC_CON17 0x888A +#define MT6332_AUXADC_CON18 0x888C +#define MT6332_AUXADC_CON19 0x888E +#define MT6332_AUXADC_CON20 0x8890 +#define MT6332_AUXADC_CON21 0x8892 +#define MT6332_AUXADC_CON22 0x8894 +#define MT6332_AUXADC_CON23 0x8896 +#define MT6332_AUXADC_CON24 0x8898 +#define MT6332_AUXADC_CON25 0x889A +#define MT6332_AUXADC_CON26 0x889C +#define MT6332_AUXADC_CON27 0x889E +#define MT6332_AUXADC_CON28 0x88A0 +#define MT6332_AUXADC_CON29 0x88A2 +#define MT6332_AUXADC_CON30 0x88A4 +#define MT6332_AUXADC_CON31 0x88A6 +#define MT6332_AUXADC_CON32 0x88A8 +#define MT6332_AUXADC_CON33 0x88AA +#define MT6332_AUXADC_CON34 0x88AC +#define MT6332_AUXADC_CON35 0x88AE +#define MT6332_AUXADC_CON36 0x88B0 +#define MT6332_AUXADC_CON37 0x88B2 +#define MT6332_AUXADC_CON38 0x88B4 +#define MT6332_AUXADC_CON39 0x88B6 +#define MT6332_AUXADC_CON40 0x88B8 +#define MT6332_AUXADC_CON41 0x88BA +#define MT6332_AUXADC_CON42 0x88BC +#define MT6332_AUXADC_CON43 0x88BE +#define MT6332_AUXADC_CON44 0x88C0 +#define MT6332_AUXADC_CON45 0x88C2 +#define MT6332_AUXADC_CON46 0x88C4 +#define MT6332_AUXADC_CON47 0x88C6 +#define MT6332_STRUP_CONA0 0x8C00 +#define MT6332_STRUP_CONA1 0x8C02 +#define MT6332_STRUP_CONA2 0x8C04 +#define MT6332_STRUP_CON0 0x8C06 +#define MT6332_STRUP_CON2 0x8C08 +#define MT6332_STRUP_CON3 0x8C0A +#define MT6332_STRUP_CON4 0x8C0C +#define MT6332_STRUP_CON5 0x8C0E +#define MT6332_STRUP_CON6 0x8C10 +#define MT6332_STRUP_CON7 0x8C12 +#define MT6332_STRUP_CON8 0x8C14 +#define MT6332_STRUP_CON9 0x8C16 +#define MT6332_STRUP_CON10 0x8C18 +#define MT6332_STRUP_CON11 0x8C1A +#define MT6332_STRUP_CON12 0x8C1C +#define MT6332_STRUP_CON13 0x8C1E +#define MT6332_STRUP_CON14 0x8C20 +#define MT6332_STRUP_CON15 0x8C22 +#define MT6332_STRUP_CON16 0x8C24 +#define MT6332_STRUP_CON17 0x8C26 +#define MT6332_FGADC_CON0 0x8C28 +#define MT6332_FGADC_CON1 0x8C2A +#define MT6332_FGADC_CON2 0x8C2C +#define MT6332_FGADC_CON3 0x8C2E +#define MT6332_FGADC_CON4 0x8C30 +#define MT6332_FGADC_CON5 0x8C32 +#define MT6332_FGADC_CON6 0x8C34 +#define MT6332_FGADC_CON7 0x8C36 +#define MT6332_FGADC_CON8 0x8C38 +#define MT6332_FGADC_CON9 0x8C3A +#define MT6332_FGADC_CON10 0x8C3C +#define MT6332_FGADC_CON11 0x8C3E +#define MT6332_FGADC_CON12 0x8C40 +#define MT6332_FGADC_CON13 0x8C42 +#define MT6332_FGADC_CON14 0x8C44 +#define MT6332_FGADC_CON15 0x8C46 +#define MT6332_FGADC_CON16 0x8C48 +#define MT6332_FGADC_CON17 0x8C4A +#define MT6332_FGADC_CON18 0x8C4C +#define MT6332_FGADC_CON19 0x8C4E +#define MT6332_FGADC_CON20 0x8C50 +#define MT6332_FGADC_CON21 0x8C52 +#define MT6332_FGADC_CON22 0x8C54 +#define MT6332_OTP_CON0 0x8C56 +#define MT6332_OTP_CON1 0x8C58 +#define MT6332_OTP_CON2 0x8C5A +#define MT6332_OTP_CON3 0x8C5C +#define MT6332_OTP_CON4 0x8C5E +#define MT6332_OTP_CON5 0x8C60 +#define MT6332_OTP_CON6 0x8C62 +#define MT6332_OTP_CON7 0x8C64 +#define MT6332_OTP_CON8 0x8C66 +#define MT6332_OTP_CON9 0x8C68 +#define MT6332_OTP_CON10 0x8C6A +#define MT6332_OTP_CON11 0x8C6C +#define MT6332_OTP_CON12 0x8C6E +#define MT6332_OTP_CON13 0x8C70 +#define MT6332_OTP_CON14 0x8C72 +#define MT6332_OTP_DOUT_0_15 0x8C74 +#define MT6332_OTP_DOUT_16_31 0x8C76 +#define MT6332_OTP_DOUT_32_47 0x8C78 +#define MT6332_OTP_DOUT_48_63 0x8C7A +#define MT6332_OTP_DOUT_64_79 0x8C7C +#define MT6332_OTP_DOUT_80_95 0x8C7E +#define MT6332_OTP_DOUT_96_111 0x8C80 +#define MT6332_OTP_DOUT_112_127 0x8C82 +#define MT6332_OTP_DOUT_128_143 0x8C84 +#define MT6332_OTP_DOUT_144_159 0x8C86 +#define MT6332_OTP_DOUT_160_175 0x8C88 +#define MT6332_OTP_DOUT_176_191 0x8C8A +#define MT6332_OTP_DOUT_192_207 0x8C8C +#define MT6332_OTP_DOUT_208_223 0x8C8E +#define MT6332_OTP_DOUT_224_239 0x8C90 +#define MT6332_OTP_DOUT_240_255 0x8C92 +#define MT6332_OTP_VAL_0_15 0x8C94 +#define MT6332_OTP_VAL_16_31 0x8C96 +#define MT6332_OTP_VAL_32_47 0x8C98 +#define MT6332_OTP_VAL_48_63 0x8C9A +#define MT6332_OTP_VAL_64_79 0x8C9C +#define MT6332_OTP_VAL_80_95 0x8C9E +#define MT6332_OTP_VAL_96_111 0x8CA0 +#define MT6332_OTP_VAL_112_127 0x8CA2 +#define MT6332_OTP_VAL_128_143 0x8CA4 +#define MT6332_OTP_VAL_144_159 0x8CA6 +#define MT6332_OTP_VAL_160_175 0x8CA8 +#define MT6332_OTP_VAL_176_191 0x8CAA +#define MT6332_OTP_VAL_192_207 0x8CAC +#define MT6332_OTP_VAL_208_223 0x8CAE +#define MT6332_OTP_VAL_224_239 0x8CB0 +#define MT6332_OTP_VAL_240_255 0x8CB2 +#define MT6332_LDO_CON0 0x8CB4 +#define MT6332_LDO_CON1 0x8CB6 +#define MT6332_LDO_CON2 0x8CB8 +#define MT6332_LDO_CON3 0x8CBA +#define MT6332_LDO_CON5 0x8CBC +#define MT6332_LDO_CON6 0x8CBE +#define MT6332_LDO_CON7 0x8CC0 +#define MT6332_LDO_CON8 0x8CC2 +#define MT6332_LDO_CON9 0x8CC4 +#define MT6332_LDO_CON10 0x8CC6 +#define MT6332_LDO_CON11 0x8CC8 +#define MT6332_LDO_CON12 0x8CCA +#define MT6332_LDO_CON13 0x8CCC +#define MT6332_FQMTR_CON0 0x8CCE +#define MT6332_FQMTR_CON1 0x8CD0 +#define MT6332_FQMTR_CON2 0x8CD2 +#define MT6332_IWLED_CON0 0x8CD4 +#define MT6332_IWLED_DEG 0x8CD6 +#define MT6332_IWLED_STATUS 0x8CD8 +#define MT6332_IWLED_EN_CTRL 0x8CDA +#define MT6332_IWLED_CON1 0x8CDC +#define MT6332_IWLED_CON2 0x8CDE +#define MT6332_IWLED_TRIM0 0x8CE0 +#define MT6332_IWLED_TRIM1 0x8CE2 +#define MT6332_IWLED_CON3 0x8CE4 +#define MT6332_IWLED_CON4 0x8CE6 +#define MT6332_IWLED_CON5 0x8CE8 +#define MT6332_IWLED_CON6 0x8CEA +#define MT6332_IWLED_CON7 0x8CEC +#define MT6332_IWLED_CON8 0x8CEE +#define MT6332_IWLED_CON9 0x8CF0 +#define MT6332_SPK_CON0 0x8CF2 +#define MT6332_SPK_CON1 0x8CF4 +#define MT6332_SPK_CON2 0x8CF6 +#define MT6332_SPK_CON3 0x8CF8 +#define MT6332_SPK_CON4 0x8CFA +#define MT6332_SPK_CON5 0x8CFC +#define MT6332_SPK_CON6 0x8CFE +#define MT6332_SPK_CON7 0x8D00 +#define MT6332_SPK_CON8 0x8D02 +#define MT6332_SPK_CON9 0x8D04 +#define MT6332_SPK_CON10 0x8D06 +#define MT6332_SPK_CON11 0x8D08 +#define MT6332_SPK_CON12 0x8D0A +#define MT6332_SPK_CON13 0x8D0C +#define MT6332_SPK_CON14 0x8D0E +#define MT6332_SPK_CON15 0x8D10 +#define MT6332_SPK_CON16 0x8D12 +#define MT6332_TESTI_CON0 0x8D14 +#define MT6332_TESTI_CON1 0x8D16 +#define MT6332_TESTI_CON2 0x8D18 +#define MT6332_TESTI_CON3 0x8D1A +#define MT6332_TESTI_CON4 0x8D1C +#define MT6332_TESTI_CON5 0x8D1E +#define MT6332_TESTI_CON6 0x8D20 +#define MT6332_TESTI_MUX_CON0 0x8D22 +#define MT6332_TESTI_MUX_CON1 0x8D24 +#define MT6332_TESTI_MUX_CON2 0x8D26 +#define MT6332_TESTI_MUX_CON3 0x8D28 +#define MT6332_TESTI_MUX_CON4 0x8D2A +#define MT6332_TESTI_MUX_CON5 0x8D2C +#define MT6332_TESTI_MUX_CON6 0x8D2E +#define MT6332_TESTO_CON0 0x8D30 +#define MT6332_TESTO_CON1 0x8D32 +#define MT6332_TEST_OMUX_CON0 0x8D34 +#define MT6332_TEST_OMUX_CON1 0x8D36 +#define MT6332_DEBUG_CON0 0x8D38 +#define MT6332_DEBUG_CON1 0x8D3A +#define MT6332_DEBUG_CON2 0x8D3C +#define MT6332_FGADC_CON23 0x8D3E +#define MT6332_FGADC_CON24 0x8D40 +#define MT6332_FGADC_CON25 0x8D42 +#define MT6332_TOP_RST_STATUS 0x8D44 +#define MT6332_TOP_RST_STATUS_SET 0x8D46 +#define MT6332_TOP_RST_STATUS_CLR 0x8D48 +#define MT6332_VDVFS2_CON28 0x8D4A + +#endif /* __MFD_MT6332_REGISTERS_H__ */ diff --git a/include/linux/mfd/mt6397/core.h b/include/linux/mfd/mt6397/core.h index 3fecaffe5019..627487e26287 100644 --- a/include/linux/mfd/mt6397/core.h +++ b/include/linux/mfd/mt6397/core.h @@ -12,6 +12,8 @@ enum chip_id { MT6323_CHIP_ID = 0x23, + MT6331_CHIP_ID = 0x20, + MT6332_CHIP_ID = 0x20, MT6357_CHIP_ID = 0x57, MT6358_CHIP_ID = 0x58, MT6359_CHIP_ID = 0x59, -- cgit v1.2.3 From 32f02a211b0a192553075803b0b819027f96bb43 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 19 Jul 2022 13:57:48 +0200 Subject: Revert "platform/chrome: Add Type-C mux set command definitions" This reverts commit 28a6ed8e39f77f6ac613ec9b7461aa75e85fa79a. The chrome platform driver changes need to come in through the platform tree due to some api changes that showed up there that cause build errors in linux-next Reported-by: Stephen Rothwell Link: https://lore.kernel.org/r/20220719160821.5e68e30b@oak.ozlabs.ibm.com Cc: Prashant Malani Signed-off-by: Greg Kroah-Hartman --- include/linux/platform_data/cros_ec_commands.h | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index a3945c5e7f50..8cfa8cfca77e 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -5722,21 +5722,8 @@ enum typec_control_command { TYPEC_CONTROL_COMMAND_EXIT_MODES, TYPEC_CONTROL_COMMAND_CLEAR_EVENTS, TYPEC_CONTROL_COMMAND_ENTER_MODE, - TYPEC_CONTROL_COMMAND_TBT_UFP_REPLY, - TYPEC_CONTROL_COMMAND_USB_MUX_SET, }; -/* Replies the AP may specify to the TBT EnterMode command as a UFP */ -enum typec_tbt_ufp_reply { - TYPEC_TBT_UFP_REPLY_NAK, - TYPEC_TBT_UFP_REPLY_ACK, -}; - -struct typec_usb_mux_set { - uint8_t mux_index; /* Index of the mux to set in the chain */ - uint8_t mux_flags; /* USB_PD_MUX_*-encoded USB mux state to set */ -} __ec_align1; - struct ec_params_typec_control { uint8_t port; uint8_t command; /* enum typec_control_command */ @@ -5750,8 +5737,6 @@ struct ec_params_typec_control { union { uint32_t clear_events_mask; uint8_t mode_to_enter; /* enum typec_mode */ - uint8_t tbt_ufp_reply; /* enum typec_tbt_ufp_reply */ - struct typec_usb_mux_set mux_params; uint8_t placeholder[128]; }; } __ec_align1; @@ -5830,9 +5815,6 @@ enum tcpc_cc_polarity { #define PD_STATUS_EVENT_SOP_DISC_DONE BIT(0) #define PD_STATUS_EVENT_SOP_PRIME_DISC_DONE BIT(1) #define PD_STATUS_EVENT_HARD_RESET BIT(2) -#define PD_STATUS_EVENT_DISCONNECTED BIT(3) -#define PD_STATUS_EVENT_MUX_0_SET_DONE BIT(4) -#define PD_STATUS_EVENT_MUX_1_SET_DONE BIT(5) struct ec_params_typec_status { uint8_t port; -- cgit v1.2.3 From 2b3416ceff5e6bd4922f6d1c61fb68113dd82302 Mon Sep 17 00:00:00 2001 From: Yang Xu Date: Thu, 14 Jul 2022 14:11:25 +0800 Subject: fs: add mode_strip_sgid() helper Add a dedicated helper to handle the setgid bit when creating a new file in a setgid directory. This is a preparatory patch for moving setgid stripping into the vfs. The patch contains no functional changes. Currently the setgid stripping logic is open-coded directly in inode_init_owner() and the individual filesystems are responsible for handling setgid inheritance. Since this has proven to be brittle as evidenced by old issues we uncovered over the last months (see [1] to [3] below) we will try to move this logic into the vfs. Link: e014f37db1a2 ("xfs: use setattr_copy to set vfs inode attributes") [1] Link: 01ea173e103e ("xfs: fix up non-directory creation in SGID directories") [2] Link: fd84bfdddd16 ("ceph: fix up non-directory creation in SGID directories") [3] Link: https://lore.kernel.org/r/1657779088-2242-1-git-send-email-xuyang2018.jy@fujitsu.com Reviewed-by: Darrick J. Wong Reviewed-by: Christian Brauner (Microsoft) Reviewed-and-Tested-by: Jeff Layton Signed-off-by: Yang Xu Signed-off-by: Christian Brauner (Microsoft) --- fs/inode.c | 36 ++++++++++++++++++++++++++++++++---- include/linux/fs.h | 2 ++ 2 files changed, 34 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/inode.c b/fs/inode.c index bd4da9c5207e..71b36afc3893 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2246,10 +2246,8 @@ void inode_init_owner(struct user_namespace *mnt_userns, struct inode *inode, /* Directories are special, and always inherit S_ISGID */ if (S_ISDIR(mode)) mode |= S_ISGID; - else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) && - !in_group_p(i_gid_into_mnt(mnt_userns, dir)) && - !capable_wrt_inode_uidgid(mnt_userns, dir, CAP_FSETID)) - mode &= ~S_ISGID; + else + mode = mode_strip_sgid(mnt_userns, dir, mode); } else inode_fsgid_set(inode, mnt_userns); inode->i_mode = mode; @@ -2405,3 +2403,33 @@ struct timespec64 current_time(struct inode *inode) return timestamp_truncate(now, inode); } EXPORT_SYMBOL(current_time); + +/** + * mode_strip_sgid - handle the sgid bit for non-directories + * @mnt_userns: User namespace of the mount the inode was created from + * @dir: parent directory inode + * @mode: mode of the file to be created in @dir + * + * If the @mode of the new file has both the S_ISGID and S_IXGRP bit + * raised and @dir has the S_ISGID bit raised ensure that the caller is + * either in the group of the parent directory or they have CAP_FSETID + * in their user namespace and are privileged over the parent directory. + * In all other cases, strip the S_ISGID bit from @mode. + * + * Return: the new mode to use for the file + */ +umode_t mode_strip_sgid(struct user_namespace *mnt_userns, + const struct inode *dir, umode_t mode) +{ + if ((mode & (S_ISGID | S_IXGRP)) != (S_ISGID | S_IXGRP)) + return mode; + if (S_ISDIR(mode) || !dir || !(dir->i_mode & S_ISGID)) + return mode; + if (in_group_p(i_gid_into_mnt(mnt_userns, dir))) + return mode; + if (capable_wrt_inode_uidgid(mnt_userns, dir, CAP_FSETID)) + return mode; + + return mode & ~S_ISGID; +} +EXPORT_SYMBOL(mode_strip_sgid); diff --git a/include/linux/fs.h b/include/linux/fs.h index 9ad5e3520fae..50642668c60f 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1903,6 +1903,8 @@ extern long compat_ptr_ioctl(struct file *file, unsigned int cmd, void inode_init_owner(struct user_namespace *mnt_userns, struct inode *inode, const struct inode *dir, umode_t mode); extern bool may_open_dev(const struct path *path); +umode_t mode_strip_sgid(struct user_namespace *mnt_userns, + const struct inode *dir, umode_t mode); /* * This is the "filldir" function type, used by readdir() to let -- cgit v1.2.3 From fd1894224407c484f652ad456e1ce423e89bb3eb Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Fri, 15 Jul 2022 19:55:59 +0800 Subject: bpf: Don't redirect packets with invalid pkt_len Syzbot found an issue [1]: fq_codel_drop() try to drop a flow whitout any skbs, that is, the flow->head is null. The root cause, as the [2] says, is because that bpf_prog_test_run_skb() run a bpf prog which redirects empty skbs. So we should determine whether the length of the packet modified by bpf prog or others like bpf_prog_test is valid before forwarding it directly. LINK: [1] https://syzkaller.appspot.com/bug?id=0b84da80c2917757915afa89f7738a9d16ec96c5 LINK: [2] https://www.spinics.net/lists/netdev/msg777503.html Reported-by: syzbot+7a12909485b94426aceb@syzkaller.appspotmail.com Signed-off-by: Zhengchao Shao Reviewed-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220715115559.139691-1-shaozhengchao@huawei.com Signed-off-by: Alexei Starovoitov --- include/linux/skbuff.h | 8 ++++++++ net/bpf/test_run.c | 3 +++ net/core/dev.c | 1 + 3 files changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index f6a27ab19202..82e8368ba6e6 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2459,6 +2459,14 @@ static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) #endif /* NET_SKBUFF_DATA_USES_OFFSET */ +static inline void skb_assert_len(struct sk_buff *skb) +{ +#ifdef CONFIG_DEBUG_NET + if (WARN_ONCE(!skb->len, "%s\n", __func__)) + DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); +#endif /* CONFIG_DEBUG_NET */ +} + /* * Add data to an sk_buff */ diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 2ca96acbc50a..dc9dc0bedca0 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -955,6 +955,9 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) { struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; + if (!skb->len) + return -EINVAL; + if (!__skb) return 0; diff --git a/net/core/dev.c b/net/core/dev.c index 978ed0622d8f..e241a475036f 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4168,6 +4168,7 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) bool again = false; skb_reset_mac_header(skb); + skb_assert_len(skb); if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); -- cgit v1.2.3 From 800d6acf40e5ce676b53a1259fb4e93e56279367 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 27 May 2022 17:07:45 +0200 Subject: rcu: tiny: Record kvfree_call_rcu() call stack for KASAN When running KASAN with Tiny RCU (e.g. under ARCH=um, where a working KASAN patch is now available), we don't get any information on the original kfree_rcu() (or similar) caller when a problem is reported, as Tiny RCU doesn't record this. Add the recording, which required pulling kvfree_call_rcu() out of line for the KASAN case since the recording function (kasan_record_aux_stack_noalloc) is neither exported, nor can we include kasan.h into rcutiny.h. without KASAN, the patch has no size impact (ARCH=um kernel): text data bss dec hex filename 6151515 4423154 33148520 43723189 29b29b5 linux 6151515 4423154 33148520 43723189 29b29b5 linux + patch with KASAN, the impact on my build was minimal: text data bss dec hex filename 13915539 7388050 33282304 54585893 340ea25 linux 13911266 7392114 33282304 54585684 340e954 linux + patch -4273 +4064 +-0 -209 Acked-by: Dmitry Vyukov Signed-off-by: Johannes Berg Signed-off-by: Paul E. McKenney --- include/linux/rcutiny.h | 11 ++++++++++- kernel/rcu/tiny.c | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 5fed476f977f..d84e13f2c384 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -38,7 +38,7 @@ static inline void synchronize_rcu_expedited(void) */ extern void kvfree(const void *addr); -static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +static inline void __kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) { if (head) { call_rcu(head, func); @@ -51,6 +51,15 @@ static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) kvfree((void *) func); } +#ifdef CONFIG_KASAN_GENERIC +void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func); +#else +static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +{ + __kvfree_call_rcu(head, func); +} +#endif + void rcu_qs(void); static inline void rcu_softirq_qs(void) diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 340b3f8b090d..58ff3721d975 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -217,6 +217,20 @@ bool poll_state_synchronize_rcu(unsigned long oldstate) } EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); +#ifdef CONFIG_KASAN_GENERIC +void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +{ + if (head) { + void *ptr = (void *) head - (unsigned long) func; + + kasan_record_aux_stack_noalloc(ptr); + } + + __kvfree_call_rcu(head, func); +} +EXPORT_SYMBOL_GPL(kvfree_call_rcu); +#endif + void __init rcu_init(void) { open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); -- cgit v1.2.3 From 2e5e4185ff89ea0fc44c9dead8dccf306a3b3bbb Mon Sep 17 00:00:00 2001 From: Aya Levin Date: Mon, 4 Jul 2022 19:34:08 +0300 Subject: net/mlx5: Expose ts_cqe_metadata_size2wqe_counter Add capability field which indicates the mask for wqe_counter which connects between loopback CQE and the original WQE. With this connection the driver can identify lost of the loopback CQE and reply PTP synchronization with timestamp given in the original CQE. Signed-off-by: Aya Levin Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 254cc22f5eec..51b4e71017ee 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1833,7 +1833,11 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 sw_vhca_id[0xe]; u8 reserved_at_230[0x10]; - u8 reserved_at_240[0x5c0]; + u8 reserved_at_240[0xb]; + u8 ts_cqe_metadata_size2wqe_counter[0x5]; + u8 reserved_at_250[0x10]; + + u8 reserved_at_260[0x5a0]; }; enum mlx5_ifc_flow_destination_type { -- cgit v1.2.3 From 7c701d92b2b5e5175dbfec875816474b802b0c45 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Jul 2022 21:52:29 +0100 Subject: skbuff: carry external ubuf_info in msghdr Make possible for network in-kernel callers like io_uring to pass in a custom ubuf_info by setting it in a new field of struct msghdr. Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- include/linux/socket.h | 1 + net/compat.c | 1 + net/socket.c | 2 ++ 3 files changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index 17311ad9f9af..7bac9fc1cee0 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -69,6 +69,7 @@ struct msghdr { unsigned int msg_flags; /* flags on received message */ __kernel_size_t msg_controllen; /* ancillary data buffer length */ struct kiocb *msg_iocb; /* ptr to iocb for async requests */ + struct ubuf_info *msg_ubuf; }; struct user_msghdr { diff --git a/net/compat.c b/net/compat.c index 210fc3b4d0d8..6cd2e7683dd0 100644 --- a/net/compat.c +++ b/net/compat.c @@ -80,6 +80,7 @@ int __get_compat_msghdr(struct msghdr *kmsg, return -EMSGSIZE; kmsg->msg_iocb = NULL; + kmsg->msg_ubuf = NULL; *ptr = msg.msg_iov; *len = msg.msg_iovlen; return 0; diff --git a/net/socket.c b/net/socket.c index 96300cdc0625..82af3882b876 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2106,6 +2106,7 @@ int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags, msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_namelen = 0; + msg.msg_ubuf = NULL; if (addr) { err = move_addr_to_kernel(addr, addr_len, &address); if (err < 0) @@ -2405,6 +2406,7 @@ int __copy_msghdr_from_user(struct msghdr *kmsg, return -EMSGSIZE; kmsg->msg_iocb = NULL; + kmsg->msg_ubuf = NULL; *uiov = msg.msg_iov; *nsegs = msg.msg_iovlen; return 0; -- cgit v1.2.3 From ebe73a284f4de8c5d401adeccd9b8fe3183b6e95 Mon Sep 17 00:00:00 2001 From: David Ahern Date: Tue, 12 Jul 2022 21:52:30 +0100 Subject: net: Allow custom iter handler in msghdr Add support for custom iov_iter handling to msghdr. The idea is that in-kernel subsystems want control over how an SG is split. Signed-off-by: David Ahern [pavel: move callback into msghdr] Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 7 ++++--- include/linux/socket.h | 4 ++++ net/core/datagram.c | 14 ++++++++++---- net/core/skbuff.c | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 8e12b3b9ad6c..a8a2dd4cfdfd 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1776,13 +1776,14 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, bool success); -int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb, - struct iov_iter *from, size_t length); +int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, + struct sk_buff *skb, struct iov_iter *from, + size_t length); static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len) { - return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len); + return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len); } int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, diff --git a/include/linux/socket.h b/include/linux/socket.h index 7bac9fc1cee0..3c11ef18a9cf 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -14,6 +14,8 @@ struct file; struct pid; struct cred; struct socket; +struct sock; +struct sk_buff; #define __sockaddr_check_size(size) \ BUILD_BUG_ON(((size) > sizeof(struct __kernel_sockaddr_storage))) @@ -70,6 +72,8 @@ struct msghdr { __kernel_size_t msg_controllen; /* ancillary data buffer length */ struct kiocb *msg_iocb; /* ptr to iocb for async requests */ struct ubuf_info *msg_ubuf; + int (*sg_from_iter)(struct sock *sk, struct sk_buff *skb, + struct iov_iter *from, size_t length); }; struct user_msghdr { diff --git a/net/core/datagram.c b/net/core/datagram.c index 50f4faeea76c..28cdb79df74d 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -613,10 +613,16 @@ fault: } EXPORT_SYMBOL(skb_copy_datagram_from_iter); -int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb, - struct iov_iter *from, size_t length) +int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, + struct sk_buff *skb, struct iov_iter *from, + size_t length) { - int frag = skb_shinfo(skb)->nr_frags; + int frag; + + if (msg && msg->sg_from_iter) + return msg->sg_from_iter(sk, skb, from, length); + + frag = skb_shinfo(skb)->nr_frags; while (length && iov_iter_count(from)) { struct page *pages[MAX_SKB_FRAGS]; @@ -702,7 +708,7 @@ int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *from) if (skb_copy_datagram_from_iter(skb, 0, from, copy)) return -EFAULT; - return __zerocopy_sg_from_iter(NULL, skb, from, ~0U); + return __zerocopy_sg_from_iter(NULL, NULL, skb, from, ~0U); } EXPORT_SYMBOL(zerocopy_sg_from_iter); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index fc22b3d32052..f5a3ebbc1f7e 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1358,7 +1358,7 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, if (orig_uarg && uarg != orig_uarg) return -EEXIST; - err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); + err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len); if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { struct sock *save_sk = skb->sk; -- cgit v1.2.3 From 753f1ca4e1e50248a1b760c9774d6d6b354562cc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Jul 2022 21:52:31 +0100 Subject: net: introduce managed frags infrastructure Some users like io_uring can do page pinning more efficiently, so we want a way to delegate referencing to other subsystems. For that add a new flag called SKBFL_MANAGED_FRAG_REFS. When set, skb doesn't hold page references and upper layers are responsivle to managing page lifetime. It's allowed to convert skbs from managed to normal by calling skb_zcopy_downgrade_managed(). The function will take all needed page references and clear the flag. It's needed, for instance, to avoid mixing managed modes. Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 25 +++++++++++++++++++++++-- net/core/skbuff.c | 29 +++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index a8a2dd4cfdfd..07004593d7ca 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -688,11 +688,16 @@ enum { SKBFL_PURE_ZEROCOPY = BIT(2), SKBFL_DONT_ORPHAN = BIT(3), + + /* page references are managed by the ubuf_info, so it's safe to + * use frags only up until ubuf_info is released + */ + SKBFL_MANAGED_FRAG_REFS = BIT(4), }; #define SKBFL_ZEROCOPY_FRAG (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG) #define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \ - SKBFL_DONT_ORPHAN) + SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS) /* * The callback notifies userspace to release buffers when skb DMA is done in @@ -1810,6 +1815,11 @@ static inline bool skb_zcopy_pure(const struct sk_buff *skb) return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY; } +static inline bool skb_zcopy_managed(const struct sk_buff *skb) +{ + return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS; +} + static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1, const struct sk_buff *skb2) { @@ -1884,6 +1894,14 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) } } +void __skb_zcopy_downgrade_managed(struct sk_buff *skb); + +static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb) +{ + if (unlikely(skb_zcopy_managed(skb))) + __skb_zcopy_downgrade_managed(skb); +} + static inline void skb_mark_not_on_list(struct sk_buff *skb) { skb->next = NULL; @@ -3499,7 +3517,10 @@ static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle) */ static inline void skb_frag_unref(struct sk_buff *skb, int f) { - __skb_frag_unref(&skb_shinfo(skb)->frags[f], skb->pp_recycle); + struct skb_shared_info *shinfo = skb_shinfo(skb); + + if (!skb_zcopy_managed(skb)) + __skb_frag_unref(&shinfo->frags[f], skb->pp_recycle); } /** diff --git a/net/core/skbuff.c b/net/core/skbuff.c index f5a3ebbc1f7e..cf4107d80bc4 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -666,11 +666,18 @@ static void skb_release_data(struct sk_buff *skb) &shinfo->dataref)) goto exit; - skb_zcopy_clear(skb, true); + if (skb_zcopy(skb)) { + bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; + + skb_zcopy_clear(skb, true); + if (skip_unref) + goto free_head; + } for (i = 0; i < shinfo->nr_frags; i++) __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle); +free_head: if (shinfo->frag_list) kfree_skb_list(shinfo->frag_list); @@ -895,7 +902,10 @@ EXPORT_SYMBOL(skb_dump); */ void skb_tx_error(struct sk_buff *skb) { - skb_zcopy_clear(skb, true); + if (skb) { + skb_zcopy_downgrade_managed(skb); + skb_zcopy_clear(skb, true); + } } EXPORT_SYMBOL(skb_tx_error); @@ -1375,6 +1385,16 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, } EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); +void __skb_zcopy_downgrade_managed(struct sk_buff *skb) +{ + int i; + + skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) + skb_frag_ref(skb, i); +} +EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); + static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, gfp_t gfp_mask) { @@ -1692,6 +1712,8 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, BUG_ON(skb_shared(skb)); + skb_zcopy_downgrade_managed(skb); + size = SKB_DATA_ALIGN(size); if (skb_pfmemalloc(skb)) @@ -3488,6 +3510,8 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) int pos = skb_headlen(skb); const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; + skb_zcopy_downgrade_managed(skb); + skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; skb_zerocopy_clone(skb1, skb, 0); if (len < pos) /* Split line is inside header. */ @@ -3841,6 +3865,7 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page, if (skb_can_coalesce(skb, i, page, offset)) { skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); } else if (i < MAX_SKB_FRAGS) { + skb_zcopy_downgrade_managed(skb); get_page(page); skb_fill_page_desc(skb, i, page, offset, size); } else { -- cgit v1.2.3 From 84ce071e38a6e25ea3ea91188e5482ac1f17b3af Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Jul 2022 21:52:32 +0100 Subject: net: introduce __skb_fill_page_desc_noacc Managed pages contain pinned userspace pages and controlled by upper layers, there is no need in tracking skb->pfmemalloc for them. Introduce a helper for filling frags but ignoring page tracking, it'll be needed later. Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 07004593d7ca..1111adefd906 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2550,6 +2550,22 @@ static inline unsigned int skb_pagelen(const struct sk_buff *skb) return skb_headlen(skb) + __skb_pagelen(skb); } +static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo, + int i, struct page *page, + int off, int size) +{ + skb_frag_t *frag = &shinfo->frags[i]; + + /* + * Propagate page pfmemalloc to the skb if we can. The problem is + * that not all callers have unique ownership of the page but rely + * on page_is_pfmemalloc doing the right thing(tm). + */ + frag->bv_page = page; + frag->bv_offset = off; + skb_frag_size_set(frag, size); +} + /** * __skb_fill_page_desc - initialise a paged fragment in an skb * @skb: buffer containing fragment to be initialised @@ -2566,17 +2582,7 @@ static inline unsigned int skb_pagelen(const struct sk_buff *skb) static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, int off, int size) { - skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - - /* - * Propagate page pfmemalloc to the skb if we can. The problem is - * that not all callers have unique ownership of the page but rely - * on page_is_pfmemalloc doing the right thing(tm). - */ - frag->bv_page = page; - frag->bv_offset = off; - skb_frag_size_set(frag, size); - + __skb_fill_page_desc_noacc(skb_shinfo(skb), i, page, off, size); page = compound_head(page); if (page_is_pfmemalloc(page)) skb->pfmemalloc = true; -- cgit v1.2.3 From e6b8a0a5e7f688e092d1c639d438ccd0b323213c Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 19 Jul 2022 13:52:44 -0700 Subject: PCI: Add vendor ID for the PCI SIG This ID is used in DOE headers to identify protocols that are defined within the PCI Express Base Specification, PCIe r6.0, sec 6.30.1.1 table 6-32. Acked-by: Bjorn Helgaas Reviewed-by: Davidlohr Bueso Reviewed-by: Dan Williams Signed-off-by: Jonathan Cameron Link: https://lore.kernel.org/r/20220719205249.566684-2-ira.weiny@intel.com Signed-off-by: Dan Williams --- include/linux/pci_ids.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 0178823ce8c2..8af3b86206b1 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -151,6 +151,7 @@ #define PCI_CLASS_OTHERS 0xff /* Vendors and devices. Sort key: vendor first, device next. */ +#define PCI_VENDOR_ID_PCI_SIG 0x0001 #define PCI_VENDOR_ID_LOONGSON 0x0014 -- cgit v1.2.3 From 9d24322e887b6a3d3f9f9c3e76937a646102c8c1 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 19 Jul 2022 13:52:46 -0700 Subject: PCI/DOE: Add DOE mailbox support functions Introduced in a PCIe r6.0, sec 6.30, DOE provides a config space based mailbox with standard protocol discovery. Each mailbox is accessed through a DOE Extended Capability. Each DOE mailbox must support the DOE discovery protocol in addition to any number of additional protocols. Define core PCIe functionality to manage a single PCIe DOE mailbox at a defined config space offset. Functionality includes iterating, creating, query of supported protocol, and task submission. Destruction of the mailboxes is device managed. Cc: "Li, Ming" Cc: Bjorn Helgaas Cc: Matthew Wilcox Acked-by: Bjorn Helgaas Signed-off-by: Jonathan Cameron Co-developed-by: Ira Weiny Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20220719205249.566684-4-ira.weiny@intel.com Signed-off-by: Dan Williams --- .clang-format | 1 + drivers/pci/Kconfig | 3 + drivers/pci/Makefile | 1 + drivers/pci/doe.c | 536 ++++++++++++++++++++++++++++++++++++++++++ include/linux/pci-doe.h | 77 ++++++ include/uapi/linux/pci_regs.h | 29 ++- 6 files changed, 646 insertions(+), 1 deletion(-) create mode 100644 drivers/pci/doe.c create mode 100644 include/linux/pci-doe.h (limited to 'include/linux') diff --git a/.clang-format b/.clang-format index 9b87ea1fc16e..1247d54f9e49 100644 --- a/.clang-format +++ b/.clang-format @@ -516,6 +516,7 @@ ForEachMacros: - 'of_property_for_each_string' - 'of_property_for_each_u32' - 'pci_bus_for_each_resource' + - 'pci_doe_for_each_off' - 'pcl_for_each_chunk' - 'pcl_for_each_segment' - 'pcm_for_each_format' diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 133c73207782..b2f2e588a817 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -121,6 +121,9 @@ config XEN_PCIDEV_FRONTEND config PCI_ATS bool +config PCI_DOE + bool + config PCI_ECAM bool diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 0da6b1ebc694..2680e4c92f0a 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_PCI_ECAM) += ecam.o obj-$(CONFIG_PCI_P2PDMA) += p2pdma.o obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o obj-$(CONFIG_VGA_ARB) += vgaarb.o +obj-$(CONFIG_PCI_DOE) += doe.o # Endpoint library must be initialized before its users obj-$(CONFIG_PCI_ENDPOINT) += endpoint/ diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c new file mode 100644 index 000000000000..e402f05068a5 --- /dev/null +++ b/drivers/pci/doe.c @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Data Object Exchange + * PCIe r6.0, sec 6.30 DOE + * + * Copyright (C) 2021 Huawei + * Jonathan Cameron + * + * Copyright (C) 2022 Intel Corporation + * Ira Weiny + */ + +#define dev_fmt(fmt) "DOE: " fmt + +#include +#include +#include +#include +#include +#include +#include + +#define PCI_DOE_PROTOCOL_DISCOVERY 0 + +/* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */ +#define PCI_DOE_TIMEOUT HZ +#define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128) + +#define PCI_DOE_FLAG_CANCEL 0 +#define PCI_DOE_FLAG_DEAD 1 + +/** + * struct pci_doe_mb - State for a single DOE mailbox + * + * This state is used to manage a single DOE mailbox capability. All fields + * should be considered opaque to the consumers and the structure passed into + * the helpers below after being created by devm_pci_doe_create() + * + * @pdev: PCI device this mailbox belongs to + * @cap_offset: Capability offset + * @prots: Array of protocols supported (encoded as long values) + * @wq: Wait queue for work item + * @work_queue: Queue of pci_doe_work items + * @flags: Bit array of PCI_DOE_FLAG_* flags + */ +struct pci_doe_mb { + struct pci_dev *pdev; + u16 cap_offset; + struct xarray prots; + + wait_queue_head_t wq; + struct workqueue_struct *work_queue; + unsigned long flags; +}; + +static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout) +{ + if (wait_event_timeout(doe_mb->wq, + test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags), + timeout)) + return -EIO; + return 0; +} + +static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val) +{ + struct pci_dev *pdev = doe_mb->pdev; + int offset = doe_mb->cap_offset; + + pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val); +} + +static int pci_doe_abort(struct pci_doe_mb *doe_mb) +{ + struct pci_dev *pdev = doe_mb->pdev; + int offset = doe_mb->cap_offset; + unsigned long timeout_jiffies; + + pci_dbg(pdev, "[%x] Issuing Abort\n", offset); + + timeout_jiffies = jiffies + PCI_DOE_TIMEOUT; + pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT); + + do { + int rc; + u32 val; + + rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL); + if (rc) + return rc; + pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); + + /* Abort success! */ + if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) && + !FIELD_GET(PCI_DOE_STATUS_BUSY, val)) + return 0; + + } while (!time_after(jiffies, timeout_jiffies)); + + /* Abort has timed out and the MB is dead */ + pci_err(pdev, "[%x] ABORT timed out\n", offset); + return -EIO; +} + +static int pci_doe_send_req(struct pci_doe_mb *doe_mb, + struct pci_doe_task *task) +{ + struct pci_dev *pdev = doe_mb->pdev; + int offset = doe_mb->cap_offset; + u32 val; + int i; + + /* + * Check the DOE busy bit is not set. If it is set, this could indicate + * someone other than Linux (e.g. firmware) is using the mailbox. Note + * it is expected that firmware and OS will negotiate access rights via + * an, as yet to be defined, method. + */ + pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); + if (FIELD_GET(PCI_DOE_STATUS_BUSY, val)) + return -EBUSY; + + if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) + return -EIO; + + /* Write DOE Header */ + val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) | + FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type); + pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val); + /* Length is 2 DW of header + length of payload in DW */ + pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, + FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, + 2 + task->request_pl_sz / + sizeof(u32))); + for (i = 0; i < task->request_pl_sz / sizeof(u32); i++) + pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, + task->request_pl[i]); + + pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO); + + return 0; +} + +static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb) +{ + struct pci_dev *pdev = doe_mb->pdev; + int offset = doe_mb->cap_offset; + u32 val; + + pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); + if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) + return true; + return false; +} + +static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) +{ + struct pci_dev *pdev = doe_mb->pdev; + int offset = doe_mb->cap_offset; + size_t length, payload_length; + u32 val; + int i; + + /* Read the first dword to get the protocol */ + pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); + if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) || + (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) { + dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n", + doe_mb->cap_offset, task->prot.vid, task->prot.type, + FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val), + FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val)); + return -EIO; + } + + pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); + /* Read the second dword to get the length */ + pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); + pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); + + length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val); + if (length > SZ_1M || length < 2) + return -EIO; + + /* First 2 dwords have already been read */ + length -= 2; + payload_length = min(length, task->response_pl_sz / sizeof(u32)); + /* Read the rest of the response payload */ + for (i = 0; i < payload_length; i++) { + pci_read_config_dword(pdev, offset + PCI_DOE_READ, + &task->response_pl[i]); + /* Prior to the last ack, ensure Data Object Ready */ + if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb)) + return -EIO; + pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); + } + + /* Flush excess length */ + for (; i < length; i++) { + pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); + pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); + } + + /* Final error check to pick up on any since Data Object Ready */ + pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); + if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) + return -EIO; + + return min(length, task->response_pl_sz / sizeof(u32)) * sizeof(u32); +} + +static void signal_task_complete(struct pci_doe_task *task, int rv) +{ + task->rv = rv; + task->complete(task); +} + +static void signal_task_abort(struct pci_doe_task *task, int rv) +{ + struct pci_doe_mb *doe_mb = task->doe_mb; + struct pci_dev *pdev = doe_mb->pdev; + + if (pci_doe_abort(doe_mb)) { + /* + * If the device can't process an abort; set the mailbox dead + * - no more submissions + */ + pci_err(pdev, "[%x] Abort failed marking mailbox dead\n", + doe_mb->cap_offset); + set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags); + } + signal_task_complete(task, rv); +} + +static void doe_statemachine_work(struct work_struct *work) +{ + struct pci_doe_task *task = container_of(work, struct pci_doe_task, + work); + struct pci_doe_mb *doe_mb = task->doe_mb; + struct pci_dev *pdev = doe_mb->pdev; + int offset = doe_mb->cap_offset; + unsigned long timeout_jiffies; + u32 val; + int rc; + + if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) { + signal_task_complete(task, -EIO); + return; + } + + /* Send request */ + rc = pci_doe_send_req(doe_mb, task); + if (rc) { + /* + * The specification does not provide any guidance on how to + * resolve conflicting requests from other entities. + * Furthermore, it is likely that busy will not be detected + * most of the time. Flag any detection of status busy with an + * error. + */ + if (rc == -EBUSY) + dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n", + offset); + signal_task_abort(task, rc); + return; + } + + timeout_jiffies = jiffies + PCI_DOE_TIMEOUT; + /* Poll for response */ +retry_resp: + pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); + if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) { + signal_task_abort(task, -EIO); + return; + } + + if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) { + if (time_after(jiffies, timeout_jiffies)) { + signal_task_abort(task, -EIO); + return; + } + rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL); + if (rc) { + signal_task_abort(task, rc); + return; + } + goto retry_resp; + } + + rc = pci_doe_recv_resp(doe_mb, task); + if (rc < 0) { + signal_task_abort(task, rc); + return; + } + + signal_task_complete(task, rc); +} + +static void pci_doe_task_complete(struct pci_doe_task *task) +{ + complete(task->private); +} + +static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid, + u8 *protocol) +{ + u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX, + *index); + u32 response_pl; + DECLARE_COMPLETION_ONSTACK(c); + struct pci_doe_task task = { + .prot.vid = PCI_VENDOR_ID_PCI_SIG, + .prot.type = PCI_DOE_PROTOCOL_DISCOVERY, + .request_pl = &request_pl, + .request_pl_sz = sizeof(request_pl), + .response_pl = &response_pl, + .response_pl_sz = sizeof(response_pl), + .complete = pci_doe_task_complete, + .private = &c, + }; + int rc; + + rc = pci_doe_submit_task(doe_mb, &task); + if (rc < 0) + return rc; + + wait_for_completion(&c); + + if (task.rv != sizeof(response_pl)) + return -EIO; + + *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl); + *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL, + response_pl); + *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX, + response_pl); + + return 0; +} + +static void *pci_doe_xa_prot_entry(u16 vid, u8 prot) +{ + return xa_mk_value((vid << 8) | prot); +} + +static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb) +{ + u8 index = 0; + u8 xa_idx = 0; + + do { + int rc; + u16 vid; + u8 prot; + + rc = pci_doe_discovery(doe_mb, &index, &vid, &prot); + if (rc) + return rc; + + pci_dbg(doe_mb->pdev, + "[%x] Found protocol %d vid: %x prot: %x\n", + doe_mb->cap_offset, xa_idx, vid, prot); + + rc = xa_insert(&doe_mb->prots, xa_idx++, + pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL); + if (rc) + return rc; + } while (index); + + return 0; +} + +static void pci_doe_xa_destroy(void *mb) +{ + struct pci_doe_mb *doe_mb = mb; + + xa_destroy(&doe_mb->prots); +} + +static void pci_doe_destroy_workqueue(void *mb) +{ + struct pci_doe_mb *doe_mb = mb; + + destroy_workqueue(doe_mb->work_queue); +} + +static void pci_doe_flush_mb(void *mb) +{ + struct pci_doe_mb *doe_mb = mb; + + /* Stop all pending work items from starting */ + set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags); + + /* Cancel an in progress work item, if necessary */ + set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags); + wake_up(&doe_mb->wq); + + /* Flush all work items */ + flush_workqueue(doe_mb->work_queue); +} + +/** + * pcim_doe_create_mb() - Create a DOE mailbox object + * + * @pdev: PCI device to create the DOE mailbox for + * @cap_offset: Offset of the DOE mailbox + * + * Create a single mailbox object to manage the mailbox protocol at the + * cap_offset specified. + * + * RETURNS: created mailbox object on success + * ERR_PTR(-errno) on failure + */ +struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset) +{ + struct pci_doe_mb *doe_mb; + struct device *dev = &pdev->dev; + int rc; + + doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL); + if (!doe_mb) + return ERR_PTR(-ENOMEM); + + doe_mb->pdev = pdev; + doe_mb->cap_offset = cap_offset; + init_waitqueue_head(&doe_mb->wq); + + xa_init(&doe_mb->prots); + rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb); + if (rc) + return ERR_PTR(rc); + + doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0, + dev_driver_string(&pdev->dev), + pci_name(pdev), + doe_mb->cap_offset); + if (!doe_mb->work_queue) { + pci_err(pdev, "[%x] failed to allocate work queue\n", + doe_mb->cap_offset); + return ERR_PTR(-ENOMEM); + } + rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb); + if (rc) + return ERR_PTR(rc); + + /* Reset the mailbox by issuing an abort */ + rc = pci_doe_abort(doe_mb); + if (rc) { + pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n", + doe_mb->cap_offset, rc); + return ERR_PTR(rc); + } + + /* + * The state machine and the mailbox should be in sync now; + * Set up mailbox flush prior to using the mailbox to query protocols. + */ + rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb); + if (rc) + return ERR_PTR(rc); + + rc = pci_doe_cache_protocols(doe_mb); + if (rc) { + pci_err(pdev, "[%x] failed to cache protocols : %d\n", + doe_mb->cap_offset, rc); + return ERR_PTR(rc); + } + + return doe_mb; +} +EXPORT_SYMBOL_GPL(pcim_doe_create_mb); + +/** + * pci_doe_supports_prot() - Return if the DOE instance supports the given + * protocol + * @doe_mb: DOE mailbox capability to query + * @vid: Protocol Vendor ID + * @type: Protocol type + * + * RETURNS: True if the DOE mailbox supports the protocol specified + */ +bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type) +{ + unsigned long index; + void *entry; + + /* The discovery protocol must always be supported */ + if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY) + return true; + + xa_for_each(&doe_mb->prots, index, entry) + if (entry == pci_doe_xa_prot_entry(vid, type)) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(pci_doe_supports_prot); + +/** + * pci_doe_submit_task() - Submit a task to be processed by the state machine + * + * @doe_mb: DOE mailbox capability to submit to + * @task: task to be queued + * + * Submit a DOE task (request/response) to the DOE mailbox to be processed. + * Returns upon queueing the task object. If the queue is full this function + * will sleep until there is room in the queue. + * + * task->complete will be called when the state machine is done processing this + * task. + * + * Excess data will be discarded. + * + * RETURNS: 0 when task has been successfully queued, -ERRNO on error + */ +int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) +{ + if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type)) + return -EINVAL; + + /* + * DOE requests must be a whole number of DW and the response needs to + * be big enough for at least 1 DW + */ + if (task->request_pl_sz % sizeof(u32) || + task->response_pl_sz < sizeof(u32)) + return -EINVAL; + + if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) + return -EIO; + + task->doe_mb = doe_mb; + INIT_WORK(&task->work, doe_statemachine_work); + queue_work(doe_mb->work_queue, &task->work); + return 0; +} +EXPORT_SYMBOL_GPL(pci_doe_submit_task); diff --git a/include/linux/pci-doe.h b/include/linux/pci-doe.h new file mode 100644 index 000000000000..ed9b4df792b8 --- /dev/null +++ b/include/linux/pci-doe.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Data Object Exchange + * PCIe r6.0, sec 6.30 DOE + * + * Copyright (C) 2021 Huawei + * Jonathan Cameron + * + * Copyright (C) 2022 Intel Corporation + * Ira Weiny + */ + +#ifndef LINUX_PCI_DOE_H +#define LINUX_PCI_DOE_H + +struct pci_doe_protocol { + u16 vid; + u8 type; +}; + +struct pci_doe_mb; + +/** + * struct pci_doe_task - represents a single query/response + * + * @prot: DOE Protocol + * @request_pl: The request payload + * @request_pl_sz: Size of the request payload (bytes) + * @response_pl: The response payload + * @response_pl_sz: Size of the response payload (bytes) + * @rv: Return value. Length of received response or error (bytes) + * @complete: Called when task is complete + * @private: Private data for the consumer + * @work: Used internally by the mailbox + * @doe_mb: Used internally by the mailbox + * + * The payload sizes and rv are specified in bytes with the following + * restrictions concerning the protocol. + * + * 1) The request_pl_sz must be a multiple of double words (4 bytes) + * 2) The response_pl_sz must be >= a single double word (4 bytes) + * 3) rv is returned as bytes but it will be a multiple of double words + * + * NOTE there is no need for the caller to initialize work or doe_mb. + */ +struct pci_doe_task { + struct pci_doe_protocol prot; + u32 *request_pl; + size_t request_pl_sz; + u32 *response_pl; + size_t response_pl_sz; + int rv; + void (*complete)(struct pci_doe_task *task); + void *private; + + /* No need for the user to initialize these fields */ + struct work_struct work; + struct pci_doe_mb *doe_mb; +}; + +/** + * pci_doe_for_each_off - Iterate each DOE capability + * @pdev: struct pci_dev to iterate + * @off: u16 of config space offset of each mailbox capability found + */ +#define pci_doe_for_each_off(pdev, off) \ + for (off = pci_find_next_ext_capability(pdev, off, \ + PCI_EXT_CAP_ID_DOE); \ + off > 0; \ + off = pci_find_next_ext_capability(pdev, off, \ + PCI_EXT_CAP_ID_DOE)) + +struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset); +bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type); +int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task); + +#endif diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 108f8523fa04..57b8e2ffb1dd 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -737,7 +737,8 @@ #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ -#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_PL_16GT +#define PCI_EXT_CAP_ID_DOE 0x2E /* Data Object Exchange */ +#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_DOE #define PCI_EXT_CAP_DSN_SIZEOF 12 #define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40 @@ -1103,4 +1104,30 @@ #define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK 0x000000F0 #define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT 4 +/* Data Object Exchange */ +#define PCI_DOE_CAP 0x04 /* DOE Capabilities Register */ +#define PCI_DOE_CAP_INT_SUP 0x00000001 /* Interrupt Support */ +#define PCI_DOE_CAP_INT_MSG_NUM 0x00000ffe /* Interrupt Message Number */ +#define PCI_DOE_CTRL 0x08 /* DOE Control Register */ +#define PCI_DOE_CTRL_ABORT 0x00000001 /* DOE Abort */ +#define PCI_DOE_CTRL_INT_EN 0x00000002 /* DOE Interrupt Enable */ +#define PCI_DOE_CTRL_GO 0x80000000 /* DOE Go */ +#define PCI_DOE_STATUS 0x0c /* DOE Status Register */ +#define PCI_DOE_STATUS_BUSY 0x00000001 /* DOE Busy */ +#define PCI_DOE_STATUS_INT_STATUS 0x00000002 /* DOE Interrupt Status */ +#define PCI_DOE_STATUS_ERROR 0x00000004 /* DOE Error */ +#define PCI_DOE_STATUS_DATA_OBJECT_READY 0x80000000 /* Data Object Ready */ +#define PCI_DOE_WRITE 0x10 /* DOE Write Data Mailbox Register */ +#define PCI_DOE_READ 0x14 /* DOE Read Data Mailbox Register */ + +/* DOE Data Object - note not actually registers */ +#define PCI_DOE_DATA_OBJECT_HEADER_1_VID 0x0000ffff +#define PCI_DOE_DATA_OBJECT_HEADER_1_TYPE 0x00ff0000 +#define PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH 0x0003ffff + +#define PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX 0x000000ff +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID 0x0000ffff +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL 0x00ff0000 +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX 0xff000000 + #endif /* LINUX_PCI_REGS_H */ -- cgit v1.2.3 From 9d6794feeb90903b10c34bddd9c74c992447ce83 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Tue, 19 Jul 2022 13:52:48 -0700 Subject: driver-core: Introduce BIN_ATTR_ADMIN_{RO,RW} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many binary attributes need to limit access to CAP_SYS_ADMIN only; ie many binary attributes specify is_visible with 0400 or 0600. Make setting the permissions of such attributes more explicit by defining BIN_ATTR_ADMIN_{RO,RW}. Cc: Bjorn Helgaas Suggested-by: Dan Williams Suggested-by: Krzysztof Wilczyński Reviewed-by: Jonathan Cameron Reviewed-by: Greg Kroah-Hartman Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20220719205249.566684-6-ira.weiny@intel.com Signed-off-by: Dan Williams --- include/linux/sysfs.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h index e3f1e8ac1f85..fd3fe5c8c17f 100644 --- a/include/linux/sysfs.h +++ b/include/linux/sysfs.h @@ -235,6 +235,22 @@ struct bin_attribute bin_attr_##_name = __BIN_ATTR_WO(_name, _size) #define BIN_ATTR_RW(_name, _size) \ struct bin_attribute bin_attr_##_name = __BIN_ATTR_RW(_name, _size) + +#define __BIN_ATTR_ADMIN_RO(_name, _size) { \ + .attr = { .name = __stringify(_name), .mode = 0400 }, \ + .read = _name##_read, \ + .size = _size, \ +} + +#define __BIN_ATTR_ADMIN_RW(_name, _size) \ + __BIN_ATTR(_name, 0600, _name##_read, _name##_write, _size) + +#define BIN_ATTR_ADMIN_RO(_name, _size) \ +struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RO(_name, _size) + +#define BIN_ATTR_ADMIN_RW(_name, _size) \ +struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RW(_name, _size) + struct sysfs_ops { ssize_t (*show)(struct kobject *, struct attribute *, char *); ssize_t (*store)(struct kobject *, struct attribute *, const char *, size_t); -- cgit v1.2.3 From ce4b4657ff18925c315855aa290e93c5fa652d96 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 19 Jul 2022 21:02:48 -0300 Subject: vfio: Replace the DMA unmapping notifier with a callback Instead of having drivers register the notifier with explicit code just have them provide a dma_unmap callback op in their driver ops and rely on the core code to wire it up. Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Reviewed-by: Kevin Tian Reviewed-by: Tony Krowiak Reviewed-by: Eric Farman Reviewed-by: Zhenyu Wang Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/1-v4-681e038e30fd+78-vfio_unmap_notif_jgg@nvidia.com Signed-off-by: Alex Williamson --- drivers/gpu/drm/i915/gvt/gvt.h | 1 - drivers/gpu/drm/i915/gvt/kvmgt.c | 75 +++++--------------- drivers/s390/cio/vfio_ccw_ops.c | 39 +++------- drivers/s390/cio/vfio_ccw_private.h | 2 - drivers/s390/crypto/vfio_ap_ops.c | 53 +++----------- drivers/s390/crypto/vfio_ap_private.h | 3 - drivers/vfio/vfio.c | 129 ++++++++++++---------------------- drivers/vfio/vfio.h | 3 + include/linux/vfio.h | 21 ++---- 9 files changed, 86 insertions(+), 240 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index aee1a45da74b..705689e64011 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -226,7 +226,6 @@ struct intel_vgpu { unsigned long nr_cache_entries; struct mutex cache_lock; - struct notifier_block iommu_notifier; atomic_t released; struct kvm_page_track_notifier_node track_node; diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index e2f6c56ab342..ecd5bb37b63a 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -729,34 +729,25 @@ int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num) return ret; } -static int intel_vgpu_iommu_notifier(struct notifier_block *nb, - unsigned long action, void *data) +static void intel_vgpu_dma_unmap(struct vfio_device *vfio_dev, u64 iova, + u64 length) { - struct intel_vgpu *vgpu = - container_of(nb, struct intel_vgpu, iommu_notifier); - - if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { - struct vfio_iommu_type1_dma_unmap *unmap = data; - struct gvt_dma *entry; - unsigned long iov_pfn, end_iov_pfn; + struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); + struct gvt_dma *entry; + u64 iov_pfn = iova >> PAGE_SHIFT; + u64 end_iov_pfn = iov_pfn + length / PAGE_SIZE; - iov_pfn = unmap->iova >> PAGE_SHIFT; - end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE; + mutex_lock(&vgpu->cache_lock); + for (; iov_pfn < end_iov_pfn; iov_pfn++) { + entry = __gvt_cache_find_gfn(vgpu, iov_pfn); + if (!entry) + continue; - mutex_lock(&vgpu->cache_lock); - for (; iov_pfn < end_iov_pfn; iov_pfn++) { - entry = __gvt_cache_find_gfn(vgpu, iov_pfn); - if (!entry) - continue; - - gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr, - entry->size); - __gvt_cache_remove_entry(vgpu, entry); - } - mutex_unlock(&vgpu->cache_lock); + gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr, + entry->size); + __gvt_cache_remove_entry(vgpu, entry); } - - return NOTIFY_OK; + mutex_unlock(&vgpu->cache_lock); } static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu) @@ -783,36 +774,20 @@ out: static int intel_vgpu_open_device(struct vfio_device *vfio_dev) { struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); - unsigned long events; - int ret; - - vgpu->iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; - events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; - ret = vfio_register_notifier(vfio_dev, VFIO_IOMMU_NOTIFY, &events, - &vgpu->iommu_notifier); - if (ret != 0) { - gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n", - ret); - goto out; - } - - ret = -EEXIST; if (vgpu->attached) - goto undo_iommu; + return -EEXIST; - ret = -ESRCH; if (!vgpu->vfio_device.kvm || vgpu->vfio_device.kvm->mm != current->mm) { gvt_vgpu_err("KVM is required to use Intel vGPU\n"); - goto undo_iommu; + return -ESRCH; } kvm_get_kvm(vgpu->vfio_device.kvm); - ret = -EEXIST; if (__kvmgt_vgpu_exist(vgpu)) - goto undo_iommu; + return -EEXIST; vgpu->attached = true; @@ -831,12 +806,6 @@ static int intel_vgpu_open_device(struct vfio_device *vfio_dev) atomic_set(&vgpu->released, 0); return 0; - -undo_iommu: - vfio_unregister_notifier(vfio_dev, VFIO_IOMMU_NOTIFY, - &vgpu->iommu_notifier); -out: - return ret; } static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu) @@ -853,8 +822,6 @@ static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu) static void intel_vgpu_close_device(struct vfio_device *vfio_dev) { struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); - struct drm_i915_private *i915 = vgpu->gvt->gt->i915; - int ret; if (!vgpu->attached) return; @@ -864,11 +831,6 @@ static void intel_vgpu_close_device(struct vfio_device *vfio_dev) intel_gvt_release_vgpu(vgpu); - ret = vfio_unregister_notifier(&vgpu->vfio_device, VFIO_IOMMU_NOTIFY, - &vgpu->iommu_notifier); - drm_WARN(&i915->drm, ret, - "vfio_unregister_notifier for iommu failed: %d\n", ret); - debugfs_remove(debugfs_lookup(KVMGT_DEBUGFS_FILENAME, vgpu->debugfs)); kvm_page_track_unregister_notifier(vgpu->vfio_device.kvm, @@ -1610,6 +1572,7 @@ static const struct vfio_device_ops intel_vgpu_dev_ops = { .write = intel_vgpu_write, .mmap = intel_vgpu_mmap, .ioctl = intel_vgpu_ioctl, + .dma_unmap = intel_vgpu_dma_unmap, }; static int intel_vgpu_probe(struct mdev_device *mdev) diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index bc2176421dc5..0047fd88f938 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -33,30 +33,16 @@ static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) return 0; } -static int vfio_ccw_mdev_notifier(struct notifier_block *nb, - unsigned long action, - void *data) +static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) { struct vfio_ccw_private *private = - container_of(nb, struct vfio_ccw_private, nb); - - /* - * Vendor drivers MUST unpin pages in response to an - * invalidation. - */ - if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { - struct vfio_iommu_type1_dma_unmap *unmap = data; - - if (!cp_iova_pinned(&private->cp, unmap->iova)) - return NOTIFY_OK; - - if (vfio_ccw_mdev_reset(private)) - return NOTIFY_BAD; + container_of(vdev, struct vfio_ccw_private, vdev); - return NOTIFY_OK; - } + /* Drivers MUST unpin pages in response to an invalidation. */ + if (!cp_iova_pinned(&private->cp, iova)) + return; - return NOTIFY_DONE; + vfio_ccw_mdev_reset(private); } static ssize_t name_show(struct mdev_type *mtype, @@ -154,23 +140,15 @@ static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) { struct vfio_ccw_private *private = container_of(vdev, struct vfio_ccw_private, vdev); - unsigned long events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; int ret; /* Device cannot simply be opened again from this state */ if (private->state == VFIO_CCW_STATE_NOT_OPER) return -EINVAL; - private->nb.notifier_call = vfio_ccw_mdev_notifier; - - ret = vfio_register_notifier(vdev, VFIO_IOMMU_NOTIFY, - &events, &private->nb); - if (ret) - return ret; - ret = vfio_ccw_register_async_dev_regions(private); if (ret) - goto out_unregister; + return ret; ret = vfio_ccw_register_schib_dev_regions(private); if (ret) @@ -190,7 +168,6 @@ static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) out_unregister: vfio_ccw_unregister_dev_regions(private); - vfio_unregister_notifier(vdev, VFIO_IOMMU_NOTIFY, &private->nb); return ret; } @@ -201,7 +178,6 @@ static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); vfio_ccw_unregister_dev_regions(private); - vfio_unregister_notifier(vdev, VFIO_IOMMU_NOTIFY, &private->nb); } static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, @@ -624,6 +600,7 @@ static const struct vfio_device_ops vfio_ccw_dev_ops = { .write = vfio_ccw_mdev_write, .ioctl = vfio_ccw_mdev_ioctl, .request = vfio_ccw_mdev_request, + .dma_unmap = vfio_ccw_dma_unmap, }; struct mdev_driver vfio_ccw_mdev_driver = { diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index abac532bf03e..cd24b7fada91 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -73,7 +73,6 @@ struct vfio_ccw_crw { * @state: internal state of the device * @completion: synchronization helper of the I/O completion * @avail: available for creating a mediated device - * @nb: notifier for vfio events * @io_region: MMIO region to input/output I/O arguments/results * @io_mutex: protect against concurrent update of I/O regions * @region: additional regions for other subchannel operations @@ -96,7 +95,6 @@ struct vfio_ccw_private { int state; struct completion *completion; atomic_t avail; - struct notifier_block nb; struct ccw_io_region *io_region; struct mutex io_mutex; struct vfio_ccw_region *region; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index a7d2a95796d3..bb1a1677c5c2 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1226,34 +1226,14 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, return 0; } -/** - * vfio_ap_mdev_iommu_notifier - IOMMU notifier callback - * - * @nb: The notifier block - * @action: Action to be taken - * @data: data associated with the request - * - * For an UNMAP request, unpin the guest IOVA (the NIB guest address we - * pinned before). Other requests are ignored. - * - * Return: for an UNMAP request, NOFITY_OK; otherwise NOTIFY_DONE. - */ -static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, - unsigned long action, void *data) +static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova, + u64 length) { - struct ap_matrix_mdev *matrix_mdev; - - matrix_mdev = container_of(nb, struct ap_matrix_mdev, iommu_notifier); - - if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { - struct vfio_iommu_type1_dma_unmap *unmap = data; - unsigned long g_pfn = unmap->iova >> PAGE_SHIFT; - - vfio_unpin_pages(&matrix_mdev->vdev, &g_pfn, 1); - return NOTIFY_OK; - } + struct ap_matrix_mdev *matrix_mdev = + container_of(vdev, struct ap_matrix_mdev, vdev); + unsigned long g_pfn = iova >> PAGE_SHIFT; - return NOTIFY_DONE; + vfio_unpin_pages(&matrix_mdev->vdev, &g_pfn, 1); } /** @@ -1380,27 +1360,11 @@ static int vfio_ap_mdev_open_device(struct vfio_device *vdev) { struct ap_matrix_mdev *matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev); - unsigned long events; - int ret; if (!vdev->kvm) return -EINVAL; - ret = vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm); - if (ret) - return ret; - - matrix_mdev->iommu_notifier.notifier_call = vfio_ap_mdev_iommu_notifier; - events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; - ret = vfio_register_notifier(vdev, VFIO_IOMMU_NOTIFY, &events, - &matrix_mdev->iommu_notifier); - if (ret) - goto err_kvm; - return 0; - -err_kvm: - vfio_ap_mdev_unset_kvm(matrix_mdev); - return ret; + return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm); } static void vfio_ap_mdev_close_device(struct vfio_device *vdev) @@ -1408,8 +1372,6 @@ static void vfio_ap_mdev_close_device(struct vfio_device *vdev) struct ap_matrix_mdev *matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev); - vfio_unregister_notifier(vdev, VFIO_IOMMU_NOTIFY, - &matrix_mdev->iommu_notifier); vfio_ap_mdev_unset_kvm(matrix_mdev); } @@ -1461,6 +1423,7 @@ static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { .open_device = vfio_ap_mdev_open_device, .close_device = vfio_ap_mdev_close_device, .ioctl = vfio_ap_mdev_ioctl, + .dma_unmap = vfio_ap_mdev_dma_unmap, }; static struct mdev_driver vfio_ap_matrix_driver = { diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index a26efd804d0d..abb59d59f81b 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -81,8 +81,6 @@ struct ap_matrix { * @node: allows the ap_matrix_mdev struct to be added to a list * @matrix: the adapters, usage domains and control domains assigned to the * mediated matrix device. - * @iommu_notifier: notifier block used for specifying callback function for - * handling the VFIO_IOMMU_NOTIFY_DMA_UNMAP even * @kvm: the struct holding guest's state * @pqap_hook: the function pointer to the interception handler for the * PQAP(AQIC) instruction. @@ -92,7 +90,6 @@ struct ap_matrix_mdev { struct vfio_device vdev; struct list_head node; struct ap_matrix matrix; - struct notifier_block iommu_notifier; struct kvm *kvm; crypto_hook pqap_hook; struct mdev_device *mdev; diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index bd84ca7c5e35..83c375fa2421 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -231,6 +231,9 @@ int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops) { struct vfio_iommu_driver *driver, *tmp; + if (WARN_ON(!ops->register_notifier != !ops->unregister_notifier)) + return -EINVAL; + driver = kzalloc(sizeof(*driver), GFP_KERNEL); if (!driver) return -ENOMEM; @@ -1079,8 +1082,20 @@ static void vfio_device_unassign_container(struct vfio_device *device) up_write(&device->group->group_rwsem); } +static int vfio_iommu_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct vfio_device *vfio_device = + container_of(nb, struct vfio_device, iommu_nb); + struct vfio_iommu_type1_dma_unmap *unmap = data; + + vfio_device->ops->dma_unmap(vfio_device, unmap->iova, unmap->size); + return NOTIFY_OK; +} + static struct file *vfio_device_open(struct vfio_device *device) { + struct vfio_iommu_driver *iommu_driver; struct file *filep; int ret; @@ -1111,6 +1126,18 @@ static struct file *vfio_device_open(struct vfio_device *device) if (ret) goto err_undo_count; } + + iommu_driver = device->group->container->iommu_driver; + if (device->ops->dma_unmap && iommu_driver && + iommu_driver->ops->register_notifier) { + unsigned long events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; + + device->iommu_nb.notifier_call = vfio_iommu_notifier; + iommu_driver->ops->register_notifier( + device->group->container->iommu_data, &events, + &device->iommu_nb); + } + up_read(&device->group->group_rwsem); } mutex_unlock(&device->dev_set->lock); @@ -1145,8 +1172,16 @@ static struct file *vfio_device_open(struct vfio_device *device) err_close_device: mutex_lock(&device->dev_set->lock); down_read(&device->group->group_rwsem); - if (device->open_count == 1 && device->ops->close_device) + if (device->open_count == 1 && device->ops->close_device) { device->ops->close_device(device); + + iommu_driver = device->group->container->iommu_driver; + if (device->ops->dma_unmap && iommu_driver && + iommu_driver->ops->unregister_notifier) + iommu_driver->ops->unregister_notifier( + device->group->container->iommu_data, + &device->iommu_nb); + } err_undo_count: up_read(&device->group->group_rwsem); device->open_count--; @@ -1341,12 +1376,20 @@ static const struct file_operations vfio_group_fops = { static int vfio_device_fops_release(struct inode *inode, struct file *filep) { struct vfio_device *device = filep->private_data; + struct vfio_iommu_driver *iommu_driver; mutex_lock(&device->dev_set->lock); vfio_assert_device_open(device); down_read(&device->group->group_rwsem); if (device->open_count == 1 && device->ops->close_device) device->ops->close_device(device); + + iommu_driver = device->group->container->iommu_driver; + if (device->ops->dma_unmap && iommu_driver && + iommu_driver->ops->unregister_notifier) + iommu_driver->ops->unregister_notifier( + device->group->container->iommu_data, + &device->iommu_nb); up_read(&device->group->group_rwsem); device->open_count--; if (device->open_count == 0) @@ -2029,90 +2072,6 @@ int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, void *data, } EXPORT_SYMBOL(vfio_dma_rw); -static int vfio_register_iommu_notifier(struct vfio_group *group, - unsigned long *events, - struct notifier_block *nb) -{ - struct vfio_container *container; - struct vfio_iommu_driver *driver; - int ret; - - lockdep_assert_held_read(&group->group_rwsem); - - container = group->container; - driver = container->iommu_driver; - if (likely(driver && driver->ops->register_notifier)) - ret = driver->ops->register_notifier(container->iommu_data, - events, nb); - else - ret = -ENOTTY; - - return ret; -} - -static int vfio_unregister_iommu_notifier(struct vfio_group *group, - struct notifier_block *nb) -{ - struct vfio_container *container; - struct vfio_iommu_driver *driver; - int ret; - - lockdep_assert_held_read(&group->group_rwsem); - - container = group->container; - driver = container->iommu_driver; - if (likely(driver && driver->ops->unregister_notifier)) - ret = driver->ops->unregister_notifier(container->iommu_data, - nb); - else - ret = -ENOTTY; - - return ret; -} - -int vfio_register_notifier(struct vfio_device *device, - enum vfio_notify_type type, unsigned long *events, - struct notifier_block *nb) -{ - struct vfio_group *group = device->group; - int ret; - - if (!nb || !events || (*events == 0) || - !vfio_assert_device_open(device)) - return -EINVAL; - - switch (type) { - case VFIO_IOMMU_NOTIFY: - ret = vfio_register_iommu_notifier(group, events, nb); - break; - default: - ret = -EINVAL; - } - return ret; -} -EXPORT_SYMBOL(vfio_register_notifier); - -int vfio_unregister_notifier(struct vfio_device *device, - enum vfio_notify_type type, - struct notifier_block *nb) -{ - struct vfio_group *group = device->group; - int ret; - - if (!nb || !vfio_assert_device_open(device)) - return -EINVAL; - - switch (type) { - case VFIO_IOMMU_NOTIFY: - ret = vfio_unregister_iommu_notifier(group, nb); - break; - default: - ret = -EINVAL; - } - return ret; -} -EXPORT_SYMBOL(vfio_unregister_notifier); - /* * Module/class support */ diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index a67130221151..25da02ca1568 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -33,6 +33,9 @@ enum vfio_iommu_notify_type { VFIO_IOMMU_CONTAINER_CLOSE = 0, }; +/* events for register_notifier() */ +#define VFIO_IOMMU_NOTIFY_DMA_UNMAP BIT(0) + /** * struct vfio_iommu_driver_ops - VFIO IOMMU driver callbacks */ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 4d26e149db81..1f9fc7a9be9e 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -49,6 +49,7 @@ struct vfio_device { unsigned int open_count; struct completion comp; struct list_head group_next; + struct notifier_block iommu_nb; }; /** @@ -65,6 +66,8 @@ struct vfio_device { * @match: Optional device name match callback (return: 0 for no-match, >0 for * match, -errno for abort (ex. match with insufficient or incorrect * additional args) + * @dma_unmap: Called when userspace unmaps IOVA from the container + * this device is attached to. * @device_feature: Optional, fill in the VFIO_DEVICE_FEATURE ioctl */ struct vfio_device_ops { @@ -80,6 +83,7 @@ struct vfio_device_ops { int (*mmap)(struct vfio_device *vdev, struct vm_area_struct *vma); void (*request)(struct vfio_device *vdev, unsigned int count); int (*match)(struct vfio_device *vdev, char *buf); + void (*dma_unmap)(struct vfio_device *vdev, u64 iova, u64 length); int (*device_feature)(struct vfio_device *device, u32 flags, void __user *arg, size_t argsz); }; @@ -164,23 +168,6 @@ int vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, void *data, size_t len, bool write); -/* each type has independent events */ -enum vfio_notify_type { - VFIO_IOMMU_NOTIFY = 0, -}; - -/* events for VFIO_IOMMU_NOTIFY */ -#define VFIO_IOMMU_NOTIFY_DMA_UNMAP BIT(0) - -int vfio_register_notifier(struct vfio_device *device, - enum vfio_notify_type type, - unsigned long *required_events, - struct notifier_block *nb); -int vfio_unregister_notifier(struct vfio_device *device, - enum vfio_notify_type type, - struct notifier_block *nb); - - /* * Sub-module helpers */ -- cgit v1.2.3 From 8cfc5b60751bcf9b4c6bbab3f6a72d59e0156a89 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 19 Jul 2022 21:02:49 -0300 Subject: vfio: Replace the iommu notifier with a device list Instead of bouncing the function call to the driver op through a blocking notifier just have the iommu layer call it directly. Register each device that is being attached to the iommu with the lower driver which then threads them on a linked list and calls the appropriate driver op at the right time. Currently the only use is if dma_unmap() is defined. Also, fully lock all the debugging tests on the pinning path that a dma_unmap is registered. Reviewed-by: Christoph Hellwig Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/2-v4-681e038e30fd+78-vfio_unmap_notif_jgg@nvidia.com Signed-off-by: Alex Williamson --- drivers/vfio/vfio.c | 41 ++++------------ drivers/vfio/vfio.h | 12 ++--- drivers/vfio/vfio_iommu_type1.c | 103 +++++++++++++++++++++++++--------------- include/linux/vfio.h | 2 +- 4 files changed, 81 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 83c375fa2421..b3ce8073cfb1 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -231,7 +231,7 @@ int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops) { struct vfio_iommu_driver *driver, *tmp; - if (WARN_ON(!ops->register_notifier != !ops->unregister_notifier)) + if (WARN_ON(!ops->register_device != !ops->unregister_device)) return -EINVAL; driver = kzalloc(sizeof(*driver), GFP_KERNEL); @@ -1082,17 +1082,6 @@ static void vfio_device_unassign_container(struct vfio_device *device) up_write(&device->group->group_rwsem); } -static int vfio_iommu_notifier(struct notifier_block *nb, unsigned long action, - void *data) -{ - struct vfio_device *vfio_device = - container_of(nb, struct vfio_device, iommu_nb); - struct vfio_iommu_type1_dma_unmap *unmap = data; - - vfio_device->ops->dma_unmap(vfio_device, unmap->iova, unmap->size); - return NOTIFY_OK; -} - static struct file *vfio_device_open(struct vfio_device *device) { struct vfio_iommu_driver *iommu_driver; @@ -1128,15 +1117,9 @@ static struct file *vfio_device_open(struct vfio_device *device) } iommu_driver = device->group->container->iommu_driver; - if (device->ops->dma_unmap && iommu_driver && - iommu_driver->ops->register_notifier) { - unsigned long events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; - - device->iommu_nb.notifier_call = vfio_iommu_notifier; - iommu_driver->ops->register_notifier( - device->group->container->iommu_data, &events, - &device->iommu_nb); - } + if (iommu_driver && iommu_driver->ops->register_device) + iommu_driver->ops->register_device( + device->group->container->iommu_data, device); up_read(&device->group->group_rwsem); } @@ -1176,11 +1159,9 @@ err_close_device: device->ops->close_device(device); iommu_driver = device->group->container->iommu_driver; - if (device->ops->dma_unmap && iommu_driver && - iommu_driver->ops->unregister_notifier) - iommu_driver->ops->unregister_notifier( - device->group->container->iommu_data, - &device->iommu_nb); + if (iommu_driver && iommu_driver->ops->unregister_device) + iommu_driver->ops->unregister_device( + device->group->container->iommu_data, device); } err_undo_count: up_read(&device->group->group_rwsem); @@ -1385,11 +1366,9 @@ static int vfio_device_fops_release(struct inode *inode, struct file *filep) device->ops->close_device(device); iommu_driver = device->group->container->iommu_driver; - if (device->ops->dma_unmap && iommu_driver && - iommu_driver->ops->unregister_notifier) - iommu_driver->ops->unregister_notifier( - device->group->container->iommu_data, - &device->iommu_nb); + if (iommu_driver && iommu_driver->ops->unregister_device) + iommu_driver->ops->unregister_device( + device->group->container->iommu_data, device); up_read(&device->group->group_rwsem); device->open_count--; if (device->open_count == 0) diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index 25da02ca1568..4a7db1f3c33e 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -33,9 +33,6 @@ enum vfio_iommu_notify_type { VFIO_IOMMU_CONTAINER_CLOSE = 0, }; -/* events for register_notifier() */ -#define VFIO_IOMMU_NOTIFY_DMA_UNMAP BIT(0) - /** * struct vfio_iommu_driver_ops - VFIO IOMMU driver callbacks */ @@ -58,11 +55,10 @@ struct vfio_iommu_driver_ops { unsigned long *phys_pfn); int (*unpin_pages)(void *iommu_data, unsigned long *user_pfn, int npage); - int (*register_notifier)(void *iommu_data, - unsigned long *events, - struct notifier_block *nb); - int (*unregister_notifier)(void *iommu_data, - struct notifier_block *nb); + void (*register_device)(void *iommu_data, + struct vfio_device *vdev); + void (*unregister_device)(void *iommu_data, + struct vfio_device *vdev); int (*dma_rw)(void *iommu_data, dma_addr_t user_iova, void *data, size_t count, bool write); struct iommu_domain *(*group_iommu_domain)(void *iommu_data, diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index db24062fb343..026a1d2553a2 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -67,7 +67,8 @@ struct vfio_iommu { struct list_head iova_list; struct mutex lock; struct rb_root dma_list; - struct blocking_notifier_head notifier; + struct list_head device_list; + struct mutex device_list_lock; unsigned int dma_avail; unsigned int vaddr_invalid_count; uint64_t pgsize_bitmap; @@ -865,8 +866,8 @@ again: } } - /* Fail if notifier list is empty */ - if (!iommu->notifier.head) { + /* Fail if no dma_umap notifier is registered */ + if (list_empty(&iommu->device_list)) { ret = -EINVAL; goto pin_done; } @@ -1287,6 +1288,35 @@ static int verify_bitmap_size(uint64_t npages, uint64_t bitmap_size) return 0; } +/* + * Notify VFIO drivers using vfio_register_emulated_iommu_dev() to invalidate + * and unmap iovas within the range we're about to unmap. Drivers MUST unpin + * pages in response to an invalidation. + */ +static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, + struct vfio_dma *dma) +{ + struct vfio_device *device; + + if (list_empty(&iommu->device_list)) + return; + + /* + * The device is expected to call vfio_unpin_pages() for any IOVA it has + * pinned within the range. Since vfio_unpin_pages() will eventually + * call back down to this code and try to obtain the iommu->lock we must + * drop it. + */ + mutex_lock(&iommu->device_list_lock); + mutex_unlock(&iommu->lock); + + list_for_each_entry(device, &iommu->device_list, iommu_entry) + device->ops->dma_unmap(device, dma->iova, dma->size); + + mutex_unlock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); +} + static int vfio_dma_do_unmap(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_unmap *unmap, struct vfio_bitmap *bitmap) @@ -1400,8 +1430,6 @@ again: } if (!RB_EMPTY_ROOT(&dma->pfn_list)) { - struct vfio_iommu_type1_dma_unmap nb_unmap; - if (dma_last == dma) { BUG_ON(++retries > 10); } else { @@ -1409,20 +1437,7 @@ again: retries = 0; } - nb_unmap.iova = dma->iova; - nb_unmap.size = dma->size; - - /* - * Notify anyone (mdev vendor drivers) to invalidate and - * unmap iovas within the range we're about to unmap. - * Vendor drivers MUST unpin pages in response to an - * invalidation. - */ - mutex_unlock(&iommu->lock); - blocking_notifier_call_chain(&iommu->notifier, - VFIO_IOMMU_NOTIFY_DMA_UNMAP, - &nb_unmap); - mutex_lock(&iommu->lock); + vfio_notify_dma_unmap(iommu, dma); goto again; } @@ -2475,7 +2490,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, if (list_empty(&iommu->emulated_iommu_groups) && list_empty(&iommu->domain_list)) { - WARN_ON(iommu->notifier.head); + WARN_ON(!list_empty(&iommu->device_list)); vfio_iommu_unmap_unpin_all(iommu); } goto detach_group_done; @@ -2507,7 +2522,8 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, if (list_empty(&domain->group_list)) { if (list_is_singular(&iommu->domain_list)) { if (list_empty(&iommu->emulated_iommu_groups)) { - WARN_ON(iommu->notifier.head); + WARN_ON(!list_empty( + &iommu->device_list)); vfio_iommu_unmap_unpin_all(iommu); } else { vfio_iommu_unmap_unpin_reaccount(iommu); @@ -2568,7 +2584,8 @@ static void *vfio_iommu_type1_open(unsigned long arg) iommu->dma_avail = dma_entry_limit; iommu->container_open = true; mutex_init(&iommu->lock); - BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier); + mutex_init(&iommu->device_list_lock); + INIT_LIST_HEAD(&iommu->device_list); init_waitqueue_head(&iommu->vaddr_wait); iommu->pgsize_bitmap = PAGE_MASK; INIT_LIST_HEAD(&iommu->emulated_iommu_groups); @@ -3005,28 +3022,40 @@ static long vfio_iommu_type1_ioctl(void *iommu_data, } } -static int vfio_iommu_type1_register_notifier(void *iommu_data, - unsigned long *events, - struct notifier_block *nb) +static void vfio_iommu_type1_register_device(void *iommu_data, + struct vfio_device *vdev) { struct vfio_iommu *iommu = iommu_data; - /* clear known events */ - *events &= ~VFIO_IOMMU_NOTIFY_DMA_UNMAP; - - /* refuse to register if still events remaining */ - if (*events) - return -EINVAL; + if (!vdev->ops->dma_unmap) + return; - return blocking_notifier_chain_register(&iommu->notifier, nb); + /* + * list_empty(&iommu->device_list) is tested under the iommu->lock while + * iteration for dma_unmap must be done under the device_list_lock. + * Holding both locks here allows avoiding the device_list_lock in + * several fast paths. See vfio_notify_dma_unmap() + */ + mutex_lock(&iommu->lock); + mutex_lock(&iommu->device_list_lock); + list_add(&vdev->iommu_entry, &iommu->device_list); + mutex_unlock(&iommu->device_list_lock); + mutex_unlock(&iommu->lock); } -static int vfio_iommu_type1_unregister_notifier(void *iommu_data, - struct notifier_block *nb) +static void vfio_iommu_type1_unregister_device(void *iommu_data, + struct vfio_device *vdev) { struct vfio_iommu *iommu = iommu_data; - return blocking_notifier_chain_unregister(&iommu->notifier, nb); + if (!vdev->ops->dma_unmap) + return; + + mutex_lock(&iommu->lock); + mutex_lock(&iommu->device_list_lock); + list_del(&vdev->iommu_entry); + mutex_unlock(&iommu->device_list_lock); + mutex_unlock(&iommu->lock); } static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, @@ -3160,8 +3189,8 @@ static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { .detach_group = vfio_iommu_type1_detach_group, .pin_pages = vfio_iommu_type1_pin_pages, .unpin_pages = vfio_iommu_type1_unpin_pages, - .register_notifier = vfio_iommu_type1_register_notifier, - .unregister_notifier = vfio_iommu_type1_unregister_notifier, + .register_device = vfio_iommu_type1_register_device, + .unregister_device = vfio_iommu_type1_unregister_device, .dma_rw = vfio_iommu_type1_dma_rw, .group_iommu_domain = vfio_iommu_type1_group_iommu_domain, .notify = vfio_iommu_type1_notify, diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 1f9fc7a9be9e..19cefbaa3d06 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -49,7 +49,7 @@ struct vfio_device { unsigned int open_count; struct completion comp; struct list_head group_next; - struct notifier_block iommu_nb; + struct list_head iommu_entry; }; /** -- cgit v1.2.3 From 9cb61fda8c71baaff423fe5be2e609100860fa78 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Wed, 20 Jul 2022 08:52:20 -0700 Subject: bpf: Fix bpf_trampoline_{,un}link_cgroup_shim ifdef guards They were updated in kernel/bpf/trampoline.c to fix another build issue. We should to do the same for include/linux/bpf.h header. Fixes: 3908fcddc65d ("bpf: fix lsm_cgroup build errors on esoteric configs") Reported-by: Stephen Rothwell Signed-off-by: Stanislav Fomichev Signed-off-by: Daniel Borkmann Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20220720155220.4087433-1-sdf@google.com --- include/linux/bpf.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a5bf00649995..11950029284f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1254,9 +1254,6 @@ struct bpf_dummy_ops { int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); #endif -int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, - int cgroup_atype); -void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); #else static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) { @@ -1280,6 +1277,13 @@ static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, { return -EINVAL; } +#endif + +#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) +int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, + int cgroup_atype); +void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); +#else static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, int cgroup_atype) { -- cgit v1.2.3 From 5588d628027092e66195097bdf6835ddf64418b3 Mon Sep 17 00:00:00 2001 From: Łukasz Spintzyk Date: Wed, 20 Jul 2022 08:05:18 +0200 Subject: net/cdc_ncm: Increase NTB max RX/TX values to 64kb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DisplayLink ethernet devices require NTB buffers larger then 32kb in order to run with highest performance. This patch is changing upper limit of the rx and tx buffers. Those buffers are initialized with CDC_NCM_NTB_DEF_SIZE_RX and CDC_NCM_NTB_DEF_SIZE_TX which is 16kb so by default no device is affected by increased limit. Rx and tx buffer is increased under two conditions: - Device need to advertise that it supports higher buffer size in dwNtbMaxInMaxSize and dwNtbMaxOutMaxSize. - cdc_ncm/rx_max and cdc_ncm/tx_max driver parameters must be adjusted with udev rule or ethtool. Summary of testing and performance results: Tests were performed on following devices: - DisplayLink DL-3xxx family device - DisplayLink DL-6xxx family device - ASUS USB-C2500 2.5G USB3 ethernet adapter - Plugable USB3 1G USB3 ethernet adapter - EDIMAX EU-4307 USB-C ethernet adapter - Dell DBQBCBC064 USB-C ethernet adapter Performance measurements were done with: - iperf3 between two linux boxes - http://openspeedtest.com/ instance running on local test machine Insights from tests results: - All except one from third party usb adapters were not affected by increased buffer size to their advertised dwNtbOutMaxSize and dwNtbInMaxSize. Devices were generally reaching 912-940Mbps both download and upload. Only EDIMAX adapter experienced decreased download size from 929Mbps to 827Mbps with iper3, with openspeedtest decrease was from 968Mbps to 886Mbps. - DisplayLink DL-3xxx family devices experienced performance increase with iperf3 download from 300Mbps to 870Mbps and upload from 782Mbps to 844Mbps. With openspeedtest download increased from 556Mbps to 873Mbps and upload from 727Mbps to 973Mbps - DiplayLink DL-6xxx family devices are not affected by increased buffer size. Signed-off-by: Łukasz Spintzyk Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20220720060518.541-2-lukasz.spintzyk@synaptics.com Signed-off-by: Paolo Abeni --- include/linux/usb/cdc_ncm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/cdc_ncm.h b/include/linux/usb/cdc_ncm.h index f7cb3ddce7fb..2d207cb4837d 100644 --- a/include/linux/usb/cdc_ncm.h +++ b/include/linux/usb/cdc_ncm.h @@ -53,8 +53,8 @@ #define USB_CDC_NCM_NDP32_LENGTH_MIN 0x20 /* Maximum NTB length */ -#define CDC_NCM_NTB_MAX_SIZE_TX 32768 /* bytes */ -#define CDC_NCM_NTB_MAX_SIZE_RX 32768 /* bytes */ +#define CDC_NCM_NTB_MAX_SIZE_TX 65536 /* bytes */ +#define CDC_NCM_NTB_MAX_SIZE_RX 65536 /* bytes */ /* Initial NTB length */ #define CDC_NCM_NTB_DEF_SIZE_TX 16384 /* bytes */ -- cgit v1.2.3 From e0c7ea83f006ce8c3264ef8b6508a891d886ad4f Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Thu, 7 Jul 2022 11:00:29 +0800 Subject: dmaengine: imx-sdma: Add FIFO stride support for multi FIFO script The peripheral may have several FIFOs, but some case just select some FIFOs from them for data transfer, which means FIFO0 and FIFO2 may be selected. So add FIFO address stride support, 0 means all FIFOs are continuous, 1 means 1 word stride between FIFOs. All stride between FIFOs should be same. Another option words_per_fifo means how many audio channel data copied to one FIFO one time, 1 means one channel per FIFO, 2 means 2 channels per FIFO. If 'n_fifos_src = 4' and 'words_per_fifo = 2', it means the first two words(channels) fetch from FIFO0 and then jump to FIFO1 for next two words, and so on after the last FIFO3 fetched, roll back to FIFO0. Signed-off-by: Joy Zou Signed-off-by: Shengjiu Wang Link: https://lore.kernel.org/r/1657162829-9273-1-git-send-email-shengjiu.wang@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 27 +++++++++++++++++++++++++-- include/linux/dma/imx-dma.h | 13 +++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 8a5d4d4a4dff..e302089784ed 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -183,6 +183,8 @@ BIT(DMA_DEV_TO_DEV)) #define SDMA_WATERMARK_LEVEL_N_FIFOS GENMASK(15, 12) +#define SDMA_WATERMARK_LEVEL_OFF_FIFOS GENMASK(19, 16) +#define SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO GENMASK(31, 28) #define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23) #define SDMA_DONE0_CONFIG_DONE_SEL BIT(7) @@ -429,6 +431,9 @@ struct sdma_desc { * @n_fifos_src: number of source device fifos * @n_fifos_dst: number of destination device fifos * @sw_done: software done flag + * @stride_fifos_src: stride for source device FIFOs + * @stride_fifos_dst: stride for destination device FIFOs + * @words_per_fifo: copy number of words one time for one FIFO */ struct sdma_channel { struct virt_dma_chan vc; @@ -456,6 +461,9 @@ struct sdma_channel { bool is_ram_script; unsigned int n_fifos_src; unsigned int n_fifos_dst; + unsigned int stride_fifos_src; + unsigned int stride_fifos_dst; + unsigned int words_per_fifo; bool sw_done; }; @@ -1245,17 +1253,29 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) { unsigned int n_fifos; + unsigned int stride_fifos; + unsigned int words_per_fifo; if (sdmac->sw_done) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE; - if (sdmac->direction == DMA_DEV_TO_MEM) + if (sdmac->direction == DMA_DEV_TO_MEM) { n_fifos = sdmac->n_fifos_src; - else + stride_fifos = sdmac->stride_fifos_src; + } else { n_fifos = sdmac->n_fifos_dst; + stride_fifos = sdmac->stride_fifos_dst; + } + + words_per_fifo = sdmac->words_per_fifo; sdmac->watermark_level |= FIELD_PREP(SDMA_WATERMARK_LEVEL_N_FIFOS, n_fifos); + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_OFF_FIFOS, stride_fifos); + if (words_per_fifo) + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO, (words_per_fifo - 1)); } static int sdma_config_channel(struct dma_chan *chan) @@ -1769,6 +1789,9 @@ static int sdma_config(struct dma_chan *chan, } sdmac->n_fifos_src = sdmacfg->n_fifos_src; sdmac->n_fifos_dst = sdmacfg->n_fifos_dst; + sdmac->stride_fifos_src = sdmacfg->stride_fifos_src; + sdmac->stride_fifos_dst = sdmacfg->stride_fifos_dst; + sdmac->words_per_fifo = sdmacfg->words_per_fifo; sdmac->sw_done = sdmacfg->sw_done; } diff --git a/include/linux/dma/imx-dma.h b/include/linux/dma/imx-dma.h index 8887762360d4..f487a4fa103a 100644 --- a/include/linux/dma/imx-dma.h +++ b/include/linux/dma/imx-dma.h @@ -70,6 +70,16 @@ static inline int imx_dma_is_general_purpose(struct dma_chan *chan) * struct sdma_peripheral_config - SDMA config for audio * @n_fifos_src: Number of FIFOs for recording * @n_fifos_dst: Number of FIFOs for playback + * @stride_fifos_src: FIFO address stride for recording, 0 means all FIFOs are + * continuous, 1 means 1 word stride between FIFOs. All stride + * between FIFOs should be same. + * @stride_fifos_dst: FIFO address stride for playback + * @words_per_fifo: numbers of words per FIFO fetch/fill, 1 means + * one channel per FIFO, 2 means 2 channels per FIFO.. + * If 'n_fifos_src = 4' and 'words_per_fifo = 2', it + * means the first two words(channels) fetch from FIFO0 + * and then jump to FIFO1 for next two words, and so on + * after the last FIFO3 fetched, roll back to FIFO0. * @sw_done: Use software done. Needed for PDM (micfil) * * Some i.MX Audio devices (SAI, micfil) have multiple successive FIFO @@ -82,6 +92,9 @@ static inline int imx_dma_is_general_purpose(struct dma_chan *chan) struct sdma_peripheral_config { int n_fifos_src; int n_fifos_dst; + int stride_fifos_src; + int stride_fifos_dst; + int words_per_fifo; bool sw_done; }; -- cgit v1.2.3 From 974854ab0728532600c72e41a44d6ce1cf8f20a4 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 12 Jul 2022 18:37:54 -0700 Subject: cxl/acpi: Track CXL resources in iomem_resource Recall that CXL capable address ranges, on ACPI platforms, are published in the CEDT.CFMWS (CXL Early Discovery Table: CXL Fixed Memory Window Structures). These windows represent both the actively mapped capacity and the potential address space that can be dynamically assigned to a new CXL decode configuration (region / interleave-set). CXL endpoints like DDR DIMMs can be mapped at any physical address including 0 and legacy ranges. There is an expectation and requirement that the /proc/iomem interface and the iomem_resource tree in the kernel reflect the full set of platform address ranges. I.e. that every address range that platform firmware and bus drivers enumerate be reflected as an iomem_resource entry. The hard requirement to do this for CXL arises from the fact that facilities like CONFIG_DEVICE_PRIVATE expect to be able to treat empty iomem_resource ranges as free for software to use as proxy address space. Without CXL publishing its potential address ranges in iomem_resource, the CONFIG_DEVICE_PRIVATE mechanism may inadvertently steal capacity reserved for runtime provisioning of new CXL regions. So, iomem_resource needs to know about both active and potential CXL resource ranges. The active CXL resources might already be reflected in iomem_resource as "System RAM". insert_resource_expand_to_fit() handles re-parenting "System RAM" underneath a CXL window. The "_expand_to_fit()" behavior handles cases where a CXL window is not a strict superset of an existing entry in the iomem_resource tree. The "_expand_to_fit()" behavior is acceptable from the perspective of resource allocation. The expansion happens because a conflicting resource range is already populated, which means the resource boundary expansion does not result in any additional free CXL address space being made available. CXL address space allocation is always bounded by the orginal unexpanded address range. However, the potential for expansion does mean that something like walk_iomem_res_desc(IORES_DESC_CXL...) can only return fuzzy answers on corner case platforms that cause the resource tree to expand a CXL window resource over a range that is not decoded by CXL. This would be an odd platform configuration, but if it becomes a problem in practice the CXL subsytem could just publish an API that returns definitive answers. Cc: Andrew Morton Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Tony Luck Cc: Christoph Hellwig Reviewed-by: Jonathan Cameron Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/165784325943.1758207.5310344844375305118.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- drivers/cxl/acpi.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++-- include/linux/ioport.h | 1 + kernel/resource.c | 7 +++ 3 files changed, 149 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 62bf22ffb7aa..e2b6cbd04846 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -73,6 +73,8 @@ static int cxl_acpi_cfmws_verify(struct device *dev, struct cxl_cfmws_context { struct device *dev; struct cxl_port *root_port; + struct resource *cxl_res; + int id; }; static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, @@ -81,11 +83,13 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, int target_map[CXL_DECODER_MAX_INTERLEAVE]; struct cxl_cfmws_context *ctx = arg; struct cxl_port *root_port = ctx->root_port; + struct resource *cxl_res = ctx->cxl_res; struct cxl_switch_decoder *cxlsd; struct device *dev = ctx->dev; struct acpi_cedt_cfmws *cfmws; struct cxl_decoder *cxld; unsigned int ways, i, ig; + struct resource *res; int rc; cfmws = (struct acpi_cedt_cfmws *) header; @@ -107,6 +111,23 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, for (i = 0; i < ways; i++) target_map[i] = cfmws->interleave_targets[i]; + res = kzalloc(sizeof(*res), GFP_KERNEL); + if (!res) + return -ENOMEM; + + res->name = kasprintf(GFP_KERNEL, "CXL Window %d", ctx->id++); + if (!res->name) + goto err_name; + + res->start = cfmws->base_hpa; + res->end = cfmws->base_hpa + cfmws->window_size - 1; + res->flags = IORESOURCE_MEM; + + /* add to the local resource tracking to establish a sort order */ + rc = insert_resource(cxl_res, res); + if (rc) + goto err_insert; + cxlsd = cxl_root_decoder_alloc(root_port, ways); if (IS_ERR(cxld)) return 0; @@ -115,8 +136,8 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions); cxld->target_type = CXL_DECODER_EXPANDER; cxld->hpa_range = (struct range) { - .start = cfmws->base_hpa, - .end = cfmws->base_hpa + cfmws->window_size - 1, + .start = res->start, + .end = res->end, }; cxld->interleave_ways = ways; cxld->interleave_granularity = ig; @@ -137,6 +158,12 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, cxld->hpa_range.start, cxld->hpa_range.end); return 0; + +err_insert: + kfree(res->name); +err_name: + kfree(res); + return -ENOMEM; } __mock struct acpi_device *to_cxl_host_bridge(struct device *host, @@ -291,9 +318,101 @@ static void cxl_acpi_lock_reset_class(void *dev) device_lock_reset_class(dev); } +static void del_cxl_resource(struct resource *res) +{ + kfree(res->name); + kfree(res); +} + +static void cxl_set_public_resource(struct resource *priv, struct resource *pub) +{ + priv->desc = (unsigned long) pub; +} + +static struct resource *cxl_get_public_resource(struct resource *priv) +{ + return (struct resource *) priv->desc; +} + +static void remove_cxl_resources(void *data) +{ + struct resource *res, *next, *cxl = data; + + for (res = cxl->child; res; res = next) { + struct resource *victim = cxl_get_public_resource(res); + + next = res->sibling; + remove_resource(res); + + if (victim) { + remove_resource(victim); + kfree(victim); + } + + del_cxl_resource(res); + } +} + +/** + * add_cxl_resources() - reflect CXL fixed memory windows in iomem_resource + * @cxl_res: A standalone resource tree where each CXL window is a sibling + * + * Walk each CXL window in @cxl_res and add it to iomem_resource potentially + * expanding its boundaries to ensure that any conflicting resources become + * children. If a window is expanded it may then conflict with a another window + * entry and require the window to be truncated or trimmed. Consider this + * situation: + * + * |-- "CXL Window 0" --||----- "CXL Window 1" -----| + * |--------------- "System RAM" -------------| + * + * ...where platform firmware has established as System RAM resource across 2 + * windows, but has left some portion of window 1 for dynamic CXL region + * provisioning. In this case "Window 0" will span the entirety of the "System + * RAM" span, and "CXL Window 1" is truncated to the remaining tail past the end + * of that "System RAM" resource. + */ +static int add_cxl_resources(struct resource *cxl_res) +{ + struct resource *res, *new, *next; + + for (res = cxl_res->child; res; res = next) { + new = kzalloc(sizeof(*new), GFP_KERNEL); + if (!new) + return -ENOMEM; + new->name = res->name; + new->start = res->start; + new->end = res->end; + new->flags = IORESOURCE_MEM; + new->desc = IORES_DESC_CXL; + + /* + * Record the public resource in the private cxl_res tree for + * later removal. + */ + cxl_set_public_resource(res, new); + + insert_resource_expand_to_fit(&iomem_resource, new); + + next = res->sibling; + while (next && resource_overlaps(new, next)) { + if (resource_contains(new, next)) { + struct resource *_next = next->sibling; + + remove_resource(next); + del_cxl_resource(next); + next = _next; + } else + next->start = new->end + 1; + } + } + return 0; +} + static int cxl_acpi_probe(struct platform_device *pdev) { int rc; + struct resource *cxl_res; struct cxl_port *root_port; struct device *host = &pdev->dev; struct acpi_device *adev = ACPI_COMPANION(host); @@ -305,6 +424,14 @@ static int cxl_acpi_probe(struct platform_device *pdev) if (rc) return rc; + cxl_res = devm_kzalloc(host, sizeof(*cxl_res), GFP_KERNEL); + if (!cxl_res) + return -ENOMEM; + cxl_res->name = "CXL mem"; + cxl_res->start = 0; + cxl_res->end = -1; + cxl_res->flags = IORESOURCE_MEM; + root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); if (IS_ERR(root_port)) return PTR_ERR(root_port); @@ -315,11 +442,22 @@ static int cxl_acpi_probe(struct platform_device *pdev) if (rc < 0) return rc; + rc = devm_add_action_or_reset(host, remove_cxl_resources, cxl_res); + if (rc) + return rc; + ctx = (struct cxl_cfmws_context) { .dev = host, .root_port = root_port, + .cxl_res = cxl_res, }; - acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, cxl_parse_cfmws, &ctx); + rc = acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, cxl_parse_cfmws, &ctx); + if (rc < 0) + return -ENXIO; + + rc = add_cxl_resources(cxl_res); + if (rc) + return rc; /* * Root level scanned with host-bridge as dports, now scan host-bridges diff --git a/include/linux/ioport.h b/include/linux/ioport.h index ec5f71f7135b..79d1ad6d6275 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -141,6 +141,7 @@ enum { IORES_DESC_DEVICE_PRIVATE_MEMORY = 6, IORES_DESC_RESERVED = 7, IORES_DESC_SOFT_RESERVED = 8, + IORES_DESC_CXL = 9, }; /* diff --git a/kernel/resource.c b/kernel/resource.c index 34eaee179689..53a534db350e 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -891,6 +891,13 @@ void insert_resource_expand_to_fit(struct resource *root, struct resource *new) } write_unlock(&resource_lock); } +/* + * Not for general consumption, only early boot memory map parsing, PCI + * resource discovery, and late discovery of CXL resources are expected + * to use this interface. The former are built-in and only the latter, + * CXL, is a module. + */ +EXPORT_SYMBOL_NS_GPL(insert_resource_expand_to_fit, CXL); /** * remove_resource - Remove a resource in the resource tree -- cgit v1.2.3 From 14b80582c43e4f550acfd93c2b2cadbe36ea0874 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 20 May 2022 13:41:24 -0700 Subject: resource: Introduce alloc_free_mem_region() The core of devm_request_free_mem_region() is a helper that searches for free space in iomem_resource and performs __request_region_locked() on the result of that search. The policy choices of the implementation conform to what CONFIG_DEVICE_PRIVATE users want which is memory that is immediately marked busy, and a preference to search for the first-fit free range in descending order from the top of the physical address space. CXL has a need for a similar allocator, but with the following tweaks: 1/ Search for free space in ascending order 2/ Search for free space relative to a given CXL window 3/ 'insert' rather than 'request' the new resource given downstream drivers from the CXL Region driver (like the pmem or dax drivers) are responsible for request_mem_region() when they activate the memory range. Rework __request_free_mem_region() into get_free_mem_region() which takes a set of GFR_* (Get Free Region) flags to control the allocation policy (ascending vs descending), and "busy" policy (insert_resource() vs request_region()). As part of the consolidation of the legacy GFR_REQUEST_REGION case with the new default of just inserting a new resource into the free space some minor cleanups like not checking for NULL before calling devres_free() (which does its own check) is included. Suggested-by: Jason Gunthorpe Link: https://lore.kernel.org/linux-cxl/20220420143406.GY2120790@nvidia.com/ Cc: Matthew Wilcox Cc: Christoph Hellwig Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/165784333333.1758207.13703329337805274043.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- include/linux/ioport.h | 2 + kernel/resource.c | 178 +++++++++++++++++++++++++++++++++++++++---------- mm/Kconfig | 5 ++ 3 files changed, 150 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 79d1ad6d6275..616b683563a9 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -330,6 +330,8 @@ struct resource *devm_request_free_mem_region(struct device *dev, struct resource *base, unsigned long size); struct resource *request_free_mem_region(struct resource *base, unsigned long size, const char *name); +struct resource *alloc_free_mem_region(struct resource *base, + unsigned long size, unsigned long align, const char *name); static inline void irqresource_disabled(struct resource *res, u32 irq) { diff --git a/kernel/resource.c b/kernel/resource.c index 53a534db350e..4c5e80b92f2f 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -489,8 +489,9 @@ int __weak page_is_ram(unsigned long pfn) } EXPORT_SYMBOL_GPL(page_is_ram); -static int __region_intersects(resource_size_t start, size_t size, - unsigned long flags, unsigned long desc) +static int __region_intersects(struct resource *parent, resource_size_t start, + size_t size, unsigned long flags, + unsigned long desc) { struct resource res; int type = 0; int other = 0; @@ -499,7 +500,7 @@ static int __region_intersects(resource_size_t start, size_t size, res.start = start; res.end = start + size - 1; - for (p = iomem_resource.child; p ; p = p->sibling) { + for (p = parent->child; p ; p = p->sibling) { bool is_type = (((p->flags & flags) == flags) && ((desc == IORES_DESC_NONE) || (desc == p->desc))); @@ -543,7 +544,7 @@ int region_intersects(resource_size_t start, size_t size, unsigned long flags, int ret; read_lock(&resource_lock); - ret = __region_intersects(start, size, flags, desc); + ret = __region_intersects(&iomem_resource, start, size, flags, desc); read_unlock(&resource_lock); return ret; @@ -1780,62 +1781,139 @@ void resource_list_free(struct list_head *head) } EXPORT_SYMBOL(resource_list_free); -#ifdef CONFIG_DEVICE_PRIVATE -static struct resource *__request_free_mem_region(struct device *dev, - struct resource *base, unsigned long size, const char *name) +#ifdef CONFIG_GET_FREE_REGION +#define GFR_DESCENDING (1UL << 0) +#define GFR_REQUEST_REGION (1UL << 1) +#define GFR_DEFAULT_ALIGN (1UL << PA_SECTION_SHIFT) + +static resource_size_t gfr_start(struct resource *base, resource_size_t size, + resource_size_t align, unsigned long flags) +{ + if (flags & GFR_DESCENDING) { + resource_size_t end; + + end = min_t(resource_size_t, base->end, + (1ULL << MAX_PHYSMEM_BITS) - 1); + return end - size + 1; + } + + return ALIGN(base->start, align); +} + +static bool gfr_continue(struct resource *base, resource_size_t addr, + resource_size_t size, unsigned long flags) +{ + if (flags & GFR_DESCENDING) + return addr > size && addr >= base->start; + /* + * In the ascend case be careful that the last increment by + * @size did not wrap 0. + */ + return addr > addr - size && + addr <= min_t(resource_size_t, base->end, + (1ULL << MAX_PHYSMEM_BITS) - 1); +} + +static resource_size_t gfr_next(resource_size_t addr, resource_size_t size, + unsigned long flags) +{ + if (flags & GFR_DESCENDING) + return addr - size; + return addr + size; +} + +static void remove_free_mem_region(void *_res) +{ + struct resource *res = _res; + + if (res->parent) + remove_resource(res); + free_resource(res); +} + +static struct resource * +get_free_mem_region(struct device *dev, struct resource *base, + resource_size_t size, const unsigned long align, + const char *name, const unsigned long desc, + const unsigned long flags) { - resource_size_t end, addr; + resource_size_t addr; struct resource *res; struct region_devres *dr = NULL; - size = ALIGN(size, 1UL << PA_SECTION_SHIFT); - end = min_t(unsigned long, base->end, (1UL << MAX_PHYSMEM_BITS) - 1); - addr = end - size + 1UL; + size = ALIGN(size, align); res = alloc_resource(GFP_KERNEL); if (!res) return ERR_PTR(-ENOMEM); - if (dev) { + if (dev && (flags & GFR_REQUEST_REGION)) { dr = devres_alloc(devm_region_release, sizeof(struct region_devres), GFP_KERNEL); if (!dr) { free_resource(res); return ERR_PTR(-ENOMEM); } + } else if (dev) { + if (devm_add_action_or_reset(dev, remove_free_mem_region, res)) + return ERR_PTR(-ENOMEM); } write_lock(&resource_lock); - for (; addr > size && addr >= base->start; addr -= size) { - if (__region_intersects(addr, size, 0, IORES_DESC_NONE) != - REGION_DISJOINT) + for (addr = gfr_start(base, size, align, flags); + gfr_continue(base, addr, size, flags); + addr = gfr_next(addr, size, flags)) { + if (__region_intersects(base, addr, size, 0, IORES_DESC_NONE) != + REGION_DISJOINT) continue; - if (__request_region_locked(res, &iomem_resource, addr, size, - name, 0)) - break; + if (flags & GFR_REQUEST_REGION) { + if (__request_region_locked(res, &iomem_resource, addr, + size, name, 0)) + break; - if (dev) { - dr->parent = &iomem_resource; - dr->start = addr; - dr->n = size; - devres_add(dev, dr); - } + if (dev) { + dr->parent = &iomem_resource; + dr->start = addr; + dr->n = size; + devres_add(dev, dr); + } - res->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY; - write_unlock(&resource_lock); + res->desc = desc; + write_unlock(&resource_lock); + + + /* + * A driver is claiming this region so revoke any + * mappings. + */ + revoke_iomem(res); + } else { + res->start = addr; + res->end = addr + size - 1; + res->name = name; + res->desc = desc; + res->flags = IORESOURCE_MEM; + + /* + * Only succeed if the resource hosts an exclusive + * range after the insert + */ + if (__insert_resource(base, res) || res->child) + break; + + write_unlock(&resource_lock); + } - /* - * A driver is claiming this region so revoke any mappings. - */ - revoke_iomem(res); return res; } write_unlock(&resource_lock); - free_resource(res); - if (dr) + if (flags & GFR_REQUEST_REGION) { + free_resource(res); devres_free(dr); + } else if (dev) + devm_release_action(dev, remove_free_mem_region, res); return ERR_PTR(-ERANGE); } @@ -1854,18 +1932,48 @@ static struct resource *__request_free_mem_region(struct device *dev, struct resource *devm_request_free_mem_region(struct device *dev, struct resource *base, unsigned long size) { - return __request_free_mem_region(dev, base, size, dev_name(dev)); + unsigned long flags = GFR_DESCENDING | GFR_REQUEST_REGION; + + return get_free_mem_region(dev, base, size, GFR_DEFAULT_ALIGN, + dev_name(dev), + IORES_DESC_DEVICE_PRIVATE_MEMORY, flags); } EXPORT_SYMBOL_GPL(devm_request_free_mem_region); struct resource *request_free_mem_region(struct resource *base, unsigned long size, const char *name) { - return __request_free_mem_region(NULL, base, size, name); + unsigned long flags = GFR_DESCENDING | GFR_REQUEST_REGION; + + return get_free_mem_region(NULL, base, size, GFR_DEFAULT_ALIGN, name, + IORES_DESC_DEVICE_PRIVATE_MEMORY, flags); } EXPORT_SYMBOL_GPL(request_free_mem_region); -#endif /* CONFIG_DEVICE_PRIVATE */ +/** + * alloc_free_mem_region - find a free region relative to @base + * @base: resource that will parent the new resource + * @size: size in bytes of memory to allocate from @base + * @align: alignment requirements for the allocation + * @name: resource name + * + * Buses like CXL, that can dynamically instantiate new memory regions, + * need a method to allocate physical address space for those regions. + * Allocate and insert a new resource to cover a free, unclaimed by a + * descendant of @base, range in the span of @base. + */ +struct resource *alloc_free_mem_region(struct resource *base, + unsigned long size, unsigned long align, + const char *name) +{ + /* Default of ascending direction and insert resource */ + unsigned long flags = 0; + + return get_free_mem_region(NULL, base, size, align, name, + IORES_DESC_NONE, flags); +} +EXPORT_SYMBOL_NS_GPL(alloc_free_mem_region, CXL); +#endif /* CONFIG_GET_FREE_REGION */ static int __init strict_iomem(char *str) { diff --git a/mm/Kconfig b/mm/Kconfig index 169e64192e48..a5b4fee2e3fd 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -994,9 +994,14 @@ config HMM_MIRROR bool depends on MMU +config GET_FREE_REGION + depends on SPARSEMEM + bool + config DEVICE_PRIVATE bool "Unaddressable device memory (GPU memory, ...)" depends on ZONE_DEVICE + select GET_FREE_REGION help Allows creation of struct pages to represent unaddressable device -- cgit v1.2.3 From d96c52fe4907c68adc5e61a0bef7aec0933223d5 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 15 Apr 2022 10:55:42 -0700 Subject: rcu: Add polled expedited grace-period primitives This commit adds expedited grace-period functionality to RCU's polled grace-period API, adding start_poll_synchronize_rcu_expedited() and cond_synchronize_rcu_expedited(), which are similar to the existing start_poll_synchronize_rcu() and cond_synchronize_rcu() functions, respectively. Note that although start_poll_synchronize_rcu_expedited() can be invoked very early, the resulting expedited grace periods are not guaranteed to start until after workqueues are fully initialized. On the other hand, both synchronize_rcu() and synchronize_rcu_expedited() can also be invoked very early, and the resulting grace periods will be taken into account as they occur. [ paulmck: Apply feedback from Neeraj Upadhyay. ] Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- include/linux/rcutiny.h | 10 ++++++ include/linux/rcutree.h | 2 ++ kernel/rcu/tree.c | 17 +++++++--- kernel/rcu/tree.h | 7 ++++ kernel/rcu/tree_exp.h | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 5fed476f977f..ab7e20dfb07b 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -23,6 +23,16 @@ static inline void cond_synchronize_rcu(unsigned long oldstate) might_sleep(); } +static inline unsigned long start_poll_synchronize_rcu_expedited(void) +{ + return start_poll_synchronize_rcu(); +} + +static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) +{ + cond_synchronize_rcu(oldstate); +} + extern void rcu_barrier(void); static inline void synchronize_rcu_expedited(void) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9c6cfb742504..20dbaa9a3882 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -40,6 +40,8 @@ bool rcu_eqs_special_set(int cpu); void rcu_momentary_dyntick_idle(void); void kfree_rcu_scheduler_running(void); bool rcu_gp_might_be_stalled(void); +unsigned long start_poll_synchronize_rcu_expedited(void); +void cond_synchronize_rcu_expedited(unsigned long oldstate); unsigned long get_state_synchronize_rcu(void); unsigned long start_poll_synchronize_rcu(void); bool poll_state_synchronize_rcu(unsigned long oldstate); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 1505b02b4e53..6cf5b51622cd 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4027,20 +4027,20 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); /** * cond_synchronize_rcu - Conditionally wait for an RCU grace period * - * @oldstate: value from get_state_synchronize_rcu() or start_poll_synchronize_rcu() + * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited() * * If a full RCU grace period has elapsed since the earlier call to * get_state_synchronize_rcu() or start_poll_synchronize_rcu(), just return. * Otherwise, invoke synchronize_rcu() to wait for a full grace period. * - * Yes, this function does not take counter wrap into account. But - * counter wrap is harmless. If the counter wraps, we have waited for + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited for * more than 2 billion grace periods (and way more on a 64-bit system!), - * so waiting for one additional grace period should be just fine. + * so waiting for a couple of additional grace periods should be just fine. * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @oldstate, and that returned at the end + * to the function that provided @oldstate and that returned at the end * of this function. */ void cond_synchronize_rcu(unsigned long oldstate) @@ -4793,6 +4793,9 @@ static void __init rcu_init_one(void) init_waitqueue_head(&rnp->exp_wq[3]); spin_lock_init(&rnp->exp_lock); mutex_init(&rnp->boost_kthread_mutex); + raw_spin_lock_init(&rnp->exp_poll_lock); + rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; + INIT_WORK(&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } } @@ -5018,6 +5021,10 @@ void __init rcu_init(void) qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark; else qovld_calc = qovld; + + // Kick-start any polled grace periods that started early. + if (!(per_cpu_ptr(&rcu_data, cpu)->mynode->exp_seq_poll_rq & 0x1)) + (void)start_poll_synchronize_rcu_expedited(); } #include "tree_stall.h" diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 5634e76106c4..fb77deca5f5c 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -133,6 +133,10 @@ struct rcu_node { wait_queue_head_t exp_wq[4]; struct rcu_exp_work rew; bool exp_need_flush; /* Need to flush workitem? */ + raw_spinlock_t exp_poll_lock; + /* Lock and data for polled expedited grace periods. */ + unsigned long exp_seq_poll_rq; + struct work_struct exp_poll_wq; } ____cacheline_internodealigned_in_smp; /* @@ -484,3 +488,6 @@ static void rcu_iw_handler(struct irq_work *iwp); static void check_cpu_stall(struct rcu_data *rdp); static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, const unsigned long gpssdelay); + +/* Forward declarations for tree_exp.h. */ +static void sync_rcu_do_polled_gp(struct work_struct *wp); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index e0258066b881..571b0a700cce 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -962,3 +962,88 @@ void synchronize_rcu_expedited(void) synchronize_rcu_expedited_destroy_work(&rew); } EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); + +/* + * Ensure that start_poll_synchronize_rcu_expedited() has the expedited + * RCU grace periods that it needs. + */ +static void sync_rcu_do_polled_gp(struct work_struct *wp) +{ + unsigned long flags; + struct rcu_node *rnp = container_of(wp, struct rcu_node, exp_poll_wq); + unsigned long s; + + raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); + s = rnp->exp_seq_poll_rq; + rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; + raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); + if (s == RCU_GET_STATE_COMPLETED) + return; + while (!poll_state_synchronize_rcu(s)) + synchronize_rcu_expedited(); + raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); + s = rnp->exp_seq_poll_rq; + if (poll_state_synchronize_rcu(s)) + rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; + raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); +} + +/** + * start_poll_synchronize_rcu_expedited - Snapshot current RCU state and start expedited grace period + * + * Returns a cookie to pass to a call to cond_synchronize_rcu(), + * cond_synchronize_rcu_expedited(), or poll_state_synchronize_rcu(), + * allowing them to determine whether or not any sort of grace period has + * elapsed in the meantime. If the needed expedited grace period is not + * already slated to start, initiates that grace period. + */ +unsigned long start_poll_synchronize_rcu_expedited(void) +{ + unsigned long flags; + struct rcu_data *rdp; + struct rcu_node *rnp; + unsigned long s; + + s = get_state_synchronize_rcu(); + rdp = per_cpu_ptr(&rcu_data, raw_smp_processor_id()); + rnp = rdp->mynode; + if (rcu_init_invoked()) + raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); + if (!poll_state_synchronize_rcu(s)) { + rnp->exp_seq_poll_rq = s; + if (rcu_init_invoked()) + queue_work(rcu_gp_wq, &rnp->exp_poll_wq); + } + if (rcu_init_invoked()) + raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); + + return s; +} +EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited); + +/** + * cond_synchronize_rcu_expedited - Conditionally wait for an expedited RCU grace period + * + * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited() + * + * If any type of full RCU grace period has elapsed since the earlier + * call to get_state_synchronize_rcu(), start_poll_synchronize_rcu(), + * or start_poll_synchronize_rcu_expedited(), just return. Otherwise, + * invoke synchronize_rcu_expedited() to wait for a full grace period. + * + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited for + * more than 2 billion grace periods (and way more on a 64-bit system!), + * so waiting for a couple of additional grace periods should be just fine. + * + * This function provides the same memory-ordering guarantees that + * would be provided by a synchronize_rcu() that was invoked at the call + * to the function that provided @oldstate and that returned at the end + * of this function. + */ +void cond_synchronize_rcu_expedited(unsigned long oldstate) +{ + if (!poll_state_synchronize_rcu(oldstate)) + synchronize_rcu_expedited(); +} +EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); -- cgit v1.2.3 From ab21d6063c01180a8e9b22a37b847e5819525d9f Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 21 Jul 2022 15:42:33 +0200 Subject: bpf: Introduce 8-byte BTF set Introduce support for defining flags for kfuncs using a new set of macros, BTF_SET8_START/BTF_SET8_END, which define a set which contains 8 byte elements (each of which consists of a pair of BTF ID and flags), using a new BTF_ID_FLAGS macro. This will be used to tag kfuncs registered for a certain program type as acquire, release, sleepable, ret_null, etc. without having to create more and more sets which was proving to be an unscalable solution. Now, when looking up whether a kfunc is allowed for a certain program, we can also obtain its kfunc flags in the same call and avoid further lookups. The resolve_btfids change is split into a separate patch. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20220721134245.2450-2-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/btf_ids.h | 68 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 252a4befeab1..3cb0741e71d7 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -8,6 +8,15 @@ struct btf_id_set { u32 ids[]; }; +struct btf_id_set8 { + u32 cnt; + u32 flags; + struct { + u32 id; + u32 flags; + } pairs[]; +}; + #ifdef CONFIG_DEBUG_INFO_BTF #include /* for __PASTE */ @@ -25,7 +34,7 @@ struct btf_id_set { #define BTF_IDS_SECTION ".BTF_ids" -#define ____BTF_ID(symbol) \ +#define ____BTF_ID(symbol, word) \ asm( \ ".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ ".local " #symbol " ; \n" \ @@ -33,10 +42,11 @@ asm( \ ".size " #symbol ", 4; \n" \ #symbol ": \n" \ ".zero 4 \n" \ +word \ ".popsection; \n"); -#define __BTF_ID(symbol) \ - ____BTF_ID(symbol) +#define __BTF_ID(symbol, word) \ + ____BTF_ID(symbol, word) #define __ID(prefix) \ __PASTE(prefix, __COUNTER__) @@ -46,7 +56,14 @@ asm( \ * to 4 zero bytes. */ #define BTF_ID(prefix, name) \ - __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__)) + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), "") + +#define ____BTF_ID_FLAGS(prefix, name, flags) \ + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), ".long " #flags "\n") +#define __BTF_ID_FLAGS(prefix, name, flags, ...) \ + ____BTF_ID_FLAGS(prefix, name, flags) +#define BTF_ID_FLAGS(prefix, name, ...) \ + __BTF_ID_FLAGS(prefix, name, ##__VA_ARGS__, 0) /* * The BTF_ID_LIST macro defines pure (unsorted) list @@ -145,10 +162,51 @@ asm( \ ".popsection; \n"); \ extern struct btf_id_set name; +/* + * The BTF_SET8_START/END macros pair defines sorted list of + * BTF IDs and their flags plus its members count, with the + * following layout: + * + * BTF_SET8_START(list) + * BTF_ID_FLAGS(type1, name1, flags) + * BTF_ID_FLAGS(type2, name2, flags) + * BTF_SET8_END(list) + * + * __BTF_ID__set8__list: + * .zero 8 + * list: + * __BTF_ID__type1__name1__3: + * .zero 4 + * .word (1 << 0) | (1 << 2) + * __BTF_ID__type2__name2__5: + * .zero 4 + * .word (1 << 3) | (1 << 1) | (1 << 2) + * + */ +#define __BTF_SET8_START(name, scope) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +"." #scope " __BTF_ID__set8__" #name "; \n" \ +"__BTF_ID__set8__" #name ":; \n" \ +".zero 8 \n" \ +".popsection; \n"); + +#define BTF_SET8_START(name) \ +__BTF_ID_LIST(name, local) \ +__BTF_SET8_START(name, local) + +#define BTF_SET8_END(name) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +".size __BTF_ID__set8__" #name ", .-" #name " \n" \ +".popsection; \n"); \ +extern struct btf_id_set8 name; + #else #define BTF_ID_LIST(name) static u32 __maybe_unused name[5]; #define BTF_ID(prefix, name) +#define BTF_ID_FLAGS(prefix, name, flags) #define BTF_ID_UNUSED #define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1]; @@ -156,6 +214,8 @@ extern struct btf_id_set name; #define BTF_SET_START(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_END(name) +#define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 }; +#define BTF_SET8_END(name) static struct btf_id_set8 __maybe_unused name = { 0 }; #endif /* CONFIG_DEBUG_INFO_BTF */ -- cgit v1.2.3 From a4703e3184320d6e15e2bc81d2ccf1c8c883f9d1 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 21 Jul 2022 15:42:35 +0200 Subject: bpf: Switch to new kfunc flags infrastructure Instead of populating multiple sets to indicate some attribute and then researching the same BTF ID in them, prepare a single unified BTF set which indicates whether a kfunc is allowed to be called, and also its attributes if any at the same time. Now, only one call is needed to perform the lookup for both kfunc availability and its attributes. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20220721134245.2450-4-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 3 +- include/linux/btf.h | 33 ++----- kernel/bpf/btf.c | 106 ++++++++++----------- kernel/bpf/verifier.c | 14 ++- net/bpf/test_run.c | 70 +++++--------- net/ipv4/bpf_tcp_ca.c | 18 ++-- net/ipv4/tcp_bbr.c | 24 ++--- net/ipv4/tcp_cubic.c | 20 ++-- net/ipv4/tcp_dctcp.c | 20 ++-- net/netfilter/nf_conntrack_bpf.c | 49 +++------- .../selftests/bpf/bpf_testmod/bpf_testmod.c | 10 +- 11 files changed, 145 insertions(+), 222 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 11950029284f..a97751d845c9 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1924,7 +1924,8 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, - struct bpf_reg_state *regs); + struct bpf_reg_state *regs, + u32 kfunc_flags); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *reg); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, diff --git a/include/linux/btf.h b/include/linux/btf.h index 1bfed7fa0428..6dfc6eaf7f8c 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -12,14 +12,11 @@ #define BTF_TYPE_EMIT(type) ((void)(type *)0) #define BTF_TYPE_EMIT_ENUM(enum_val) ((void)enum_val) -enum btf_kfunc_type { - BTF_KFUNC_TYPE_CHECK, - BTF_KFUNC_TYPE_ACQUIRE, - BTF_KFUNC_TYPE_RELEASE, - BTF_KFUNC_TYPE_RET_NULL, - BTF_KFUNC_TYPE_KPTR_ACQUIRE, - BTF_KFUNC_TYPE_MAX, -}; +/* These need to be macros, as the expressions are used in assembler input */ +#define KF_ACQUIRE (1 << 0) /* kfunc is an acquire function */ +#define KF_RELEASE (1 << 1) /* kfunc is a release function */ +#define KF_RET_NULL (1 << 2) /* kfunc returns a pointer that may be NULL */ +#define KF_KPTR_GET (1 << 3) /* kfunc returns reference to a kptr */ struct btf; struct btf_member; @@ -30,16 +27,7 @@ struct btf_id_set; struct btf_kfunc_id_set { struct module *owner; - union { - struct { - struct btf_id_set *check_set; - struct btf_id_set *acquire_set; - struct btf_id_set *release_set; - struct btf_id_set *ret_null_set; - struct btf_id_set *kptr_acquire_set; - }; - struct btf_id_set *sets[BTF_KFUNC_TYPE_MAX]; - }; + struct btf_id_set8 *set; }; struct btf_id_dtor_kfunc { @@ -378,9 +366,9 @@ const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); const char *btf_name_by_offset(const struct btf *btf, u32 offset); struct btf *btf_parse_vmlinux(void); struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog); -bool btf_kfunc_id_set_contains(const struct btf *btf, +u32 *btf_kfunc_id_set_contains(const struct btf *btf, enum bpf_prog_type prog_type, - enum btf_kfunc_type type, u32 kfunc_btf_id); + u32 kfunc_btf_id); int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, const struct btf_kfunc_id_set *s); s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id); @@ -397,12 +385,11 @@ static inline const char *btf_name_by_offset(const struct btf *btf, { return NULL; } -static inline bool btf_kfunc_id_set_contains(const struct btf *btf, +static inline u32 *btf_kfunc_id_set_contains(const struct btf *btf, enum bpf_prog_type prog_type, - enum btf_kfunc_type type, u32 kfunc_btf_id) { - return false; + return NULL; } static inline int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, const struct btf_kfunc_id_set *s) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 5869f03bcb6e..4d9c2d88720f 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -213,7 +213,7 @@ enum { }; struct btf_kfunc_set_tab { - struct btf_id_set *sets[BTF_KFUNC_HOOK_MAX][BTF_KFUNC_TYPE_MAX]; + struct btf_id_set8 *sets[BTF_KFUNC_HOOK_MAX]; }; struct btf_id_dtor_kfunc_tab { @@ -1616,7 +1616,7 @@ static void btf_free_id(struct btf *btf) static void btf_free_kfunc_set_tab(struct btf *btf) { struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; - int hook, type; + int hook; if (!tab) return; @@ -1625,10 +1625,8 @@ static void btf_free_kfunc_set_tab(struct btf *btf) */ if (btf_is_module(btf)) goto free_tab; - for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) { - for (type = 0; type < ARRAY_SIZE(tab->sets[0]); type++) - kfree(tab->sets[hook][type]); - } + for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) + kfree(tab->sets[hook]); free_tab: kfree(tab); btf->kfunc_set_tab = NULL; @@ -6172,7 +6170,8 @@ static bool is_kfunc_arg_mem_size(const struct btf *btf, static int btf_check_func_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, struct bpf_reg_state *regs, - bool ptr_to_mem_ok) + bool ptr_to_mem_ok, + u32 kfunc_flags) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); struct bpf_verifier_log *log = &env->log; @@ -6210,10 +6209,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, if (is_kfunc) { /* Only kfunc can be release func */ - rel = btf_kfunc_id_set_contains(btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_RELEASE, func_id); - kptr_get = btf_kfunc_id_set_contains(btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_KPTR_ACQUIRE, func_id); + rel = kfunc_flags & KF_RELEASE; + kptr_get = kfunc_flags & KF_KPTR_GET; } /* check that BTF function arguments match actual types that the @@ -6442,7 +6439,7 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, return -EINVAL; is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; - err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global); + err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, 0); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. @@ -6455,9 +6452,10 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, - struct bpf_reg_state *regs) + struct bpf_reg_state *regs, + u32 kfunc_flags) { - return btf_check_func_arg_match(env, btf, func_id, regs, true); + return btf_check_func_arg_match(env, btf, func_id, regs, true, kfunc_flags); } /* Convert BTF of a function into bpf_reg_state if possible @@ -6854,6 +6852,11 @@ bool btf_id_set_contains(const struct btf_id_set *set, u32 id) return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL; } +static void *btf_id_set8_contains(const struct btf_id_set8 *set, u32 id) +{ + return bsearch(&id, set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func); +} + enum { BTF_MODULE_F_LIVE = (1 << 0), }; @@ -7102,16 +7105,16 @@ BTF_TRACING_TYPE_xxx /* Kernel Function (kfunc) BTF ID set registration API */ -static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, - enum btf_kfunc_type type, - struct btf_id_set *add_set, bool vmlinux_set) +static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, + struct btf_id_set8 *add_set) { + bool vmlinux_set = !btf_is_module(btf); struct btf_kfunc_set_tab *tab; - struct btf_id_set *set; + struct btf_id_set8 *set; u32 set_cnt; int ret; - if (hook >= BTF_KFUNC_HOOK_MAX || type >= BTF_KFUNC_TYPE_MAX) { + if (hook >= BTF_KFUNC_HOOK_MAX) { ret = -EINVAL; goto end; } @@ -7127,7 +7130,7 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, btf->kfunc_set_tab = tab; } - set = tab->sets[hook][type]; + set = tab->sets[hook]; /* Warn when register_btf_kfunc_id_set is called twice for the same hook * for module sets. */ @@ -7141,7 +7144,7 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, * pointer and return. */ if (!vmlinux_set) { - tab->sets[hook][type] = add_set; + tab->sets[hook] = add_set; return 0; } @@ -7150,7 +7153,7 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, * and concatenate all individual sets being registered. While each set * is individually sorted, they may become unsorted when concatenated, * hence re-sorting the final set again is required to make binary - * searching the set using btf_id_set_contains function work. + * searching the set using btf_id_set8_contains function work. */ set_cnt = set ? set->cnt : 0; @@ -7165,8 +7168,8 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, } /* Grow set */ - set = krealloc(tab->sets[hook][type], - offsetof(struct btf_id_set, ids[set_cnt + add_set->cnt]), + set = krealloc(tab->sets[hook], + offsetof(struct btf_id_set8, pairs[set_cnt + add_set->cnt]), GFP_KERNEL | __GFP_NOWARN); if (!set) { ret = -ENOMEM; @@ -7174,15 +7177,15 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, } /* For newly allocated set, initialize set->cnt to 0 */ - if (!tab->sets[hook][type]) + if (!tab->sets[hook]) set->cnt = 0; - tab->sets[hook][type] = set; + tab->sets[hook] = set; /* Concatenate the two sets */ - memcpy(set->ids + set->cnt, add_set->ids, add_set->cnt * sizeof(set->ids[0])); + memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0])); set->cnt += add_set->cnt; - sort(set->ids, set->cnt, sizeof(set->ids[0]), btf_id_cmp_func, NULL); + sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL); return 0; end: @@ -7190,38 +7193,25 @@ end: return ret; } -static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, - const struct btf_kfunc_id_set *kset) -{ - bool vmlinux_set = !btf_is_module(btf); - int type, ret = 0; - - for (type = 0; type < ARRAY_SIZE(kset->sets); type++) { - if (!kset->sets[type]) - continue; - - ret = __btf_populate_kfunc_set(btf, hook, type, kset->sets[type], vmlinux_set); - if (ret) - break; - } - return ret; -} - -static bool __btf_kfunc_id_set_contains(const struct btf *btf, +static u32 *__btf_kfunc_id_set_contains(const struct btf *btf, enum btf_kfunc_hook hook, - enum btf_kfunc_type type, u32 kfunc_btf_id) { - struct btf_id_set *set; + struct btf_id_set8 *set; + u32 *id; - if (hook >= BTF_KFUNC_HOOK_MAX || type >= BTF_KFUNC_TYPE_MAX) - return false; + if (hook >= BTF_KFUNC_HOOK_MAX) + return NULL; if (!btf->kfunc_set_tab) - return false; - set = btf->kfunc_set_tab->sets[hook][type]; + return NULL; + set = btf->kfunc_set_tab->sets[hook]; if (!set) - return false; - return btf_id_set_contains(set, kfunc_btf_id); + return NULL; + id = btf_id_set8_contains(set, kfunc_btf_id); + if (!id) + return NULL; + /* The flags for BTF ID are located next to it */ + return id + 1; } static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) @@ -7249,14 +7239,14 @@ static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) * keeping the reference for the duration of the call provides the necessary * protection for looking up a well-formed btf->kfunc_set_tab. */ -bool btf_kfunc_id_set_contains(const struct btf *btf, +u32 *btf_kfunc_id_set_contains(const struct btf *btf, enum bpf_prog_type prog_type, - enum btf_kfunc_type type, u32 kfunc_btf_id) + u32 kfunc_btf_id) { enum btf_kfunc_hook hook; hook = bpf_prog_type_to_kfunc_hook(prog_type); - return __btf_kfunc_id_set_contains(btf, hook, type, kfunc_btf_id); + return __btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id); } /* This function must be invoked only from initcalls/module init functions */ @@ -7283,7 +7273,7 @@ int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, return PTR_ERR(btf); hook = bpf_prog_type_to_kfunc_hook(prog_type); - ret = btf_populate_kfunc_set(btf, hook, kset); + ret = btf_populate_kfunc_set(btf, hook, kset->set); btf_put(btf); return ret; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7c1e056624f9..096fdac70165 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7562,6 +7562,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int err, insn_idx = *insn_idx_p; const struct btf_param *args; struct btf *desc_btf; + u32 *kfunc_flags; bool acq; /* skip for now, but return error when we find this in fixup_kfunc_call */ @@ -7577,18 +7578,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, func_name = btf_name_by_offset(desc_btf, func->name_off); func_proto = btf_type_by_id(desc_btf, func->type); - if (!btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_CHECK, func_id)) { + kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); + if (!kfunc_flags) { verbose(env, "calling kernel function %s is not allowed\n", func_name); return -EACCES; } - - acq = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_ACQUIRE, func_id); + acq = *kfunc_flags & KF_ACQUIRE; /* Check the arguments */ - err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs); + err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, *kfunc_flags); if (err < 0) return err; /* In case of release function, we get register number of refcounted @@ -7632,8 +7631,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].type = PTR_TO_BTF_ID; regs[BPF_REG_0].btf_id = ptr_type_id; - if (btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_RET_NULL, func_id)) { + if (*kfunc_flags & KF_RET_NULL) { regs[BPF_REG_0].type |= PTR_MAYBE_NULL; /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ regs[BPF_REG_0].id = ++env->id_gen; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index dc9dc0bedca0..ca5b7234a350 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -695,48 +695,26 @@ __diag_pop(); ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); -BTF_SET_START(test_sk_check_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test1) -BTF_ID(func, bpf_kfunc_call_test2) -BTF_ID(func, bpf_kfunc_call_test3) -BTF_ID(func, bpf_kfunc_call_test_acquire) -BTF_ID(func, bpf_kfunc_call_memb_acquire) -BTF_ID(func, bpf_kfunc_call_test_release) -BTF_ID(func, bpf_kfunc_call_memb_release) -BTF_ID(func, bpf_kfunc_call_memb1_release) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_ID(func, bpf_kfunc_call_test_pass_ctx) -BTF_ID(func, bpf_kfunc_call_test_pass1) -BTF_ID(func, bpf_kfunc_call_test_pass2) -BTF_ID(func, bpf_kfunc_call_test_fail1) -BTF_ID(func, bpf_kfunc_call_test_fail2) -BTF_ID(func, bpf_kfunc_call_test_fail3) -BTF_ID(func, bpf_kfunc_call_test_mem_len_pass1) -BTF_ID(func, bpf_kfunc_call_test_mem_len_fail1) -BTF_ID(func, bpf_kfunc_call_test_mem_len_fail2) -BTF_SET_END(test_sk_check_kfunc_ids) - -BTF_SET_START(test_sk_acquire_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_acquire) -BTF_ID(func, bpf_kfunc_call_memb_acquire) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_SET_END(test_sk_acquire_kfunc_ids) - -BTF_SET_START(test_sk_release_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_release) -BTF_ID(func, bpf_kfunc_call_memb_release) -BTF_ID(func, bpf_kfunc_call_memb1_release) -BTF_SET_END(test_sk_release_kfunc_ids) - -BTF_SET_START(test_sk_ret_null_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_acquire) -BTF_ID(func, bpf_kfunc_call_memb_acquire) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_SET_END(test_sk_ret_null_kfunc_ids) - -BTF_SET_START(test_sk_kptr_acquire_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_SET_END(test_sk_kptr_acquire_kfunc_ids) +BTF_SET8_START(test_sk_check_kfunc_ids) +BTF_ID_FLAGS(func, bpf_kfunc_call_test1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test3) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_acquire, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_call_memb_acquire, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_kfunc_call_memb1_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_kptr_get, KF_ACQUIRE | KF_RET_NULL | KF_KPTR_GET) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2) +BTF_SET8_END(test_sk_check_kfunc_ids) static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, u32 size, u32 headroom, u32 tailroom) @@ -1620,12 +1598,8 @@ out: } static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &test_sk_check_kfunc_ids, - .acquire_set = &test_sk_acquire_kfunc_ids, - .release_set = &test_sk_release_kfunc_ids, - .ret_null_set = &test_sk_ret_null_kfunc_ids, - .kptr_acquire_set = &test_sk_kptr_acquire_kfunc_ids + .owner = THIS_MODULE, + .set = &test_sk_check_kfunc_ids, }; BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids) diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 7a181631b995..85a9e500c42d 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -197,17 +197,17 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, } } -BTF_SET_START(bpf_tcp_ca_check_kfunc_ids) -BTF_ID(func, tcp_reno_ssthresh) -BTF_ID(func, tcp_reno_cong_avoid) -BTF_ID(func, tcp_reno_undo_cwnd) -BTF_ID(func, tcp_slow_start) -BTF_ID(func, tcp_cong_avoid_ai) -BTF_SET_END(bpf_tcp_ca_check_kfunc_ids) +BTF_SET8_START(bpf_tcp_ca_check_kfunc_ids) +BTF_ID_FLAGS(func, tcp_reno_ssthresh) +BTF_ID_FLAGS(func, tcp_reno_cong_avoid) +BTF_ID_FLAGS(func, tcp_reno_undo_cwnd) +BTF_ID_FLAGS(func, tcp_slow_start) +BTF_ID_FLAGS(func, tcp_cong_avoid_ai) +BTF_SET8_END(bpf_tcp_ca_check_kfunc_ids) static const struct btf_kfunc_id_set bpf_tcp_ca_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &bpf_tcp_ca_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &bpf_tcp_ca_check_kfunc_ids, }; static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = { diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 075e744bfb48..54eec33c6e1c 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -1154,24 +1154,24 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { .set_state = bbr_set_state, }; -BTF_SET_START(tcp_bbr_check_kfunc_ids) +BTF_SET8_START(tcp_bbr_check_kfunc_ids) #ifdef CONFIG_X86 #ifdef CONFIG_DYNAMIC_FTRACE -BTF_ID(func, bbr_init) -BTF_ID(func, bbr_main) -BTF_ID(func, bbr_sndbuf_expand) -BTF_ID(func, bbr_undo_cwnd) -BTF_ID(func, bbr_cwnd_event) -BTF_ID(func, bbr_ssthresh) -BTF_ID(func, bbr_min_tso_segs) -BTF_ID(func, bbr_set_state) +BTF_ID_FLAGS(func, bbr_init) +BTF_ID_FLAGS(func, bbr_main) +BTF_ID_FLAGS(func, bbr_sndbuf_expand) +BTF_ID_FLAGS(func, bbr_undo_cwnd) +BTF_ID_FLAGS(func, bbr_cwnd_event) +BTF_ID_FLAGS(func, bbr_ssthresh) +BTF_ID_FLAGS(func, bbr_min_tso_segs) +BTF_ID_FLAGS(func, bbr_set_state) #endif #endif -BTF_SET_END(tcp_bbr_check_kfunc_ids) +BTF_SET8_END(tcp_bbr_check_kfunc_ids) static const struct btf_kfunc_id_set tcp_bbr_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &tcp_bbr_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &tcp_bbr_check_kfunc_ids, }; static int __init bbr_register(void) diff --git a/net/ipv4/tcp_cubic.c b/net/ipv4/tcp_cubic.c index 68178e7280ce..768c10c1f649 100644 --- a/net/ipv4/tcp_cubic.c +++ b/net/ipv4/tcp_cubic.c @@ -485,22 +485,22 @@ static struct tcp_congestion_ops cubictcp __read_mostly = { .name = "cubic", }; -BTF_SET_START(tcp_cubic_check_kfunc_ids) +BTF_SET8_START(tcp_cubic_check_kfunc_ids) #ifdef CONFIG_X86 #ifdef CONFIG_DYNAMIC_FTRACE -BTF_ID(func, cubictcp_init) -BTF_ID(func, cubictcp_recalc_ssthresh) -BTF_ID(func, cubictcp_cong_avoid) -BTF_ID(func, cubictcp_state) -BTF_ID(func, cubictcp_cwnd_event) -BTF_ID(func, cubictcp_acked) +BTF_ID_FLAGS(func, cubictcp_init) +BTF_ID_FLAGS(func, cubictcp_recalc_ssthresh) +BTF_ID_FLAGS(func, cubictcp_cong_avoid) +BTF_ID_FLAGS(func, cubictcp_state) +BTF_ID_FLAGS(func, cubictcp_cwnd_event) +BTF_ID_FLAGS(func, cubictcp_acked) #endif #endif -BTF_SET_END(tcp_cubic_check_kfunc_ids) +BTF_SET8_END(tcp_cubic_check_kfunc_ids) static const struct btf_kfunc_id_set tcp_cubic_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &tcp_cubic_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &tcp_cubic_check_kfunc_ids, }; static int __init cubictcp_register(void) diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c index ab034a4e9324..2a6c0dd665a4 100644 --- a/net/ipv4/tcp_dctcp.c +++ b/net/ipv4/tcp_dctcp.c @@ -239,22 +239,22 @@ static struct tcp_congestion_ops dctcp_reno __read_mostly = { .name = "dctcp-reno", }; -BTF_SET_START(tcp_dctcp_check_kfunc_ids) +BTF_SET8_START(tcp_dctcp_check_kfunc_ids) #ifdef CONFIG_X86 #ifdef CONFIG_DYNAMIC_FTRACE -BTF_ID(func, dctcp_init) -BTF_ID(func, dctcp_update_alpha) -BTF_ID(func, dctcp_cwnd_event) -BTF_ID(func, dctcp_ssthresh) -BTF_ID(func, dctcp_cwnd_undo) -BTF_ID(func, dctcp_state) +BTF_ID_FLAGS(func, dctcp_init) +BTF_ID_FLAGS(func, dctcp_update_alpha) +BTF_ID_FLAGS(func, dctcp_cwnd_event) +BTF_ID_FLAGS(func, dctcp_ssthresh) +BTF_ID_FLAGS(func, dctcp_cwnd_undo) +BTF_ID_FLAGS(func, dctcp_state) #endif #endif -BTF_SET_END(tcp_dctcp_check_kfunc_ids) +BTF_SET8_END(tcp_dctcp_check_kfunc_ids) static const struct btf_kfunc_id_set tcp_dctcp_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &tcp_dctcp_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &tcp_dctcp_check_kfunc_ids, }; static int __init dctcp_register(void) diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c index bc4d5cd63a94..cf2096f65d0e 100644 --- a/net/netfilter/nf_conntrack_bpf.c +++ b/net/netfilter/nf_conntrack_bpf.c @@ -219,48 +219,21 @@ void bpf_ct_release(struct nf_conn *nfct) __diag_pop() -BTF_SET_START(nf_ct_xdp_check_kfunc_ids) -BTF_ID(func, bpf_xdp_ct_lookup) -BTF_ID(func, bpf_ct_release) -BTF_SET_END(nf_ct_xdp_check_kfunc_ids) - -BTF_SET_START(nf_ct_tc_check_kfunc_ids) -BTF_ID(func, bpf_skb_ct_lookup) -BTF_ID(func, bpf_ct_release) -BTF_SET_END(nf_ct_tc_check_kfunc_ids) - -BTF_SET_START(nf_ct_acquire_kfunc_ids) -BTF_ID(func, bpf_xdp_ct_lookup) -BTF_ID(func, bpf_skb_ct_lookup) -BTF_SET_END(nf_ct_acquire_kfunc_ids) - -BTF_SET_START(nf_ct_release_kfunc_ids) -BTF_ID(func, bpf_ct_release) -BTF_SET_END(nf_ct_release_kfunc_ids) - -/* Both sets are identical */ -#define nf_ct_ret_null_kfunc_ids nf_ct_acquire_kfunc_ids - -static const struct btf_kfunc_id_set nf_conntrack_xdp_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &nf_ct_xdp_check_kfunc_ids, - .acquire_set = &nf_ct_acquire_kfunc_ids, - .release_set = &nf_ct_release_kfunc_ids, - .ret_null_set = &nf_ct_ret_null_kfunc_ids, -}; - -static const struct btf_kfunc_id_set nf_conntrack_tc_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &nf_ct_tc_check_kfunc_ids, - .acquire_set = &nf_ct_acquire_kfunc_ids, - .release_set = &nf_ct_release_kfunc_ids, - .ret_null_set = &nf_ct_ret_null_kfunc_ids, +BTF_SET8_START(nf_ct_kfunc_set) +BTF_ID_FLAGS(func, bpf_xdp_ct_lookup, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_skb_ct_lookup, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_ct_release, KF_RELEASE) +BTF_SET8_END(nf_ct_kfunc_set) + +static const struct btf_kfunc_id_set nf_conntrack_kfunc_set = { + .owner = THIS_MODULE, + .set = &nf_ct_kfunc_set, }; int register_nf_conntrack_bpf(void) { int ret; - ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_xdp_kfunc_set); - return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_tc_kfunc_set); + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_kfunc_set); + return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set); } diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c index e585e1cefc77..792cb15bac40 100644 --- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c @@ -148,13 +148,13 @@ static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = { .write = bpf_testmod_test_write, }; -BTF_SET_START(bpf_testmod_check_kfunc_ids) -BTF_ID(func, bpf_testmod_test_mod_kfunc) -BTF_SET_END(bpf_testmod_check_kfunc_ids) +BTF_SET8_START(bpf_testmod_check_kfunc_ids) +BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc) +BTF_SET8_END(bpf_testmod_check_kfunc_ids) static const struct btf_kfunc_id_set bpf_testmod_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &bpf_testmod_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &bpf_testmod_check_kfunc_ids, }; extern int bpf_fentry_test1(int a); -- cgit v1.2.3 From 56e948ffc098a780fefb6c1784a3a2c7b81100a1 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 21 Jul 2022 15:42:36 +0200 Subject: bpf: Add support for forcing kfunc args to be trusted Teach the verifier to detect a new KF_TRUSTED_ARGS kfunc flag, which means each pointer argument must be trusted, which we define as a pointer that is referenced (has non-zero ref_obj_id) and also needs to have its offset unchanged, similar to how release functions expect their argument. This allows a kfunc to receive pointer arguments unchanged from the result of the acquire kfunc. This is required to ensure that kfunc that operate on some object only work on acquired pointers and not normal PTR_TO_BTF_ID with same type which can be obtained by pointer walking. The restrictions applied to release arguments also apply to trusted arguments. This implies that strict type matching (not deducing type by recursively following members at offset) and OBJ_RELEASE offset checks (ensuring they are zero) are used for trusted pointer arguments. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20220721134245.2450-5-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/btf.h | 32 ++++++++++++++++++++++++++++++++ kernel/bpf/btf.c | 17 ++++++++++++++--- net/bpf/test_run.c | 5 +++++ 3 files changed, 51 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index 6dfc6eaf7f8c..cdb376d53238 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -17,6 +17,38 @@ #define KF_RELEASE (1 << 1) /* kfunc is a release function */ #define KF_RET_NULL (1 << 2) /* kfunc returns a pointer that may be NULL */ #define KF_KPTR_GET (1 << 3) /* kfunc returns reference to a kptr */ +/* Trusted arguments are those which are meant to be referenced arguments with + * unchanged offset. It is used to enforce that pointers obtained from acquire + * kfuncs remain unmodified when being passed to helpers taking trusted args. + * + * Consider + * struct foo { + * int data; + * struct foo *next; + * }; + * + * struct bar { + * int data; + * struct foo f; + * }; + * + * struct foo *f = alloc_foo(); // Acquire kfunc + * struct bar *b = alloc_bar(); // Acquire kfunc + * + * If a kfunc set_foo_data() wants to operate only on the allocated object, it + * will set the KF_TRUSTED_ARGS flag, which will prevent unsafe usage like: + * + * set_foo_data(f, 42); // Allowed + * set_foo_data(f->next, 42); // Rejected, non-referenced pointer + * set_foo_data(&f->next, 42);// Rejected, referenced, but wrong type + * set_foo_data(&b->f, 42); // Rejected, referenced, but bad offset + * + * In the final case, usually for the purposes of type matching, it is deduced + * by looking at the type of the member at the offset, but due to the + * requirement of trusted argument, this deduction will be strict and not done + * for this case. + */ +#define KF_TRUSTED_ARGS (1 << 4) /* kfunc only takes trusted pointer arguments */ struct btf; struct btf_member; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4d9c2d88720f..7ac971ea98d1 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6174,10 +6174,10 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, u32 kfunc_flags) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); + bool rel = false, kptr_get = false, trusted_arg = false; struct bpf_verifier_log *log = &env->log; u32 i, nargs, ref_id, ref_obj_id = 0; bool is_kfunc = btf_is_kernel(btf); - bool rel = false, kptr_get = false; const char *func_name, *ref_tname; const struct btf_type *t, *ref_t; const struct btf_param *args; @@ -6211,6 +6211,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, /* Only kfunc can be release func */ rel = kfunc_flags & KF_RELEASE; kptr_get = kfunc_flags & KF_KPTR_GET; + trusted_arg = kfunc_flags & KF_TRUSTED_ARGS; } /* check that BTF function arguments match actual types that the @@ -6235,10 +6236,19 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, return -EINVAL; } + /* Check if argument must be a referenced pointer, args + i has + * been verified to be a pointer (after skipping modifiers). + */ + if (is_kfunc && trusted_arg && !reg->ref_obj_id) { + bpf_log(log, "R%d must be referenced\n", regno); + return -EINVAL; + } + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - if (rel && reg->ref_obj_id) + /* Trusted args have the same offset checks as release arguments */ + if (trusted_arg || (rel && reg->ref_obj_id)) arg_type |= OBJ_RELEASE; ret = check_func_arg_reg_off(env, reg, regno, arg_type); if (ret < 0) @@ -6336,7 +6346,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); if (!btf_struct_ids_match(log, reg_btf, reg_ref_id, - reg->off, btf, ref_id, rel && reg->ref_obj_id)) { + reg->off, btf, ref_id, + trusted_arg || (rel && reg->ref_obj_id))) { bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", func_name, i, btf_type_str(ref_t), ref_tname, diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index ca5b7234a350..cbc9cd5058cb 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -691,6 +691,10 @@ noinline void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len) { } +noinline void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p) +{ +} + __diag_pop(); ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); @@ -714,6 +718,7 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS) BTF_SET8_END(test_sk_check_kfunc_ids) static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, -- cgit v1.2.3 From 949d6b405e6160ae44baea39192d67b39cb7eeac Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 20 Jul 2022 16:57:58 -0700 Subject: net: add missing includes and forward declarations under net/ This patch adds missing includes to headers under include/net. All these problems are currently masked by the existing users including the missing dependency before the broken header. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/linux/lapb.h | 5 +++++ include/net/af_vsock.h | 1 + include/net/amt.h | 3 +++ include/net/ax88796.h | 2 ++ include/net/bond_options.h | 8 ++++++++ include/net/codel_qdisc.h | 1 + include/net/datalink.h | 7 +++++++ include/net/dcbevent.h | 2 ++ include/net/dcbnl.h | 2 ++ include/net/dn_dev.h | 1 + include/net/dn_fib.h | 2 ++ include/net/dn_neigh.h | 2 ++ include/net/dn_nsp.h | 6 ++++++ include/net/dn_route.h | 3 +++ include/net/erspan.h | 3 +++ include/net/esp.h | 1 + include/net/ethoc.h | 3 +++ include/net/firewire.h | 2 ++ include/net/fq.h | 4 ++++ include/net/garp.h | 2 ++ include/net/gtp.h | 4 ++++ include/net/gue.h | 3 +++ include/net/hwbm.h | 2 ++ include/net/ila.h | 2 ++ include/net/inet6_connection_sock.h | 2 ++ include/net/inet_common.h | 6 ++++++ include/net/inet_frag.h | 3 +++ include/net/ip6_route.h | 20 ++++++++++---------- include/net/ipcomp.h | 2 ++ include/net/ipconfig.h | 2 ++ include/net/llc_c_ac.h | 7 +++++++ include/net/llc_c_st.h | 4 ++++ include/net/llc_s_ac.h | 4 ++++ include/net/llc_s_ev.h | 1 + include/net/mpls_iptunnel.h | 3 +++ include/net/mrp.h | 4 ++++ include/net/ncsi.h | 2 ++ include/net/netevent.h | 1 + include/net/netns/can.h | 1 + include/net/netns/core.h | 2 ++ include/net/netns/generic.h | 1 + include/net/netns/ipv4.h | 1 + include/net/netns/mctp.h | 1 + include/net/netns/mpls.h | 2 ++ include/net/netns/nexthop.h | 1 + include/net/netns/sctp.h | 3 +++ include/net/netns/unix.h | 2 ++ include/net/netrom.h | 1 + include/net/p8022.h | 5 +++++ include/net/phonet/pep.h | 3 +++ include/net/phonet/phonet.h | 4 ++++ include/net/phonet/pn_dev.h | 5 +++++ include/net/pptp.h | 3 +++ include/net/psnap.h | 5 +++++ include/net/regulatory.h | 3 +++ include/net/rose.h | 1 + include/net/secure_seq.h | 2 ++ include/net/smc.h | 7 +++++++ include/net/stp.h | 2 ++ include/net/transp_v6.h | 2 ++ include/net/tun_proto.h | 3 ++- include/net/udplite.h | 1 + include/net/xdp_priv.h | 1 + 63 files changed, 183 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lapb.h b/include/linux/lapb.h index eb56472f23b2..b5333f9413dc 100644 --- a/include/linux/lapb.h +++ b/include/linux/lapb.h @@ -6,6 +6,11 @@ #ifndef LAPB_KERNEL_H #define LAPB_KERNEL_H +#include +#include + +struct net_device; + #define LAPB_OK 0 #define LAPB_BADTOKEN 1 #define LAPB_INVALUE 2 diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index f742e50207fb..1c53c4c4d88f 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -10,6 +10,7 @@ #include #include +#include #include #include "vsock_addr.h" diff --git a/include/net/amt.h b/include/net/amt.h index 08fc30cf2f34..c881bc8b673b 100644 --- a/include/net/amt.h +++ b/include/net/amt.h @@ -7,6 +7,9 @@ #include #include +#include +#include +#include enum amt_msg_type { AMT_MSG_DISCOVERY = 1, diff --git a/include/net/ax88796.h b/include/net/ax88796.h index 2ed23a368602..b658471f97f0 100644 --- a/include/net/ax88796.h +++ b/include/net/ax88796.h @@ -8,6 +8,8 @@ #ifndef __NET_AX88796_PLAT_H #define __NET_AX88796_PLAT_H +#include + struct sk_buff; struct net_device; struct platform_device; diff --git a/include/net/bond_options.h b/include/net/bond_options.h index d2aea5cf1e41..69292ecc0325 100644 --- a/include/net/bond_options.h +++ b/include/net/bond_options.h @@ -7,6 +7,14 @@ #ifndef _NET_BOND_OPTIONS_H #define _NET_BOND_OPTIONS_H +#include +#include +#include +#include + +struct netlink_ext_ack; +struct nlattr; + #define BOND_OPT_MAX_NAMELEN 32 #define BOND_OPT_VALID(opt) ((opt) < BOND_OPT_LAST) #define BOND_MODE_ALL_EX(x) (~(x)) diff --git a/include/net/codel_qdisc.h b/include/net/codel_qdisc.h index 58b6d0ebea10..7d3d9219f4fe 100644 --- a/include/net/codel_qdisc.h +++ b/include/net/codel_qdisc.h @@ -49,6 +49,7 @@ * Implemented on linux by Dave Taht and Eric Dumazet */ +#include #include /* Qdiscs using codel plugin must use codel_skb_cb in their own cb[] */ diff --git a/include/net/datalink.h b/include/net/datalink.h index d9b7faaa539f..c837ffc7ebf8 100644 --- a/include/net/datalink.h +++ b/include/net/datalink.h @@ -2,6 +2,13 @@ #ifndef _NET_INET_DATALINK_H_ #define _NET_INET_DATALINK_H_ +#include + +struct llc_sap; +struct net_device; +struct packet_type; +struct sk_buff; + struct datalink_proto { unsigned char type[8]; diff --git a/include/net/dcbevent.h b/include/net/dcbevent.h index 43e34131a53f..02700262f71a 100644 --- a/include/net/dcbevent.h +++ b/include/net/dcbevent.h @@ -8,6 +8,8 @@ #ifndef _DCB_EVENT_H #define _DCB_EVENT_H +struct notifier_block; + enum dcbevent_notif_type { DCB_APP_EVENT = 1, }; diff --git a/include/net/dcbnl.h b/include/net/dcbnl.h index e4ad58c4062c..2b2d86fb3131 100644 --- a/include/net/dcbnl.h +++ b/include/net/dcbnl.h @@ -10,6 +10,8 @@ #include +struct net_device; + struct dcb_app_type { int ifindex; struct dcb_app app; diff --git a/include/net/dn_dev.h b/include/net/dn_dev.h index 595b4f6c1eb1..bec303ea8367 100644 --- a/include/net/dn_dev.h +++ b/include/net/dn_dev.h @@ -2,6 +2,7 @@ #ifndef _NET_DN_DEV_H #define _NET_DN_DEV_H +#include struct dn_dev; diff --git a/include/net/dn_fib.h b/include/net/dn_fib.h index ddd6565957b3..1929a3cd5ebe 100644 --- a/include/net/dn_fib.h +++ b/include/net/dn_fib.h @@ -4,6 +4,8 @@ #include #include +#include +#include extern const struct nla_policy rtm_dn_policy[]; diff --git a/include/net/dn_neigh.h b/include/net/dn_neigh.h index 2e3e7793973a..1f7df98bfc33 100644 --- a/include/net/dn_neigh.h +++ b/include/net/dn_neigh.h @@ -2,6 +2,8 @@ #ifndef _NET_DN_NEIGH_H #define _NET_DN_NEIGH_H +#include + /* * The position of the first two fields of * this structure are critical - SJW diff --git a/include/net/dn_nsp.h b/include/net/dn_nsp.h index f83932b864a9..a4a18fee0b7c 100644 --- a/include/net/dn_nsp.h +++ b/include/net/dn_nsp.h @@ -6,6 +6,12 @@ *******************************************************************************/ /* dn_nsp.c functions prototyping */ +#include +#include +#include + +struct sk_buff; +struct sk_buff_head; void dn_nsp_send_data_ack(struct sock *sk); void dn_nsp_send_oth_ack(struct sock *sk); diff --git a/include/net/dn_route.h b/include/net/dn_route.h index 6f1e94ac0bdf..88c0300236cc 100644 --- a/include/net/dn_route.h +++ b/include/net/dn_route.h @@ -7,6 +7,9 @@ *******************************************************************************/ +#include +#include + struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri); int dn_route_output_sock(struct dst_entry __rcu **pprt, struct flowidn *, struct sock *sk, int flags); diff --git a/include/net/erspan.h b/include/net/erspan.h index 0d9e86bd9893..6cb4cbd6a48f 100644 --- a/include/net/erspan.h +++ b/include/net/erspan.h @@ -58,6 +58,9 @@ * GRE proto ERSPAN type I/II = 0x88BE, type III = 0x22EB */ +#include +#include +#include #include #define ERSPAN_VERSION 0x1 /* ERSPAN type II */ diff --git a/include/net/esp.h b/include/net/esp.h index 9c5637d41d95..322950727dd0 100644 --- a/include/net/esp.h +++ b/include/net/esp.h @@ -5,6 +5,7 @@ #include struct ip_esp_hdr; +struct xfrm_state; static inline struct ip_esp_hdr *ip_esp_hdr(const struct sk_buff *skb) { diff --git a/include/net/ethoc.h b/include/net/ethoc.h index 78519ed42ab4..73810f3ca492 100644 --- a/include/net/ethoc.h +++ b/include/net/ethoc.h @@ -10,6 +10,9 @@ #ifndef LINUX_NET_ETHOC_H #define LINUX_NET_ETHOC_H 1 +#include +#include + struct ethoc_platform_data { u8 hwaddr[IFHWADDRLEN]; s8 phy_id; diff --git a/include/net/firewire.h b/include/net/firewire.h index 299e5df38552..2442d645e412 100644 --- a/include/net/firewire.h +++ b/include/net/firewire.h @@ -2,6 +2,8 @@ #ifndef _NET_FIREWIRE_H #define _NET_FIREWIRE_H +#include + /* Pseudo L2 address */ #define FWNET_ALEN 16 union fwnet_hwaddr { diff --git a/include/net/fq.h b/include/net/fq.h index 2eccbbd2b559..07b5aff6ec58 100644 --- a/include/net/fq.h +++ b/include/net/fq.h @@ -7,6 +7,10 @@ #ifndef __NET_SCHED_FQ_H #define __NET_SCHED_FQ_H +#include +#include +#include + struct fq_tin; /** diff --git a/include/net/garp.h b/include/net/garp.h index 4d9a0c6a2e5f..59a07b171def 100644 --- a/include/net/garp.h +++ b/include/net/garp.h @@ -2,6 +2,8 @@ #ifndef _NET_GARP_H #define _NET_GARP_H +#include +#include #include #define GARP_PROTOCOL_ID 0x1 diff --git a/include/net/gtp.h b/include/net/gtp.h index c1d6169df331..2a503f035d18 100644 --- a/include/net/gtp.h +++ b/include/net/gtp.h @@ -2,6 +2,10 @@ #ifndef _GTP_H_ #define _GTP_H_ +#include +#include +#include + /* General GTP protocol related definitions. */ #define GTP0_PORT 3386 diff --git a/include/net/gue.h b/include/net/gue.h index e42402f180b7..dfca298bec9c 100644 --- a/include/net/gue.h +++ b/include/net/gue.h @@ -30,6 +30,9 @@ * may refer to options placed after this field. */ +#include +#include + struct guehdr { union { struct { diff --git a/include/net/hwbm.h b/include/net/hwbm.h index c81444611a22..aa495decec35 100644 --- a/include/net/hwbm.h +++ b/include/net/hwbm.h @@ -2,6 +2,8 @@ #ifndef _HWBM_H #define _HWBM_H +#include + struct hwbm_pool { /* Capacity of the pool */ int size; diff --git a/include/net/ila.h b/include/net/ila.h index f98dcd5791b0..73ebe5eab272 100644 --- a/include/net/ila.h +++ b/include/net/ila.h @@ -8,6 +8,8 @@ #ifndef _NET_ILA_H #define _NET_ILA_H +struct sk_buff; + int ila_xlat_outgoing(struct sk_buff *skb); int ila_xlat_incoming(struct sk_buff *skb); diff --git a/include/net/inet6_connection_sock.h b/include/net/inet6_connection_sock.h index 7392f959a405..025bd8d3c769 100644 --- a/include/net/inet6_connection_sock.h +++ b/include/net/inet6_connection_sock.h @@ -11,6 +11,8 @@ #include +struct flowi; +struct flowi6; struct request_sock; struct sk_buff; struct sock; diff --git a/include/net/inet_common.h b/include/net/inet_common.h index cad2a611efde..cec453c18f1d 100644 --- a/include/net/inet_common.h +++ b/include/net/inet_common.h @@ -3,6 +3,10 @@ #define _INET_COMMON_H #include +#include +#include +#include +#include extern const struct proto_ops inet_stream_ops; extern const struct proto_ops inet_dgram_ops; @@ -12,6 +16,8 @@ extern const struct proto_ops inet_dgram_ops; */ struct msghdr; +struct net; +struct page; struct sock; struct sockaddr; struct socket; diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h index 911ad930867d..0b0876610553 100644 --- a/include/net/inet_frag.h +++ b/include/net/inet_frag.h @@ -4,6 +4,9 @@ #include #include +#include +#include +#include /* Per netns frag queues directory */ struct fqdir { diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index ca2d6b60e1ec..035d61d50a98 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -2,6 +2,16 @@ #ifndef _NET_IP6_ROUTE_H #define _NET_IP6_ROUTE_H +#include +#include +#include +#include +#include +#include +#include +#include +#include + struct route_info { __u8 type; __u8 length; @@ -19,16 +29,6 @@ struct route_info { __u8 prefix[]; /* 0,8 or 16 */ }; -#include -#include -#include -#include -#include -#include -#include -#include -#include - #define RT6_LOOKUP_F_IFACE 0x00000001 #define RT6_LOOKUP_F_REACHABLE 0x00000002 #define RT6_LOOKUP_F_HAS_SADDR 0x00000004 diff --git a/include/net/ipcomp.h b/include/net/ipcomp.h index fee6fc451597..c31108295079 100644 --- a/include/net/ipcomp.h +++ b/include/net/ipcomp.h @@ -2,11 +2,13 @@ #ifndef _NET_IPCOMP_H #define _NET_IPCOMP_H +#include #include #define IPCOMP_SCRATCH_SIZE 65400 struct crypto_comp; +struct ip_comp_hdr; struct ipcomp_data { u16 threshold; diff --git a/include/net/ipconfig.h b/include/net/ipconfig.h index e3534299bd2a..8276897d0c2e 100644 --- a/include/net/ipconfig.h +++ b/include/net/ipconfig.h @@ -7,6 +7,8 @@ /* The following are initdata: */ +#include + extern int ic_proto_enabled; /* Protocols enabled (see IC_xxx) */ extern int ic_set_manually; /* IPconfig parameters set manually */ diff --git a/include/net/llc_c_ac.h b/include/net/llc_c_ac.h index e766300b3e99..3e1f76786d7b 100644 --- a/include/net/llc_c_ac.h +++ b/include/net/llc_c_ac.h @@ -16,6 +16,13 @@ * Connection state transition actions * (Fb = F bit; Pb = P bit; Xb = X bit) */ + +#include + +struct sk_buff; +struct sock; +struct timer_list; + #define LLC_CONN_AC_CLR_REMOTE_BUSY 1 #define LLC_CONN_AC_CONN_IND 2 #define LLC_CONN_AC_CONN_CONFIRM 3 diff --git a/include/net/llc_c_st.h b/include/net/llc_c_st.h index 48f3f891b2f9..53823d61d8b6 100644 --- a/include/net/llc_c_st.h +++ b/include/net/llc_c_st.h @@ -11,6 +11,10 @@ * * See the GNU General Public License for more details. */ + +#include +#include + /* Connection component state management */ /* connection states */ #define LLC_CONN_OUT_OF_SVC 0 /* prior to allocation */ diff --git a/include/net/llc_s_ac.h b/include/net/llc_s_ac.h index a61b98c108ee..f71790305bc9 100644 --- a/include/net/llc_s_ac.h +++ b/include/net/llc_s_ac.h @@ -11,6 +11,10 @@ * * See the GNU General Public License for more details. */ + +struct llc_sap; +struct sk_buff; + /* SAP component actions */ #define SAP_ACT_UNITDATA_IND 1 #define SAP_ACT_SEND_UI 2 diff --git a/include/net/llc_s_ev.h b/include/net/llc_s_ev.h index 84db3a59ed28..fb7df1d70af3 100644 --- a/include/net/llc_s_ev.h +++ b/include/net/llc_s_ev.h @@ -13,6 +13,7 @@ */ #include +#include /* Defines SAP component events */ /* Types of events (possible values in 'ev->type') */ diff --git a/include/net/mpls_iptunnel.h b/include/net/mpls_iptunnel.h index 9deb3a3735da..0c71c27979fb 100644 --- a/include/net/mpls_iptunnel.h +++ b/include/net/mpls_iptunnel.h @@ -6,6 +6,9 @@ #ifndef _NET_MPLS_IPTUNNEL_H #define _NET_MPLS_IPTUNNEL_H 1 +#include +#include + struct mpls_iptunnel_encap { u8 labels; u8 ttl_propagate; diff --git a/include/net/mrp.h b/include/net/mrp.h index 1c308c034e1a..92cd3fb6cf9d 100644 --- a/include/net/mrp.h +++ b/include/net/mrp.h @@ -2,6 +2,10 @@ #ifndef _NET_MRP_H #define _NET_MRP_H +#include +#include +#include + #define MRP_END_MARK 0x0 struct mrp_pdu_hdr { diff --git a/include/net/ncsi.h b/include/net/ncsi.h index fbefe80361ee..08a50d9acb0a 100644 --- a/include/net/ncsi.h +++ b/include/net/ncsi.h @@ -2,6 +2,8 @@ #ifndef __NET_NCSI_H #define __NET_NCSI_H +#include + /* * The NCSI device states seen from external. More NCSI device states are * only visible internally (in net/ncsi/internal.h). When the NCSI device diff --git a/include/net/netevent.h b/include/net/netevent.h index 4107016c3bb4..1be3757a8b7f 100644 --- a/include/net/netevent.h +++ b/include/net/netevent.h @@ -14,6 +14,7 @@ struct dst_entry; struct neighbour; +struct notifier_block ; struct netevent_redirect { struct dst_entry *old; diff --git a/include/net/netns/can.h b/include/net/netns/can.h index 52fbd8291a96..48b79f7e6236 100644 --- a/include/net/netns/can.h +++ b/include/net/netns/can.h @@ -7,6 +7,7 @@ #define __NETNS_CAN_H__ #include +#include struct can_dev_rcv_lists; struct can_pkg_stats; diff --git a/include/net/netns/core.h b/include/net/netns/core.h index 388244e315e7..8249060cf5d0 100644 --- a/include/net/netns/core.h +++ b/include/net/netns/core.h @@ -2,6 +2,8 @@ #ifndef __NETNS_CORE_H__ #define __NETNS_CORE_H__ +#include + struct ctl_table_header; struct prot_inuse; diff --git a/include/net/netns/generic.h b/include/net/netns/generic.h index 8a1ab47c3fb3..7ce68183f6e1 100644 --- a/include/net/netns/generic.h +++ b/include/net/netns/generic.h @@ -8,6 +8,7 @@ #include #include +#include /* * Generic net pointers are to be used by modules to put some private diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index ce0cc4e8d8c7..c7320ef356d9 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -9,6 +9,7 @@ #include #include #include +#include #include struct ctl_table_header; diff --git a/include/net/netns/mctp.h b/include/net/netns/mctp.h index acedef12a35e..1db8f9aaddb4 100644 --- a/include/net/netns/mctp.h +++ b/include/net/netns/mctp.h @@ -6,6 +6,7 @@ #ifndef __NETNS_MCTP_H__ #define __NETNS_MCTP_H__ +#include #include struct netns_mctp { diff --git a/include/net/netns/mpls.h b/include/net/netns/mpls.h index a7bdcfbb0b28..19ad2574b267 100644 --- a/include/net/netns/mpls.h +++ b/include/net/netns/mpls.h @@ -6,6 +6,8 @@ #ifndef __NETNS_MPLS_H__ #define __NETNS_MPLS_H__ +#include + struct mpls_route; struct ctl_table_header; diff --git a/include/net/netns/nexthop.h b/include/net/netns/nexthop.h index 1849e77eb68a..434239b37014 100644 --- a/include/net/netns/nexthop.h +++ b/include/net/netns/nexthop.h @@ -6,6 +6,7 @@ #ifndef __NETNS_NEXTHOP_H__ #define __NETNS_NEXTHOP_H__ +#include #include struct netns_nexthop { diff --git a/include/net/netns/sctp.h b/include/net/netns/sctp.h index 40240722cdca..a681147aecd8 100644 --- a/include/net/netns/sctp.h +++ b/include/net/netns/sctp.h @@ -2,6 +2,9 @@ #ifndef __NETNS_SCTP_H__ #define __NETNS_SCTP_H__ +#include +#include + struct sock; struct proc_dir_entry; struct sctp_mib; diff --git a/include/net/netns/unix.h b/include/net/netns/unix.h index 6f1a33df061d..9859d134d5a8 100644 --- a/include/net/netns/unix.h +++ b/include/net/netns/unix.h @@ -5,6 +5,8 @@ #ifndef __NETNS_UNIX_H__ #define __NETNS_UNIX_H__ +#include + struct unix_table { spinlock_t *locks; struct hlist_head *buckets; diff --git a/include/net/netrom.h b/include/net/netrom.h index 80f15b1c1a48..f0565a5987d1 100644 --- a/include/net/netrom.h +++ b/include/net/netrom.h @@ -14,6 +14,7 @@ #include #include #include +#include #define NR_NETWORK_LEN 15 #define NR_TRANSPORT_LEN 5 diff --git a/include/net/p8022.h b/include/net/p8022.h index c2bacc66bfbc..b690ffcad66b 100644 --- a/include/net/p8022.h +++ b/include/net/p8022.h @@ -1,6 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _NET_P8022_H #define _NET_P8022_H + +struct net_device; +struct packet_type; +struct sk_buff; + struct datalink_proto * register_8022_client(unsigned char type, int (*func)(struct sk_buff *skb, diff --git a/include/net/phonet/pep.h b/include/net/phonet/pep.h index 27b1ab5e4e6d..645dddf5ce77 100644 --- a/include/net/phonet/pep.h +++ b/include/net/phonet/pep.h @@ -10,6 +10,9 @@ #ifndef NET_PHONET_PEP_H #define NET_PHONET_PEP_H +#include +#include + struct pep_sock { struct pn_sock pn_sk; diff --git a/include/net/phonet/phonet.h b/include/net/phonet/phonet.h index a27bdc6cfeab..862f1719b523 100644 --- a/include/net/phonet/phonet.h +++ b/include/net/phonet/phonet.h @@ -10,6 +10,10 @@ #ifndef AF_PHONET_H #define AF_PHONET_H +#include +#include +#include + /* * The lower layers may not require more space, ever. Make sure it's * enough. diff --git a/include/net/phonet/pn_dev.h b/include/net/phonet/pn_dev.h index 05b49d4d2b11..e9dc8dca5817 100644 --- a/include/net/phonet/pn_dev.h +++ b/include/net/phonet/pn_dev.h @@ -10,6 +10,11 @@ #ifndef PN_DEV_H #define PN_DEV_H +#include +#include + +struct net; + struct phonet_device_list { struct list_head list; struct mutex lock; diff --git a/include/net/pptp.h b/include/net/pptp.h index 383e25ca53a7..e63176bdd4c8 100644 --- a/include/net/pptp.h +++ b/include/net/pptp.h @@ -2,6 +2,9 @@ #ifndef _NET_PPTP_H #define _NET_PPTP_H +#include +#include + #define PPP_LCP_ECHOREQ 0x09 #define PPP_LCP_ECHOREP 0x0A #define SC_RCV_BITS (SC_RCV_B7_1|SC_RCV_B7_0|SC_RCV_ODDP|SC_RCV_EVNP) diff --git a/include/net/psnap.h b/include/net/psnap.h index 7cb0c8ab4171..88802b0754ad 100644 --- a/include/net/psnap.h +++ b/include/net/psnap.h @@ -2,6 +2,11 @@ #ifndef _NET_PSNAP_H #define _NET_PSNAP_H +struct datalink_proto; +struct sk_buff; +struct packet_type; +struct net_device; + struct datalink_proto * register_snap_client(const unsigned char *desc, int (*rcvfunc)(struct sk_buff *, struct net_device *, diff --git a/include/net/regulatory.h b/include/net/regulatory.h index 47f06f6f5a67..896191f420d5 100644 --- a/include/net/regulatory.h +++ b/include/net/regulatory.h @@ -1,3 +1,4 @@ + #ifndef __NET_REGULATORY_H #define __NET_REGULATORY_H /* @@ -19,6 +20,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include +#include #include /** diff --git a/include/net/rose.h b/include/net/rose.h index 0f0a4ce0fee7..f192a64ddef2 100644 --- a/include/net/rose.h +++ b/include/net/rose.h @@ -9,6 +9,7 @@ #define _ROSE_H #include +#include #include #define ROSE_ADDR_LEN 5 diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h index dac91aa38c5a..21e7fa2a1813 100644 --- a/include/net/secure_seq.h +++ b/include/net/secure_seq.h @@ -4,6 +4,8 @@ #include +struct net; + u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport); diff --git a/include/net/smc.h b/include/net/smc.h index e441aa97ad61..37f829d9c6e5 100644 --- a/include/net/smc.h +++ b/include/net/smc.h @@ -11,6 +11,13 @@ #ifndef _SMC_H #define _SMC_H +#include +#include +#include +#include + +struct sock; + #define SMC_MAX_PNETID_LEN 16 /* Max. length of PNET id */ struct smc_hashinfo { diff --git a/include/net/stp.h b/include/net/stp.h index 2914e6d53490..528103fce2c0 100644 --- a/include/net/stp.h +++ b/include/net/stp.h @@ -2,6 +2,8 @@ #ifndef _NET_STP_H #define _NET_STP_H +#include + struct stp_proto { unsigned char group_address[ETH_ALEN]; void (*rcv)(const struct stp_proto *, struct sk_buff *, diff --git a/include/net/transp_v6.h b/include/net/transp_v6.h index da06613c9603..b830463e3dff 100644 --- a/include/net/transp_v6.h +++ b/include/net/transp_v6.h @@ -3,6 +3,7 @@ #define _TRANSP_V6_H #include +#include /* IPv6 transport protocols */ extern struct proto rawv6_prot; @@ -12,6 +13,7 @@ extern struct proto tcpv6_prot; extern struct proto pingv6_prot; struct flowi6; +struct ipcm6_cookie; /* extension headers */ int ipv6_exthdrs_init(void); diff --git a/include/net/tun_proto.h b/include/net/tun_proto.h index 2ea3deba4c99..7b0de7852908 100644 --- a/include/net/tun_proto.h +++ b/include/net/tun_proto.h @@ -1,7 +1,8 @@ #ifndef __NET_TUN_PROTO_H #define __NET_TUN_PROTO_H -#include +#include +#include /* One byte protocol values as defined by VXLAN-GPE and NSH. These will * hopefully get a shared IANA registry. diff --git a/include/net/udplite.h b/include/net/udplite.h index a3c53110d30b..0143b373602e 100644 --- a/include/net/udplite.h +++ b/include/net/udplite.h @@ -6,6 +6,7 @@ #define _UDPLITE_H #include +#include /* UDP-Lite socket options */ #define UDPLITE_SEND_CSCOV 10 /* sender partial coverage (as sent) */ diff --git a/include/net/xdp_priv.h b/include/net/xdp_priv.h index a2d58b1a12e1..c9df68d5f258 100644 --- a/include/net/xdp_priv.h +++ b/include/net/xdp_priv.h @@ -3,6 +3,7 @@ #define __LINUX_NET_XDP_PRIV_H__ #include +#include /* Private to net/core/xdp.c, but used by trace/events/xdp.h */ struct xdp_mem_allocator { -- cgit v1.2.3 From 0903f899418ee0235e4ad756f5502162f29b49b9 Mon Sep 17 00:00:00 2001 From: Avraham Stern Date: Thu, 27 Jan 2022 14:39:46 +0200 Subject: wifi: ieee80211: add helper functions for detecting TM/FTM frames Add helper functions for detection timing measurement and fine timing measurement frames. Signed-off-by: Avraham Stern Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index cca564372d16..55e6f4ad0ca6 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -1332,6 +1332,15 @@ struct ieee80211_mgmt { u8 action_code; u8 variable[]; } __packed s1g; + struct { + u8 action_code; + u8 dialog_token; + u8 follow_up; + u32 tod; + u32 toa; + u8 max_tod_error; + u8 max_toa_error; + } __packed wnm_timing_msr; } u; } __packed action; } u; @@ -3522,6 +3531,17 @@ enum ieee80211_mesh_actioncode { WLAN_MESH_ACTION_TBTT_ADJUSTMENT_RESPONSE, }; +/* Unprotected WNM action codes */ +enum ieee80211_unprotected_wnm_actioncode { + WLAN_UNPROTECTED_WNM_ACTION_TIM = 0, + WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE = 1, +}; + +/* Public action codes */ +enum ieee80211_public_actioncode { + WLAN_PUBLIC_ACTION_FTM_RESPONSE = 33, +}; + /* Security key length */ enum ieee80211_key_len { WLAN_KEY_LEN_WEP40 = 5, @@ -4311,6 +4331,40 @@ static inline bool ieee80211_action_contains_tpc(struct sk_buff *skb) return true; } +static inline bool ieee80211_is_timing_measurement(struct sk_buff *skb) +{ + struct ieee80211_mgmt *mgmt = (void *)skb->data; + + if (skb->len < IEEE80211_MIN_ACTION_SIZE) + return false; + + if (!ieee80211_is_action(mgmt->frame_control)) + return false; + + if (mgmt->u.action.category == WLAN_CATEGORY_WNM_UNPROTECTED && + mgmt->u.action.u.wnm_timing_msr.action_code == + WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE && + skb->len >= offsetofend(typeof(*mgmt), u.action.u.wnm_timing_msr)) + return true; + + return false; +} + +static inline bool ieee80211_is_ftm(struct sk_buff *skb) +{ + struct ieee80211_mgmt *mgmt = (void *)skb->data; + + if (!ieee80211_is_public_action((void *)mgmt, skb->len)) + return false; + + if (mgmt->u.action.u.ftm.action_code == + WLAN_PUBLIC_ACTION_FTM_RESPONSE && + skb->len >= offsetofend(typeof(*mgmt), u.action.u.ftm)) + return true; + + return false; +} + struct element { u8 id; u8 datalen; -- cgit v1.2.3 From dea997733575c5793ca77e166bbbf89097987eb4 Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Fri, 22 Jul 2022 10:48:50 +0100 Subject: firmware: cs_dsp: Add pre_stop callback The code already has a post_stop callback, add a matching pre_stop callback to the client_ops that is called before execution is stopped. This callback provides a convenient place for the client code to communicate with the DSP before it is stopped. Signed-off-by: Charles Keepax Link: https://lore.kernel.org/r/20220722094851.92521-1-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 3 +++ include/linux/firmware/cirrus/cs_dsp.h | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 7dad6f57d970..b402f841d72c 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -2725,6 +2725,9 @@ void cs_dsp_stop(struct cs_dsp *dsp) mutex_lock(&dsp->pwr_lock); + if (dsp->client_ops->pre_stop) + dsp->client_ops->pre_stop(dsp); + dsp->running = false; if (dsp->ops->stop_core) diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 30055706cce2..6ab230218df0 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -189,7 +189,8 @@ struct cs_dsp { * @control_remove: Called under the pwr_lock when a control is destroyed * @pre_run: Called under the pwr_lock by cs_dsp_run() before the core is started * @post_run: Called under the pwr_lock by cs_dsp_run() after the core is started - * @post_stop: Called under the pwr_lock by cs_dsp_stop() + * @pre_stop: Called under the pwr_lock by cs_dsp_stop() before the core is stopped + * @post_stop: Called under the pwr_lock by cs_dsp_stop() after the core is stopped * @watchdog_expired: Called when a watchdog expiry is detected * * These callbacks give the cs_dsp client an opportunity to respond to events @@ -200,6 +201,7 @@ struct cs_dsp_client_ops { void (*control_remove)(struct cs_dsp_coeff_ctl *ctl); int (*pre_run)(struct cs_dsp *dsp); int (*post_run)(struct cs_dsp *dsp); + void (*pre_stop)(struct cs_dsp *dsp); void (*post_stop)(struct cs_dsp *dsp); void (*watchdog_expired)(struct cs_dsp *dsp); }; -- cgit v1.2.3 From a4b976552f122ea851f556769874022cf097741e Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Fri, 22 Jul 2022 10:48:51 +0100 Subject: firmware: cs_dsp: Add memory chunk helpers Add helpers that can be layered on top of a buffer read from or to be written to the DSP to faciliate accessing datastructures within the DSP memory. These functions handle adding the padding bytes for the DSP, converting to big endian, and packing arbitrary length data. Signed-off-by: Charles Keepax Link: https://lore.kernel.org/r/20220722094851.92521-2-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 104 +++++++++++++++++++++++++++++++++ include/linux/firmware/cirrus/cs_dsp.h | 73 +++++++++++++++++++++++ 2 files changed, 177 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index b402f841d72c..81cc3d0f6eec 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -3180,6 +3180,110 @@ static const struct cs_dsp_ops cs_dsp_halo_ops = { .stop_core = cs_dsp_halo_stop_core, }; +/** + * cs_dsp_chunk_write() - Format data to a DSP memory chunk + * @ch: Pointer to the chunk structure + * @nbits: Number of bits to write + * @val: Value to write + * + * This function sequentially writes values into the format required for DSP + * memory, it handles both inserting of the padding bytes and converting to + * big endian. Note that data is only committed to the chunk when a whole DSP + * words worth of data is available. + * + * Return: Zero for success, a negative number on error. + */ +int cs_dsp_chunk_write(struct cs_dsp_chunk *ch, int nbits, u32 val) +{ + int nwrite, i; + + nwrite = min(CS_DSP_DATA_WORD_BITS - ch->cachebits, nbits); + + ch->cache <<= nwrite; + ch->cache |= val >> (nbits - nwrite); + ch->cachebits += nwrite; + nbits -= nwrite; + + if (ch->cachebits == CS_DSP_DATA_WORD_BITS) { + if (cs_dsp_chunk_end(ch)) + return -ENOSPC; + + ch->cache &= 0xFFFFFF; + for (i = 0; i < sizeof(ch->cache); i++, ch->cache <<= BITS_PER_BYTE) + *ch->data++ = (ch->cache & 0xFF000000) >> CS_DSP_DATA_WORD_BITS; + + ch->bytes += sizeof(ch->cache); + ch->cachebits = 0; + } + + if (nbits) + return cs_dsp_chunk_write(ch, nbits, val); + + return 0; +} +EXPORT_SYMBOL_GPL(cs_dsp_chunk_write); + +/** + * cs_dsp_chunk_flush() - Pad remaining data with zero and commit to chunk + * @ch: Pointer to the chunk structure + * + * As cs_dsp_chunk_write only writes data when a whole DSP word is ready to + * be written out it is possible that some data will remain in the cache, this + * function will pad that data with zeros upto a whole DSP word and write out. + * + * Return: Zero for success, a negative number on error. + */ +int cs_dsp_chunk_flush(struct cs_dsp_chunk *ch) +{ + if (!ch->cachebits) + return 0; + + return cs_dsp_chunk_write(ch, CS_DSP_DATA_WORD_BITS - ch->cachebits, 0); +} +EXPORT_SYMBOL_GPL(cs_dsp_chunk_flush); + +/** + * cs_dsp_chunk_read() - Parse data from a DSP memory chunk + * @ch: Pointer to the chunk structure + * @nbits: Number of bits to read + * + * This function sequentially reads values from a DSP memory formatted buffer, + * it handles both removing of the padding bytes and converting from big endian. + * + * Return: A negative number is returned on error, otherwise the read value. + */ +int cs_dsp_chunk_read(struct cs_dsp_chunk *ch, int nbits) +{ + int nread, i; + u32 result; + + if (!ch->cachebits) { + if (cs_dsp_chunk_end(ch)) + return -ENOSPC; + + ch->cache = 0; + ch->cachebits = CS_DSP_DATA_WORD_BITS; + + for (i = 0; i < sizeof(ch->cache); i++, ch->cache <<= BITS_PER_BYTE) + ch->cache |= *ch->data++; + + ch->bytes += sizeof(ch->cache); + } + + nread = min(ch->cachebits, nbits); + nbits -= nread; + + result = ch->cache >> ((sizeof(ch->cache) * BITS_PER_BYTE) - nread); + ch->cache <<= nread; + ch->cachebits -= nread; + + if (nbits) + result = (result << nbits) | cs_dsp_chunk_read(ch, nbits); + + return result; +} +EXPORT_SYMBOL_GPL(cs_dsp_chunk_read); + MODULE_DESCRIPTION("Cirrus Logic DSP Support"); MODULE_AUTHOR("Simon Trimmer "); MODULE_LICENSE("GPL v2"); diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 6ab230218df0..cad828e21c72 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -11,6 +11,7 @@ #ifndef __CS_DSP_H #define __CS_DSP_H +#include #include #include #include @@ -34,6 +35,7 @@ #define CS_ADSP2_REGION_ALL (CS_ADSP2_REGION_0 | CS_ADSP2_REGION_1_9) #define CS_DSP_DATA_WORD_SIZE 3 +#define CS_DSP_DATA_WORD_BITS (3 * BITS_PER_BYTE) #define CS_DSP_ACKED_CTL_TIMEOUT_MS 100 #define CS_DSP_ACKED_CTL_N_QUICKPOLLS 10 @@ -252,4 +254,75 @@ struct cs_dsp_alg_region *cs_dsp_find_alg_region(struct cs_dsp *dsp, const char *cs_dsp_mem_region_name(unsigned int type); +/** + * struct cs_dsp_chunk - Describes a buffer holding data formatted for the DSP + * @data: Pointer to underlying buffer memory + * @max: Pointer to end of the buffer memory + * @bytes: Number of bytes read/written into the memory chunk + * @cache: Temporary holding data as it is formatted + * @cachebits: Number of bits of data currently in cache + */ +struct cs_dsp_chunk { + u8 *data; + u8 *max; + int bytes; + + u32 cache; + int cachebits; +}; + +/** + * cs_dsp_chunk() - Create a DSP memory chunk + * @data: Pointer to the buffer that will be used to store data + * @size: Size of the buffer in bytes + * + * Return: A cs_dsp_chunk structure + */ +static inline struct cs_dsp_chunk cs_dsp_chunk(void *data, int size) +{ + struct cs_dsp_chunk ch = { + .data = data, + .max = data + size, + }; + + return ch; +} + +/** + * cs_dsp_chunk_end() - Check if a DSP memory chunk is full + * @ch: Pointer to the chunk structure + * + * Return: True if the whole buffer has been read/written + */ +static inline bool cs_dsp_chunk_end(struct cs_dsp_chunk *ch) +{ + return ch->data == ch->max; +} + +/** + * cs_dsp_chunk_bytes() - Number of bytes written/read from a DSP memory chunk + * @ch: Pointer to the chunk structure + * + * Return: Number of bytes read/written to the buffer + */ +static inline int cs_dsp_chunk_bytes(struct cs_dsp_chunk *ch) +{ + return ch->bytes; +} + +/** + * cs_dsp_chunk_valid_addr() - Check if an address is in a DSP memory chunk + * @ch: Pointer to the chunk structure + * + * Return: True if the given address is within the buffer + */ +static inline bool cs_dsp_chunk_valid_addr(struct cs_dsp_chunk *ch, void *addr) +{ + return (u8 *)addr >= ch->data && (u8 *)addr < ch->max; +} + +int cs_dsp_chunk_write(struct cs_dsp_chunk *ch, int nbits, u32 val); +int cs_dsp_chunk_flush(struct cs_dsp_chunk *ch); +int cs_dsp_chunk_read(struct cs_dsp_chunk *ch, int nbits); + #endif -- cgit v1.2.3 From e423414375866a399ebbe55ed044acb39846e8bf Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 22 Jul 2022 13:36:05 +0200 Subject: bpf: Fix build error in case of !CONFIG_DEBUG_INFO_BTF BTF_ID_FLAGS macro needs to be able to take 0 or 1 args, so make it a variable argument. BTF_SET8_END is incorrect, it should just be empty. Reported-by: kernel test robot Fixes: ab21d6063c01 ("bpf: Introduce 8-byte BTF set") Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Jiri Olsa Link: https://lore.kernel.org/r/20220722113605.6513-1-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/btf_ids.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 3cb0741e71d7..2aea877d644f 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -206,7 +206,7 @@ extern struct btf_id_set8 name; #define BTF_ID_LIST(name) static u32 __maybe_unused name[5]; #define BTF_ID(prefix, name) -#define BTF_ID_FLAGS(prefix, name, flags) +#define BTF_ID_FLAGS(prefix, name, ...) #define BTF_ID_UNUSED #define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1]; @@ -215,7 +215,7 @@ extern struct btf_id_set8 name; #define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_END(name) #define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 }; -#define BTF_SET8_END(name) static struct btf_id_set8 __maybe_unused name = { 0 }; +#define BTF_SET8_END(name) #endif /* CONFIG_DEBUG_INFO_BTF */ -- cgit v1.2.3 From 478af190cb6c501efaa8de2b9c9418ece2e4d0bd Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Jul 2022 10:59:17 -0700 Subject: iomap: remove iomap_writepage Unused now. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Chaitanya Kulkarni Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/iomap/buffered-io.c | 15 --------------- include/linux/iomap.h | 3 --- 2 files changed, 18 deletions(-) (limited to 'include/linux') diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index afd260632836..7c4a56b3ac01 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -1519,21 +1519,6 @@ unlock: return 0; } -int -iomap_writepage(struct page *page, struct writeback_control *wbc, - struct iomap_writepage_ctx *wpc, - const struct iomap_writeback_ops *ops) -{ - int ret; - - wpc->ops = ops; - ret = iomap_do_writepage(page, wbc, wpc); - if (!wpc->ioend) - return ret; - return iomap_submit_ioend(wpc, wpc->ioend, ret); -} -EXPORT_SYMBOL_GPL(iomap_writepage); - int iomap_writepages(struct address_space *mapping, struct writeback_control *wbc, struct iomap_writepage_ctx *wpc, diff --git a/include/linux/iomap.h b/include/linux/iomap.h index e552097c67e0..911888560d3e 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -303,9 +303,6 @@ void iomap_finish_ioends(struct iomap_ioend *ioend, int error); void iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends); void iomap_sort_ioends(struct list_head *ioend_list); -int iomap_writepage(struct page *page, struct writeback_control *wbc, - struct iomap_writepage_ctx *wpc, - const struct iomap_writeback_ops *ops); int iomap_writepages(struct address_space *mapping, struct writeback_control *wbc, struct iomap_writepage_ctx *wpc, const struct iomap_writeback_ops *ops); -- cgit v1.2.3 From f96f644ab97abeed3f7007c953836a574ce928cc Mon Sep 17 00:00:00 2001 From: Song Liu Date: Tue, 19 Jul 2022 17:21:23 -0700 Subject: ftrace: Add modify_ftrace_direct_multi_nolock This is similar to modify_ftrace_direct_multi, but does not acquire direct_mutex. This is useful when direct_mutex is already locked by the user. Signed-off-by: Song Liu Signed-off-by: Daniel Borkmann Reviewed-by: Steven Rostedt (Google) Link: https://lore.kernel.org/bpf/20220720002126.803253-2-song@kernel.org --- include/linux/ftrace.h | 5 +++ kernel/trace/ftrace.c | 86 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 67 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 979f6bfa2c25..acb35243ce5d 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -340,6 +340,7 @@ unsigned long ftrace_find_rec_direct(unsigned long ip); int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr); int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr); int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr); +int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr); #else struct ftrace_ops; @@ -384,6 +385,10 @@ static inline int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned lo { return -ENODEV; } +static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr) +{ + return -ENODEV; +} #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 601ccf1b2f09..5d67dc12231d 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5691,22 +5691,8 @@ int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) } EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi); -/** - * modify_ftrace_direct_multi - Modify an existing direct 'multi' call - * to call something else - * @ops: The address of the struct ftrace_ops object - * @addr: The address of the new trampoline to call at @ops functions - * - * This is used to unregister currently registered direct caller and - * register new one @addr on functions registered in @ops object. - * - * Note there's window between ftrace_shutdown and ftrace_startup calls - * where there will be no callbacks called. - * - * Returns: zero on success. Non zero on error, which includes: - * -EINVAL - The @ops object was not properly registered. - */ -int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) +static int +__modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) { struct ftrace_hash *hash; struct ftrace_func_entry *entry, *iter; @@ -5717,12 +5703,7 @@ int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) int i, size; int err; - if (check_direct_multi(ops)) - return -EINVAL; - if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) - return -EINVAL; - - mutex_lock(&direct_mutex); + lockdep_assert_held_once(&direct_mutex); /* Enable the tmp_ops to have the same functions as the direct ops */ ftrace_ops_init(&tmp_ops); @@ -5730,7 +5711,7 @@ int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) err = register_ftrace_function(&tmp_ops); if (err) - goto out_direct; + return err; /* * Now the ftrace_ops_list_func() is called to do the direct callers. @@ -5754,7 +5735,64 @@ int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) /* Removing the tmp_ops will add the updated direct callers to the functions */ unregister_ftrace_function(&tmp_ops); - out_direct: + return err; +} + +/** + * modify_ftrace_direct_multi_nolock - Modify an existing direct 'multi' call + * to call something else + * @ops: The address of the struct ftrace_ops object + * @addr: The address of the new trampoline to call at @ops functions + * + * This is used to unregister currently registered direct caller and + * register new one @addr on functions registered in @ops object. + * + * Note there's window between ftrace_shutdown and ftrace_startup calls + * where there will be no callbacks called. + * + * Caller should already have direct_mutex locked, so we don't lock + * direct_mutex here. + * + * Returns: zero on success. Non zero on error, which includes: + * -EINVAL - The @ops object was not properly registered. + */ +int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr) +{ + if (check_direct_multi(ops)) + return -EINVAL; + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) + return -EINVAL; + + return __modify_ftrace_direct_multi(ops, addr); +} +EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi_nolock); + +/** + * modify_ftrace_direct_multi - Modify an existing direct 'multi' call + * to call something else + * @ops: The address of the struct ftrace_ops object + * @addr: The address of the new trampoline to call at @ops functions + * + * This is used to unregister currently registered direct caller and + * register new one @addr on functions registered in @ops object. + * + * Note there's window between ftrace_shutdown and ftrace_startup calls + * where there will be no callbacks called. + * + * Returns: zero on success. Non zero on error, which includes: + * -EINVAL - The @ops object was not properly registered. + */ +int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) +{ + int err; + + if (check_direct_multi(ops)) + return -EINVAL; + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) + return -EINVAL; + + mutex_lock(&direct_mutex); + err = __modify_ftrace_direct_multi(ops, addr); mutex_unlock(&direct_mutex); return err; } -- cgit v1.2.3 From 53cd885bc5c3ea283cc9c00ca6446c778f00bfba Mon Sep 17 00:00:00 2001 From: Song Liu Date: Tue, 19 Jul 2022 17:21:24 -0700 Subject: ftrace: Allow IPMODIFY and DIRECT ops on the same function IPMODIFY (livepatch) and DIRECT (bpf trampoline) ops are both important users of ftrace. It is necessary to allow them work on the same function at the same time. First, DIRECT ops no longer specify IPMODIFY flag. Instead, DIRECT flag is handled together with IPMODIFY flag in __ftrace_hash_update_ipmodify(). Then, a callback function, ops_func, is added to ftrace_ops. This is used by ftrace core code to understand whether the DIRECT ops can share with an IPMODIFY ops. To share with IPMODIFY ops, the DIRECT ops need to implement the callback function and adjust the direct trampoline accordingly. If DIRECT ops is attached before the IPMODIFY ops, ftrace core code calls ENABLE_SHARE_IPMODIFY_PEER on the DIRECT ops before registering the IPMODIFY ops. If IPMODIFY ops is attached before the DIRECT ops, ftrace core code calls ENABLE_SHARE_IPMODIFY_SELF in __ftrace_hash_update_ipmodify. Owner of the DIRECT ops may return 0 if the DIRECT trampoline can share with IPMODIFY, so error code otherwise. The error code is propagated to register_ftrace_direct_multi so that onwer of the DIRECT trampoline can handle it properly. For more details, please refer to comment before enum ftrace_ops_cmd. Signed-off-by: Song Liu Signed-off-by: Daniel Borkmann Reviewed-by: Steven Rostedt (Google) Link: https://lore.kernel.org/all/20220602193706.2607681-2-song@kernel.org/ Link: https://lore.kernel.org/all/20220718055449.3960512-1-song@kernel.org/ Link: https://lore.kernel.org/bpf/20220720002126.803253-3-song@kernel.org --- include/linux/ftrace.h | 38 ++++++++ kernel/trace/ftrace.c | 242 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 254 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index acb35243ce5d..0b61371e287b 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -208,6 +208,43 @@ enum { FTRACE_OPS_FL_DIRECT = BIT(17), }; +/* + * FTRACE_OPS_CMD_* commands allow the ftrace core logic to request changes + * to a ftrace_ops. Note, the requests may fail. + * + * ENABLE_SHARE_IPMODIFY_SELF - enable a DIRECT ops to work on the same + * function as an ops with IPMODIFY. Called + * when the DIRECT ops is being registered. + * This is called with both direct_mutex and + * ftrace_lock are locked. + * + * ENABLE_SHARE_IPMODIFY_PEER - enable a DIRECT ops to work on the same + * function as an ops with IPMODIFY. Called + * when the other ops (the one with IPMODIFY) + * is being registered. + * This is called with direct_mutex locked. + * + * DISABLE_SHARE_IPMODIFY_PEER - disable a DIRECT ops to work on the same + * function as an ops with IPMODIFY. Called + * when the other ops (the one with IPMODIFY) + * is being unregistered. + * This is called with direct_mutex locked. + */ +enum ftrace_ops_cmd { + FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF, + FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER, + FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER, +}; + +/* + * For most ftrace_ops_cmd, + * Returns: + * 0 - Success. + * Negative on failure. The return value is dependent on the + * callback. + */ +typedef int (*ftrace_ops_func_t)(struct ftrace_ops *op, enum ftrace_ops_cmd cmd); + #ifdef CONFIG_DYNAMIC_FTRACE /* The hash used to know what functions callbacks trace */ struct ftrace_ops_hash { @@ -250,6 +287,7 @@ struct ftrace_ops { unsigned long trampoline; unsigned long trampoline_size; struct list_head list; + ftrace_ops_func_t ops_func; #endif }; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 5d67dc12231d..bc921a3f7ea8 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1861,6 +1861,8 @@ static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, ftrace_hash_rec_update_modify(ops, filter_hash, 1); } +static bool ops_references_ip(struct ftrace_ops *ops, unsigned long ip); + /* * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK * or no-needed to update, -EBUSY if it detects a conflict of the flag @@ -1869,6 +1871,13 @@ static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) * - If the hash is EMPTY_HASH, it hits nothing * - Anything else hits the recs which match the hash entries. + * + * DIRECT ops does not have IPMODIFY flag, but we still need to check it + * against functions with FTRACE_FL_IPMODIFY. If there is any overlap, call + * ops_func(SHARE_IPMODIFY_SELF) to make sure current ops can share with + * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate + * the return value to the caller and eventually to the owner of the DIRECT + * ops. */ static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, struct ftrace_hash *old_hash, @@ -1877,17 +1886,26 @@ static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, struct ftrace_page *pg; struct dyn_ftrace *rec, *end = NULL; int in_old, in_new; + bool is_ipmodify, is_direct; /* Only update if the ops has been registered */ if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) return 0; - if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) + is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY; + is_direct = ops->flags & FTRACE_OPS_FL_DIRECT; + + /* neither IPMODIFY nor DIRECT, skip */ + if (!is_ipmodify && !is_direct) + return 0; + + if (WARN_ON_ONCE(is_ipmodify && is_direct)) return 0; /* - * Since the IPMODIFY is a very address sensitive action, we do not - * allow ftrace_ops to set all functions to new hash. + * Since the IPMODIFY and DIRECT are very address sensitive + * actions, we do not allow ftrace_ops to set all functions to new + * hash. */ if (!new_hash || !old_hash) return -EINVAL; @@ -1905,12 +1923,32 @@ static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, continue; if (in_new) { - /* New entries must ensure no others are using it */ - if (rec->flags & FTRACE_FL_IPMODIFY) - goto rollback; - rec->flags |= FTRACE_FL_IPMODIFY; - } else /* Removed entry */ + if (rec->flags & FTRACE_FL_IPMODIFY) { + int ret; + + /* Cannot have two ipmodify on same rec */ + if (is_ipmodify) + goto rollback; + + FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT); + + /* + * Another ops with IPMODIFY is already + * attached. We are now attaching a direct + * ops. Run SHARE_IPMODIFY_SELF, to check + * whether sharing is supported. + */ + if (!ops->ops_func) + return -EBUSY; + ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); + if (ret) + return ret; + } else if (is_ipmodify) { + rec->flags |= FTRACE_FL_IPMODIFY; + } + } else if (is_ipmodify) { rec->flags &= ~FTRACE_FL_IPMODIFY; + } } while_for_each_ftrace_rec(); return 0; @@ -2454,8 +2492,7 @@ static void call_direct_funcs(unsigned long ip, unsigned long pip, struct ftrace_ops direct_ops = { .func = call_direct_funcs, - .flags = FTRACE_OPS_FL_IPMODIFY - | FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS + .flags = FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_PERMANENT, /* * By declaring the main trampoline as this trampoline @@ -3072,14 +3109,14 @@ static inline int ops_traces_mod(struct ftrace_ops *ops) } /* - * Check if the current ops references the record. + * Check if the current ops references the given ip. * * If the ops traces all functions, then it was already accounted for. * If the ops does not trace the current record function, skip it. * If the ops ignores the function via notrace filter, skip it. */ -static inline bool -ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) +static bool +ops_references_ip(struct ftrace_ops *ops, unsigned long ip) { /* If ops isn't enabled, ignore it */ if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) @@ -3091,16 +3128,29 @@ ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) /* The function must be in the filter */ if (!ftrace_hash_empty(ops->func_hash->filter_hash) && - !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) + !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) return false; /* If in notrace hash, we ignore it too */ - if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) + if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) return false; return true; } +/* + * Check if the current ops references the record. + * + * If the ops traces all functions, then it was already accounted for. + * If the ops does not trace the current record function, skip it. + * If the ops ignores the function via notrace filter, skip it. + */ +static bool +ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) +{ + return ops_references_ip(ops, rec->ip); +} + static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) { bool init_nop = ftrace_need_init_nop(); @@ -5215,6 +5265,8 @@ static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr) return direct; } +static int register_ftrace_function_nolock(struct ftrace_ops *ops); + /** * register_ftrace_direct - Call a custom trampoline directly * @ip: The address of the nop at the beginning of a function @@ -5286,7 +5338,7 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr) ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0); if (!ret && !(direct_ops.flags & FTRACE_OPS_FL_ENABLED)) { - ret = register_ftrace_function(&direct_ops); + ret = register_ftrace_function_nolock(&direct_ops); if (ret) ftrace_set_filter_ip(&direct_ops, ip, 1, 0); } @@ -5545,8 +5597,7 @@ int modify_ftrace_direct(unsigned long ip, } EXPORT_SYMBOL_GPL(modify_ftrace_direct); -#define MULTI_FLAGS (FTRACE_OPS_FL_IPMODIFY | FTRACE_OPS_FL_DIRECT | \ - FTRACE_OPS_FL_SAVE_REGS) +#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS) static int check_direct_multi(struct ftrace_ops *ops) { @@ -5639,7 +5690,7 @@ int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) ops->flags = MULTI_FLAGS; ops->trampoline = FTRACE_REGS_ADDR; - err = register_ftrace_function(ops); + err = register_ftrace_function_nolock(ops); out_remove: if (err) @@ -5709,7 +5760,7 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) ftrace_ops_init(&tmp_ops); tmp_ops.func_hash = ops->func_hash; - err = register_ftrace_function(&tmp_ops); + err = register_ftrace_function_nolock(&tmp_ops); if (err) return err; @@ -8003,6 +8054,143 @@ int ftrace_is_dead(void) return ftrace_disabled; } +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +/* + * When registering ftrace_ops with IPMODIFY, it is necessary to make sure + * it doesn't conflict with any direct ftrace_ops. If there is existing + * direct ftrace_ops on a kernel function being patched, call + * FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER on it to enable sharing. + * + * @ops: ftrace_ops being registered. + * + * Returns: + * 0 on success; + * Negative on failure. + */ +static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) +{ + struct ftrace_func_entry *entry; + struct ftrace_hash *hash; + struct ftrace_ops *op; + int size, i, ret; + + lockdep_assert_held_once(&direct_mutex); + + if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) + return 0; + + hash = ops->func_hash->filter_hash; + size = 1 << hash->size_bits; + for (i = 0; i < size; i++) { + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { + unsigned long ip = entry->ip; + bool found_op = false; + + mutex_lock(&ftrace_lock); + do_for_each_ftrace_op(op, ftrace_ops_list) { + if (!(op->flags & FTRACE_OPS_FL_DIRECT)) + continue; + if (ops_references_ip(op, ip)) { + found_op = true; + break; + } + } while_for_each_ftrace_op(op); + mutex_unlock(&ftrace_lock); + + if (found_op) { + if (!op->ops_func) + return -EBUSY; + + ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER); + if (ret) + return ret; + } + } + } + + return 0; +} + +/* + * Similar to prepare_direct_functions_for_ipmodify, clean up after ops + * with IPMODIFY is unregistered. The cleanup is optional for most DIRECT + * ops. + */ +static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) +{ + struct ftrace_func_entry *entry; + struct ftrace_hash *hash; + struct ftrace_ops *op; + int size, i; + + if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) + return; + + mutex_lock(&direct_mutex); + + hash = ops->func_hash->filter_hash; + size = 1 << hash->size_bits; + for (i = 0; i < size; i++) { + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { + unsigned long ip = entry->ip; + bool found_op = false; + + mutex_lock(&ftrace_lock); + do_for_each_ftrace_op(op, ftrace_ops_list) { + if (!(op->flags & FTRACE_OPS_FL_DIRECT)) + continue; + if (ops_references_ip(op, ip)) { + found_op = true; + break; + } + } while_for_each_ftrace_op(op); + mutex_unlock(&ftrace_lock); + + /* The cleanup is optional, ignore any errors */ + if (found_op && op->ops_func) + op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); + } + } + mutex_unlock(&direct_mutex); +} + +#define lock_direct_mutex() mutex_lock(&direct_mutex) +#define unlock_direct_mutex() mutex_unlock(&direct_mutex) + +#else /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + +static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) +{ + return 0; +} + +static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) +{ +} + +#define lock_direct_mutex() do { } while (0) +#define unlock_direct_mutex() do { } while (0) + +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + +/* + * Similar to register_ftrace_function, except we don't lock direct_mutex. + */ +static int register_ftrace_function_nolock(struct ftrace_ops *ops) +{ + int ret; + + ftrace_ops_init(ops); + + mutex_lock(&ftrace_lock); + + ret = ftrace_startup(ops, 0); + + mutex_unlock(&ftrace_lock); + + return ret; +} + /** * register_ftrace_function - register a function for profiling * @ops: ops structure that holds the function for profiling. @@ -8018,14 +8206,15 @@ int register_ftrace_function(struct ftrace_ops *ops) { int ret; - ftrace_ops_init(ops); - - mutex_lock(&ftrace_lock); - - ret = ftrace_startup(ops, 0); + lock_direct_mutex(); + ret = prepare_direct_functions_for_ipmodify(ops); + if (ret < 0) + goto out_unlock; - mutex_unlock(&ftrace_lock); + ret = register_ftrace_function_nolock(ops); +out_unlock: + unlock_direct_mutex(); return ret; } EXPORT_SYMBOL_GPL(register_ftrace_function); @@ -8044,6 +8233,7 @@ int unregister_ftrace_function(struct ftrace_ops *ops) ret = ftrace_shutdown(ops, 0); mutex_unlock(&ftrace_lock); + cleanup_direct_functions_after_ipmodify(ops); return ret; } EXPORT_SYMBOL_GPL(unregister_ftrace_function); -- cgit v1.2.3 From 316cba62dfb7878b7353177e6a7da9cc0c979cde Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 19 Jul 2022 17:21:25 -0700 Subject: bpf, x64: Allow to use caller address from stack Currently we call the original function by using the absolute address given at the JIT generation. That's not usable when having trampoline attached to multiple functions, or the target address changes dynamically (in case of live patch). In such cases we need to take the return address from the stack. Adding support to retrieve the original function address from the stack by adding new BPF_TRAMP_F_ORIG_STACK flag for arch_prepare_bpf_trampoline function. Basically we take the return address of the 'fentry' call: function + 0: call fentry # stores 'function + 5' address on stack function + 5: ... The 'function + 5' address will be used as the address for the original function to call. Signed-off-by: Jiri Olsa Signed-off-by: Song Liu Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220720002126.803253-4-song@kernel.org --- arch/x86/net/bpf_jit_comp.c | 13 +++++++++---- include/linux/bpf.h | 5 +++++ 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 54c7f46c453f..e1b0c5ed0b7c 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2119,10 +2119,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (flags & BPF_TRAMP_F_CALL_ORIG) { restore_regs(m, &prog, nr_args, regs_off); - /* call original function */ - if (emit_call(&prog, orig_call, prog)) { - ret = -EINVAL; - goto cleanup; + if (flags & BPF_TRAMP_F_ORIG_STACK) { + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); + EMIT2(0xff, 0xd0); /* call *rax */ + } else { + /* call original function */ + if (emit_call(&prog, orig_call, prog)) { + ret = -EINVAL; + goto cleanup; + } } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a97751d845c9..1d22df8bf306 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -751,6 +751,11 @@ struct btf_func_model { /* Return the return value of fentry prog. Only used by bpf_struct_ops. */ #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4) +/* Get original function from stack instead of from provided direct address. + * Makes sense for trampolines with fexit or fmod_ret programs. + */ +#define BPF_TRAMP_F_ORIG_STACK BIT(5) + /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. */ -- cgit v1.2.3 From 00963a2e75a872e5fce4d0115ac2786ec86b57a6 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Tue, 19 Jul 2022 17:21:26 -0700 Subject: bpf: Support bpf_trampoline on functions with IPMODIFY (e.g. livepatch) When tracing a function with IPMODIFY ftrace_ops (livepatch), the bpf trampoline must follow the instruction pointer saved on stack. This needs extra handling for bpf trampolines with BPF_TRAMP_F_CALL_ORIG flag. Implement bpf_tramp_ftrace_ops_func and use it for the ftrace_ops used by BPF trampoline. This enables tracing functions with livepatch. This also requires moving bpf trampoline to *_ftrace_direct_mult APIs. Signed-off-by: Song Liu Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/all/20220602193706.2607681-2-song@kernel.org/ Link: https://lore.kernel.org/bpf/20220720002126.803253-5-song@kernel.org --- include/linux/bpf.h | 8 +++ kernel/bpf/trampoline.c | 158 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 149 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1d22df8bf306..20c26aed7896 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -47,6 +47,7 @@ struct kobject; struct mem_cgroup; struct module; struct bpf_func_state; +struct ftrace_ops; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; @@ -756,6 +757,11 @@ struct btf_func_model { */ #define BPF_TRAMP_F_ORIG_STACK BIT(5) +/* This trampoline is on a function with another ftrace_ops with IPMODIFY, + * e.g., a live patch. This flag is set and cleared by ftrace call backs, + */ +#define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6) + /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. */ @@ -838,9 +844,11 @@ struct bpf_tramp_image { struct bpf_trampoline { /* hlist for trampoline_table */ struct hlist_node hlist; + struct ftrace_ops *fops; /* serializes access to fields of this trampoline */ struct mutex mutex; refcount_t refcnt; + u32 flags; u64 key; struct { struct btf_func_model model; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 6691dbf9e467..42e387a12694 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -13,6 +13,7 @@ #include #include #include +#include /* dummy _ops. The verifier will operate on target program's ops. */ const struct bpf_verifier_ops bpf_extension_verifier_ops = { @@ -29,6 +30,81 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; /* serializes access to trampoline_table */ static DEFINE_MUTEX(trampoline_mutex); +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex); + +static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd) +{ + struct bpf_trampoline *tr = ops->private; + int ret = 0; + + if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) { + /* This is called inside register_ftrace_direct_multi(), so + * tr->mutex is already locked. + */ + lockdep_assert_held_once(&tr->mutex); + + /* Instead of updating the trampoline here, we propagate + * -EAGAIN to register_ftrace_direct_multi(). Then we can + * retry register_ftrace_direct_multi() after updating the + * trampoline. + */ + if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && + !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) { + if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY)) + return -EBUSY; + + tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; + return -EAGAIN; + } + + return 0; + } + + /* The normal locking order is + * tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c) + * + * The following two commands are called from + * + * prepare_direct_functions_for_ipmodify + * cleanup_direct_functions_after_ipmodify + * + * In both cases, direct_mutex is already locked. Use + * mutex_trylock(&tr->mutex) to avoid deadlock in race condition + * (something else is making changes to this same trampoline). + */ + if (!mutex_trylock(&tr->mutex)) { + /* sleep 1 ms to make sure whatever holding tr->mutex makes + * some progress. + */ + msleep(1); + return -EAGAIN; + } + + switch (cmd) { + case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER: + tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; + + if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && + !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) + ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); + break; + case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER: + tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY; + + if (tr->flags & BPF_TRAMP_F_ORIG_STACK) + ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); + break; + default: + ret = -EINVAL; + break; + }; + + mutex_unlock(&tr->mutex); + return ret; +} +#endif + bool bpf_prog_has_trampoline(const struct bpf_prog *prog) { enum bpf_attach_type eatype = prog->expected_attach_type; @@ -89,6 +165,16 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) tr = kzalloc(sizeof(*tr), GFP_KERNEL); if (!tr) goto out; +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL); + if (!tr->fops) { + kfree(tr); + tr = NULL; + goto out; + } + tr->fops->private = tr; + tr->fops->ops_func = bpf_tramp_ftrace_ops_func; +#endif tr->key = key; INIT_HLIST_NODE(&tr->hlist); @@ -128,7 +214,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) int ret; if (tr->func.ftrace_managed) - ret = unregister_ftrace_direct((long)ip, (long)old_addr); + ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr); else ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); @@ -137,15 +223,20 @@ static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) return ret; } -static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr) +static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr, + bool lock_direct_mutex) { void *ip = tr->func.addr; int ret; - if (tr->func.ftrace_managed) - ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr); - else + if (tr->func.ftrace_managed) { + if (lock_direct_mutex) + ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr); + else + ret = modify_ftrace_direct_multi_nolock(tr->fops, (long)new_addr); + } else { ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); + } return ret; } @@ -163,10 +254,12 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) if (bpf_trampoline_module_get(tr)) return -ENOENT; - if (tr->func.ftrace_managed) - ret = register_ftrace_direct((long)ip, (long)new_addr); - else + if (tr->func.ftrace_managed) { + ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 0); + ret = register_ftrace_direct_multi(tr->fops, (long)new_addr); + } else { ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); + } if (ret) bpf_trampoline_module_put(tr); @@ -332,11 +425,11 @@ out: return ERR_PTR(err); } -static int bpf_trampoline_update(struct bpf_trampoline *tr) +static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex) { struct bpf_tramp_image *im; struct bpf_tramp_links *tlinks; - u32 flags = BPF_TRAMP_F_RESTORE_REGS; + u32 orig_flags = tr->flags; bool ip_arg = false; int err, total; @@ -358,18 +451,31 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) goto out; } + /* clear all bits except SHARE_IPMODIFY */ + tr->flags &= BPF_TRAMP_F_SHARE_IPMODIFY; + if (tlinks[BPF_TRAMP_FEXIT].nr_links || - tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) + tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) { /* NOTE: BPF_TRAMP_F_RESTORE_REGS and BPF_TRAMP_F_SKIP_FRAME * should not be set together. */ - flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; + tr->flags |= BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; + } else { + tr->flags |= BPF_TRAMP_F_RESTORE_REGS; + } if (ip_arg) - flags |= BPF_TRAMP_F_IP_ARG; + tr->flags |= BPF_TRAMP_F_IP_ARG; + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +again: + if ((tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) && + (tr->flags & BPF_TRAMP_F_CALL_ORIG)) + tr->flags |= BPF_TRAMP_F_ORIG_STACK; +#endif err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE, - &tr->func.model, flags, tlinks, + &tr->func.model, tr->flags, tlinks, tr->func.addr); if (err < 0) goto out; @@ -378,17 +484,34 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) WARN_ON(!tr->cur_image && tr->selector); if (tr->cur_image) /* progs already running at this address */ - err = modify_fentry(tr, tr->cur_image->image, im->image); + err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex); else /* first time registering */ err = register_fentry(tr, im->image); + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + if (err == -EAGAIN) { + /* -EAGAIN from bpf_tramp_ftrace_ops_func. Now + * BPF_TRAMP_F_SHARE_IPMODIFY is set, we can generate the + * trampoline again, and retry register. + */ + /* reset fops->func and fops->trampoline for re-register */ + tr->fops->func = NULL; + tr->fops->trampoline = 0; + goto again; + } +#endif if (err) goto out; + if (tr->cur_image) bpf_tramp_image_put(tr->cur_image); tr->cur_image = im; tr->selector++; out: + /* If any error happens, restore previous flags */ + if (err) + tr->flags = orig_flags; kfree(tlinks); return err; } @@ -454,7 +577,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]); tr->progs_cnt[kind]++; - err = bpf_trampoline_update(tr); + err = bpf_trampoline_update(tr, true /* lock_direct_mutex */); if (err) { hlist_del_init(&link->tramp_hlist); tr->progs_cnt[kind]--; @@ -487,7 +610,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ } hlist_del_init(&link->tramp_hlist); tr->progs_cnt[kind]--; - return bpf_trampoline_update(tr); + return bpf_trampoline_update(tr, true /* lock_direct_mutex */); } /* bpf_trampoline_unlink_prog() should never fail. */ @@ -715,6 +838,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) * multiple rcu callbacks. */ hlist_del(&tr->hlist); + kfree(tr->fops); kfree(tr); out: mutex_unlock(&trampoline_mutex); -- cgit v1.2.3 From 189c6c33ff421def040b904fb14ef76c5bf5af4c Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Tue, 28 Jun 2022 16:30:59 +0200 Subject: PCI: Extend isolated function probing to s390 Like the jailhouse hypervisor, s390's PCI architecture allows passing isolated PCI functions to a guest OS instance. As of now this is was not utilized even with multi-function support as the s390 PCI code makes sure that only virtual PCI busses including a function with devfn 0 are presented to the PCI subsystem. A subsequent change will remove this restriction. Allow probing such functions by replacing the existing check for jailhouse_paravirt() with a new hypervisor_isolated_pci_functions() helper. Link: https://lore.kernel.org/r/20220628143100.3228092-5-schnelle@linux.ibm.com Signed-off-by: Niklas Schnelle Signed-off-by: Bjorn Helgaas Reviewed-by: Pierre Morel Cc: Jan Kiszka --- drivers/pci/probe.c | 2 +- include/linux/hypervisor.h | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 1f91ed67b5c5..4948531481d1 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2668,7 +2668,7 @@ int pci_scan_slot(struct pci_bus *bus, int devfn) * a hypervisor that passes through individual PCI * functions. */ - if (!jailhouse_paravirt()) + if (!hypervisor_isolated_pci_functions()) break; } fn = next_fn(bus, dev, fn); diff --git a/include/linux/hypervisor.h b/include/linux/hypervisor.h index fc08b433c856..9efbc54e35e5 100644 --- a/include/linux/hypervisor.h +++ b/include/linux/hypervisor.h @@ -32,4 +32,12 @@ static inline bool jailhouse_paravirt(void) #endif /* !CONFIG_X86 */ +static inline bool hypervisor_isolated_pci_functions(void) +{ + if (IS_ENABLED(CONFIG_S390)) + return true; + + return jailhouse_paravirt(); +} + #endif /* __LINUX_HYPEVISOR_H */ -- cgit v1.2.3 From abb4970ac33514c84b143516583eaf8cc47abd67 Mon Sep 17 00:00:00 2001 From: Stafford Horne Date: Sat, 23 Jul 2022 06:49:42 +0900 Subject: PCI: Move isa_dma_bridge_buggy out of asm/dma.h The isa_dma_bridge_buggy symbol is only used for x86_32, and only x86_32 platforms or quirks ever set it. Add a new linux/isa-dma.h header that #defines isa_dma_bridge_buggy to 0 except on x86_32, where we keep it as a variable, and remove all the arch- specific definitions. [bhelgaas: commit log] Suggested-by: Arnd Bergmann Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220722214944.831438-3-shorne@gmail.com Signed-off-by: Stafford Horne Signed-off-by: Bjorn Helgaas Reviewed-by: Christoph Hellwig Acked-by: Geert Uytterhoeven --- arch/alpha/include/asm/dma.h | 9 --------- arch/arc/include/asm/dma.h | 5 ----- arch/arm/include/asm/dma.h | 6 ------ arch/arm64/include/asm/pci.h | 2 -- arch/csky/include/asm/pci.h | 2 -- arch/ia64/include/asm/dma.h | 2 -- arch/m68k/include/asm/dma.h | 6 ------ arch/microblaze/include/asm/dma.h | 6 ------ arch/mips/include/asm/dma.h | 8 -------- arch/parisc/include/asm/dma.h | 6 ------ arch/powerpc/include/asm/dma.h | 6 ------ arch/riscv/include/asm/pci.h | 2 -- arch/s390/include/asm/dma.h | 6 ------ arch/sh/include/asm/dma.h | 6 ------ arch/sparc/include/asm/dma.h | 8 -------- arch/um/include/asm/pci.h | 2 -- arch/x86/include/asm/dma.h | 8 -------- arch/xtensa/include/asm/dma.h | 7 ------- drivers/comedi/drivers/comedi_isadma.c | 2 +- drivers/pci/pci.c | 2 ++ drivers/pci/quirks.c | 4 +++- include/linux/isa-dma.h | 14 ++++++++++++++ sound/core/isadma.c | 2 +- 23 files changed, 21 insertions(+), 100 deletions(-) create mode 100644 include/linux/isa-dma.h (limited to 'include/linux') diff --git a/arch/alpha/include/asm/dma.h b/arch/alpha/include/asm/dma.h index 28610ea7786d..a04d76b96089 100644 --- a/arch/alpha/include/asm/dma.h +++ b/arch/alpha/include/asm/dma.h @@ -365,13 +365,4 @@ extern void free_dma(unsigned int dmanr); /* release it again */ #define KERNEL_HAVE_CHECK_DMA extern int check_dma(unsigned int dmanr); -/* From PCI */ - -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - - #endif /* _ASM_DMA_H */ diff --git a/arch/arc/include/asm/dma.h b/arch/arc/include/asm/dma.h index 5b744f4b10a7..02431027ed2f 100644 --- a/arch/arc/include/asm/dma.h +++ b/arch/arc/include/asm/dma.h @@ -7,10 +7,5 @@ #define ASM_ARC_DMA_H #define MAX_DMA_ADDRESS 0xC0000000 -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy 0 -#endif #endif diff --git a/arch/arm/include/asm/dma.h b/arch/arm/include/asm/dma.h index a81dda65c576..907d139be431 100644 --- a/arch/arm/include/asm/dma.h +++ b/arch/arm/include/asm/dma.h @@ -143,10 +143,4 @@ extern int get_dma_residue(unsigned int chan); #endif /* CONFIG_ISA_DMA_API */ -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* __ASM_ARM_DMA_H */ diff --git a/arch/arm64/include/asm/pci.h b/arch/arm64/include/asm/pci.h index 0aebc3488c32..682c922b5658 100644 --- a/arch/arm64/include/asm/pci.h +++ b/arch/arm64/include/asm/pci.h @@ -20,8 +20,6 @@ #define arch_can_pci_mmap_wc() 1 #define ARCH_GENERIC_PCI_MMAP_RESOURCE 1 -extern int isa_dma_bridge_buggy; - #ifdef CONFIG_PCI static inline int pci_proc_domain(struct pci_bus *bus) { diff --git a/arch/csky/include/asm/pci.h b/arch/csky/include/asm/pci.h index 0535f1aaae38..5c02454ec724 100644 --- a/arch/csky/include/asm/pci.h +++ b/arch/csky/include/asm/pci.h @@ -15,8 +15,6 @@ /* C-SKY shim does not initialize PCI bus */ #define pcibios_assign_all_busses() 1 -extern int isa_dma_bridge_buggy; - #ifdef CONFIG_PCI static inline int pci_proc_domain(struct pci_bus *bus) { diff --git a/arch/ia64/include/asm/dma.h b/arch/ia64/include/asm/dma.h index 59625e9c1f9c..eaed2626ffda 100644 --- a/arch/ia64/include/asm/dma.h +++ b/arch/ia64/include/asm/dma.h @@ -12,8 +12,6 @@ extern unsigned long MAX_DMA_ADDRESS; -extern int isa_dma_bridge_buggy; - #define free_dma(x) #endif /* _ASM_IA64_DMA_H */ diff --git a/arch/m68k/include/asm/dma.h b/arch/m68k/include/asm/dma.h index f6c5e0dfb4e5..1c8d9c5bc2fa 100644 --- a/arch/m68k/include/asm/dma.h +++ b/arch/m68k/include/asm/dma.h @@ -6,10 +6,4 @@ bootmem allocator (but this should do it for this) */ #define MAX_DMA_ADDRESS PAGE_OFFSET -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* _M68K_DMA_H */ diff --git a/arch/microblaze/include/asm/dma.h b/arch/microblaze/include/asm/dma.h index f801582be912..7484c9eb66c4 100644 --- a/arch/microblaze/include/asm/dma.h +++ b/arch/microblaze/include/asm/dma.h @@ -9,10 +9,4 @@ /* Virtual address corresponding to last available physical memory address. */ #define MAX_DMA_ADDRESS (CONFIG_KERNEL_START + memory_size - 1) -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* _ASM_MICROBLAZE_DMA_H */ diff --git a/arch/mips/include/asm/dma.h b/arch/mips/include/asm/dma.h index be726b943530..d6186e6bea7e 100644 --- a/arch/mips/include/asm/dma.h +++ b/arch/mips/include/asm/dma.h @@ -307,12 +307,4 @@ static __inline__ int get_dma_residue(unsigned int dmanr) extern int request_dma(unsigned int dmanr, const char * device_id); /* reserve a DMA channel */ extern void free_dma(unsigned int dmanr); /* release it again */ -/* From PCI */ - -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* _ASM_DMA_H */ diff --git a/arch/parisc/include/asm/dma.h b/arch/parisc/include/asm/dma.h index eea80ed34e6d..9e8c101de902 100644 --- a/arch/parisc/include/asm/dma.h +++ b/arch/parisc/include/asm/dma.h @@ -176,10 +176,4 @@ static __inline__ void set_dma_count(unsigned int dmanr, unsigned int count) #define free_dma(dmanr) -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* _ASM_DMA_H */ diff --git a/arch/powerpc/include/asm/dma.h b/arch/powerpc/include/asm/dma.h index 6161a9596196..d97c66d9ae34 100644 --- a/arch/powerpc/include/asm/dma.h +++ b/arch/powerpc/include/asm/dma.h @@ -340,11 +340,5 @@ extern int request_dma(unsigned int dmanr, const char *device_id); /* release it again */ extern void free_dma(unsigned int dmanr); -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_DMA_H */ diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h index a7b8f0d0df7f..f904df586c03 100644 --- a/arch/riscv/include/asm/pci.h +++ b/arch/riscv/include/asm/pci.h @@ -20,8 +20,6 @@ #define ARCH_GENERIC_PCI_MMAP_RESOURCE 1 -extern int isa_dma_bridge_buggy; - #ifdef CONFIG_PCI static inline int pci_proc_domain(struct pci_bus *bus) { diff --git a/arch/s390/include/asm/dma.h b/arch/s390/include/asm/dma.h index 6f26f35d4a71..dec1c4ce628c 100644 --- a/arch/s390/include/asm/dma.h +++ b/arch/s390/include/asm/dma.h @@ -11,10 +11,4 @@ */ #define MAX_DMA_ADDRESS 0x80000000 -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* _ASM_S390_DMA_H */ diff --git a/arch/sh/include/asm/dma.h b/arch/sh/include/asm/dma.h index 17d23ae98c77..c8bee3f985a2 100644 --- a/arch/sh/include/asm/dma.h +++ b/arch/sh/include/asm/dma.h @@ -137,10 +137,4 @@ extern int register_chan_caps(const char *dmac, struct dma_chan_caps *capslist); extern int dma_create_sysfs_files(struct dma_channel *, struct dma_info *); extern void dma_remove_sysfs_files(struct dma_channel *, struct dma_info *); -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* __ASM_SH_DMA_H */ diff --git a/arch/sparc/include/asm/dma.h b/arch/sparc/include/asm/dma.h index 462e7c794a09..08043f35b110 100644 --- a/arch/sparc/include/asm/dma.h +++ b/arch/sparc/include/asm/dma.h @@ -82,14 +82,6 @@ #define DMA_BURST64 0x40 #define DMA_BURSTBITS 0x7f -/* From PCI */ - -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #ifdef CONFIG_SPARC32 struct device; diff --git a/arch/um/include/asm/pci.h b/arch/um/include/asm/pci.h index 26b96c02ef61..1211855aff34 100644 --- a/arch/um/include/asm/pci.h +++ b/arch/um/include/asm/pci.h @@ -9,8 +9,6 @@ #define pcibios_assign_all_busses() 1 -extern int isa_dma_bridge_buggy; - #ifdef CONFIG_PCI_DOMAINS static inline int pci_proc_domain(struct pci_bus *bus) { diff --git a/arch/x86/include/asm/dma.h b/arch/x86/include/asm/dma.h index 8e95aa4b0d17..8ae6e0e11b8b 100644 --- a/arch/x86/include/asm/dma.h +++ b/arch/x86/include/asm/dma.h @@ -307,12 +307,4 @@ extern int request_dma(unsigned int dmanr, const char *device_id); extern void free_dma(unsigned int dmanr); #endif -/* From PCI */ - -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* _ASM_X86_DMA_H */ diff --git a/arch/xtensa/include/asm/dma.h b/arch/xtensa/include/asm/dma.h index bb099a373b5a..172644539032 100644 --- a/arch/xtensa/include/asm/dma.h +++ b/arch/xtensa/include/asm/dma.h @@ -52,11 +52,4 @@ extern int request_dma(unsigned int dmanr, const char * device_id); extern void free_dma(unsigned int dmanr); -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - - #endif diff --git a/drivers/comedi/drivers/comedi_isadma.c b/drivers/comedi/drivers/comedi_isadma.c index 700982464c53..020b3d1e1ac0 100644 --- a/drivers/comedi/drivers/comedi_isadma.c +++ b/drivers/comedi/drivers/comedi_isadma.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index cfaf40a540a8..60c55d2cb2cc 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -41,8 +41,10 @@ const char *pci_power_names[] = { }; EXPORT_SYMBOL_GPL(pci_power_names); +#ifdef CONFIG_X86_32 int isa_dma_bridge_buggy; EXPORT_SYMBOL(isa_dma_bridge_buggy); +#endif int pci_pci_problems; EXPORT_SYMBOL(pci_pci_problems); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 41aeaa235132..6fc64509eee7 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -17,6 +17,7 @@ #include #include #include +#include /* isa_dma_bridge_buggy */ #include #include #include @@ -30,7 +31,6 @@ #include #include #include -#include /* isa_dma_bridge_buggy */ #include "pci.h" static ktime_t fixup_debug_start(struct pci_dev *dev, @@ -239,6 +239,7 @@ static void quirk_passive_release(struct pci_dev *dev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82441, quirk_passive_release); DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82441, quirk_passive_release); +#ifdef CONFIG_X86_32 /* * The VIA VP2/VP3/MVP3 seem to have some 'features'. There may be a * workaround but VIA don't answer queries. If you happen to have good @@ -265,6 +266,7 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, quirk_isa_dma DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NEC, PCI_DEVICE_ID_NEC_CBUS_1, quirk_isa_dma_hangs); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NEC, PCI_DEVICE_ID_NEC_CBUS_2, quirk_isa_dma_hangs); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NEC, PCI_DEVICE_ID_NEC_CBUS_3, quirk_isa_dma_hangs); +#endif /* * Intel NM10 "TigerPoint" LPC PM1a_STS.BM_STS must be clear diff --git a/include/linux/isa-dma.h b/include/linux/isa-dma.h new file mode 100644 index 000000000000..61504a8c1b9e --- /dev/null +++ b/include/linux/isa-dma.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __LINUX_ISA_DMA_H +#define __LINUX_ISA_DMA_H + +#include + +#if defined(CONFIG_PCI) && defined(CONFIG_X86_32) +extern int isa_dma_bridge_buggy; +#else +#define isa_dma_bridge_buggy (0) +#endif + +#endif /* __LINUX_ISA_DMA_H */ diff --git a/sound/core/isadma.c b/sound/core/isadma.c index 1f45ede023b4..18a86212e3a8 100644 --- a/sound/core/isadma.c +++ b/sound/core/isadma.c @@ -12,8 +12,8 @@ #undef HAVE_REALLY_SLOW_DMA_CONTROLLER #include +#include #include -#include /** * snd_dma_program - program an ISA DMA transfer -- cgit v1.2.3 From e8f90717ed3b58e81c480b3aa38e641c0da5a456 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Fri, 22 Jul 2022 19:02:47 -0700 Subject: vfio: Make vfio_unpin_pages() return void There's only one caller that checks its return value with a WARN_ON_ONCE, while all other callers don't check the return value at all. Above that, an undo function should not fail. So, simplify the API to return void by embedding similar WARN_ONs. Also for users to pinpoint which condition fails, separate WARN_ON lines, yet remove the "driver->ops->unpin_pages" check, since it's unreasonable for callers to unpin on something totally random that wasn't even pinned. And remove NULL pointer checks for they would trigger oops vs. warnings. Note that npage is already validated in the vfio core, thus drop the same check in the type1 code. Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Reviewed-by: Jason Gunthorpe Reviewed-by: Kirti Wankhede Tested-by: Terrence Xu Signed-off-by: Nicolin Chen Link: https://lore.kernel.org/r/20220723020256.30081-2-nicolinc@nvidia.com Signed-off-by: Alex Williamson --- Documentation/driver-api/vfio-mediated-device.rst | 2 +- drivers/gpu/drm/i915/gvt/kvmgt.c | 5 +---- drivers/vfio/vfio.c | 21 +++++++-------------- drivers/vfio/vfio.h | 2 +- drivers/vfio/vfio_iommu_type1.c | 15 ++++++--------- include/linux/vfio.h | 4 ++-- 6 files changed, 18 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst index 1c57815619fd..b0fdf76b339a 100644 --- a/Documentation/driver-api/vfio-mediated-device.rst +++ b/Documentation/driver-api/vfio-mediated-device.rst @@ -265,7 +265,7 @@ driver:: int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, int npage, int prot, unsigned long *phys_pfn); - int vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, + void vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, int npage); These functions call back into the back-end IOMMU module by using the pin_pages diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index ecd5bb37b63a..4d32a2748958 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -231,18 +231,15 @@ static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, unsigned long size) { - struct drm_i915_private *i915 = vgpu->gvt->gt->i915; int total_pages; int npage; - int ret; total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE; for (npage = 0; npage < total_pages; npage++) { unsigned long cur_gfn = gfn + npage; - ret = vfio_unpin_pages(&vgpu->vfio_device, &cur_gfn, 1); - drm_WARN_ON(&i915->drm, ret != 1); + vfio_unpin_pages(&vgpu->vfio_device, &cur_gfn, 1); } } diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index b3ce8073cfb1..92b10aafae28 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1983,31 +1983,24 @@ EXPORT_SYMBOL(vfio_pin_pages); * PFNs should not be greater than VFIO_PIN_PAGES_MAX_ENTRIES. * @npage [in] : count of elements in user_pfn array. This count should not * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. - * Return error or number of pages unpinned. */ -int vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, - int npage) +void vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, + int npage) { struct vfio_container *container; struct vfio_iommu_driver *driver; - int ret; - if (!user_pfn || !npage || !vfio_assert_device_open(device)) - return -EINVAL; + if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES)) + return; - if (npage > VFIO_PIN_PAGES_MAX_ENTRIES) - return -E2BIG; + if (WARN_ON(!vfio_assert_device_open(device))) + return; /* group->container cannot change while a vfio device is open */ container = device->group->container; driver = container->iommu_driver; - if (likely(driver && driver->ops->unpin_pages)) - ret = driver->ops->unpin_pages(container->iommu_data, user_pfn, - npage); - else - ret = -ENOTTY; - return ret; + driver->ops->unpin_pages(container->iommu_data, user_pfn, npage); } EXPORT_SYMBOL(vfio_unpin_pages); diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index 4a7db1f3c33e..6a8424b407c7 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -53,7 +53,7 @@ struct vfio_iommu_driver_ops { unsigned long *user_pfn, int npage, int prot, unsigned long *phys_pfn); - int (*unpin_pages)(void *iommu_data, + void (*unpin_pages)(void *iommu_data, unsigned long *user_pfn, int npage); void (*register_device)(void *iommu_data, struct vfio_device *vdev); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 026a1d2553a2..e49fbe9968ef 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -949,20 +949,16 @@ pin_done: return ret; } -static int vfio_iommu_type1_unpin_pages(void *iommu_data, - unsigned long *user_pfn, - int npage) +static void vfio_iommu_type1_unpin_pages(void *iommu_data, + unsigned long *user_pfn, int npage) { struct vfio_iommu *iommu = iommu_data; bool do_accounting; int i; - if (!iommu || !user_pfn || npage <= 0) - return -EINVAL; - /* Supported for v2 version only */ - if (!iommu->v2) - return -EACCES; + if (WARN_ON(!iommu->v2)) + return; mutex_lock(&iommu->lock); @@ -980,7 +976,8 @@ static int vfio_iommu_type1_unpin_pages(void *iommu_data, } mutex_unlock(&iommu->lock); - return i > 0 ? i : -EINVAL; + + WARN_ON(i != npage); } static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 19cefbaa3d06..9f7d74c24925 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -163,8 +163,8 @@ bool vfio_file_has_dev(struct file *file, struct vfio_device *device); int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, int npage, int prot, unsigned long *phys_pfn); -int vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, - int npage); +void vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, + int npage); int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, void *data, size_t len, bool write); -- cgit v1.2.3 From 6a010a49b63ac8465851a79185d8deff966f8e1a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 23 Jul 2022 04:28:28 -1000 Subject: cgroup: Make !percpu threadgroup_rwsem operations optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3942a9bd7b58 ("locking, rcu, cgroup: Avoid synchronize_sched() in __cgroup_procs_write()") disabled percpu operations on threadgroup_rwsem because the impiled synchronize_rcu() on write locking was pushing up the latencies too much for android which constantly moves processes between cgroups. This makes the hotter paths - fork and exit - slower as they're always forced into the slow path. There is no reason to force this on everyone especially given that more common static usage pattern can now completely avoid write-locking the rwsem. Write-locking is elided when turning on and off controllers on empty sub-trees and CLONE_INTO_CGROUP enables seeding a cgroup without grabbing the rwsem. Restore the default percpu operations and introduce the mount option "favordynmods" and config option CGROUP_FAVOR_DYNMODS for users who need lower latencies for the dynamic operations. Signed-off-by: Tejun Heo Cc: Christian Brauner Cc: Michal Koutn� Cc: Peter Zijlstra Cc: John Stultz Cc: Dmitry Shmidt Cc: Oleg Nesterov --- Documentation/admin-guide/cgroup-v2.rst | 8 ++++++ include/linux/cgroup-defs.h | 19 ++++++++++++--- init/Kconfig | 10 ++++++++ kernel/cgroup/cgroup-internal.h | 1 + kernel/cgroup/cgroup-v1.c | 17 ++++++++++++- kernel/cgroup/cgroup.c | 43 +++++++++++++++++++++++++++------ 6 files changed, 87 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index f0f03d5470b5..4bd0bee22a0d 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -184,6 +184,14 @@ cgroup v2 currently supports the following mount options. ignored on non-init namespace mounts. Please refer to the Delegation section for details. + [no]favordynmods + Reduce the latencies of dynamic cgroup modifications such as + task migrations and controller on/offs at the cost of making + hot path operations such as forks and exits more expensive. + The static usage pattern of creating a cgroup, enabling + controllers, and then seeding it with CLONE_INTO_CGROUP is + not affected by this option. + memory_[no]localevents Only populate memory.events with data for the current cgroup, and not any subtrees. This is legacy behaviour, the default diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 672de25e3ec8..63bf43c7ca3b 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -88,20 +88,33 @@ enum { */ CGRP_ROOT_NS_DELEGATE = (1 << 3), + /* + * Reduce latencies on dynamic cgroup modifications such as task + * migrations and controller on/offs by disabling percpu operation on + * cgroup_threadgroup_rwsem. This makes hot path operations such as + * forks and exits into the slow path and more expensive. + * + * The static usage pattern of creating a cgroup, enabling controllers, + * and then seeding it with CLONE_INTO_CGROUP doesn't require write + * locking cgroup_threadgroup_rwsem and thus doesn't benefit from + * favordynmod. + */ + CGRP_ROOT_FAVOR_DYNMODS = (1 << 4), + /* * Enable cpuset controller in v1 cgroup to use v2 behavior. */ - CGRP_ROOT_CPUSET_V2_MODE = (1 << 4), + CGRP_ROOT_CPUSET_V2_MODE = (1 << 16), /* * Enable legacy local memory.events. */ - CGRP_ROOT_MEMORY_LOCAL_EVENTS = (1 << 5), + CGRP_ROOT_MEMORY_LOCAL_EVENTS = (1 << 17), /* * Enable recursive subtree protection */ - CGRP_ROOT_MEMORY_RECURSIVE_PROT = (1 << 6), + CGRP_ROOT_MEMORY_RECURSIVE_PROT = (1 << 18), }; /* cftype->flags */ diff --git a/init/Kconfig b/init/Kconfig index c984afc489de..c93b10b3de3f 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -936,6 +936,16 @@ if CGROUPS config PAGE_COUNTER bool +config CGROUP_FAVOR_DYNMODS + bool "Favor dynamic modification latency reduction by default" + help + This option enables the "favordynmods" mount option by default + which reduces the latencies of dynamic cgroup modifications such + as task migrations and controller on/offs at the cost of making + hot path operations such as forks and exits more expensive. + + Say N if unsure. + config MEMCG bool "Memory controller" select PAGE_COUNTER diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 5da09c74228d..36b740cb3d59 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -233,6 +233,7 @@ void cgroup_kn_unlock(struct kernfs_node *kn); int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, struct cgroup_namespace *ns); +void cgroup_favor_dynmods(struct cgroup_root *root, bool favor); void cgroup_free_root(struct cgroup_root *root); void init_cgroup_root(struct cgroup_fs_context *ctx); int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask); diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index afc6c0e9c966..2ade21b54dc4 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -875,6 +875,8 @@ static int cgroup1_show_options(struct seq_file *seq, struct kernfs_root *kf_roo seq_puts(seq, ",xattr"); if (root->flags & CGRP_ROOT_CPUSET_V2_MODE) seq_puts(seq, ",cpuset_v2_mode"); + if (root->flags & CGRP_ROOT_FAVOR_DYNMODS) + seq_puts(seq, ",favordynmods"); spin_lock(&release_agent_path_lock); if (strlen(root->release_agent_path)) @@ -898,6 +900,8 @@ enum cgroup1_param { Opt_noprefix, Opt_release_agent, Opt_xattr, + Opt_favordynmods, + Opt_nofavordynmods, }; const struct fs_parameter_spec cgroup1_fs_parameters[] = { @@ -909,6 +913,8 @@ const struct fs_parameter_spec cgroup1_fs_parameters[] = { fsparam_flag ("noprefix", Opt_noprefix), fsparam_string("release_agent", Opt_release_agent), fsparam_flag ("xattr", Opt_xattr), + fsparam_flag ("favordynmods", Opt_favordynmods), + fsparam_flag ("nofavordynmods", Opt_nofavordynmods), {} }; @@ -960,6 +966,12 @@ int cgroup1_parse_param(struct fs_context *fc, struct fs_parameter *param) case Opt_xattr: ctx->flags |= CGRP_ROOT_XATTR; break; + case Opt_favordynmods: + ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS; + break; + case Opt_nofavordynmods: + ctx->flags &= ~CGRP_ROOT_FAVOR_DYNMODS; + break; case Opt_release_agent: /* Specifying two release agents is forbidden */ if (ctx->release_agent) @@ -1211,8 +1223,11 @@ static int cgroup1_root_to_use(struct fs_context *fc) init_cgroup_root(ctx); ret = cgroup_setup_root(root, ctx->subsys_mask); - if (ret) + if (!ret) + cgroup_favor_dynmods(root, ctx->flags & CGRP_ROOT_FAVOR_DYNMODS); + else cgroup_free_root(root); + return ret; } diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 9ce24d5cf2d5..7d023d42a6a5 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -1305,6 +1305,20 @@ struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root) return root_cgrp->root; } +void cgroup_favor_dynmods(struct cgroup_root *root, bool favor) +{ + bool favoring = root->flags & CGRP_ROOT_FAVOR_DYNMODS; + + /* see the comment above CGRP_ROOT_FAVOR_DYNMODS definition */ + if (favor && !favoring) { + rcu_sync_enter(&cgroup_threadgroup_rwsem.rss); + root->flags |= CGRP_ROOT_FAVOR_DYNMODS; + } else if (!favor && favoring) { + rcu_sync_exit(&cgroup_threadgroup_rwsem.rss); + root->flags &= ~CGRP_ROOT_FAVOR_DYNMODS; + } +} + static int cgroup_init_root_id(struct cgroup_root *root) { int id; @@ -1365,6 +1379,7 @@ static void cgroup_destroy_root(struct cgroup_root *root) cgroup_root_count--; } + cgroup_favor_dynmods(root, false); cgroup_exit_root_id(root); mutex_unlock(&cgroup_mutex); @@ -1858,6 +1873,7 @@ int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node, enum cgroup2_param { Opt_nsdelegate, Opt_nonsdelegate, + Opt_favordynmods, Opt_nofavordynmods, Opt_memory_localevents, Opt_memory_nolocalevents, Opt_memory_recursiveprot, Opt_memory_norecursiveprot, nr__cgroup2_params @@ -1866,6 +1882,8 @@ enum cgroup2_param { static const struct fs_parameter_spec cgroup2_fs_parameters[] = { fsparam_flag("nsdelegate", Opt_nsdelegate), fsparam_flag("nonsdelegate", Opt_nonsdelegate), + fsparam_flag("favordynmods", Opt_favordynmods), + fsparam_flag("nofavordynmods", Opt_nofavordynmods), fsparam_flag("memory_localevents", Opt_memory_localevents), fsparam_flag("memory_nolocalevents", Opt_memory_nolocalevents), fsparam_flag("memory_recursiveprot", Opt_memory_recursiveprot), @@ -1890,6 +1908,12 @@ static int cgroup2_parse_param(struct fs_context *fc, struct fs_parameter *param case Opt_nonsdelegate: ctx->flags &= ~CGRP_ROOT_NS_DELEGATE; return 0; + case Opt_favordynmods: + ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS; + return 0; + case Opt_nofavordynmods: + ctx->flags &= ~CGRP_ROOT_FAVOR_DYNMODS; + return 0; case Opt_memory_localevents: ctx->flags |= CGRP_ROOT_MEMORY_LOCAL_EVENTS; return 0; @@ -1914,6 +1938,9 @@ static void apply_cgroup_root_flags(unsigned int root_flags) else cgrp_dfl_root.flags &= ~CGRP_ROOT_NS_DELEGATE; + cgroup_favor_dynmods(&cgrp_dfl_root, + root_flags & CGRP_ROOT_FAVOR_DYNMODS); + if (root_flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS) cgrp_dfl_root.flags |= CGRP_ROOT_MEMORY_LOCAL_EVENTS; else @@ -1930,6 +1957,8 @@ static int cgroup_show_options(struct seq_file *seq, struct kernfs_root *kf_root { if (cgrp_dfl_root.flags & CGRP_ROOT_NS_DELEGATE) seq_puts(seq, ",nsdelegate"); + if (cgrp_dfl_root.flags & CGRP_ROOT_FAVOR_DYNMODS) + seq_puts(seq, ",favordynmods"); if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS) seq_puts(seq, ",memory_localevents"); if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT) @@ -1980,7 +2009,8 @@ void init_cgroup_root(struct cgroup_fs_context *ctx) cgrp->root = root; init_cgroup_housekeeping(cgrp); - root->flags = ctx->flags; + /* DYNMODS must be modified through cgroup_favor_dynmods() */ + root->flags = ctx->flags & ~CGRP_ROOT_FAVOR_DYNMODS; if (ctx->release_agent) strscpy(root->release_agent_path, ctx->release_agent, PATH_MAX); if (ctx->name) @@ -2202,6 +2232,10 @@ static int cgroup_init_fs_context(struct fs_context *fc) put_user_ns(fc->user_ns); fc->user_ns = get_user_ns(ctx->ns->user_ns); fc->global = true; + +#ifdef CONFIG_CGROUP_FAVOR_DYNMODS + ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS; +#endif return 0; } @@ -5854,12 +5888,6 @@ int __init cgroup_init(void) cgroup_rstat_boot(); - /* - * The latency of the synchronize_rcu() is too high for cgroups, - * avoid it at the cost of forcing all readers into the slow path. - */ - rcu_sync_enter_start(&cgroup_threadgroup_rwsem.rss); - get_user_ns(init_cgroup_ns.user_ns); mutex_lock(&cgroup_mutex); @@ -6771,6 +6799,7 @@ static ssize_t features_show(struct kobject *kobj, struct kobj_attribute *attr, { return snprintf(buf, PAGE_SIZE, "nsdelegate\n" + "favordynmods\n" "memory_localevents\n" "memory_recursiveprot\n"); } -- cgit v1.2.3 From ba8ec7a607e98e8491a1fcf924a2e6c96ac9d413 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 23 Jul 2022 14:47:28 -0400 Subject: SUNRPC: Shrink size of struct rpc_task Move the field 'tk_rpc_status' so that we eliminate a 4 byte hole in the structure. For x86_64, this shrinks the size of the struct by 8 bytes. 'pahole' output before the change: /* size: 232, cachelines: 4, members: 27 */ /* sum members: 222, holes: 1, sum holes: 4 */ /* sum bitfield members: 8 bits (1 bytes) */ /* padding: 5 */ /* last cacheline: 40 bytes */ 'pahole' output after the change: /* size: 224, cachelines: 4, members: 27 */ /* padding: 1 */ /* last cacheline: 32 bytes */ Signed-off-by: Trond Myklebust --- include/linux/sunrpc/sched.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index 1d7a3e51b795..acc62647317c 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -61,8 +61,6 @@ struct rpc_task { struct rpc_wait tk_wait; /* RPC wait */ } u; - int tk_rpc_status; /* Result of last RPC operation */ - /* * RPC call state */ @@ -82,6 +80,8 @@ struct rpc_task { ktime_t tk_start; /* RPC task init timestamp */ pid_t tk_owner; /* Process id for batching tasks */ + + int tk_rpc_status; /* Result of last RPC operation */ unsigned short tk_flags; /* misc flags */ unsigned short tk_timeouts; /* maj timeouts */ -- cgit v1.2.3 From 69d966510d9f5de81588b37d23a9ee8ccc477b23 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 22 Jul 2022 14:12:20 -0400 Subject: nfs: only issue commit in DIO codepath if we have uncommitted data Currently, we try to determine whether to issue a commit based on nfs_write_need_commit which looks at the current verifier. In the case where we got a short write and then tried to follow it up with one that failed, the verifier can't be trusted. What we really want to know is whether the pgio request had any successful writes that came back as UNSTABLE. Add a new flag to the pgio request, and use that to indicate that we've had a successful unstable write. Only issue a commit if that flag is set. Signed-off-by: Jeff Layton Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 2 +- fs/nfs/write.c | 48 ++++++++++++++++++++++++++++++------------------ include/linux/nfs_xdr.h | 1 + 3 files changed, 32 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index a47d13296194..86df66bb14c5 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -690,7 +690,7 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) } nfs_direct_count_bytes(dreq, hdr); - if (hdr->good_bytes != 0 && nfs_write_need_commit(hdr)) { + if (test_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags)) { if (!dreq->flags) dreq->flags = NFS_ODIRECT_DO_COMMIT; flags = dreq->flags; diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 1c706465d090..16d166bc4099 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1576,25 +1576,37 @@ static int nfs_writeback_done(struct rpc_task *task, nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count); trace_nfs_writeback_done(task, hdr); - if (hdr->res.verf->committed < hdr->args.stable && - task->tk_status >= 0) { - /* We tried a write call, but the server did not - * commit data to stable storage even though we - * requested it. - * Note: There is a known bug in Tru64 < 5.0 in which - * the server reports NFS_DATA_SYNC, but performs - * NFS_FILE_SYNC. We therefore implement this checking - * as a dprintk() in order to avoid filling syslog. - */ - static unsigned long complain; + if (task->tk_status >= 0) { + enum nfs3_stable_how committed = hdr->res.verf->committed; + + if (committed == NFS_UNSTABLE) { + /* + * We have some uncommitted data on the server at + * this point, so ensure that we keep track of that + * fact irrespective of what later writes do. + */ + set_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags); + } - /* Note this will print the MDS for a DS write */ - if (time_before(complain, jiffies)) { - dprintk("NFS: faulty NFS server %s:" - " (committed = %d) != (stable = %d)\n", - NFS_SERVER(inode)->nfs_client->cl_hostname, - hdr->res.verf->committed, hdr->args.stable); - complain = jiffies + 300 * HZ; + if (committed < hdr->args.stable) { + /* We tried a write call, but the server did not + * commit data to stable storage even though we + * requested it. + * Note: There is a known bug in Tru64 < 5.0 in which + * the server reports NFS_DATA_SYNC, but performs + * NFS_FILE_SYNC. We therefore implement this checking + * as a dprintk() in order to avoid filling syslog. + */ + static unsigned long complain; + + /* Note this will print the MDS for a DS write */ + if (time_before(complain, jiffies)) { + dprintk("NFS: faulty NFS server %s:" + " (committed = %d) != (stable = %d)\n", + NFS_SERVER(inode)->nfs_client->cl_hostname, + committed, hdr->args.stable); + complain = jiffies + 300 * HZ; + } } } diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 0e3aa0f5f324..e86cf6642d21 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1600,6 +1600,7 @@ enum { NFS_IOHDR_STAT, NFS_IOHDR_RESEND_PNFS, NFS_IOHDR_RESEND_MDS, + NFS_IOHDR_UNSTABLE_WRITES, }; struct nfs_io_completion; -- cgit v1.2.3 From 4f5f3b6028343d687d0533329b130e4b8280ab32 Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Thu, 21 Jul 2022 14:21:31 -0400 Subject: SUNRPC: Introduce xdr_stream_move_subsegment() I do this by creating an xdr subsegment for the range we will be operating over. This lets me shift data to the correct place without potentially overwriting anything already there. Signed-off-by: Anna Schumaker Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xdr.h | 2 ++ net/sunrpc/xdr.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 5860f32e3958..7dcc6c31fe29 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -262,6 +262,8 @@ extern unsigned int xdr_align_data(struct xdr_stream *, unsigned int offset, uns extern unsigned int xdr_expand_hole(struct xdr_stream *, unsigned int offset, unsigned int length); extern bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf, unsigned int len); +extern unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int offset, + unsigned int target, unsigned int length); /** * xdr_set_scratch_buffer - Attach a scratch buffer for decoding data. diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index 5d2b3e6979fb..8ba11a754297 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -775,6 +775,34 @@ static void xdr_buf_pages_shift_left(const struct xdr_buf *buf, xdr_buf_tail_copy_left(buf, 0, len - buf->page_len, shift); } +static void xdr_buf_head_shift_left(const struct xdr_buf *buf, + unsigned int base, unsigned int len, + unsigned int shift) +{ + const struct kvec *head = buf->head; + unsigned int bytes; + + if (!shift || !len) + return; + + if (shift > base) { + bytes = (shift - base); + if (bytes >= len) + return; + base += bytes; + len -= bytes; + } + + if (base < head->iov_len) { + bytes = min_t(unsigned int, len, head->iov_len - base); + memmove(head->iov_base + (base - shift), + head->iov_base + base, bytes); + base += bytes; + len -= bytes; + } + xdr_buf_pages_shift_left(buf, base - head->iov_len, len, shift); +} + /** * xdr_shrink_bufhead * @buf: xdr_buf @@ -1680,6 +1708,37 @@ bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf, } EXPORT_SYMBOL_GPL(xdr_stream_subsegment); +/** + * xdr_stream_move_subsegment - Move part of a stream to another position + * @xdr: the source xdr_stream + * @offset: the source offset of the segment + * @target: the target offset of the segment + * @length: the number of bytes to move + * + * Moves @length bytes from @offset to @target in the xdr_stream, overwriting + * anything in its space. Returns the number of bytes in the segment. + */ +unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int offset, + unsigned int target, unsigned int length) +{ + struct xdr_buf buf; + unsigned int shift; + + if (offset < target) { + shift = target - offset; + if (xdr_buf_subsegment(xdr->buf, &buf, offset, shift + length) < 0) + return 0; + xdr_buf_head_shift_right(&buf, 0, length, shift); + } else if (offset > target) { + shift = offset - target; + if (xdr_buf_subsegment(xdr->buf, &buf, target, shift + length) < 0) + return 0; + xdr_buf_head_shift_left(&buf, shift, length, shift); + } + return length; +} +EXPORT_SYMBOL_GPL(xdr_stream_move_subsegment); + /** * xdr_buf_trim - lop at most "len" bytes off the end of "buf" * @buf: buf to be trimmed -- cgit v1.2.3 From 7c4cd5f4d2dd4a028a46bfb696b0cd387caadf33 Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Thu, 21 Jul 2022 14:21:32 -0400 Subject: SUNRPC: Add a function for directly setting the xdr page len We need to do this step during READ_PLUS decoding so that we know pages are the right length and any extra data has been preserved in the tail. Signed-off-by: Anna Schumaker Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xdr.h | 1 + net/sunrpc/xdr.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 7dcc6c31fe29..8cd38a9994ca 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -258,6 +258,7 @@ extern __be32 *xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes); extern unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len); extern void xdr_enter_page(struct xdr_stream *xdr, unsigned int len); extern int xdr_process_buf(const struct xdr_buf *buf, unsigned int offset, unsigned int len, int (*actor)(struct scatterlist *, void *), void *data); +extern void xdr_set_pagelen(struct xdr_stream *, unsigned int len); extern unsigned int xdr_align_data(struct xdr_stream *, unsigned int offset, unsigned int length); extern unsigned int xdr_expand_hole(struct xdr_stream *, unsigned int offset, unsigned int length); extern bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf, diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index 8ba11a754297..e4ac700ca554 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -1500,6 +1500,36 @@ unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len) } EXPORT_SYMBOL_GPL(xdr_read_pages); +/** + * xdr_set_pagelen - Sets the length of the XDR pages + * @xdr: pointer to xdr_stream struct + * @len: new length of the XDR page data + * + * Either grows or shrinks the length of the xdr pages by setting pagelen to + * @len bytes. When shrinking, any extra data is moved into buf->tail, whereas + * when growing any data beyond the current pointer is moved into the tail. + * + * Returns True if the operation was successful, and False otherwise. + */ +void xdr_set_pagelen(struct xdr_stream *xdr, unsigned int len) +{ + struct xdr_buf *buf = xdr->buf; + size_t remaining = xdr_stream_remaining(xdr); + size_t base = 0; + + if (len < buf->page_len) { + base = buf->page_len - len; + xdr_shrink_pagelen(buf, len); + } else { + xdr_buf_head_shift_right(buf, xdr_stream_pos(xdr), + buf->page_len, remaining); + if (len > buf->page_len) + xdr_buf_try_expand(buf, len - buf->page_len); + } + xdr_set_tail_base(xdr, base, remaining); +} +EXPORT_SYMBOL_GPL(xdr_set_pagelen); + unsigned int xdr_align_data(struct xdr_stream *xdr, unsigned int offset, unsigned int length) { -- cgit v1.2.3 From e1bd87608d4b6f87813f79b91e834de610f1049b Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Thu, 21 Jul 2022 14:21:33 -0400 Subject: SUNRPC: Add a function for zeroing out a portion of an xdr_stream This will be used during READ_PLUS decoding for handling HOLE segments. Signed-off-by: Anna Schumaker Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xdr.h | 2 ++ net/sunrpc/xdr.c | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 8cd38a9994ca..f0ab06acab61 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -265,6 +265,8 @@ extern bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf unsigned int len); extern unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int offset, unsigned int target, unsigned int length); +extern unsigned int xdr_stream_zero(struct xdr_stream *xdr, unsigned int offset, + unsigned int length); /** * xdr_set_scratch_buffer - Attach a scratch buffer for decoding data. diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index e4ac700ca554..f09a7ab1a82b 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -1769,6 +1769,29 @@ unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int off } EXPORT_SYMBOL_GPL(xdr_stream_move_subsegment); +/** + * xdr_stream_zero - zero out a portion of an xdr_stream + * @xdr: an xdr_stream to zero out + * @offset: the starting point in the stream + * @length: the number of bytes to zero + */ +unsigned int xdr_stream_zero(struct xdr_stream *xdr, unsigned int offset, + unsigned int length) +{ + struct xdr_buf buf; + + if (xdr_buf_subsegment(xdr->buf, &buf, offset, length) < 0) + return 0; + if (buf.head[0].iov_len) + xdr_buf_iov_zero(buf.head, 0, buf.head[0].iov_len); + if (buf.page_len > 0) + xdr_buf_pages_zero(&buf, 0, buf.page_len); + if (buf.tail[0].iov_len) + xdr_buf_iov_zero(buf.tail, 0, buf.tail[0].iov_len); + return length; +} +EXPORT_SYMBOL_GPL(xdr_stream_zero); + /** * xdr_buf_trim - lop at most "len" bytes off the end of "buf" * @buf: buf to be trimmed -- cgit v1.2.3 From 29946fbcb2c31a2a367887dc58a2e7e5b012e285 Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Thu, 21 Jul 2022 14:21:35 -0400 Subject: SUNRPC: Remove xdr_align_data() and xdr_expand_hole() These functions are no longer needed now that the NFS client places data and hole segments directly. Signed-off-by: Anna Schumaker Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xdr.h | 2 -- net/sunrpc/xdr.c | 66 ---------------------------------------------- 2 files changed, 68 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index f0ab06acab61..f38c97f45354 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -259,8 +259,6 @@ extern unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len); extern void xdr_enter_page(struct xdr_stream *xdr, unsigned int len); extern int xdr_process_buf(const struct xdr_buf *buf, unsigned int offset, unsigned int len, int (*actor)(struct scatterlist *, void *), void *data); extern void xdr_set_pagelen(struct xdr_stream *, unsigned int len); -extern unsigned int xdr_align_data(struct xdr_stream *, unsigned int offset, unsigned int length); -extern unsigned int xdr_expand_hole(struct xdr_stream *, unsigned int offset, unsigned int length); extern bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf, unsigned int len); extern unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int offset, diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index f09a7ab1a82b..482586c23fdd 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -1530,72 +1530,6 @@ void xdr_set_pagelen(struct xdr_stream *xdr, unsigned int len) } EXPORT_SYMBOL_GPL(xdr_set_pagelen); -unsigned int xdr_align_data(struct xdr_stream *xdr, unsigned int offset, - unsigned int length) -{ - struct xdr_buf *buf = xdr->buf; - unsigned int from, bytes, len; - unsigned int shift; - - xdr_realign_pages(xdr); - from = xdr_page_pos(xdr); - - if (from >= buf->page_len + buf->tail->iov_len) - return 0; - if (from + buf->head->iov_len >= buf->len) - return 0; - - len = buf->len - buf->head->iov_len; - - /* We only shift data left! */ - if (WARN_ONCE(from < offset, "SUNRPC: misaligned data src=%u dst=%u\n", - from, offset)) - return 0; - if (WARN_ONCE(offset > buf->page_len, - "SUNRPC: buffer overflow. offset=%u, page_len=%u\n", - offset, buf->page_len)) - return 0; - - /* Move page data to the left */ - shift = from - offset; - xdr_buf_pages_shift_left(buf, from, len, shift); - - bytes = xdr_stream_remaining(xdr); - if (length > bytes) - length = bytes; - bytes -= length; - - xdr->buf->len -= shift; - xdr_set_page(xdr, offset + length, bytes); - return length; -} -EXPORT_SYMBOL_GPL(xdr_align_data); - -unsigned int xdr_expand_hole(struct xdr_stream *xdr, unsigned int offset, - unsigned int length) -{ - struct xdr_buf *buf = xdr->buf; - unsigned int from, to, shift; - - xdr_realign_pages(xdr); - from = xdr_page_pos(xdr); - to = xdr_align_size(offset + length); - - /* Could the hole be behind us? */ - if (to > from) { - unsigned int buflen = buf->len - buf->head->iov_len; - shift = to - from; - xdr_buf_try_expand(buf, shift); - xdr_buf_pages_shift_right(buf, from, buflen, shift); - xdr_set_page(xdr, to, xdr_stream_remaining(xdr)); - } else if (to != from) - xdr_align_data(xdr, to, 0); - xdr_buf_pages_zero(buf, offset, length); - - return length; -} -EXPORT_SYMBOL_GPL(xdr_expand_hole); - /** * xdr_enter_page - decode data from the XDR page * @xdr: pointer to xdr_stream struct -- cgit v1.2.3 From ab1c84d855cf2c284fa0f4b17fc04063659c54a1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 16 Jun 2022 13:57:19 +0100 Subject: io_uring: make io_uring_types.h public Move io_uring types to linux/include, need them public so tracing can see the definitions and we can clean trace/events/io_uring.h Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a15f12e8cb7289b2de0deaddcc7518d98a132d17.1655384063.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 554 +++++++++++++++++++++++++++++++++++++++++ io_uring/filetable.h | 11 - io_uring/io-wq.h | 17 +- io_uring/io_uring.h | 4 +- io_uring/io_uring_types.h | 530 --------------------------------------- io_uring/refs.h | 2 +- 6 files changed, 559 insertions(+), 559 deletions(-) create mode 100644 include/linux/io_uring_types.h delete mode 100644 io_uring/io_uring_types.h (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h new file mode 100644 index 000000000000..779c72da5b8f --- /dev/null +++ b/include/linux/io_uring_types.h @@ -0,0 +1,554 @@ +#ifndef IO_URING_TYPES_H +#define IO_URING_TYPES_H + +#include +#include +#include +#include + +struct io_wq_work_node { + struct io_wq_work_node *next; +}; + +struct io_wq_work_list { + struct io_wq_work_node *first; + struct io_wq_work_node *last; +}; + +struct io_wq_work { + struct io_wq_work_node list; + unsigned flags; + /* place it here instead of io_kiocb as it fills padding and saves 4B */ + int cancel_seq; +}; + +struct io_fixed_file { + /* file * with additional FFS_* flags */ + unsigned long file_ptr; +}; + +struct io_file_table { + struct io_fixed_file *files; + unsigned long *bitmap; + unsigned int alloc_hint; +}; + +struct io_hash_bucket { + spinlock_t lock; + struct hlist_head list; +} ____cacheline_aligned_in_smp; + +struct io_hash_table { + struct io_hash_bucket *hbs; + unsigned hash_bits; +}; + +struct io_uring { + u32 head ____cacheline_aligned_in_smp; + u32 tail ____cacheline_aligned_in_smp; +}; + +/* + * This data is shared with the application through the mmap at offsets + * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. + * + * The offsets to the member fields are published through struct + * io_sqring_offsets when calling io_uring_setup. + */ +struct io_rings { + /* + * Head and tail offsets into the ring; the offsets need to be + * masked to get valid indices. + * + * The kernel controls head of the sq ring and the tail of the cq ring, + * and the application controls tail of the sq ring and the head of the + * cq ring. + */ + struct io_uring sq, cq; + /* + * Bitmasks to apply to head and tail offsets (constant, equals + * ring_entries - 1) + */ + u32 sq_ring_mask, cq_ring_mask; + /* Ring sizes (constant, power of 2) */ + u32 sq_ring_entries, cq_ring_entries; + /* + * Number of invalid entries dropped by the kernel due to + * invalid index stored in array + * + * Written by the kernel, shouldn't be modified by the + * application (i.e. get number of "new events" by comparing to + * cached value). + * + * After a new SQ head value was read by the application this + * counter includes all submissions that were dropped reaching + * the new SQ head (and possibly more). + */ + u32 sq_dropped; + /* + * Runtime SQ flags + * + * Written by the kernel, shouldn't be modified by the + * application. + * + * The application needs a full memory barrier before checking + * for IORING_SQ_NEED_WAKEUP after updating the sq tail. + */ + atomic_t sq_flags; + /* + * Runtime CQ flags + * + * Written by the application, shouldn't be modified by the + * kernel. + */ + u32 cq_flags; + /* + * Number of completion events lost because the queue was full; + * this should be avoided by the application by making sure + * there are not more requests pending than there is space in + * the completion queue. + * + * Written by the kernel, shouldn't be modified by the + * application (i.e. get number of "new events" by comparing to + * cached value). + * + * As completion events come in out of order this counter is not + * ordered with any other data. + */ + u32 cq_overflow; + /* + * Ring buffer of completion events. + * + * The kernel writes completion events fresh every time they are + * produced, so the application is allowed to modify pending + * entries. + */ + struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; +}; + +struct io_restriction { + DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); + DECLARE_BITMAP(sqe_op, IORING_OP_LAST); + u8 sqe_flags_allowed; + u8 sqe_flags_required; + bool registered; +}; + +struct io_submit_link { + struct io_kiocb *head; + struct io_kiocb *last; +}; + +struct io_submit_state { + /* inline/task_work completion list, under ->uring_lock */ + struct io_wq_work_node free_list; + /* batch completion logic */ + struct io_wq_work_list compl_reqs; + struct io_submit_link link; + + bool plug_started; + bool need_plug; + bool flush_cqes; + unsigned short submit_nr; + struct blk_plug plug; +}; + +struct io_ev_fd { + struct eventfd_ctx *cq_ev_fd; + unsigned int eventfd_async: 1; + struct rcu_head rcu; +}; + +struct io_ring_ctx { + /* const or read-mostly hot data */ + struct { + struct percpu_ref refs; + + struct io_rings *rings; + unsigned int flags; + enum task_work_notify_mode notify_method; + unsigned int compat: 1; + unsigned int drain_next: 1; + unsigned int restricted: 1; + unsigned int off_timeout_used: 1; + unsigned int drain_active: 1; + unsigned int drain_disabled: 1; + unsigned int has_evfd: 1; + unsigned int syscall_iopoll: 1; + } ____cacheline_aligned_in_smp; + + /* submission data */ + struct { + struct mutex uring_lock; + + /* + * Ring buffer of indices into array of io_uring_sqe, which is + * mmapped by the application using the IORING_OFF_SQES offset. + * + * This indirection could e.g. be used to assign fixed + * io_uring_sqe entries to operations and only submit them to + * the queue when needed. + * + * The kernel modifies neither the indices array nor the entries + * array. + */ + u32 *sq_array; + struct io_uring_sqe *sq_sqes; + unsigned cached_sq_head; + unsigned sq_entries; + + /* + * Fixed resources fast path, should be accessed only under + * uring_lock, and updated through io_uring_register(2) + */ + struct io_rsrc_node *rsrc_node; + int rsrc_cached_refs; + atomic_t cancel_seq; + struct io_file_table file_table; + unsigned nr_user_files; + unsigned nr_user_bufs; + struct io_mapped_ubuf **user_bufs; + + struct io_submit_state submit_state; + + struct io_buffer_list *io_bl; + struct xarray io_bl_xa; + struct list_head io_buffers_cache; + + struct io_hash_table cancel_table_locked; + struct list_head cq_overflow_list; + struct list_head apoll_cache; + struct xarray personalities; + u32 pers_next; + } ____cacheline_aligned_in_smp; + + /* IRQ completion list, under ->completion_lock */ + struct io_wq_work_list locked_free_list; + unsigned int locked_free_nr; + + const struct cred *sq_creds; /* cred used for __io_sq_thread() */ + struct io_sq_data *sq_data; /* if using sq thread polling */ + + struct wait_queue_head sqo_sq_wait; + struct list_head sqd_list; + + unsigned long check_cq; + + struct { + /* + * We cache a range of free CQEs we can use, once exhausted it + * should go through a slower range setup, see __io_get_cqe() + */ + struct io_uring_cqe *cqe_cached; + struct io_uring_cqe *cqe_sentinel; + + unsigned cached_cq_tail; + unsigned cq_entries; + struct io_ev_fd __rcu *io_ev_fd; + struct wait_queue_head cq_wait; + unsigned cq_extra; + } ____cacheline_aligned_in_smp; + + struct { + spinlock_t completion_lock; + + /* + * ->iopoll_list is protected by the ctx->uring_lock for + * io_uring instances that don't use IORING_SETUP_SQPOLL. + * For SQPOLL, only the single threaded io_sq_thread() will + * manipulate the list, hence no extra locking is needed there. + */ + struct io_wq_work_list iopoll_list; + struct io_hash_table cancel_table; + bool poll_multi_queue; + + struct list_head io_buffers_comp; + } ____cacheline_aligned_in_smp; + + /* timeouts */ + struct { + spinlock_t timeout_lock; + atomic_t cq_timeouts; + struct list_head timeout_list; + struct list_head ltimeout_list; + unsigned cq_last_tm_flush; + } ____cacheline_aligned_in_smp; + + /* Keep this last, we don't need it for the fast path */ + + struct io_restriction restrictions; + struct task_struct *submitter_task; + + /* slow path rsrc auxilary data, used by update/register */ + struct io_rsrc_node *rsrc_backup_node; + struct io_mapped_ubuf *dummy_ubuf; + struct io_rsrc_data *file_data; + struct io_rsrc_data *buf_data; + + struct delayed_work rsrc_put_work; + struct llist_head rsrc_put_llist; + struct list_head rsrc_ref_list; + spinlock_t rsrc_ref_lock; + + struct list_head io_buffers_pages; + + #if defined(CONFIG_UNIX) + struct socket *ring_sock; + #endif + /* hashed buffered write serialization */ + struct io_wq_hash *hash_map; + + /* Only used for accounting purposes */ + struct user_struct *user; + struct mm_struct *mm_account; + + /* ctx exit and cancelation */ + struct llist_head fallback_llist; + struct delayed_work fallback_work; + struct work_struct exit_work; + struct list_head tctx_list; + struct completion ref_comp; + + /* io-wq management, e.g. thread count */ + u32 iowq_limits[2]; + bool iowq_limits_set; + + struct list_head defer_list; + unsigned sq_thread_idle; +}; + +enum { + REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT, + REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT, + REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT, + REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT, + REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT, + REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT, + REQ_F_CQE_SKIP_BIT = IOSQE_CQE_SKIP_SUCCESS_BIT, + + /* first byte is taken by user flags, shift it to not overlap */ + REQ_F_FAIL_BIT = 8, + REQ_F_INFLIGHT_BIT, + REQ_F_CUR_POS_BIT, + REQ_F_NOWAIT_BIT, + REQ_F_LINK_TIMEOUT_BIT, + REQ_F_NEED_CLEANUP_BIT, + REQ_F_POLLED_BIT, + REQ_F_BUFFER_SELECTED_BIT, + REQ_F_BUFFER_RING_BIT, + REQ_F_REISSUE_BIT, + REQ_F_CREDS_BIT, + REQ_F_REFCOUNT_BIT, + REQ_F_ARM_LTIMEOUT_BIT, + REQ_F_ASYNC_DATA_BIT, + REQ_F_SKIP_LINK_CQES_BIT, + REQ_F_SINGLE_POLL_BIT, + REQ_F_DOUBLE_POLL_BIT, + REQ_F_PARTIAL_IO_BIT, + REQ_F_CQE32_INIT_BIT, + REQ_F_APOLL_MULTISHOT_BIT, + REQ_F_CLEAR_POLLIN_BIT, + REQ_F_HASH_LOCKED_BIT, + /* keep async read/write and isreg together and in order */ + REQ_F_SUPPORT_NOWAIT_BIT, + REQ_F_ISREG_BIT, + + /* not a real bit, just to check we're not overflowing the space */ + __REQ_F_LAST_BIT, +}; + +enum { + /* ctx owns file */ + REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT), + /* drain existing IO first */ + REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT), + /* linked sqes */ + REQ_F_LINK = BIT(REQ_F_LINK_BIT), + /* doesn't sever on completion < 0 */ + REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT), + /* IOSQE_ASYNC */ + REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT), + /* IOSQE_BUFFER_SELECT */ + REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT), + /* IOSQE_CQE_SKIP_SUCCESS */ + REQ_F_CQE_SKIP = BIT(REQ_F_CQE_SKIP_BIT), + + /* fail rest of links */ + REQ_F_FAIL = BIT(REQ_F_FAIL_BIT), + /* on inflight list, should be cancelled and waited on exit reliably */ + REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), + /* read/write uses file position */ + REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), + /* must not punt to workers */ + REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT), + /* has or had linked timeout */ + REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT), + /* needs cleanup */ + REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT), + /* already went through poll handler */ + REQ_F_POLLED = BIT(REQ_F_POLLED_BIT), + /* buffer already selected */ + REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), + /* buffer selected from ring, needs commit */ + REQ_F_BUFFER_RING = BIT(REQ_F_BUFFER_RING_BIT), + /* caller should reissue async */ + REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT), + /* supports async reads/writes */ + REQ_F_SUPPORT_NOWAIT = BIT(REQ_F_SUPPORT_NOWAIT_BIT), + /* regular file */ + REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), + /* has creds assigned */ + REQ_F_CREDS = BIT(REQ_F_CREDS_BIT), + /* skip refcounting if not set */ + REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT), + /* there is a linked timeout that has to be armed */ + REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT), + /* ->async_data allocated */ + REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT), + /* don't post CQEs while failing linked requests */ + REQ_F_SKIP_LINK_CQES = BIT(REQ_F_SKIP_LINK_CQES_BIT), + /* single poll may be active */ + REQ_F_SINGLE_POLL = BIT(REQ_F_SINGLE_POLL_BIT), + /* double poll may active */ + REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT), + /* request has already done partial IO */ + REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT), + /* fast poll multishot mode */ + REQ_F_APOLL_MULTISHOT = BIT(REQ_F_APOLL_MULTISHOT_BIT), + /* ->extra1 and ->extra2 are initialised */ + REQ_F_CQE32_INIT = BIT(REQ_F_CQE32_INIT_BIT), + /* recvmsg special flag, clear EPOLLIN */ + REQ_F_CLEAR_POLLIN = BIT(REQ_F_CLEAR_POLLIN_BIT), + /* hashed into ->cancel_hash_locked, protected by ->uring_lock */ + REQ_F_HASH_LOCKED = BIT(REQ_F_HASH_LOCKED_BIT), +}; + +typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); + +struct io_task_work { + union { + struct io_wq_work_node node; + struct llist_node fallback_node; + }; + io_req_tw_func_t func; +}; + +struct io_cqe { + __u64 user_data; + __s32 res; + /* fd initially, then cflags for completion */ + union { + __u32 flags; + int fd; + }; +}; + +/* + * Each request type overlays its private data structure on top of this one. + * They must not exceed this one in size. + */ +struct io_cmd_data { + struct file *file; + /* each command gets 56 bytes of data */ + __u8 data[56]; +}; + +#define io_kiocb_to_cmd(req) ((void *) &(req)->cmd) +#define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr) + +struct io_kiocb { + union { + /* + * NOTE! Each of the io_kiocb union members has the file pointer + * as the first entry in their struct definition. So you can + * access the file pointer through any of the sub-structs, + * or directly as just 'file' in this struct. + */ + struct file *file; + struct io_cmd_data cmd; + }; + + u8 opcode; + /* polled IO has completed */ + u8 iopoll_completed; + /* + * Can be either a fixed buffer index, or used with provided buffers. + * For the latter, before issue it points to the buffer group ID, + * and after selection it points to the buffer ID itself. + */ + u16 buf_index; + unsigned int flags; + + struct io_cqe cqe; + + struct io_ring_ctx *ctx; + struct task_struct *task; + + struct io_rsrc_node *rsrc_node; + + union { + /* store used ubuf, so we can prevent reloading */ + struct io_mapped_ubuf *imu; + + /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ + struct io_buffer *kbuf; + + /* + * stores buffer ID for ring provided buffers, valid IFF + * REQ_F_BUFFER_RING is set. + */ + struct io_buffer_list *buf_list; + }; + + union { + /* used by request caches, completion batching and iopoll */ + struct io_wq_work_node comp_list; + /* cache ->apoll->events */ + __poll_t apoll_events; + }; + atomic_t refs; + atomic_t poll_refs; + struct io_task_work io_task_work; + /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ + union { + struct hlist_node hash_node; + struct { + u64 extra1; + u64 extra2; + }; + }; + /* internal polling, see IORING_FEAT_FAST_POLL */ + struct async_poll *apoll; + /* opcode allocated if it needs to store data for async defer */ + void *async_data; + /* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */ + struct io_kiocb *link; + /* custom credentials, valid IFF REQ_F_CREDS is set */ + const struct cred *creds; + struct io_wq_work work; +}; + +struct io_cancel_data { + struct io_ring_ctx *ctx; + union { + u64 data; + struct file *file; + }; + u32 flags; + int seq; +}; + +struct io_overflow_cqe { + struct list_head list; + struct io_uring_cqe cqe; +}; + +struct io_mapped_ubuf { + u64 ubuf; + u64 ubuf_end; + unsigned int nr_bvecs; + unsigned long acct_pages; + struct bio_vec bvec[]; +}; + +#endif diff --git a/io_uring/filetable.h b/io_uring/filetable.h index c404360f7090..6b58aa48bc45 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -22,17 +22,6 @@ struct io_kiocb; #endif #define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG|FFS_SCM) -struct io_fixed_file { - /* file * with additional FFS_* flags */ - unsigned long file_ptr; -}; - -struct io_file_table { - struct io_fixed_file *files; - unsigned long *bitmap; - unsigned int alloc_hint; -}; - bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files); void io_free_file_tables(struct io_file_table *table); diff --git a/io_uring/io-wq.h b/io_uring/io-wq.h index 3f54ee2a8eeb..10b80ef78bb8 100644 --- a/io_uring/io-wq.h +++ b/io_uring/io-wq.h @@ -2,6 +2,7 @@ #define INTERNAL_IO_WQ_H #include +#include struct io_wq; @@ -20,15 +21,6 @@ enum io_wq_cancel { IO_WQ_CANCEL_NOTFOUND, /* work not found */ }; -struct io_wq_work_node { - struct io_wq_work_node *next; -}; - -struct io_wq_work_list { - struct io_wq_work_node *first; - struct io_wq_work_node *last; -}; - #define wq_list_for_each(pos, prv, head) \ for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) @@ -152,13 +144,6 @@ struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack) return node; } -struct io_wq_work { - struct io_wq_work_node list; - unsigned flags; - /* place it here instead of io_kiocb as it fills padding and saves 4B */ - int cancel_seq; -}; - static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) { if (!work->list.next) diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 558a860a93fc..5eaa01c4697c 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -3,7 +3,9 @@ #include #include -#include "io_uring_types.h" +#include +#include "io-wq.h" +#include "filetable.h" #ifndef CREATE_TRACE_POINTS #include diff --git a/io_uring/io_uring_types.h b/io_uring/io_uring_types.h deleted file mode 100644 index 65ac7cdaaa73..000000000000 --- a/io_uring/io_uring_types.h +++ /dev/null @@ -1,530 +0,0 @@ -#ifndef IO_URING_TYPES_H -#define IO_URING_TYPES_H - -#include -#include -#include -#include - -#include "io-wq.h" -#include "filetable.h" - -struct io_hash_bucket { - spinlock_t lock; - struct hlist_head list; -} ____cacheline_aligned_in_smp; - -struct io_hash_table { - struct io_hash_bucket *hbs; - unsigned hash_bits; -}; - -struct io_uring { - u32 head ____cacheline_aligned_in_smp; - u32 tail ____cacheline_aligned_in_smp; -}; - -/* - * This data is shared with the application through the mmap at offsets - * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. - * - * The offsets to the member fields are published through struct - * io_sqring_offsets when calling io_uring_setup. - */ -struct io_rings { - /* - * Head and tail offsets into the ring; the offsets need to be - * masked to get valid indices. - * - * The kernel controls head of the sq ring and the tail of the cq ring, - * and the application controls tail of the sq ring and the head of the - * cq ring. - */ - struct io_uring sq, cq; - /* - * Bitmasks to apply to head and tail offsets (constant, equals - * ring_entries - 1) - */ - u32 sq_ring_mask, cq_ring_mask; - /* Ring sizes (constant, power of 2) */ - u32 sq_ring_entries, cq_ring_entries; - /* - * Number of invalid entries dropped by the kernel due to - * invalid index stored in array - * - * Written by the kernel, shouldn't be modified by the - * application (i.e. get number of "new events" by comparing to - * cached value). - * - * After a new SQ head value was read by the application this - * counter includes all submissions that were dropped reaching - * the new SQ head (and possibly more). - */ - u32 sq_dropped; - /* - * Runtime SQ flags - * - * Written by the kernel, shouldn't be modified by the - * application. - * - * The application needs a full memory barrier before checking - * for IORING_SQ_NEED_WAKEUP after updating the sq tail. - */ - atomic_t sq_flags; - /* - * Runtime CQ flags - * - * Written by the application, shouldn't be modified by the - * kernel. - */ - u32 cq_flags; - /* - * Number of completion events lost because the queue was full; - * this should be avoided by the application by making sure - * there are not more requests pending than there is space in - * the completion queue. - * - * Written by the kernel, shouldn't be modified by the - * application (i.e. get number of "new events" by comparing to - * cached value). - * - * As completion events come in out of order this counter is not - * ordered with any other data. - */ - u32 cq_overflow; - /* - * Ring buffer of completion events. - * - * The kernel writes completion events fresh every time they are - * produced, so the application is allowed to modify pending - * entries. - */ - struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; -}; - -struct io_restriction { - DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); - DECLARE_BITMAP(sqe_op, IORING_OP_LAST); - u8 sqe_flags_allowed; - u8 sqe_flags_required; - bool registered; -}; - -struct io_submit_link { - struct io_kiocb *head; - struct io_kiocb *last; -}; - -struct io_submit_state { - /* inline/task_work completion list, under ->uring_lock */ - struct io_wq_work_node free_list; - /* batch completion logic */ - struct io_wq_work_list compl_reqs; - struct io_submit_link link; - - bool plug_started; - bool need_plug; - bool flush_cqes; - unsigned short submit_nr; - struct blk_plug plug; -}; - -struct io_ev_fd { - struct eventfd_ctx *cq_ev_fd; - unsigned int eventfd_async: 1; - struct rcu_head rcu; -}; - -struct io_ring_ctx { - /* const or read-mostly hot data */ - struct { - struct percpu_ref refs; - - struct io_rings *rings; - unsigned int flags; - enum task_work_notify_mode notify_method; - unsigned int compat: 1; - unsigned int drain_next: 1; - unsigned int restricted: 1; - unsigned int off_timeout_used: 1; - unsigned int drain_active: 1; - unsigned int drain_disabled: 1; - unsigned int has_evfd: 1; - unsigned int syscall_iopoll: 1; - } ____cacheline_aligned_in_smp; - - /* submission data */ - struct { - struct mutex uring_lock; - - /* - * Ring buffer of indices into array of io_uring_sqe, which is - * mmapped by the application using the IORING_OFF_SQES offset. - * - * This indirection could e.g. be used to assign fixed - * io_uring_sqe entries to operations and only submit them to - * the queue when needed. - * - * The kernel modifies neither the indices array nor the entries - * array. - */ - u32 *sq_array; - struct io_uring_sqe *sq_sqes; - unsigned cached_sq_head; - unsigned sq_entries; - - /* - * Fixed resources fast path, should be accessed only under - * uring_lock, and updated through io_uring_register(2) - */ - struct io_rsrc_node *rsrc_node; - int rsrc_cached_refs; - atomic_t cancel_seq; - struct io_file_table file_table; - unsigned nr_user_files; - unsigned nr_user_bufs; - struct io_mapped_ubuf **user_bufs; - - struct io_submit_state submit_state; - - struct io_buffer_list *io_bl; - struct xarray io_bl_xa; - struct list_head io_buffers_cache; - - struct io_hash_table cancel_table_locked; - struct list_head cq_overflow_list; - struct list_head apoll_cache; - struct xarray personalities; - u32 pers_next; - } ____cacheline_aligned_in_smp; - - /* IRQ completion list, under ->completion_lock */ - struct io_wq_work_list locked_free_list; - unsigned int locked_free_nr; - - const struct cred *sq_creds; /* cred used for __io_sq_thread() */ - struct io_sq_data *sq_data; /* if using sq thread polling */ - - struct wait_queue_head sqo_sq_wait; - struct list_head sqd_list; - - unsigned long check_cq; - - struct { - /* - * We cache a range of free CQEs we can use, once exhausted it - * should go through a slower range setup, see __io_get_cqe() - */ - struct io_uring_cqe *cqe_cached; - struct io_uring_cqe *cqe_sentinel; - - unsigned cached_cq_tail; - unsigned cq_entries; - struct io_ev_fd __rcu *io_ev_fd; - struct wait_queue_head cq_wait; - unsigned cq_extra; - } ____cacheline_aligned_in_smp; - - struct { - spinlock_t completion_lock; - - /* - * ->iopoll_list is protected by the ctx->uring_lock for - * io_uring instances that don't use IORING_SETUP_SQPOLL. - * For SQPOLL, only the single threaded io_sq_thread() will - * manipulate the list, hence no extra locking is needed there. - */ - struct io_wq_work_list iopoll_list; - struct io_hash_table cancel_table; - bool poll_multi_queue; - - struct list_head io_buffers_comp; - } ____cacheline_aligned_in_smp; - - /* timeouts */ - struct { - spinlock_t timeout_lock; - atomic_t cq_timeouts; - struct list_head timeout_list; - struct list_head ltimeout_list; - unsigned cq_last_tm_flush; - } ____cacheline_aligned_in_smp; - - /* Keep this last, we don't need it for the fast path */ - - struct io_restriction restrictions; - struct task_struct *submitter_task; - - /* slow path rsrc auxilary data, used by update/register */ - struct io_rsrc_node *rsrc_backup_node; - struct io_mapped_ubuf *dummy_ubuf; - struct io_rsrc_data *file_data; - struct io_rsrc_data *buf_data; - - struct delayed_work rsrc_put_work; - struct llist_head rsrc_put_llist; - struct list_head rsrc_ref_list; - spinlock_t rsrc_ref_lock; - - struct list_head io_buffers_pages; - - #if defined(CONFIG_UNIX) - struct socket *ring_sock; - #endif - /* hashed buffered write serialization */ - struct io_wq_hash *hash_map; - - /* Only used for accounting purposes */ - struct user_struct *user; - struct mm_struct *mm_account; - - /* ctx exit and cancelation */ - struct llist_head fallback_llist; - struct delayed_work fallback_work; - struct work_struct exit_work; - struct list_head tctx_list; - struct completion ref_comp; - - /* io-wq management, e.g. thread count */ - u32 iowq_limits[2]; - bool iowq_limits_set; - - struct list_head defer_list; - unsigned sq_thread_idle; -}; - -enum { - REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT, - REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT, - REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT, - REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT, - REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT, - REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT, - REQ_F_CQE_SKIP_BIT = IOSQE_CQE_SKIP_SUCCESS_BIT, - - /* first byte is taken by user flags, shift it to not overlap */ - REQ_F_FAIL_BIT = 8, - REQ_F_INFLIGHT_BIT, - REQ_F_CUR_POS_BIT, - REQ_F_NOWAIT_BIT, - REQ_F_LINK_TIMEOUT_BIT, - REQ_F_NEED_CLEANUP_BIT, - REQ_F_POLLED_BIT, - REQ_F_BUFFER_SELECTED_BIT, - REQ_F_BUFFER_RING_BIT, - REQ_F_REISSUE_BIT, - REQ_F_CREDS_BIT, - REQ_F_REFCOUNT_BIT, - REQ_F_ARM_LTIMEOUT_BIT, - REQ_F_ASYNC_DATA_BIT, - REQ_F_SKIP_LINK_CQES_BIT, - REQ_F_SINGLE_POLL_BIT, - REQ_F_DOUBLE_POLL_BIT, - REQ_F_PARTIAL_IO_BIT, - REQ_F_CQE32_INIT_BIT, - REQ_F_APOLL_MULTISHOT_BIT, - REQ_F_CLEAR_POLLIN_BIT, - REQ_F_HASH_LOCKED_BIT, - /* keep async read/write and isreg together and in order */ - REQ_F_SUPPORT_NOWAIT_BIT, - REQ_F_ISREG_BIT, - - /* not a real bit, just to check we're not overflowing the space */ - __REQ_F_LAST_BIT, -}; - -enum { - /* ctx owns file */ - REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT), - /* drain existing IO first */ - REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT), - /* linked sqes */ - REQ_F_LINK = BIT(REQ_F_LINK_BIT), - /* doesn't sever on completion < 0 */ - REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT), - /* IOSQE_ASYNC */ - REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT), - /* IOSQE_BUFFER_SELECT */ - REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT), - /* IOSQE_CQE_SKIP_SUCCESS */ - REQ_F_CQE_SKIP = BIT(REQ_F_CQE_SKIP_BIT), - - /* fail rest of links */ - REQ_F_FAIL = BIT(REQ_F_FAIL_BIT), - /* on inflight list, should be cancelled and waited on exit reliably */ - REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), - /* read/write uses file position */ - REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), - /* must not punt to workers */ - REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT), - /* has or had linked timeout */ - REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT), - /* needs cleanup */ - REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT), - /* already went through poll handler */ - REQ_F_POLLED = BIT(REQ_F_POLLED_BIT), - /* buffer already selected */ - REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), - /* buffer selected from ring, needs commit */ - REQ_F_BUFFER_RING = BIT(REQ_F_BUFFER_RING_BIT), - /* caller should reissue async */ - REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT), - /* supports async reads/writes */ - REQ_F_SUPPORT_NOWAIT = BIT(REQ_F_SUPPORT_NOWAIT_BIT), - /* regular file */ - REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), - /* has creds assigned */ - REQ_F_CREDS = BIT(REQ_F_CREDS_BIT), - /* skip refcounting if not set */ - REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT), - /* there is a linked timeout that has to be armed */ - REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT), - /* ->async_data allocated */ - REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT), - /* don't post CQEs while failing linked requests */ - REQ_F_SKIP_LINK_CQES = BIT(REQ_F_SKIP_LINK_CQES_BIT), - /* single poll may be active */ - REQ_F_SINGLE_POLL = BIT(REQ_F_SINGLE_POLL_BIT), - /* double poll may active */ - REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT), - /* request has already done partial IO */ - REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT), - /* fast poll multishot mode */ - REQ_F_APOLL_MULTISHOT = BIT(REQ_F_APOLL_MULTISHOT_BIT), - /* ->extra1 and ->extra2 are initialised */ - REQ_F_CQE32_INIT = BIT(REQ_F_CQE32_INIT_BIT), - /* recvmsg special flag, clear EPOLLIN */ - REQ_F_CLEAR_POLLIN = BIT(REQ_F_CLEAR_POLLIN_BIT), - /* hashed into ->cancel_hash_locked, protected by ->uring_lock */ - REQ_F_HASH_LOCKED = BIT(REQ_F_HASH_LOCKED_BIT), -}; - -typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); - -struct io_task_work { - union { - struct io_wq_work_node node; - struct llist_node fallback_node; - }; - io_req_tw_func_t func; -}; - -struct io_cqe { - __u64 user_data; - __s32 res; - /* fd initially, then cflags for completion */ - union { - __u32 flags; - int fd; - }; -}; - -/* - * Each request type overlays its private data structure on top of this one. - * They must not exceed this one in size. - */ -struct io_cmd_data { - struct file *file; - /* each command gets 56 bytes of data */ - __u8 data[56]; -}; - -#define io_kiocb_to_cmd(req) ((void *) &(req)->cmd) -#define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr) - -struct io_kiocb { - union { - /* - * NOTE! Each of the io_kiocb union members has the file pointer - * as the first entry in their struct definition. So you can - * access the file pointer through any of the sub-structs, - * or directly as just 'file' in this struct. - */ - struct file *file; - struct io_cmd_data cmd; - }; - - u8 opcode; - /* polled IO has completed */ - u8 iopoll_completed; - /* - * Can be either a fixed buffer index, or used with provided buffers. - * For the latter, before issue it points to the buffer group ID, - * and after selection it points to the buffer ID itself. - */ - u16 buf_index; - unsigned int flags; - - struct io_cqe cqe; - - struct io_ring_ctx *ctx; - struct task_struct *task; - - struct io_rsrc_node *rsrc_node; - - union { - /* store used ubuf, so we can prevent reloading */ - struct io_mapped_ubuf *imu; - - /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ - struct io_buffer *kbuf; - - /* - * stores buffer ID for ring provided buffers, valid IFF - * REQ_F_BUFFER_RING is set. - */ - struct io_buffer_list *buf_list; - }; - - union { - /* used by request caches, completion batching and iopoll */ - struct io_wq_work_node comp_list; - /* cache ->apoll->events */ - __poll_t apoll_events; - }; - atomic_t refs; - atomic_t poll_refs; - struct io_task_work io_task_work; - /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ - union { - struct hlist_node hash_node; - struct { - u64 extra1; - u64 extra2; - }; - }; - /* internal polling, see IORING_FEAT_FAST_POLL */ - struct async_poll *apoll; - /* opcode allocated if it needs to store data for async defer */ - void *async_data; - /* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */ - struct io_kiocb *link; - /* custom credentials, valid IFF REQ_F_CREDS is set */ - const struct cred *creds; - struct io_wq_work work; -}; - -struct io_cancel_data { - struct io_ring_ctx *ctx; - union { - u64 data; - struct file *file; - }; - u32 flags; - int seq; -}; - -struct io_overflow_cqe { - struct list_head list; - struct io_uring_cqe cqe; -}; - -struct io_mapped_ubuf { - u64 ubuf; - u64 ubuf_end; - unsigned int nr_bvecs; - unsigned long acct_pages; - struct bio_vec bvec[]; -}; - -#endif diff --git a/io_uring/refs.h b/io_uring/refs.h index 334c5ead4c43..1336de3f2a30 100644 --- a/io_uring/refs.h +++ b/io_uring/refs.h @@ -2,7 +2,7 @@ #define IOU_REQ_REF_H #include -#include "io_uring_types.h" +#include /* * Shamelessly stolen from the mm implementation of page reference checking, -- cgit v1.2.3 From ad163a7e2562230c77102c60f668bac440e60cce Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 18 Jun 2022 19:44:33 -0600 Subject: io_uring: move a few private types to local headers Commit 3a3d47fa9cfd ("io_uring: make io_uring_types.h public") moved a bunch of io_uring types to a kernel wide header, so we could make tracing a bit saner rather than pass in a ton of arguments. However, there are a few types in there that are not really needed to be system wide. Move the cancel data and mapped buffers back to the appropriate io_uring local headers. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 18 ------------------ io_uring/cancel.h | 13 +++++++++++++ io_uring/fdinfo.c | 1 + io_uring/poll.h | 1 + io_uring/rsrc.h | 8 ++++++++ io_uring/timeout.h | 1 + 6 files changed, 24 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 779c72da5b8f..2015f3ea7cb7 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -528,27 +528,9 @@ struct io_kiocb { struct io_wq_work work; }; -struct io_cancel_data { - struct io_ring_ctx *ctx; - union { - u64 data; - struct file *file; - }; - u32 flags; - int seq; -}; - struct io_overflow_cqe { struct list_head list; struct io_uring_cqe cqe; }; -struct io_mapped_ubuf { - u64 ubuf; - u64 ubuf_end; - unsigned int nr_bvecs; - unsigned long acct_pages; - struct bio_vec bvec[]; -}; - #endif diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 8dd259dc383e..2338012a5b06 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -1,5 +1,18 @@ // SPDX-License-Identifier: GPL-2.0 +#include + +struct io_cancel_data { + struct io_ring_ctx *ctx; + union { + u64 data; + struct file *file; + }; + u32 flags; + int seq; +}; + + int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 61c35707a6cf..b29e2d02216f 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -13,6 +13,7 @@ #include "sqpoll.h" #include "fdinfo.h" #include "cancel.h" +#include "rsrc.h" #ifdef CONFIG_PROC_FS static __cold int io_uring_show_cred(struct seq_file *m, unsigned int id, diff --git a/io_uring/poll.h b/io_uring/poll.h index fa3e19790281..c40673d7da01 100644 --- a/io_uring/poll.h +++ b/io_uring/poll.h @@ -24,6 +24,7 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags); int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags); +struct io_cancel_data; int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned issue_flags); int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 872c86312cbc..03f26516e994 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -45,6 +45,14 @@ struct io_rsrc_node { bool done; }; +struct io_mapped_ubuf { + u64 ubuf; + u64 ubuf_end; + unsigned int nr_bvecs; + unsigned long acct_pages; + struct bio_vec bvec[]; +}; + void io_rsrc_put_work(struct work_struct *work); void io_rsrc_refs_refill(struct io_ring_ctx *ctx); void io_wait_rsrc_data(struct io_rsrc_data *data); diff --git a/io_uring/timeout.h b/io_uring/timeout.h index dd7cfb0d9366..858c62644897 100644 --- a/io_uring/timeout.h +++ b/io_uring/timeout.h @@ -22,6 +22,7 @@ static inline struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req) } __cold void io_flush_timeouts(struct io_ring_ctx *ctx); +struct io_cancel_data; int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, bool cancel_all); -- cgit v1.2.3 From d9dee4302a7cbd6c0142dbdf6d150acc7459de0d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 19 Jun 2022 12:26:08 +0100 Subject: io_uring: remove ->flush_cqes optimisation It's not clear how widely used IOSQE_CQE_SKIP_SUCCESS is, and how often ->flush_cqes flag prevents from completion being flushed. Sometimes it's high level of concurrency that enables it at least for one CQE, but sometimes it doesn't save much because nobody waiting on the CQ. Remove ->flush_cqes flag and the optimisation, it should benefit the normal use case. Note, that there is no spurious eventfd problem with that as checks for spuriousness were incorporated into io_eventfd_signal(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/692e81eeddccc096f449a7960365fa7b4a18f8e6.1655637157.git.asml.silence@gmail.com [axboe: remove now dead state->flush_cqes variable] Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 1 - io_uring/io_uring.c | 23 ++++++++++------------- io_uring/io_uring.h | 2 -- 3 files changed, 10 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 2015f3ea7cb7..6bcd7bff6479 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -148,7 +148,6 @@ struct io_submit_state { bool plug_started; bool need_plug; - bool flush_cqes; unsigned short submit_nr; struct blk_plug plug; }; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 61d4e6d0731a..16a625e854ec 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1250,22 +1250,19 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) struct io_wq_work_node *node, *prev; struct io_submit_state *state = &ctx->submit_state; - if (state->flush_cqes) { - spin_lock(&ctx->completion_lock); - wq_list_for_each(node, prev, &state->compl_reqs) { - struct io_kiocb *req = container_of(node, struct io_kiocb, - comp_list); - - if (!(req->flags & REQ_F_CQE_SKIP)) - __io_fill_cqe_req(ctx, req); - } + spin_lock(&ctx->completion_lock); + wq_list_for_each(node, prev, &state->compl_reqs) { + struct io_kiocb *req = container_of(node, struct io_kiocb, + comp_list); - io_commit_cqring(ctx); - spin_unlock(&ctx->completion_lock); - io_cqring_ev_posted(ctx); - state->flush_cqes = false; + if (!(req->flags & REQ_F_CQE_SKIP)) + __io_fill_cqe_req(ctx, req); } + io_commit_cqring(ctx); + spin_unlock(&ctx->completion_lock); + io_cqring_ev_posted(ctx); + io_free_batch_list(ctx, state->compl_reqs.first); INIT_WQ_LIST(&state->compl_reqs); } diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 7b2055b342df..bdc62727638b 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -219,8 +219,6 @@ static inline void io_req_add_compl_list(struct io_kiocb *req) { struct io_submit_state *state = &req->ctx->submit_state; - if (!(req->flags & REQ_F_CQE_SKIP)) - state->flush_cqes = true; wq_list_add_tail(&req->comp_list, &state->compl_reqs); } -- cgit v1.2.3 From 305bef98870816ae58357d647521891ec558a92e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 20 Jun 2022 01:25:55 +0100 Subject: io_uring: hide eventfd assumptions in eventfd paths Some io_uring-eventfd users assume that there won't be spurious wakeups. That assumption has to be honoured by all io_cqring_ev_posted() callers, which is inconvenient and from time to time leads to problems but should be maintained to not break the userspace. Instead of making the callers track whether a CQE was posted or not, hide it inside io_eventfd_signal(). It saves ->cached_cq_tail it saw last time and triggers the eventfd only when ->cached_cq_tail changed since then. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/0ffc66bae37a2513080b601e4370e147faaa72c5.1655684496.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 2 ++ io_uring/io_uring.c | 44 +++++++++++++++++++++++++----------------- io_uring/timeout.c | 3 +-- 3 files changed, 29 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 6bcd7bff6479..5987f8acca38 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -314,6 +314,8 @@ struct io_ring_ctx { struct list_head defer_list; unsigned sq_thread_idle; + /* protected by ->completion_lock */ + unsigned evfd_last_cq_tail; }; enum { diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 707b599b9224..84f923625216 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -473,6 +473,22 @@ static __cold void io_queue_deferred(struct io_ring_ctx *ctx) static void io_eventfd_signal(struct io_ring_ctx *ctx) { struct io_ev_fd *ev_fd; + bool skip; + + spin_lock(&ctx->completion_lock); + /* + * Eventfd should only get triggered when at least one event has been + * posted. Some applications rely on the eventfd notification count only + * changing IFF a new CQE has been added to the CQ ring. There's no + * depedency on 1:1 relationship between how many times this function is + * called (and hence the eventfd count) and number of CQEs posted to the + * CQ ring. + */ + skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail; + ctx->evfd_last_cq_tail = ctx->cached_cq_tail; + spin_unlock(&ctx->completion_lock); + if (skip) + return; rcu_read_lock(); /* @@ -511,13 +527,6 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) io_eventfd_signal(ctx); } -/* - * This should only get called when at least one event has been posted. - * Some applications rely on the eventfd notification count only changing - * IFF a new CQE has been added to the CQ ring. There's no depedency on - * 1:1 relationship between how many times this function is called (and - * hence the eventfd count) and number of CQEs posted to the CQ ring. - */ void io_cqring_ev_posted(struct io_ring_ctx *ctx) { if (unlikely(ctx->off_timeout_used || ctx->drain_active || @@ -530,7 +539,7 @@ void io_cqring_ev_posted(struct io_ring_ctx *ctx) /* Returns true if there are no backlogged entries after the flush */ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) { - bool all_flushed, posted; + bool all_flushed; size_t cqe_size = sizeof(struct io_uring_cqe); if (!force && __io_cqring_events(ctx) == ctx->cq_entries) @@ -539,7 +548,6 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) if (ctx->flags & IORING_SETUP_CQE32) cqe_size <<= 1; - posted = false; spin_lock(&ctx->completion_lock); while (!list_empty(&ctx->cq_overflow_list)) { struct io_uring_cqe *cqe = io_get_cqe(ctx); @@ -554,7 +562,6 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) else io_account_cq_overflow(ctx); - posted = true; list_del(&ocqe->list); kfree(ocqe); } @@ -567,8 +574,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) io_commit_cqring(ctx); spin_unlock(&ctx->completion_lock); - if (posted) - io_cqring_ev_posted(ctx); + io_cqring_ev_posted(ctx); return all_flushed; } @@ -758,8 +764,7 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, filled = io_fill_cqe_aux(ctx, user_data, res, cflags); io_commit_cqring(ctx); spin_unlock(&ctx->completion_lock); - if (filled) - io_cqring_ev_posted(ctx); + io_cqring_ev_posted(ctx); return filled; } @@ -940,14 +945,12 @@ __cold void io_free_req(struct io_kiocb *req) static void __io_req_find_next_prep(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; - bool posted; spin_lock(&ctx->completion_lock); - posted = io_disarm_next(req); + io_disarm_next(req); io_commit_cqring(ctx); spin_unlock(&ctx->completion_lock); - if (posted) - io_cqring_ev_posted(ctx); + io_cqring_ev_posted(ctx); } static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) @@ -2428,6 +2431,11 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, kfree(ev_fd); return ret; } + + spin_lock(&ctx->completion_lock); + ctx->evfd_last_cq_tail = ctx->cached_cq_tail; + spin_unlock(&ctx->completion_lock); + ev_fd->eventfd_async = eventfd_async; ctx->has_evfd = true; rcu_assign_pointer(ctx->io_ev_fd, ev_fd); diff --git a/io_uring/timeout.c b/io_uring/timeout.c index a79a7d6ef1b3..424b2fc858b8 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -629,7 +629,6 @@ __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, spin_unlock_irq(&ctx->timeout_lock); io_commit_cqring(ctx); spin_unlock(&ctx->completion_lock); - if (canceled != 0) - io_cqring_ev_posted(ctx); + io_cqring_ev_posted(ctx); return canceled != 0; } -- cgit v1.2.3 From f88262e60bb9cb5740891672ce9f405e7f9393e5 Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Wed, 22 Jun 2022 06:40:23 -0700 Subject: io_uring: lockless task list With networking use cases we see contention on the spinlock used to protect the task_list when multiple threads try and add completions at once. Instead we can use a lockless list, and assume that the first caller to add to the list is responsible for kicking off task work. Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20220622134028.2013417-4-dylany@fb.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 2 +- io_uring/io_uring.c | 38 +++++++++----------------------------- io_uring/tctx.c | 3 +-- io_uring/tctx.h | 6 +++--- 4 files changed, 14 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 5987f8acca38..918165a20053 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -428,7 +428,7 @@ typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); struct io_task_work { union { - struct io_wq_work_node node; + struct llist_node node; struct llist_node fallback_node; }; io_req_tw_func_t func; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index bf7ca2b279d3..0124335c6d09 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -986,11 +986,12 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) percpu_ref_put(&ctx->refs); } -static void handle_tw_list(struct io_wq_work_node *node, + +static void handle_tw_list(struct llist_node *node, struct io_ring_ctx **ctx, bool *locked) { do { - struct io_wq_work_node *next = node->next; + struct llist_node *next = node->next; struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); @@ -1014,23 +1015,11 @@ void tctx_task_work(struct callback_head *cb) struct io_ring_ctx *ctx = NULL; struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); + struct llist_node *node = llist_del_all(&tctx->task_list); - while (1) { - struct io_wq_work_node *node; - - spin_lock_irq(&tctx->task_lock); - node = tctx->task_list.first; - INIT_WQ_LIST(&tctx->task_list); - if (!node) - tctx->task_running = false; - spin_unlock_irq(&tctx->task_lock); - if (!node) - break; + if (node) { handle_tw_list(node, &ctx, &uring_locked); cond_resched(); - - if (data_race(!tctx->task_list.first) && uring_locked) - io_submit_flush_completions(ctx); } ctx_flush_and_put(ctx, &uring_locked); @@ -1044,16 +1033,10 @@ void io_req_task_work_add(struct io_kiocb *req) { struct io_uring_task *tctx = req->task->io_uring; struct io_ring_ctx *ctx = req->ctx; - struct io_wq_work_node *node; - unsigned long flags; + struct llist_node *node; bool running; - spin_lock_irqsave(&tctx->task_lock, flags); - wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); - running = tctx->task_running; - if (!running) - tctx->task_running = true; - spin_unlock_irqrestore(&tctx->task_lock, flags); + running = !llist_add(&req->io_task_work.node, &tctx->task_list); /* task_work already pending, we're done */ if (running) @@ -1065,11 +1048,8 @@ void io_req_task_work_add(struct io_kiocb *req) if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) return; - spin_lock_irqsave(&tctx->task_lock, flags); - tctx->task_running = false; - node = tctx->task_list.first; - INIT_WQ_LIST(&tctx->task_list); - spin_unlock_irqrestore(&tctx->task_lock, flags); + + node = llist_del_all(&tctx->task_list); while (node) { req = container_of(node, struct io_kiocb, io_task_work.node); diff --git a/io_uring/tctx.c b/io_uring/tctx.c index 7a68ba9beec3..7f97d97fef0a 100644 --- a/io_uring/tctx.c +++ b/io_uring/tctx.c @@ -86,8 +86,7 @@ __cold int io_uring_alloc_task_context(struct task_struct *task, atomic_set(&tctx->in_idle, 0); atomic_set(&tctx->inflight_tracked, 0); task->io_uring = tctx; - spin_lock_init(&tctx->task_lock); - INIT_WQ_LIST(&tctx->task_list); + init_llist_head(&tctx->task_list); init_task_work(&tctx->task_work, tctx_task_work); return 0; } diff --git a/io_uring/tctx.h b/io_uring/tctx.h index c8566ea5dca4..8a33ff6e5d91 100644 --- a/io_uring/tctx.h +++ b/io_uring/tctx.h @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +#include + /* * Arbitrary limit, can be raised if need be */ @@ -19,9 +21,7 @@ struct io_uring_task { struct percpu_counter inflight; struct { /* task_work */ - spinlock_t task_lock; - bool task_running; - struct io_wq_work_list task_list; + struct llist_head task_list; struct callback_head task_work; } ____cacheline_aligned_in_smp; }; -- cgit v1.2.3 From 3218e5d32dbcf1b9c6dc589eca21deebb14215fa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 25 Jun 2022 11:52:59 +0100 Subject: io_uring: fuse fallback_node and normal tw node Now as both normal and fallback paths use llist, just keep one node head in struct io_task_work and kill off ->fallback_node. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d04ebde409f7b162fe247b361b4486b193293e46.1656153285.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 5 +---- io_uring/io_uring.c | 5 ++--- 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 918165a20053..3ca8f363f504 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -427,10 +427,7 @@ enum { typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); struct io_task_work { - union { - struct llist_node node; - struct llist_node fallback_node; - }; + struct llist_node node; io_req_tw_func_t func; }; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 45538b3c3a76..86a0b0c6f5bf 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -233,7 +233,7 @@ static __cold void io_fallback_req_func(struct work_struct *work) bool locked = false; percpu_ref_get(&ctx->refs); - llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node) + llist_for_each_entry_safe(req, tmp, node, io_task_work.node) req->io_task_work.func(req, &locked); if (locked) { @@ -1091,13 +1091,12 @@ void io_req_task_work_add(struct io_kiocb *req) if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) return; - node = llist_del_all(&tctx->task_list); while (node) { req = container_of(node, struct io_kiocb, io_task_work.node); node = node->next; - if (llist_add(&req->io_task_work.fallback_node, + if (llist_add(&req->io_task_work.node, &req->ctx->fallback_llist)) schedule_delayed_work(&req->ctx->fallback_work, 1); } -- cgit v1.2.3 From 6e73dffbb93cb8797cd4e42e98d837edf0f1a967 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 25 Jun 2022 11:55:38 +0100 Subject: io_uring: let to set a range for file slot allocation From recently io_uring provides an option to allocate a file index for operation registering fixed files. However, it's utterly unusable with mixed approaches when for a part of files the userspace knows better where to place it, as it may race and users don't have any sane way to pick a slot and hoping it will not be taken. Let the userspace to register a range of fixed file slots in which the auto-allocation happens. The use case is splittting the fixed table in two parts, where on of them is used for auto-allocation and another for slot-specified operations. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/66ab0394e436f38437cf7c44676e1920d09687ad.1656154403.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 3 +++ include/uapi/linux/io_uring.h | 13 +++++++++++++ io_uring/filetable.c | 24 ++++++++++++++++++++---- io_uring/filetable.h | 20 +++++++++++++++++--- io_uring/io_uring.c | 6 ++++++ io_uring/rsrc.c | 2 ++ 6 files changed, 61 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 3ca8f363f504..26ef11e978d4 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -233,6 +233,9 @@ struct io_ring_ctx { unsigned long check_cq; + unsigned int file_alloc_start; + unsigned int file_alloc_end; + struct { /* * We cache a range of free CQEs we can use, once exhausted it diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index f378eabbff21..cf95354198a3 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -449,6 +449,9 @@ enum { /* sync cancelation API */ IORING_REGISTER_SYNC_CANCEL = 24, + /* register a range of fixed file slots for automatic slot allocation */ + IORING_REGISTER_FILE_ALLOC_RANGE = 25, + /* this goes last */ IORING_REGISTER_LAST }; @@ -595,4 +598,14 @@ struct io_uring_sync_cancel_reg { __u64 pad[4]; }; +/* + * Argument for IORING_REGISTER_FILE_ALLOC_RANGE + * The range is specified as [off, off + len) + */ +struct io_uring_file_index_range { + __u32 off; + __u32 len; + __u64 resv; +}; + #endif diff --git a/io_uring/filetable.c b/io_uring/filetable.c index abaa5ba7f655..7b473259f3f4 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -16,7 +16,7 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) { struct io_file_table *table = &ctx->file_table; - unsigned long nr = ctx->nr_user_files; + unsigned long nr = ctx->file_alloc_end; int ret; do { @@ -24,11 +24,10 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) if (ret != nr) return ret; - if (!table->alloc_hint) + if (table->alloc_hint == ctx->file_alloc_start) break; - nr = table->alloc_hint; - table->alloc_hint = 0; + table->alloc_hint = ctx->file_alloc_start; } while (1); return -ENFILE; @@ -175,3 +174,20 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) io_rsrc_node_switch(ctx, ctx->file_data); return 0; } + +int io_register_file_alloc_range(struct io_ring_ctx *ctx, + struct io_uring_file_index_range __user *arg) +{ + struct io_uring_file_index_range range; + u32 end; + + if (copy_from_user(&range, arg, sizeof(range))) + return -EFAULT; + if (check_add_overflow(range.off, range.len, &end)) + return -EOVERFLOW; + if (range.resv || end > ctx->nr_user_files) + return -EINVAL; + + io_file_table_set_alloc_range(ctx, range.off, range.len); + return 0; +} diff --git a/io_uring/filetable.h b/io_uring/filetable.h index 79eb50c1980e..ff3a712e11bf 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -3,9 +3,7 @@ #define IOU_FILE_TABLE_H #include - -struct io_ring_ctx; -struct io_kiocb; +#include /* * FFS_SCM is only available on 64-bit archs, for 32-bit we just define it as 0 @@ -33,6 +31,9 @@ int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file, unsigned int file_slot); int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset); +int io_register_file_alloc_range(struct io_ring_ctx *ctx, + struct io_uring_file_index_range __user *arg); + unsigned int io_file_get_flags(struct file *file); static inline void io_file_bitmap_clear(struct io_file_table *table, int bit) @@ -71,4 +72,17 @@ static inline void io_fixed_file_set(struct io_fixed_file *file_slot, file_slot->file_ptr = file_ptr; } +static inline void io_reset_alloc_hint(struct io_ring_ctx *ctx) +{ + ctx->file_table.alloc_hint = ctx->file_alloc_start; +} + +static inline void io_file_table_set_alloc_range(struct io_ring_ctx *ctx, + unsigned off, unsigned len) +{ + ctx->file_alloc_start = off; + ctx->file_alloc_end = off + len; + io_reset_alloc_hint(ctx); +} + #endif diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 070ee9ec9ee7..745264938a48 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3866,6 +3866,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, break; ret = io_sync_cancel(ctx, arg); break; + case IORING_REGISTER_FILE_ALLOC_RANGE: + ret = -EINVAL; + if (!arg || nr_args) + break; + ret = io_register_file_alloc_range(ctx, arg); + break; default: ret = -EINVAL; break; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 706fa020505b..d2e589c703d0 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1012,6 +1012,8 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, io_file_bitmap_set(&ctx->file_table, i); } + /* default it to the whole table */ + io_file_table_set_alloc_range(ctx, 0, ctx->nr_user_files); io_rsrc_node_switch(ctx, NULL); return 0; fail: -- cgit v1.2.3 From 9b797a37c4bd83b03cedcfbd15852b836f5e562c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Jul 2022 14:16:20 -0600 Subject: io_uring: add abstraction around apoll cache In preparation for adding limits, and one more user, abstract out the core bits of the allocation+free cache. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 6 +++++- io_uring/alloc_cache.h | 41 +++++++++++++++++++++++++++++++++++++++++ io_uring/io_uring.c | 8 ++++---- io_uring/poll.c | 18 +++++------------- io_uring/poll.h | 9 +++++++-- 5 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 io_uring/alloc_cache.h (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 26ef11e978d4..b548da03b563 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -158,6 +158,10 @@ struct io_ev_fd { struct rcu_head rcu; }; +struct io_alloc_cache { + struct hlist_head list; +}; + struct io_ring_ctx { /* const or read-mostly hot data */ struct { @@ -216,7 +220,7 @@ struct io_ring_ctx { struct io_hash_table cancel_table_locked; struct list_head cq_overflow_list; - struct list_head apoll_cache; + struct io_alloc_cache apoll_cache; struct xarray personalities; u32 pers_next; } ____cacheline_aligned_in_smp; diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h new file mode 100644 index 000000000000..98f2374c37c7 --- /dev/null +++ b/io_uring/alloc_cache.h @@ -0,0 +1,41 @@ +#ifndef IOU_ALLOC_CACHE_H +#define IOU_ALLOC_CACHE_H + +struct io_cache_entry { + struct hlist_node node; +}; + +static inline void io_alloc_cache_put(struct io_alloc_cache *cache, + struct io_cache_entry *entry) +{ + hlist_add_head(&entry->node, &cache->list); +} + +static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache) +{ + if (!hlist_empty(&cache->list)) { + struct hlist_node *node = cache->list.first; + + hlist_del(node); + return container_of(node, struct io_cache_entry, node); + } + + return NULL; +} + +static inline void io_alloc_cache_init(struct io_alloc_cache *cache) +{ + INIT_HLIST_HEAD(&cache->list); +} + +static inline void io_alloc_cache_free(struct io_alloc_cache *cache, + void (*free)(struct io_cache_entry *)) +{ + while (!hlist_empty(&cache->list)) { + struct hlist_node *node = cache->list.first; + + hlist_del(node); + free(container_of(node, struct io_cache_entry, node)); + } +} +#endif diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4d1ce58b015e..a360a3d390c6 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -92,6 +92,7 @@ #include "timeout.h" #include "poll.h" +#include "alloc_cache.h" #define IORING_MAX_ENTRIES 32768 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) @@ -295,7 +296,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->sqd_list); INIT_LIST_HEAD(&ctx->cq_overflow_list); INIT_LIST_HEAD(&ctx->io_buffers_cache); - INIT_LIST_HEAD(&ctx->apoll_cache); + io_alloc_cache_init(&ctx->apoll_cache); init_completion(&ctx->ref_comp); xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); mutex_init(&ctx->uring_lock); @@ -1180,8 +1181,7 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node) if (apoll->double_poll) kfree(apoll->double_poll); - list_add(&apoll->poll.wait.entry, - &ctx->apoll_cache); + io_alloc_cache_put(&ctx->apoll_cache, &apoll->cache); req->flags &= ~REQ_F_POLLED; } if (req->flags & IO_REQ_LINK_FLAGS) @@ -2467,7 +2467,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) if (ctx->rings) __io_cqring_overflow_flush(ctx, true); io_eventfd_unregister(ctx); - io_flush_apoll_cache(ctx); + io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free); mutex_unlock(&ctx->uring_lock); io_destroy_buffers(ctx); if (ctx->sq_creds) diff --git a/io_uring/poll.c b/io_uring/poll.c index 052fcb647208..dadd293749b0 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -590,16 +590,15 @@ static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, unsigned issue_flags) { struct io_ring_ctx *ctx = req->ctx; + struct io_cache_entry *entry; struct async_poll *apoll; if (req->flags & REQ_F_POLLED) { apoll = req->apoll; kfree(apoll->double_poll); } else if (!(issue_flags & IO_URING_F_UNLOCKED) && - !list_empty(&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); - list_del_init(&apoll->poll.wait.entry); + (entry = io_alloc_cache_get(&ctx->apoll_cache)) != NULL) { + apoll = container_of(entry, struct async_poll, cache); } else { apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); if (unlikely(!apoll)) @@ -960,14 +959,7 @@ out: return IOU_OK; } -void io_flush_apoll_cache(struct io_ring_ctx *ctx) +void io_apoll_cache_free(struct io_cache_entry *entry) { - struct async_poll *apoll; - - while (!list_empty(&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); - list_del(&apoll->poll.wait.entry); - kfree(apoll); - } + kfree(container_of(entry, struct async_poll, cache)); } diff --git a/io_uring/poll.h b/io_uring/poll.h index 95f192c7babb..5f3bae50fc81 100644 --- a/io_uring/poll.h +++ b/io_uring/poll.h @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +#include "alloc_cache.h" + enum { IO_APOLL_OK, IO_APOLL_ABORTED, @@ -14,7 +16,10 @@ struct io_poll { }; struct async_poll { - struct io_poll poll; + union { + struct io_poll poll; + struct io_cache_entry cache; + }; struct io_poll *double_poll; }; @@ -31,4 +36,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags); bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, bool cancel_all); -void io_flush_apoll_cache(struct io_ring_ctx *ctx); +void io_apoll_cache_free(struct io_cache_entry *entry); -- cgit v1.2.3 From 9731bc9855dc169f27433fef3c4d0ff3496c512d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Jul 2022 14:20:54 -0600 Subject: io_uring: impose max limit on apoll cache Caches like this tend to grow to the peak size, and then never get any smaller. Impose a max limit on the size, to prevent it from growing too big. A somewhat randomly chosen 512 is the max size we'll allow the cache to get. If a batch of frees come in and would bring it over that, we simply start kfree'ing the surplus. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 1 + io_uring/alloc_cache.h | 16 ++++++++++++++-- io_uring/io_uring.c | 3 ++- 3 files changed, 17 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index b548da03b563..bf8f95332eda 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -160,6 +160,7 @@ struct io_ev_fd { struct io_alloc_cache { struct hlist_head list; + unsigned int nr_cached; }; struct io_ring_ctx { diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h index 98f2374c37c7..729793ae9712 100644 --- a/io_uring/alloc_cache.h +++ b/io_uring/alloc_cache.h @@ -1,14 +1,24 @@ #ifndef IOU_ALLOC_CACHE_H #define IOU_ALLOC_CACHE_H +/* + * Don't allow the cache to grow beyond this size. + */ +#define IO_ALLOC_CACHE_MAX 512 + struct io_cache_entry { struct hlist_node node; }; -static inline void io_alloc_cache_put(struct io_alloc_cache *cache, +static inline bool io_alloc_cache_put(struct io_alloc_cache *cache, struct io_cache_entry *entry) { - hlist_add_head(&entry->node, &cache->list); + if (cache->nr_cached < IO_ALLOC_CACHE_MAX) { + cache->nr_cached++; + hlist_add_head(&entry->node, &cache->list); + return true; + } + return false; } static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache) @@ -26,6 +36,7 @@ static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *c static inline void io_alloc_cache_init(struct io_alloc_cache *cache) { INIT_HLIST_HEAD(&cache->list); + cache->nr_cached = 0; } static inline void io_alloc_cache_free(struct io_alloc_cache *cache, @@ -37,5 +48,6 @@ static inline void io_alloc_cache_free(struct io_alloc_cache *cache, hlist_del(node); free(container_of(node, struct io_cache_entry, node)); } + cache->nr_cached = 0; } #endif diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index a360a3d390c6..c9c23e459766 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1181,7 +1181,8 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node) if (apoll->double_poll) kfree(apoll->double_poll); - io_alloc_cache_put(&ctx->apoll_cache, &apoll->cache); + if (!io_alloc_cache_put(&ctx->apoll_cache, &apoll->cache)) + kfree(apoll); req->flags &= ~REQ_F_POLLED; } if (req->flags & IO_REQ_LINK_FLAGS) -- cgit v1.2.3 From 43e0bbbd0b0e30d232fd8e9e908125b5c49a9fbc Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Jul 2022 14:30:09 -0600 Subject: io_uring: add netmsg cache For recvmsg/sendmsg, if they don't complete inline, we currently need to allocate a struct io_async_msghdr for each request. This is a somewhat large struct. Hook up sendmsg/recvmsg to use the io_alloc_cache. This reduces the alloc + free overhead considerably, yielding 4-5% of extra performance running netbench. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 6 ++-- io_uring/io_uring.c | 3 ++ io_uring/net.c | 63 ++++++++++++++++++++++++++++++++++++------ io_uring/net.h | 13 ++++++++- 4 files changed, 73 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index bf8f95332eda..d54b8b7e0746 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -222,8 +222,7 @@ struct io_ring_ctx { struct io_hash_table cancel_table_locked; struct list_head cq_overflow_list; struct io_alloc_cache apoll_cache; - struct xarray personalities; - u32 pers_next; + struct io_alloc_cache netmsg_cache; } ____cacheline_aligned_in_smp; /* IRQ completion list, under ->completion_lock */ @@ -241,6 +240,9 @@ struct io_ring_ctx { unsigned int file_alloc_start; unsigned int file_alloc_end; + struct xarray personalities; + u32 pers_next; + struct { /* * We cache a range of free CQEs we can use, once exhausted it diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c9c23e459766..f697ca4e8f55 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -89,6 +89,7 @@ #include "kbuf.h" #include "rsrc.h" #include "cancel.h" +#include "net.h" #include "timeout.h" #include "poll.h" @@ -297,6 +298,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->cq_overflow_list); INIT_LIST_HEAD(&ctx->io_buffers_cache); io_alloc_cache_init(&ctx->apoll_cache); + io_alloc_cache_init(&ctx->netmsg_cache); init_completion(&ctx->ref_comp); xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); mutex_init(&ctx->uring_lock); @@ -2469,6 +2471,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) __io_cqring_overflow_flush(ctx, true); io_eventfd_unregister(ctx); io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free); + io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free); mutex_unlock(&ctx->uring_lock); io_destroy_buffers(ctx); if (ctx->sq_creds) diff --git a/io_uring/net.c b/io_uring/net.c index 6679069eeef1..185553174437 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -12,6 +12,7 @@ #include "io_uring.h" #include "kbuf.h" +#include "alloc_cache.h" #include "net.h" #if defined(CONFIG_NET) @@ -97,18 +98,55 @@ static bool io_net_retry(struct socket *sock, int flags) return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; } +static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) +{ + struct io_async_msghdr *hdr = req->async_data; + + if (!hdr || issue_flags & IO_URING_F_UNLOCKED) + return; + + /* Let normal cleanup path reap it if we fail adding to the cache */ + if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) { + req->async_data = NULL; + req->flags &= ~REQ_F_ASYNC_DATA; + } +} + +static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req, + unsigned int issue_flags) +{ + struct io_ring_ctx *ctx = req->ctx; + struct io_cache_entry *entry; + + if (!(issue_flags & IO_URING_F_UNLOCKED) && + (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) { + struct io_async_msghdr *hdr; + + hdr = container_of(entry, struct io_async_msghdr, cache); + req->flags |= REQ_F_ASYNC_DATA; + req->async_data = hdr; + return hdr; + } + + if (!io_alloc_async_data(req)) + return req->async_data; + + return NULL; +} + static int io_setup_async_msg(struct io_kiocb *req, - struct io_async_msghdr *kmsg) + struct io_async_msghdr *kmsg, + unsigned int issue_flags) { struct io_async_msghdr *async_msg = req->async_data; if (async_msg) return -EAGAIN; - if (io_alloc_async_data(req)) { + async_msg = io_recvmsg_alloc_async(req, issue_flags); + if (!async_msg) { kfree(kmsg->free_iov); return -ENOMEM; } - async_msg = req->async_data; req->flags |= REQ_F_NEED_CLEANUP; memcpy(async_msg, kmsg, sizeof(*kmsg)); async_msg->msg.msg_name = &async_msg->addr; @@ -195,7 +233,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) if (!(req->flags & REQ_F_POLLED) && (sr->flags & IORING_RECVSEND_POLL_FIRST)) - return io_setup_async_msg(req, kmsg); + return io_setup_async_msg(req, kmsg, issue_flags); flags = sr->msg_flags; if (issue_flags & IO_URING_F_NONBLOCK) @@ -207,13 +245,13 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) if (ret < min_ret) { if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) - return io_setup_async_msg(req, kmsg); + return io_setup_async_msg(req, kmsg, issue_flags); if (ret == -ERESTARTSYS) ret = -EINTR; if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; req->flags |= REQ_F_PARTIAL_IO; - return io_setup_async_msg(req, kmsg); + return io_setup_async_msg(req, kmsg, issue_flags); } req_set_fail(req); } @@ -221,6 +259,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) if (kmsg->free_iov) kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; + io_netmsg_recycle(req, issue_flags); if (ret >= 0) ret += sr->done_io; else if (sr->done_io) @@ -495,7 +534,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) if (!(req->flags & REQ_F_POLLED) && (sr->flags & IORING_RECVSEND_POLL_FIRST)) - return io_setup_async_msg(req, kmsg); + return io_setup_async_msg(req, kmsg, issue_flags); if (io_do_buffer_select(req)) { void __user *buf; @@ -519,13 +558,13 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags); if (ret < min_ret) { if (ret == -EAGAIN && force_nonblock) - return io_setup_async_msg(req, kmsg); + return io_setup_async_msg(req, kmsg, issue_flags); if (ret == -ERESTARTSYS) ret = -EINTR; if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; req->flags |= REQ_F_PARTIAL_IO; - return io_setup_async_msg(req, kmsg); + return io_setup_async_msg(req, kmsg, issue_flags); } req_set_fail(req); } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { @@ -535,6 +574,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) /* fast path, check for non-NULL to avoid function call */ if (kmsg->free_iov) kfree(kmsg->free_iov); + io_netmsg_recycle(req, issue_flags); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret > 0) ret += sr->done_io; @@ -848,4 +888,9 @@ out: io_req_set_res(req, ret, 0); return IOU_OK; } + +void io_netmsg_cache_free(struct io_cache_entry *entry) +{ + kfree(container_of(entry, struct io_async_msghdr, cache)); +} #endif diff --git a/io_uring/net.h b/io_uring/net.h index 81d71d164770..178a6d8b76e0 100644 --- a/io_uring/net.h +++ b/io_uring/net.h @@ -3,9 +3,14 @@ #include #include +#include "alloc_cache.h" + #if defined(CONFIG_NET) struct io_async_msghdr { - struct iovec fast_iov[UIO_FASTIOV]; + union { + struct iovec fast_iov[UIO_FASTIOV]; + struct io_cache_entry cache; + }; /* points to an allocated iov, if NULL we use fast_iov instead */ struct iovec *free_iov; struct sockaddr __user *uaddr; @@ -40,4 +45,10 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags); int io_connect_prep_async(struct io_kiocb *req); int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_connect(struct io_kiocb *req, unsigned int issue_flags); + +void io_netmsg_cache_free(struct io_cache_entry *entry); +#else +static inline void io_netmsg_cache_free(struct io_cache_entry *entry) +{ +} #endif -- cgit v1.2.3 From 7fa875b8e53c288d616234b9daf417b0650ce1cc Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Thu, 14 Jul 2022 04:02:56 -0700 Subject: net: copy from user before calling __copy_msghdr this is in preparation for multishot receive from io_uring, where it needs to have access to the original struct user_msghdr. functionally this should be a no-op. Acked-by: Paolo Abeni Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20220714110258.1336200-2-dylany@fb.com Signed-off-by: Jens Axboe --- include/linux/socket.h | 7 +++---- io_uring/net.c | 17 +++++++++-------- net/socket.c | 37 ++++++++++++++++--------------------- 3 files changed, 28 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index 17311ad9f9af..be24f1c8568a 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -416,10 +416,9 @@ extern int recvmsg_copy_msghdr(struct msghdr *msg, struct user_msghdr __user *umsg, unsigned flags, struct sockaddr __user **uaddr, struct iovec **iov); -extern int __copy_msghdr_from_user(struct msghdr *kmsg, - struct user_msghdr __user *umsg, - struct sockaddr __user **save_addr, - struct iovec __user **uiov, size_t *nsegs); +extern int __copy_msghdr(struct msghdr *kmsg, + struct user_msghdr *umsg, + struct sockaddr __user **save_addr); /* helpers which do the actual work for syscalls */ extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size, diff --git a/io_uring/net.c b/io_uring/net.c index dc9190eafbe7..da7667ed3610 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -329,31 +329,32 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { struct io_sr_msg *sr = io_kiocb_to_cmd(req); - struct iovec __user *uiov; - size_t iov_len; + struct user_msghdr msg; int ret; - ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg, - &iomsg->uaddr, &uiov, &iov_len); + if (copy_from_user(&msg, sr->umsg, sizeof(*sr->umsg))) + return -EFAULT; + + ret = __copy_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); if (ret) return ret; if (req->flags & REQ_F_BUFFER_SELECT) { - if (iov_len == 0) { + if (msg.msg_iovlen == 0) { sr->len = iomsg->fast_iov[0].iov_len = 0; iomsg->fast_iov[0].iov_base = NULL; iomsg->free_iov = NULL; - } else if (iov_len > 1) { + } else if (msg.msg_iovlen > 1) { return -EINVAL; } else { - if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) + if (copy_from_user(iomsg->fast_iov, msg.msg_iov, sizeof(*msg.msg_iov))) return -EFAULT; sr->len = iomsg->fast_iov[0].iov_len; iomsg->free_iov = NULL; } } else { iomsg->free_iov = iomsg->fast_iov; - ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, + ret = __import_iovec(READ, msg.msg_iov, msg.msg_iovlen, UIO_FASTIOV, &iomsg->free_iov, &iomsg->msg.msg_iter, false); if (ret > 0) diff --git a/net/socket.c b/net/socket.c index 96300cdc0625..843545c21ec2 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2358,25 +2358,20 @@ struct used_address { unsigned int name_len; }; -int __copy_msghdr_from_user(struct msghdr *kmsg, - struct user_msghdr __user *umsg, - struct sockaddr __user **save_addr, - struct iovec __user **uiov, size_t *nsegs) +int __copy_msghdr(struct msghdr *kmsg, + struct user_msghdr *msg, + struct sockaddr __user **save_addr) { - struct user_msghdr msg; ssize_t err; - if (copy_from_user(&msg, umsg, sizeof(*umsg))) - return -EFAULT; - kmsg->msg_control_is_user = true; kmsg->msg_get_inq = 0; - kmsg->msg_control_user = msg.msg_control; - kmsg->msg_controllen = msg.msg_controllen; - kmsg->msg_flags = msg.msg_flags; + kmsg->msg_control_user = msg->msg_control; + kmsg->msg_controllen = msg->msg_controllen; + kmsg->msg_flags = msg->msg_flags; - kmsg->msg_namelen = msg.msg_namelen; - if (!msg.msg_name) + kmsg->msg_namelen = msg->msg_namelen; + if (!msg->msg_name) kmsg->msg_namelen = 0; if (kmsg->msg_namelen < 0) @@ -2386,11 +2381,11 @@ int __copy_msghdr_from_user(struct msghdr *kmsg, kmsg->msg_namelen = sizeof(struct sockaddr_storage); if (save_addr) - *save_addr = msg.msg_name; + *save_addr = msg->msg_name; - if (msg.msg_name && kmsg->msg_namelen) { + if (msg->msg_name && kmsg->msg_namelen) { if (!save_addr) { - err = move_addr_to_kernel(msg.msg_name, + err = move_addr_to_kernel(msg->msg_name, kmsg->msg_namelen, kmsg->msg_name); if (err < 0) @@ -2401,12 +2396,10 @@ int __copy_msghdr_from_user(struct msghdr *kmsg, kmsg->msg_namelen = 0; } - if (msg.msg_iovlen > UIO_MAXIOV) + if (msg->msg_iovlen > UIO_MAXIOV) return -EMSGSIZE; kmsg->msg_iocb = NULL; - *uiov = msg.msg_iov; - *nsegs = msg.msg_iovlen; return 0; } @@ -2418,8 +2411,10 @@ static int copy_msghdr_from_user(struct msghdr *kmsg, struct user_msghdr msg; ssize_t err; - err = __copy_msghdr_from_user(kmsg, umsg, save_addr, &msg.msg_iov, - &msg.msg_iovlen); + if (copy_from_user(&msg, umsg, sizeof(*umsg))) + return -EFAULT; + + err = __copy_msghdr(kmsg, &msg, save_addr); if (err) return err; -- cgit v1.2.3 From fe6c9c6e3e3e332b998393d214fba9d09ab0acb0 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 23 Jun 2022 10:51:46 -0700 Subject: mm: Add balance_dirty_pages_ratelimited_flags() function This adds the helper function balance_dirty_pages_ratelimited_flags(). It adds the parameter flags to balance_dirty_pages_ratelimited(). The flags parameter is passed to balance_dirty_pages(). For async buffered writes the flag value will be BDP_ASYNC. If balance_dirty_pages() gets called for async buffered write, we don't want to wait. Instead we need to indicate to the caller that throttling is needed so that it can stop writing and offload the rest of the write to a context that can block. The new helper function is also used by balance_dirty_pages_ratelimited(). Signed-off-by: Jan Kara Signed-off-by: Stefan Roesch Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220623175157.1715274-4-shr@fb.com [axboe: fix kerneltest bot 'ret' issue] Signed-off-by: Jens Axboe --- include/linux/writeback.h | 7 +++++++ mm/page-writeback.c | 51 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/writeback.h b/include/linux/writeback.h index da21d63f70e2..b8c9610c2313 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -364,7 +364,14 @@ void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty); unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh); void wb_update_bandwidth(struct bdi_writeback *wb); + +/* Invoke balance dirty pages in async mode. */ +#define BDP_ASYNC 0x0001 + void balance_dirty_pages_ratelimited(struct address_space *mapping); +int balance_dirty_pages_ratelimited_flags(struct address_space *mapping, + unsigned int flags); + bool wb_over_bg_thresh(struct bdi_writeback *wb); typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc, diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 90b1998c16a1..d0d466a5c804 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -1554,8 +1554,8 @@ static inline void wb_dirty_limits(struct dirty_throttle_control *dtc) * If we're over `background_thresh' then the writeback threads are woken to * perform some writeout. */ -static void balance_dirty_pages(struct bdi_writeback *wb, - unsigned long pages_dirtied) +static int balance_dirty_pages(struct bdi_writeback *wb, + unsigned long pages_dirtied, unsigned int flags) { struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; @@ -1575,6 +1575,7 @@ static void balance_dirty_pages(struct bdi_writeback *wb, struct backing_dev_info *bdi = wb->bdi; bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT; unsigned long start_time = jiffies; + int ret = 0; for (;;) { unsigned long now = jiffies; @@ -1803,6 +1804,10 @@ pause: period, pause, start_time); + if (flags & BDP_ASYNC) { + ret = -EAGAIN; + break; + } __set_current_state(TASK_KILLABLE); wb->dirty_sleep = now; io_schedule_timeout(pause); @@ -1834,6 +1839,7 @@ pause: if (fatal_signal_pending(current)) break; } + return ret; } static DEFINE_PER_CPU(int, bdp_ratelimits); @@ -1855,27 +1861,34 @@ static DEFINE_PER_CPU(int, bdp_ratelimits); DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0; /** - * balance_dirty_pages_ratelimited - balance dirty memory state - * @mapping: address_space which was dirtied + * balance_dirty_pages_ratelimited_flags - Balance dirty memory state. + * @mapping: address_space which was dirtied. + * @flags: BDP flags. * * Processes which are dirtying memory should call in here once for each page * which was newly dirtied. The function will periodically check the system's * dirty state and will initiate writeback if needed. * - * Once we're over the dirty memory limit we decrease the ratelimiting - * by a lot, to prevent individual processes from overshooting the limit - * by (ratelimit_pages) each. + * See balance_dirty_pages_ratelimited() for details. + * + * Return: If @flags contains BDP_ASYNC, it may return -EAGAIN to + * indicate that memory is out of balance and the caller must wait + * for I/O to complete. Otherwise, it will return 0 to indicate + * that either memory was already in balance, or it was able to sleep + * until the amount of dirty memory returned to balance. */ -void balance_dirty_pages_ratelimited(struct address_space *mapping) +int balance_dirty_pages_ratelimited_flags(struct address_space *mapping, + unsigned int flags) { struct inode *inode = mapping->host; struct backing_dev_info *bdi = inode_to_bdi(inode); struct bdi_writeback *wb = NULL; int ratelimit; + int ret = 0; int *p; if (!(bdi->capabilities & BDI_CAP_WRITEBACK)) - return; + return ret; if (inode_cgwb_enabled(inode)) wb = wb_get_create_current(bdi, GFP_KERNEL); @@ -1915,9 +1928,27 @@ void balance_dirty_pages_ratelimited(struct address_space *mapping) preempt_enable(); if (unlikely(current->nr_dirtied >= ratelimit)) - balance_dirty_pages(wb, current->nr_dirtied); + ret = balance_dirty_pages(wb, current->nr_dirtied, flags); wb_put(wb); + return ret; +} + +/** + * balance_dirty_pages_ratelimited - balance dirty memory state. + * @mapping: address_space which was dirtied. + * + * Processes which are dirtying memory should call in here once for each page + * which was newly dirtied. The function will periodically check the system's + * dirty state and will initiate writeback if needed. + * + * Once we're over the dirty memory limit we decrease the ratelimiting + * by a lot, to prevent individual processes from overshooting the limit + * by (ratelimit_pages) each. + */ +void balance_dirty_pages_ratelimited(struct address_space *mapping) +{ + balance_dirty_pages_ratelimited_flags(mapping, 0); } EXPORT_SYMBOL(balance_dirty_pages_ratelimited); -- cgit v1.2.3 From 8017553980d0bbfef3e66c583363828565afd6da Mon Sep 17 00:00:00 2001 From: Stefan Roesch Date: Thu, 23 Jun 2022 10:51:50 -0700 Subject: fs: add a FMODE_BUF_WASYNC flags for f_mode This introduces the flag FMODE_BUF_WASYNC. If devices support async buffered writes, this flag can be set. It also modifies the check in generic_write_checks to take async buffered writes into consideration. Signed-off-by: Stefan Roesch Reviewed-by: Christoph Hellwig Reviewed-by: Christian Brauner (Microsoft) Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20220623175157.1715274-8-shr@fb.com Signed-off-by: Jens Axboe --- fs/read_write.c | 4 +++- include/linux/fs.h | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/read_write.c b/fs/read_write.c index e0777eefd846..319d88825d1c 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -1660,7 +1660,9 @@ int generic_write_checks_count(struct kiocb *iocb, loff_t *count) if (iocb->ki_flags & IOCB_APPEND) iocb->ki_pos = i_size_read(inode); - if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT)) + if ((iocb->ki_flags & IOCB_NOWAIT) && + !((iocb->ki_flags & IOCB_DIRECT) || + (file->f_mode & FMODE_BUF_WASYNC))) return -EINVAL; return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count); diff --git a/include/linux/fs.h b/include/linux/fs.h index 9ad5e3520fae..bc84847c201e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -180,6 +180,9 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset, /* File supports async buffered reads */ #define FMODE_BUF_RASYNC ((__force fmode_t)0x40000000) +/* File supports async nowait buffered writes */ +#define FMODE_BUF_WASYNC ((__force fmode_t)0x80000000) + /* * Attribute flags. These should be or-ed together to figure out what * has been changed! -- cgit v1.2.3 From 66fa3cedf16abc82d19b943e3289c82e685419d5 Mon Sep 17 00:00:00 2001 From: Stefan Roesch Date: Thu, 23 Jun 2022 10:51:53 -0700 Subject: fs: Add async write file modification handling. This adds a file_modified_async() function to return -EAGAIN if the request either requires to remove privileges or needs to update the file modification time. This is required for async buffered writes, so the request gets handled in the io worker of io-uring. Signed-off-by: Stefan Roesch Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Reviewed-by: Christian Brauner (Microsoft) Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20220623175157.1715274-11-shr@fb.com Signed-off-by: Jens Axboe --- fs/inode.c | 45 ++++++++++++++++++++++++++++++++++++++++++--- include/linux/fs.h | 1 + 2 files changed, 43 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/fs/inode.c b/fs/inode.c index ff726d99ecc7..259ebf438893 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2116,17 +2116,21 @@ int file_update_time(struct file *file) EXPORT_SYMBOL(file_update_time); /** - * file_modified - handle mandated vfs changes when modifying a file + * file_modified_flags - handle mandated vfs changes when modifying a file * @file: file that was modified + * @flags: kiocb flags * * When file has been modified ensure that special * file privileges are removed and time settings are updated. * + * If IOCB_NOWAIT is set, special file privileges will not be removed and + * time settings will not be updated. It will return -EAGAIN. + * * Context: Caller must hold the file's inode lock. * * Return: 0 on success, negative errno on failure. */ -int file_modified(struct file *file) +static int file_modified_flags(struct file *file, int flags) { int ret; struct inode *inode = file_inode(file); @@ -2136,7 +2140,7 @@ int file_modified(struct file *file) * Clear the security bits if the process is not being run by root. * This keeps people from modifying setuid and setgid binaries. */ - ret = __file_remove_privs(file, 0); + ret = __file_remove_privs(file, flags); if (ret) return ret; @@ -2146,11 +2150,46 @@ int file_modified(struct file *file) ret = inode_needs_update_time(inode, &now); if (ret <= 0) return ret; + if (flags & IOCB_NOWAIT) + return -EAGAIN; return __file_update_time(file, &now, ret); } + +/** + * file_modified - handle mandated vfs changes when modifying a file + * @file: file that was modified + * + * When file has been modified ensure that special + * file privileges are removed and time settings are updated. + * + * Context: Caller must hold the file's inode lock. + * + * Return: 0 on success, negative errno on failure. + */ +int file_modified(struct file *file) +{ + return file_modified_flags(file, 0); +} EXPORT_SYMBOL(file_modified); +/** + * kiocb_modified - handle mandated vfs changes when modifying a file + * @iocb: iocb that was modified + * + * When file has been modified ensure that special + * file privileges are removed and time settings are updated. + * + * Context: Caller must hold the file's inode lock. + * + * Return: 0 on success, negative errno on failure. + */ +int kiocb_modified(struct kiocb *iocb) +{ + return file_modified_flags(iocb->ki_filp, iocb->ki_flags); +} +EXPORT_SYMBOL_GPL(kiocb_modified); + int inode_needs_sync(struct inode *inode) { if (IS_SYNC(inode)) diff --git a/include/linux/fs.h b/include/linux/fs.h index bc84847c201e..c0d99b5a166b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2390,6 +2390,7 @@ static inline void file_accessed(struct file *file) } extern int file_modified(struct file *file); +int kiocb_modified(struct kiocb *iocb); int sync_inode_metadata(struct inode *inode, int wait); -- cgit v1.2.3 From e70cb60893ca64b7df06864aa16c1cf6d6c671db Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Jul 2022 21:52:37 +0100 Subject: io_uring: export io_put_task() Make io_put_task() available to non-core parts of io_uring, we'll need it for notification infrastructure. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/3686807d4c03b72e389947b0e8692d4d44334ef0.1657643355.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 25 +++++++++++++++++++++++++ io_uring/io_uring.c | 11 +---------- io_uring/io_uring.h | 10 ++++++++++ io_uring/tctx.h | 26 -------------------------- 4 files changed, 36 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index d54b8b7e0746..368c34d14b13 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -4,6 +4,7 @@ #include #include #include +#include #include struct io_wq_work_node { @@ -43,6 +44,30 @@ struct io_hash_table { unsigned hash_bits; }; +/* + * Arbitrary limit, can be raised if need be + */ +#define IO_RINGFD_REG_MAX 16 + +struct io_uring_task { + /* submission side */ + int cached_refs; + const struct io_ring_ctx *last; + struct io_wq *io_wq; + struct file *registered_rings[IO_RINGFD_REG_MAX]; + + struct xarray xa; + struct wait_queue_head wait; + atomic_t in_idle; + atomic_t inflight_tracked; + struct percpu_counter inflight; + + struct { /* task_work */ + struct llist_head task_list; + struct callback_head task_work; + } ____cacheline_aligned_in_smp; +}; + struct io_uring { u32 head ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4b3fd645d023..7795cfedf6bf 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -608,7 +608,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) return ret; } -static void __io_put_task(struct task_struct *task, int nr) +void __io_put_task(struct task_struct *task, int nr) { struct io_uring_task *tctx = task->io_uring; @@ -618,15 +618,6 @@ static void __io_put_task(struct task_struct *task, int nr) put_task_struct_many(task, nr); } -/* must to be called somewhat shortly after putting a request */ -static inline void io_put_task(struct task_struct *task, int nr) -{ - if (likely(task == current)) - task->io_uring->cached_refs += nr; - else - __io_put_task(task, nr); -} - static void io_task_refs_refill(struct io_uring_task *tctx) { unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 5db0a60dc04e..b1c0c0a400d8 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -71,6 +71,7 @@ void io_wq_submit_work(struct io_wq_work *work); void io_free_req(struct io_kiocb *req); void io_queue_next(struct io_kiocb *req); +void __io_put_task(struct task_struct *task, int nr); bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, bool cancel_all); @@ -258,4 +259,13 @@ static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) __io_commit_cqring_flush(ctx); } +/* must to be called somewhat shortly after putting a request */ +static inline void io_put_task(struct task_struct *task, int nr) +{ + if (likely(task == current)) + task->io_uring->cached_refs += nr; + else + __io_put_task(task, nr); +} + #endif diff --git a/io_uring/tctx.h b/io_uring/tctx.h index 8a33ff6e5d91..25974beed4d6 100644 --- a/io_uring/tctx.h +++ b/io_uring/tctx.h @@ -1,31 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include - -/* - * Arbitrary limit, can be raised if need be - */ -#define IO_RINGFD_REG_MAX 16 - -struct io_uring_task { - /* submission side */ - int cached_refs; - const struct io_ring_ctx *last; - struct io_wq *io_wq; - struct file *registered_rings[IO_RINGFD_REG_MAX]; - - struct xarray xa; - struct wait_queue_head wait; - atomic_t in_idle; - atomic_t inflight_tracked; - struct percpu_counter inflight; - - struct { /* task_work */ - struct llist_head task_list; - struct callback_head task_work; - } ____cacheline_aligned_in_smp; -}; - struct io_tctx_node { struct list_head ctx_node; struct task_struct *task; -- cgit v1.2.3 From eb42cebb2cf24c48f60c32856a4bba93d42659c8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Jul 2022 21:52:38 +0100 Subject: io_uring: add zc notification infrastructure Add internal part of send zerocopy notifications. There are two main structures, the first one is struct io_notif, which carries inside struct ubuf_info and maps 1:1 to it. io_uring will be binding a number of zerocopy send requests to it and ask to complete (aka flush) it. When flushed and all attached requests and skbs complete, it'll generate one and only one CQE. There are intended to be passed into the network layer as struct msghdr::msg_ubuf. The second concept is notification slots. The userspace will be able to register an array of slots and subsequently addressing them by the index in the array. Slots are independent of each other. Each slot can have only one notifier at a time (called active notifier) but many notifiers during the lifetime. When active, a notifier not going to post any completion but the userspace can attach requests to it by specifying the corresponding slot while issueing send zc requests. Eventually, the userspace will want to "flush" the notifier losing any way to attach new requests to it, however it can use the next atomatically added notifier of this slot or of any other slot. When the network layer is done with all enqueued skbs attached to a notifier and doesn't need the specified in them user data, the flushed notifier will post a CQE. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/3ecf54c31a85762bf679b0a432c9f43ecf7e61cc.1657643355.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 5 ++ io_uring/Makefile | 2 +- io_uring/io_uring.c | 8 ++-- io_uring/io_uring.h | 2 + io_uring/notif.c | 102 +++++++++++++++++++++++++++++++++++++++++ io_uring/notif.h | 64 ++++++++++++++++++++++++++ 6 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 io_uring/notif.c create mode 100644 io_uring/notif.h (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 368c34d14b13..f7fab3758cb9 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -34,6 +34,9 @@ struct io_file_table { unsigned int alloc_hint; }; +struct io_notif; +struct io_notif_slot; + struct io_hash_bucket { spinlock_t lock; struct hlist_head list; @@ -237,6 +240,8 @@ struct io_ring_ctx { unsigned nr_user_files; unsigned nr_user_bufs; struct io_mapped_ubuf **user_bufs; + struct io_notif_slot *notif_slots; + unsigned nr_notif_slots; struct io_submit_state submit_state; diff --git a/io_uring/Makefile b/io_uring/Makefile index 466639c289be..8cc8e5387a75 100644 --- a/io_uring/Makefile +++ b/io_uring/Makefile @@ -7,5 +7,5 @@ obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ openclose.o uring_cmd.o epoll.o \ statx.o net.o msg_ring.o timeout.o \ sqpoll.o fdinfo.o tctx.o poll.o \ - cancel.o kbuf.o rsrc.o rw.o opdef.o + cancel.o kbuf.o rsrc.o rw.o opdef.o notif.o obj-$(CONFIG_IO_WQ) += io-wq.o diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 7795cfedf6bf..65ac407dd74e 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -90,6 +90,7 @@ #include "rsrc.h" #include "cancel.h" #include "net.h" +#include "notif.h" #include "timeout.h" #include "poll.h" @@ -732,9 +733,8 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx) return &rings->cqes[off]; } -static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, - u64 user_data, s32 res, u32 cflags, - bool allow_overflow) +bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags, + bool allow_overflow) { struct io_uring_cqe *cqe; @@ -2491,6 +2491,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) } #endif WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list)); + WARN_ON_ONCE(ctx->notif_slots || ctx->nr_notif_slots); io_mem_free(ctx->rings); io_mem_free(ctx->sq_sqes); @@ -2667,6 +2668,7 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) io_unregister_personality(ctx, index); if (ctx->rings) io_poll_remove_all(ctx, NULL, true); + io_notif_unregister(ctx); mutex_unlock(&ctx->uring_lock); /* failed during ring init, it couldn't have issued any requests */ diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index b1c0c0a400d8..66bfd880d07f 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -33,6 +33,8 @@ void io_req_complete_post(struct io_kiocb *req); void __io_req_complete_post(struct io_kiocb *req); bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags, bool allow_overflow); +bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags, + bool allow_overflow); void __io_commit_cqring_flush(struct io_ring_ctx *ctx); struct page **io_pin_pages(unsigned long ubuf, unsigned long len, int *npages); diff --git a/io_uring/notif.c b/io_uring/notif.c new file mode 100644 index 000000000000..6ee948af6a49 --- /dev/null +++ b/io_uring/notif.c @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include + +#include "io_uring.h" +#include "notif.h" + +static void __io_notif_complete_tw(struct callback_head *cb) +{ + struct io_notif *notif = container_of(cb, struct io_notif, task_work); + struct io_ring_ctx *ctx = notif->ctx; + + io_cq_lock(ctx); + io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq, true); + io_cq_unlock_post(ctx); + + percpu_ref_put(&ctx->refs); + kfree(notif); +} + +static inline void io_notif_complete(struct io_notif *notif) +{ + __io_notif_complete_tw(¬if->task_work); +} + +static void io_notif_complete_wq(struct work_struct *work) +{ + struct io_notif *notif = container_of(work, struct io_notif, commit_work); + + io_notif_complete(notif); +} + +static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, + struct ubuf_info *uarg, + bool success) +{ + struct io_notif *notif = container_of(uarg, struct io_notif, uarg); + + if (!refcount_dec_and_test(&uarg->refcnt)) + return; + INIT_WORK(¬if->commit_work, io_notif_complete_wq); + queue_work(system_unbound_wq, ¬if->commit_work); +} + +struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx, + struct io_notif_slot *slot) + __must_hold(&ctx->uring_lock) +{ + struct io_notif *notif; + + notif = kzalloc(sizeof(*notif), GFP_ATOMIC | __GFP_ACCOUNT); + if (!notif) + return NULL; + + notif->seq = slot->seq++; + notif->tag = slot->tag; + notif->ctx = ctx; + notif->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + notif->uarg.callback = io_uring_tx_zerocopy_callback; + /* master ref owned by io_notif_slot, will be dropped on flush */ + refcount_set(¬if->uarg.refcnt, 1); + percpu_ref_get(&ctx->refs); + return notif; +} + +static void io_notif_slot_flush(struct io_notif_slot *slot) + __must_hold(&ctx->uring_lock) +{ + struct io_notif *notif = slot->notif; + + slot->notif = NULL; + + if (WARN_ON_ONCE(in_interrupt())) + return; + /* drop slot's master ref */ + if (refcount_dec_and_test(¬if->uarg.refcnt)) + io_notif_complete(notif); +} + +__cold int io_notif_unregister(struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + int i; + + if (!ctx->notif_slots) + return -ENXIO; + + for (i = 0; i < ctx->nr_notif_slots; i++) { + struct io_notif_slot *slot = &ctx->notif_slots[i]; + + if (slot->notif) + io_notif_slot_flush(slot); + } + + kvfree(ctx->notif_slots); + ctx->notif_slots = NULL; + ctx->nr_notif_slots = 0; + return 0; +} \ No newline at end of file diff --git a/io_uring/notif.h b/io_uring/notif.h new file mode 100644 index 000000000000..3d7a1d242e17 --- /dev/null +++ b/io_uring/notif.h @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +struct io_notif { + struct ubuf_info uarg; + struct io_ring_ctx *ctx; + + /* cqe->user_data, io_notif_slot::tag if not overridden */ + u64 tag; + /* see struct io_notif_slot::seq */ + u32 seq; + + union { + struct callback_head task_work; + struct work_struct commit_work; + }; +}; + +struct io_notif_slot { + /* + * Current/active notifier. A slot holds only one active notifier at a + * time and keeps one reference to it. Flush releases the reference and + * lazily replaces it with a new notifier. + */ + struct io_notif *notif; + + /* + * Default ->user_data for this slot notifiers CQEs + */ + u64 tag; + /* + * Notifiers of a slot live in generations, we create a new notifier + * only after flushing the previous one. Track the sequential number + * for all notifiers and copy it into notifiers's cqe->cflags + */ + u32 seq; +}; + +int io_notif_unregister(struct io_ring_ctx *ctx); + +struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx, + struct io_notif_slot *slot); + +static inline struct io_notif *io_get_notif(struct io_ring_ctx *ctx, + struct io_notif_slot *slot) +{ + if (!slot->notif) + slot->notif = io_alloc_notif(ctx, slot); + return slot->notif; +} + +static inline struct io_notif_slot *io_get_notif_slot(struct io_ring_ctx *ctx, + int idx) + __must_hold(&ctx->uring_lock) +{ + if (idx >= ctx->nr_notif_slots) + return NULL; + idx = array_index_nospec(idx, ctx->nr_notif_slots); + return &ctx->notif_slots[idx]; +} -- cgit v1.2.3 From eb4a299b2f95437af6183946c2a2e850621cefdb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Jul 2022 21:52:39 +0100 Subject: io_uring: cache struct io_notif kmalloc'ing struct io_notif is too expensive when done frequently, cache them as many other resources in io_uring. Keep two list, the first one is from where we're getting notifiers, it's protected by ->uring_lock. The second is protected by ->completion_lock, to which we queue released notifiers. Then we splice one list into another when needed. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9dec18f7fcbab9f4bd40b96e5ae158b119945230.1657643355.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 7 ++++++ io_uring/io_uring.c | 3 +++ io_uring/notif.c | 57 ++++++++++++++++++++++++++++++++++++------ io_uring/notif.h | 5 ++++ 4 files changed, 65 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index f7fab3758cb9..144493cbadb5 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -249,6 +249,9 @@ struct io_ring_ctx { struct xarray io_bl_xa; struct list_head io_buffers_cache; + /* struct io_notif cache, protected by uring_lock */ + struct list_head notif_list; + struct io_hash_table cancel_table_locked; struct list_head cq_overflow_list; struct io_alloc_cache apoll_cache; @@ -259,6 +262,10 @@ struct io_ring_ctx { struct io_wq_work_list locked_free_list; unsigned int locked_free_nr; + /* struct io_notif cache protected by completion_lock */ + struct list_head notif_list_locked; + unsigned int notif_locked_nr; + const struct cred *sq_creds; /* cred used for __io_sq_thread() */ struct io_sq_data *sq_data; /* if using sq thread polling */ diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 65ac407dd74e..20e65d45ca1c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -321,6 +321,8 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_WQ_LIST(&ctx->locked_free_list); INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func); INIT_WQ_LIST(&ctx->submit_state.compl_reqs); + INIT_LIST_HEAD(&ctx->notif_list); + INIT_LIST_HEAD(&ctx->notif_list_locked); return ctx; err: kfree(ctx->dummy_ubuf); @@ -2493,6 +2495,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list)); WARN_ON_ONCE(ctx->notif_slots || ctx->nr_notif_slots); + io_notif_cache_purge(ctx); io_mem_free(ctx->rings); io_mem_free(ctx->sq_sqes); diff --git a/io_uring/notif.c b/io_uring/notif.c index 6ee948af6a49..b257db2120b4 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -15,10 +15,12 @@ static void __io_notif_complete_tw(struct callback_head *cb) io_cq_lock(ctx); io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq, true); + + list_add(¬if->cache_node, &ctx->notif_list_locked); + ctx->notif_locked_nr++; io_cq_unlock_post(ctx); percpu_ref_put(&ctx->refs); - kfree(notif); } static inline void io_notif_complete(struct io_notif *notif) @@ -45,21 +47,62 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, queue_work(system_unbound_wq, ¬if->commit_work); } +static void io_notif_splice_cached(struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + spin_lock(&ctx->completion_lock); + list_splice_init(&ctx->notif_list_locked, &ctx->notif_list); + ctx->notif_locked_nr = 0; + spin_unlock(&ctx->completion_lock); +} + +void io_notif_cache_purge(struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + io_notif_splice_cached(ctx); + + while (!list_empty(&ctx->notif_list)) { + struct io_notif *notif = list_first_entry(&ctx->notif_list, + struct io_notif, cache_node); + + list_del(¬if->cache_node); + kfree(notif); + } +} + +static inline bool io_notif_has_cached(struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + if (likely(!list_empty(&ctx->notif_list))) + return true; + if (data_race(READ_ONCE(ctx->notif_locked_nr) <= IO_NOTIF_SPLICE_BATCH)) + return false; + io_notif_splice_cached(ctx); + return !list_empty(&ctx->notif_list); +} + struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx, struct io_notif_slot *slot) __must_hold(&ctx->uring_lock) { struct io_notif *notif; - notif = kzalloc(sizeof(*notif), GFP_ATOMIC | __GFP_ACCOUNT); - if (!notif) - return NULL; + if (likely(io_notif_has_cached(ctx))) { + notif = list_first_entry(&ctx->notif_list, + struct io_notif, cache_node); + list_del(¬if->cache_node); + } else { + notif = kzalloc(sizeof(*notif), GFP_ATOMIC | __GFP_ACCOUNT); + if (!notif) + return NULL; + /* pre-initialise some fields */ + notif->ctx = ctx; + notif->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + notif->uarg.callback = io_uring_tx_zerocopy_callback; + } notif->seq = slot->seq++; notif->tag = slot->tag; - notif->ctx = ctx; - notif->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; - notif->uarg.callback = io_uring_tx_zerocopy_callback; /* master ref owned by io_notif_slot, will be dropped on flush */ refcount_set(¬if->uarg.refcnt, 1); percpu_ref_get(&ctx->refs); diff --git a/io_uring/notif.h b/io_uring/notif.h index 3d7a1d242e17..b23c9c0515bb 100644 --- a/io_uring/notif.h +++ b/io_uring/notif.h @@ -5,6 +5,8 @@ #include #include +#define IO_NOTIF_SPLICE_BATCH 32 + struct io_notif { struct ubuf_info uarg; struct io_ring_ctx *ctx; @@ -13,6 +15,8 @@ struct io_notif { u64 tag; /* see struct io_notif_slot::seq */ u32 seq; + /* hook into ctx->notif_list and ctx->notif_list_locked */ + struct list_head cache_node; union { struct callback_head task_work; @@ -41,6 +45,7 @@ struct io_notif_slot { }; int io_notif_unregister(struct io_ring_ctx *ctx); +void io_notif_cache_purge(struct io_ring_ctx *ctx); struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx, struct io_notif_slot *slot); -- cgit v1.2.3 From d349ab99eec7ab0f977fc4aac27aa476907acf90 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 17 Jul 2022 12:35:24 +0200 Subject: random: handle archrandom with multiple longs The archrandom interface was originally designed for x86, which supplies RDRAND/RDSEED for receiving random words into registers, resulting in one function to generate an int and another to generate a long. However, other architectures don't follow this. On arm64, the SMCCC TRNG interface can return between one and three longs. On s390, the CPACF TRNG interface can return arbitrary amounts, with four longs having the same cost as one. On UML, the os_getrandom() interface can return arbitrary amounts. So change the api signature to take a "max_longs" parameter designating the maximum number of longs requested, and then return the number of longs generated. Since callers need to check this return value and loop anyway, each arch implementation does not bother implementing its own loop to try again to fill the maximum number of longs. Additionally, all existing callers pass in a constant max_longs parameter. Taken together, these two things mean that the codegen doesn't really change much for one-word-at-a-time platforms, while performance is greatly improved on platforms such as s390. Acked-by: Heiko Carstens Acked-by: Catalin Marinas Acked-by: Mark Rutland Acked-by: Michael Ellerman Acked-by: Borislav Petkov Signed-off-by: Jason A. Donenfeld --- arch/arm64/include/asm/archrandom.h | 102 ++++++++++++++++------------------ arch/arm64/kernel/kaslr.c | 2 +- arch/powerpc/include/asm/archrandom.h | 30 ++-------- arch/powerpc/kvm/book3s_hv.c | 2 +- arch/s390/include/asm/archrandom.h | 29 +++------- arch/um/include/asm/archrandom.h | 21 +++---- arch/x86/include/asm/archrandom.h | 41 ++------------ arch/x86/kernel/espfix_64.c | 2 +- drivers/char/random.c | 45 ++++++++++----- include/asm-generic/archrandom.h | 18 ++---- include/linux/random.h | 12 ++-- 11 files changed, 116 insertions(+), 188 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index c3b9fa56af67..109e2a4454be 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -58,7 +58,7 @@ static inline bool __arm64_rndrrs(unsigned long *v) return ok; } -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { /* * Only support the generic interface after we have detected @@ -66,27 +66,15 @@ static inline bool __must_check arch_get_random_long(unsigned long *v) * cpufeature code and with potential scheduling between CPUs * with and without the feature. */ - if (cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndr(v)) - return true; - return false; + if (max_longs && cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndr(v)) + return 1; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - if (cpus_have_const_cap(ARM64_HAS_RNG)) { - unsigned long val; - - if (__arm64_rndr(&val)) { - *v = val; - return true; - } - } - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - struct arm_smccc_res res; + if (!max_longs) + return 0; /* * We prefer the SMCCC call, since its semantics (return actual @@ -95,10 +83,23 @@ static inline bool __must_check arch_get_random_seed_long(unsigned long *v) * (the output of a pseudo RNG freshly seeded by a TRNG). */ if (smccc_trng_available) { - arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, 64, &res); + struct arm_smccc_res res; + + max_longs = min_t(size_t, 3, max_longs); + arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, max_longs * 64, &res); if ((int)res.a0 >= 0) { - *v = res.a3; - return true; + switch (max_longs) { + case 3: + *v++ = res.a1; + fallthrough; + case 2: + *v++ = res.a2; + fallthrough; + case 1: + *v++ = res.a3; + break; + } + return max_longs; } } @@ -108,32 +109,9 @@ static inline bool __must_check arch_get_random_seed_long(unsigned long *v) * enough to implement this API if no other entropy source exists. */ if (cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndrrs(v)) - return true; + return 1; - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - struct arm_smccc_res res; - unsigned long val; - - if (smccc_trng_available) { - arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, 32, &res); - if ((int)res.a0 >= 0) { - *v = res.a3 & GENMASK(31, 0); - return true; - } - } - - if (cpus_have_const_cap(ARM64_HAS_RNG)) { - if (__arm64_rndrrs(&val)) { - *v = val; - return true; - } - } - - return false; + return 0; } static inline bool __init __early_cpu_has_rndr(void) @@ -143,26 +121,40 @@ static inline bool __init __early_cpu_has_rndr(void) return (ftr >> ID_AA64ISAR0_EL1_RNDR_SHIFT) & 0xf; } -static inline bool __init __must_check -arch_get_random_seed_long_early(unsigned long *v) +static inline size_t __init __must_check +arch_get_random_seed_longs_early(unsigned long *v, size_t max_longs) { WARN_ON(system_state != SYSTEM_BOOTING); + if (!max_longs) + return 0; + if (smccc_trng_available) { struct arm_smccc_res res; - arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, 64, &res); + max_longs = min_t(size_t, 3, max_longs); + arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, max_longs * 64, &res); if ((int)res.a0 >= 0) { - *v = res.a3; - return true; + switch (max_longs) { + case 3: + *v++ = res.a1; + fallthrough; + case 2: + *v++ = res.a2; + fallthrough; + case 1: + *v++ = res.a3; + break; + } + return max_longs; } } if (__early_cpu_has_rndr() && __arm64_rndr(v)) - return true; + return 1; - return false; + return 0; } -#define arch_get_random_seed_long_early arch_get_random_seed_long_early +#define arch_get_random_seed_longs_early arch_get_random_seed_longs_early #endif /* _ASM_ARCHRANDOM_H */ diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index 418b2bba1521..c5d541f358d3 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -106,7 +106,7 @@ u64 __init kaslr_early_init(void) * and supported. */ - if (arch_get_random_seed_long_early(&raw)) + if (arch_get_random_seed_longs_early(&raw, 1)) seed ^= raw; if (!seed) { diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h index 25ba65df6b1a..0e365c5b2396 100644 --- a/arch/powerpc/include/asm/archrandom.h +++ b/arch/powerpc/include/asm/archrandom.h @@ -4,34 +4,16 @@ #include -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return false; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - if (ppc_md.get_random_seed) - return ppc_md.get_random_seed(v); - - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - unsigned long val; - bool rc; - - rc = arch_get_random_seed_long(&val); - if (rc) - *v = val; - - return rc; + if (max_longs && ppc_md.get_random_seed && ppc_md.get_random_seed(v)) + return 1; + return 0; } #ifdef CONFIG_PPC_POWERNV diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index e08fb3124dca..631062cde6b4 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -1207,7 +1207,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) break; #endif case H_RANDOM: - if (!arch_get_random_seed_long(&vcpu->arch.regs.gpr[4])) + if (!arch_get_random_seed_longs(&vcpu->arch.regs.gpr[4], 1)) ret = H_HARDWARE; break; case H_RPT_INVALIDATE: diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h index 0a1c2e66c709..cf5e000df0a1 100644 --- a/arch/s390/include/asm/archrandom.h +++ b/arch/s390/include/asm/archrandom.h @@ -18,34 +18,19 @@ DECLARE_STATIC_KEY_FALSE(s390_arch_random_available); extern atomic64_t s390_arch_random_counter; -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return false; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) -{ - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - if (static_branch_likely(&s390_arch_random_available)) { - cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); - atomic64_add(sizeof(*v), &s390_arch_random_counter); - return true; - } - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { if (static_branch_likely(&s390_arch_random_available)) { - cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); - atomic64_add(sizeof(*v), &s390_arch_random_counter); - return true; + cpacf_trng(NULL, 0, (u8 *)v, max_longs * sizeof(*v)); + atomic64_add(max_longs * sizeof(*v), &s390_arch_random_counter); + return max_longs; } - return false; + return 0; } #endif /* _ASM_S390_ARCHRANDOM_H */ diff --git a/arch/um/include/asm/archrandom.h b/arch/um/include/asm/archrandom.h index 2f24cb96391d..24e16c979c51 100644 --- a/arch/um/include/asm/archrandom.h +++ b/arch/um/include/asm/archrandom.h @@ -7,24 +7,19 @@ /* This is from , but better not to #include that in a global header here. */ ssize_t os_getrandom(void *buf, size_t len, unsigned int flags); -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return os_getrandom(v, sizeof(*v), 0) == sizeof(*v); -} + ssize_t ret; -static inline bool __must_check arch_get_random_int(unsigned int *v) -{ - return os_getrandom(v, sizeof(*v), 0) == sizeof(*v); -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - return false; + ret = os_getrandom(v, max_longs * sizeof(*v), 0); + if (ret < 0) + return 0; + return ret / sizeof(*v); } -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return false; + return 0; } #endif diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h index fb235b696175..02bae8e0758b 100644 --- a/arch/x86/include/asm/archrandom.h +++ b/arch/x86/include/asm/archrandom.h @@ -31,20 +31,6 @@ static inline bool __must_check rdrand_long(unsigned long *v) return false; } -static inline bool __must_check rdrand_int(unsigned int *v) -{ - bool ok; - unsigned int retry = RDRAND_RETRY_LOOPS; - do { - asm volatile("rdrand %[out]" - CC_SET(c) - : CC_OUT(c) (ok), [out] "=r" (*v)); - if (ok) - return true; - } while (--retry); - return false; -} - static inline bool __must_check rdseed_long(unsigned long *v) { bool ok; @@ -54,38 +40,19 @@ static inline bool __must_check rdseed_long(unsigned long *v) return ok; } -static inline bool __must_check rdseed_int(unsigned int *v) -{ - bool ok; - asm volatile("rdseed %[out]" - CC_SET(c) - : CC_OUT(c) (ok), [out] "=r" (*v)); - return ok; -} - /* * These are the generic interfaces; they must not be declared if the * stubs in are to be invoked. */ -static inline bool __must_check arch_get_random_long(unsigned long *v) -{ - return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_long(v) : false; -} - -static inline bool __must_check arch_get_random_int(unsigned int *v) -{ - return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_int(v) : false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_long(v) : false; + return max_longs && static_cpu_has(X86_FEATURE_RDRAND) && rdrand_long(v) ? 1 : 0; } -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_int(v) : false; + return max_longs && static_cpu_has(X86_FEATURE_RDSEED) && rdseed_long(v) ? 1 : 0; } #ifndef CONFIG_UML diff --git a/arch/x86/kernel/espfix_64.c b/arch/x86/kernel/espfix_64.c index 4fe7af58cfe1..9417d5aa7305 100644 --- a/arch/x86/kernel/espfix_64.c +++ b/arch/x86/kernel/espfix_64.c @@ -100,7 +100,7 @@ static void init_espfix_random(void) * This is run before the entropy pools are initialized, * but this is hopefully better than nothing. */ - if (!arch_get_random_long(&rand)) { + if (!arch_get_random_longs(&rand, 1)) { /* The constant is an arbitrary large prime */ rand = rdtsc(); rand *= 0xc345c6b72fd16123UL; diff --git a/drivers/char/random.c b/drivers/char/random.c index 0c6568ae5f68..7bf11fa66265 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -596,12 +596,20 @@ static void extract_entropy(void *buf, size_t len) unsigned long rdseed[32 / sizeof(long)]; size_t counter; } block; - size_t i; + size_t i, longs; - for (i = 0; i < ARRAY_SIZE(block.rdseed); ++i) { - if (!arch_get_random_seed_long(&block.rdseed[i]) && - !arch_get_random_long(&block.rdseed[i])) - block.rdseed[i] = random_get_entropy(); + for (i = 0; i < ARRAY_SIZE(block.rdseed);) { + longs = arch_get_random_seed_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i); + if (longs) { + i += longs; + continue; + } + longs = arch_get_random_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i); + if (longs) { + i += longs; + continue; + } + block.rdseed[i++] = random_get_entropy(); } spin_lock_irqsave(&input_pool.lock, flags); @@ -776,22 +784,31 @@ static struct notifier_block pm_notifier = { .notifier_call = random_pm_notifica int __init random_init(const char *command_line) { ktime_t now = ktime_get_real(); - unsigned int i, arch_bits; - unsigned long entropy; + size_t i, longs, arch_bits; + unsigned long entropy[BLAKE2S_BLOCK_SIZE / sizeof(long)]; #if defined(LATENT_ENTROPY_PLUGIN) static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy; _mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed)); #endif - for (i = 0, arch_bits = BLAKE2S_BLOCK_SIZE * 8; - i < BLAKE2S_BLOCK_SIZE; i += sizeof(entropy)) { - if (!arch_get_random_seed_long_early(&entropy) && - !arch_get_random_long_early(&entropy)) { - entropy = random_get_entropy(); - arch_bits -= sizeof(entropy) * 8; + for (i = 0, arch_bits = sizeof(entropy) * 8; i < ARRAY_SIZE(entropy);) { + longs = arch_get_random_seed_longs(entropy, ARRAY_SIZE(entropy) - i); + if (longs) { + _mix_pool_bytes(entropy, sizeof(*entropy) * longs); + i += longs; + continue; } - _mix_pool_bytes(&entropy, sizeof(entropy)); + longs = arch_get_random_longs(entropy, ARRAY_SIZE(entropy) - i); + if (longs) { + _mix_pool_bytes(entropy, sizeof(*entropy) * longs); + i += longs; + continue; + } + entropy[0] = random_get_entropy(); + _mix_pool_bytes(entropy, sizeof(*entropy)); + arch_bits -= sizeof(*entropy) * 8; + ++i; } _mix_pool_bytes(&now, sizeof(now)); _mix_pool_bytes(utsname(), sizeof(*(utsname()))); diff --git a/include/asm-generic/archrandom.h b/include/asm-generic/archrandom.h index 3a5ee202dd86..3cd7f980cfdc 100644 --- a/include/asm-generic/archrandom.h +++ b/include/asm-generic/archrandom.h @@ -2,24 +2,14 @@ #ifndef __ASM_GENERIC_ARCHRANDOM_H__ #define __ASM_GENERIC_ARCHRANDOM_H__ -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return false; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - return false; + return 0; } #endif diff --git a/include/linux/random.h b/include/linux/random.h index 865770e29f3e..3fec206487f6 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -112,19 +112,19 @@ declare_get_random_var_wait(long, unsigned long) * Called from the boot CPU during startup; not valid to call once * secondary CPUs are up and preemption is possible. */ -#ifndef arch_get_random_seed_long_early -static inline bool __init arch_get_random_seed_long_early(unsigned long *v) +#ifndef arch_get_random_seed_longs_early +static inline size_t __init arch_get_random_seed_longs_early(unsigned long *v, size_t max_longs) { WARN_ON(system_state != SYSTEM_BOOTING); - return arch_get_random_seed_long(v); + return arch_get_random_seed_longs(v, max_longs); } #endif -#ifndef arch_get_random_long_early -static inline bool __init arch_get_random_long_early(unsigned long *v) +#ifndef arch_get_random_longs_early +static inline bool __init arch_get_random_longs_early(unsigned long *v, size_t max_longs) { WARN_ON(system_state != SYSTEM_BOOTING); - return arch_get_random_long(v); + return arch_get_random_longs(v, max_longs); } #endif -- cgit v1.2.3 From 7ffcdaa670164a2ad3844a5ef6df5423782ba290 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 25 Jul 2022 09:32:21 -0400 Subject: SUNRPC expose functions for offline remote xprt functionality Re-arrange the code that make offline transport and delete transport callable functions. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xprt.h | 3 +++ net/sunrpc/sysfs.c | 28 +++++----------------------- net/sunrpc/xprt.c | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h index 522bbf937957..0d51b9f9ea37 100644 --- a/include/linux/sunrpc/xprt.h +++ b/include/linux/sunrpc/xprt.h @@ -505,4 +505,7 @@ static inline int xprt_test_and_set_binding(struct rpc_xprt *xprt) return test_and_set_bit(XPRT_BINDING, &xprt->state); } +void xprt_set_offline_locked(struct rpc_xprt *xprt, struct rpc_xprt_switch *xps); +void xprt_set_online_locked(struct rpc_xprt *xprt, struct rpc_xprt_switch *xps); +void xprt_delete_locked(struct rpc_xprt *xprt, struct rpc_xprt_switch *xps); #endif /* _LINUX_SUNRPC_XPRT_H */ diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index a3a2f8aeb80e..7330eb9a70cf 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -314,32 +314,14 @@ static ssize_t rpc_sysfs_xprt_state_change(struct kobject *kobj, goto release_tasks; } if (offline) { - if (!test_and_set_bit(XPRT_OFFLINE, &xprt->state)) { - spin_lock(&xps->xps_lock); - xps->xps_nactive--; - spin_unlock(&xps->xps_lock); - } + xprt_set_offline_locked(xprt, xps); } else if (online) { - if (test_and_clear_bit(XPRT_OFFLINE, &xprt->state)) { - spin_lock(&xps->xps_lock); - xps->xps_nactive++; - spin_unlock(&xps->xps_lock); - } + xprt_set_online_locked(xprt, xps); } else if (remove) { - if (test_bit(XPRT_OFFLINE, &xprt->state)) { - if (!test_and_set_bit(XPRT_REMOVE, &xprt->state)) { - xprt_force_disconnect(xprt); - if (test_bit(XPRT_CONNECTED, &xprt->state)) { - if (!xprt->sending.qlen && - !xprt->pending.qlen && - !xprt->backlog.qlen && - !atomic_long_read(&xprt->queuelen)) - rpc_xprt_switch_remove_xprt(xps, xprt); - } - } - } else { + if (test_bit(XPRT_OFFLINE, &xprt->state)) + xprt_delete_locked(xprt, xps); + else count = -EINVAL; - } } release_tasks: diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index 86d62cffba0d..8f8e3c952f24 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -2152,3 +2152,35 @@ void xprt_put(struct rpc_xprt *xprt) kref_put(&xprt->kref, xprt_destroy_kref); } EXPORT_SYMBOL_GPL(xprt_put); + +void xprt_set_offline_locked(struct rpc_xprt *xprt, struct rpc_xprt_switch *xps) +{ + if (!test_and_set_bit(XPRT_OFFLINE, &xprt->state)) { + spin_lock(&xps->xps_lock); + xps->xps_nactive--; + spin_unlock(&xps->xps_lock); + } +} + +void xprt_set_online_locked(struct rpc_xprt *xprt, struct rpc_xprt_switch *xps) +{ + if (test_and_clear_bit(XPRT_OFFLINE, &xprt->state)) { + spin_lock(&xps->xps_lock); + xps->xps_nactive++; + spin_unlock(&xps->xps_lock); + } +} + +void xprt_delete_locked(struct rpc_xprt *xprt, struct rpc_xprt_switch *xps) +{ + if (test_and_set_bit(XPRT_REMOVE, &xprt->state)) + return; + + xprt_force_disconnect(xprt); + if (!test_bit(XPRT_CONNECTED, &xprt->state)) + return; + + if (!xprt->sending.qlen && !xprt->pending.qlen && + !xprt->backlog.qlen && !atomic_long_read(&xprt->queuelen)) + rpc_xprt_switch_remove_xprt(xps, xprt); +} -- cgit v1.2.3 From 895245ccea251ff54ea19bc364c9a49007918115 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 25 Jul 2022 09:32:22 -0400 Subject: SUNRPC add function to offline remove trunkable transports Iterate thru available transports in the xprt_switch for all trunkable transports offline and possibly remote them as well. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/linux/sunrpc/clnt.h | 1 + net/sunrpc/clnt.c | 46 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 90501404fa49..d14333f4947a 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -234,6 +234,7 @@ int rpc_clnt_setup_test_and_add_xprt(struct rpc_clnt *, struct rpc_xprt_switch *, struct rpc_xprt *, void *); +void rpc_clnt_manage_trunked_xprts(struct rpc_clnt *); const char *rpc_proc_name(const struct rpc_task *task); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index a97d4e06cae3..2b079c4d8af1 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -3000,6 +3000,52 @@ out_put_switch: } EXPORT_SYMBOL_GPL(rpc_clnt_add_xprt); +static int rpc_xprt_offline(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, + void *data) +{ + struct rpc_xprt *main_xprt; + struct rpc_xprt_switch *xps; + int err = 0; + + xprt_get(xprt); + + rcu_read_lock(); + main_xprt = xprt_get(rcu_dereference(clnt->cl_xprt)); + xps = xprt_switch_get(rcu_dereference(clnt->cl_xpi.xpi_xpswitch)); + err = rpc_cmp_addr_port((struct sockaddr *)&xprt->addr, + (struct sockaddr *)&main_xprt->addr); + rcu_read_unlock(); + xprt_put(main_xprt); + if (err) + goto out; + + if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) { + err = -EINTR; + goto out; + } + xprt_set_offline_locked(xprt, xps); + + xprt_release_write(xprt, NULL); +out: + xprt_put(xprt); + xprt_switch_put(xps); + return err; +} + +/* rpc_clnt_manage_trunked_xprts -- offline trunked transports + * @clnt rpc_clnt structure + * + * For each active transport found in the rpc_clnt structure call + * the function rpc_xprt_offline() which will identify trunked transports + * and will mark them offline. + */ +void rpc_clnt_manage_trunked_xprts(struct rpc_clnt *clnt) +{ + rpc_clnt_iterate_for_each_xprt(clnt, rpc_xprt_offline, NULL); +} +EXPORT_SYMBOL_GPL(rpc_clnt_manage_trunked_xprts); + struct connect_timeout_data { unsigned long connect_timeout; unsigned long reconnect_timeout; -- cgit v1.2.3 From 95d0d30c66b855f614e677b8cd0455eed0765a6f Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 25 Jul 2022 09:32:24 -0400 Subject: SUNRPC create an iterator to list only OFFLINE xprts Create a new iterator helper that will go thru the all the transports in the switch and return transports that are marked OFFLINE. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xprtmultipath.h | 3 ++ net/sunrpc/clnt.c | 11 +++- net/sunrpc/xprtmultipath.c | 99 ++++++++++++++++++++++++++++++++---- 3 files changed, 101 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xprtmultipath.h b/include/linux/sunrpc/xprtmultipath.h index bbb8a5fa0816..688ca7eb1d01 100644 --- a/include/linux/sunrpc/xprtmultipath.h +++ b/include/linux/sunrpc/xprtmultipath.h @@ -63,6 +63,9 @@ extern void xprt_iter_init(struct rpc_xprt_iter *xpi, extern void xprt_iter_init_listall(struct rpc_xprt_iter *xpi, struct rpc_xprt_switch *xps); +extern void xprt_iter_init_listoffline(struct rpc_xprt_iter *xpi, + struct rpc_xprt_switch *xps); + extern void xprt_iter_destroy(struct rpc_xprt_iter *xpi); extern struct rpc_xprt_switch *xprt_iter_xchg_switch( diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 2b079c4d8af1..68021b70340d 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -786,7 +786,8 @@ out_revert: EXPORT_SYMBOL_GPL(rpc_switch_client_transport); static -int rpc_clnt_xprt_iter_init(struct rpc_clnt *clnt, struct rpc_xprt_iter *xpi) +int _rpc_clnt_xprt_iter_init(struct rpc_clnt *clnt, struct rpc_xprt_iter *xpi, + void func(struct rpc_xprt_iter *xpi, struct rpc_xprt_switch *xps)) { struct rpc_xprt_switch *xps; @@ -795,11 +796,17 @@ int rpc_clnt_xprt_iter_init(struct rpc_clnt *clnt, struct rpc_xprt_iter *xpi) rcu_read_unlock(); if (xps == NULL) return -EAGAIN; - xprt_iter_init_listall(xpi, xps); + func(xpi, xps); xprt_switch_put(xps); return 0; } +static +int rpc_clnt_xprt_iter_init(struct rpc_clnt *clnt, struct rpc_xprt_iter *xpi) +{ + return _rpc_clnt_xprt_iter_init(clnt, xpi, xprt_iter_init_listall); +} + /** * rpc_clnt_iterate_for_each_xprt - Apply a function to all transports * @clnt: pointer to client diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c index 1693f81aae37..8def8423fc0a 100644 --- a/net/sunrpc/xprtmultipath.c +++ b/net/sunrpc/xprtmultipath.c @@ -27,6 +27,7 @@ typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct rpc_xprt_switch *xps, static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular; static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin; static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall; +static const struct rpc_xprt_iter_ops rpc_xprt_iter_listoffline; static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps, struct rpc_xprt *xprt) @@ -248,6 +249,18 @@ struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head) return NULL; } +static +struct rpc_xprt *xprt_switch_find_first_entry_offline(struct list_head *head) +{ + struct rpc_xprt *pos; + + list_for_each_entry_rcu(pos, head, xprt_switch) { + if (!xprt_is_active(pos)) + return pos; + } + return NULL; +} + static struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi) { @@ -259,8 +272,9 @@ struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi) } static -struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head, - const struct rpc_xprt *cur) +struct rpc_xprt *_xprt_switch_find_current_entry(struct list_head *head, + const struct rpc_xprt *cur, + bool find_active) { struct rpc_xprt *pos; bool found = false; @@ -268,14 +282,25 @@ struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head, list_for_each_entry_rcu(pos, head, xprt_switch) { if (cur == pos) found = true; - if (found && xprt_is_active(pos)) + if (found && ((find_active && xprt_is_active(pos)) || + (!find_active && xprt_is_active(pos)))) return pos; } return NULL; } static -struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi) +struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head, + const struct rpc_xprt *cur) +{ + return _xprt_switch_find_current_entry(head, cur, true); +} + +static +struct rpc_xprt * _xprt_iter_current_entry(struct rpc_xprt_iter *xpi, + struct rpc_xprt *first_entry(struct list_head *head), + struct rpc_xprt *current_entry(struct list_head *head, + const struct rpc_xprt *cur)) { struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); struct list_head *head; @@ -284,8 +309,30 @@ struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi) return NULL; head = &xps->xps_xprt_list; if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2) - return xprt_switch_find_first_entry(head); - return xprt_switch_find_current_entry(head, xpi->xpi_cursor); + return first_entry(head); + return current_entry(head, xpi->xpi_cursor); +} + +static +struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi) +{ + return _xprt_iter_current_entry(xpi, xprt_switch_find_first_entry, + xprt_switch_find_current_entry); +} + +static +struct rpc_xprt *xprt_switch_find_current_entry_offline(struct list_head *head, + const struct rpc_xprt *cur) +{ + return _xprt_switch_find_current_entry(head, cur, false); +} + +static +struct rpc_xprt *xprt_iter_current_entry_offline(struct rpc_xprt_iter *xpi) +{ + return _xprt_iter_current_entry(xpi, + xprt_switch_find_first_entry_offline, + xprt_switch_find_current_entry_offline); } bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps, @@ -310,7 +357,7 @@ bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps, static struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head, - const struct rpc_xprt *cur) + const struct rpc_xprt *cur, bool check_active) { struct rpc_xprt *pos, *prev = NULL; bool found = false; @@ -318,7 +365,12 @@ struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head, list_for_each_entry_rcu(pos, head, xprt_switch) { if (cur == prev) found = true; - if (found && xprt_is_active(pos)) + /* for request to return active transports return only + * active, for request to return offline transports + * return only offline + */ + if (found && ((check_active && xprt_is_active(pos)) || + (!check_active && !xprt_is_active(pos)))) return pos; prev = pos; } @@ -355,7 +407,7 @@ struct rpc_xprt *__xprt_switch_find_next_entry_roundrobin(struct list_head *head { struct rpc_xprt *ret; - ret = xprt_switch_find_next_entry(head, cur); + ret = xprt_switch_find_next_entry(head, cur, true); if (ret != NULL) return ret; return xprt_switch_find_first_entry(head); @@ -397,7 +449,14 @@ static struct rpc_xprt *xprt_switch_find_next_entry_all(struct rpc_xprt_switch *xps, const struct rpc_xprt *cur) { - return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur); + return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur, true); +} + +static +struct rpc_xprt *xprt_switch_find_next_entry_offline(struct rpc_xprt_switch *xps, + const struct rpc_xprt *cur) +{ + return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur, false); } static @@ -407,6 +466,13 @@ struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi) xprt_switch_find_next_entry_all); } +static +struct rpc_xprt *xprt_iter_next_entry_offline(struct rpc_xprt_iter *xpi) +{ + return xprt_iter_next_entry_multiple(xpi, + xprt_switch_find_next_entry_offline); +} + /* * xprt_iter_rewind - Resets the xprt iterator * @xpi: pointer to rpc_xprt_iter @@ -460,6 +526,12 @@ void xprt_iter_init_listall(struct rpc_xprt_iter *xpi, __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall); } +void xprt_iter_init_listoffline(struct rpc_xprt_iter *xpi, + struct rpc_xprt_switch *xps) +{ + __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listoffline); +} + /** * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch * @xpi: pointer to rpc_xprt_iter @@ -574,3 +646,10 @@ const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = { .xpi_xprt = xprt_iter_current_entry, .xpi_next = xprt_iter_next_entry_all, }; + +static +const struct rpc_xprt_iter_ops rpc_xprt_iter_listoffline = { + .xpi_rewind = xprt_iter_default_rewind, + .xpi_xprt = xprt_iter_current_entry_offline, + .xpi_next = xprt_iter_next_entry_offline, +}; -- cgit v1.2.3 From 9368fd6c75053630e95a6dbd17c9522e82101276 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 25 Jul 2022 09:32:25 -0400 Subject: SUNRPC enable back offline transports in trunking discovery When we are adding a transport to a xprt_switch that's already on the list but has been marked OFFLINE, then make the state ONLINE since it's been tested now. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/linux/sunrpc/clnt.h | 1 + net/sunrpc/clnt.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index d14333f4947a..71a3a1dd7e81 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -242,6 +242,7 @@ void rpc_clnt_xprt_switch_put(struct rpc_clnt *); void rpc_clnt_xprt_switch_add_xprt(struct rpc_clnt *, struct rpc_xprt *); bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt, const struct sockaddr *sap); +void rpc_clnt_xprt_set_online(struct rpc_clnt *clnt, struct rpc_xprt *xprt); void rpc_cleanup_clids(void); static inline int rpc_reply_expected(struct rpc_task *task) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 68021b70340d..9dbce3b0d3a2 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -3095,8 +3095,22 @@ void rpc_clnt_xprt_switch_put(struct rpc_clnt *clnt) } EXPORT_SYMBOL_GPL(rpc_clnt_xprt_switch_put); +void rpc_clnt_xprt_set_online(struct rpc_clnt *clnt, struct rpc_xprt *xprt) +{ + struct rpc_xprt_switch *xps; + + rcu_read_lock(); + xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch); + rcu_read_unlock(); + xprt_set_online_locked(xprt, xps); +} + void rpc_clnt_xprt_switch_add_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt) { + if (rpc_clnt_xprt_switch_has_addr(clnt, + (const struct sockaddr *)&xprt->addr)) { + return rpc_clnt_xprt_set_online(clnt, xprt); + } rcu_read_lock(); rpc_xprt_switch_add_xprt(rcu_dereference(clnt->cl_xpi.xpi_xpswitch), xprt); -- cgit v1.2.3 From 497e6464d6adcee64f071b18fc826e63cfd2f0a5 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 25 Jul 2022 09:32:26 -0400 Subject: SUNRPC create an rpc function that allows xprt removal from rpc_clnt Expose a function that allows a removal of xprt from the rpc_clnt. When called from NFS that's running a trunked transport then don't decrement the active transport counter. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/linux/sunrpc/clnt.h | 1 + include/linux/sunrpc/xprtmultipath.h | 2 +- net/sunrpc/clnt.c | 16 +++++++++++++++- net/sunrpc/xprt.c | 2 +- net/sunrpc/xprtmultipath.c | 11 ++++++----- 5 files changed, 24 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 71a3a1dd7e81..7a43fd514398 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -240,6 +240,7 @@ const char *rpc_proc_name(const struct rpc_task *task); void rpc_clnt_xprt_switch_put(struct rpc_clnt *); void rpc_clnt_xprt_switch_add_xprt(struct rpc_clnt *, struct rpc_xprt *); +void rpc_clnt_xprt_switch_remove_xprt(struct rpc_clnt *, struct rpc_xprt *); bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt, const struct sockaddr *sap); void rpc_clnt_xprt_set_online(struct rpc_clnt *clnt, struct rpc_xprt *xprt); diff --git a/include/linux/sunrpc/xprtmultipath.h b/include/linux/sunrpc/xprtmultipath.h index 688ca7eb1d01..9fff0768d942 100644 --- a/include/linux/sunrpc/xprtmultipath.h +++ b/include/linux/sunrpc/xprtmultipath.h @@ -55,7 +55,7 @@ extern void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps); extern void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps, struct rpc_xprt *xprt); extern void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps, - struct rpc_xprt *xprt); + struct rpc_xprt *xprt, bool offline); extern void xprt_iter_init(struct rpc_xprt_iter *xpi, struct rpc_xprt_switch *xps); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 9dbce3b0d3a2..26f3102500bb 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -2144,7 +2144,8 @@ call_connect_status(struct rpc_task *task) xprt_release(task); value = atomic_long_dec_return(&xprt->queuelen); if (value == 0) - rpc_xprt_switch_remove_xprt(xps, saved); + rpc_xprt_switch_remove_xprt(xps, saved, + true); xprt_put(saved); task->tk_xprt = NULL; task->tk_action = call_start; @@ -3118,6 +3119,19 @@ void rpc_clnt_xprt_switch_add_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt) } EXPORT_SYMBOL_GPL(rpc_clnt_xprt_switch_add_xprt); +void rpc_clnt_xprt_switch_remove_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt) +{ + struct rpc_xprt_switch *xps; + + rcu_read_lock(); + xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch); + rpc_xprt_switch_remove_xprt(rcu_dereference(clnt->cl_xpi.xpi_xpswitch), + xprt, 0); + xps->xps_nunique_destaddr_xprts--; + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(rpc_clnt_xprt_switch_remove_xprt); + bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt, const struct sockaddr *sap) { diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index 8f8e3c952f24..44348c9f4b00 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -2182,5 +2182,5 @@ void xprt_delete_locked(struct rpc_xprt *xprt, struct rpc_xprt_switch *xps) if (!xprt->sending.qlen && !xprt->pending.qlen && !xprt->backlog.qlen && !atomic_long_read(&xprt->queuelen)) - rpc_xprt_switch_remove_xprt(xps, xprt); + rpc_xprt_switch_remove_xprt(xps, xprt, true); } diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c index 8def8423fc0a..55da01730311 100644 --- a/net/sunrpc/xprtmultipath.c +++ b/net/sunrpc/xprtmultipath.c @@ -62,11 +62,11 @@ void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps, } static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps, - struct rpc_xprt *xprt) + struct rpc_xprt *xprt, bool offline) { if (unlikely(xprt == NULL)) return; - if (!test_bit(XPRT_OFFLINE, &xprt->state)) + if (!test_bit(XPRT_OFFLINE, &xprt->state) && offline) xps->xps_nactive--; xps->xps_nxprts--; if (xps->xps_nxprts == 0) @@ -79,14 +79,15 @@ static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps, * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch * @xps: pointer to struct rpc_xprt_switch * @xprt: pointer to struct rpc_xprt + * @offline: indicates if the xprt that's being removed is in an offline state * * Removes xprt from the list of struct rpc_xprt in xps. */ void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps, - struct rpc_xprt *xprt) + struct rpc_xprt *xprt, bool offline) { spin_lock(&xps->xps_lock); - xprt_switch_remove_xprt_locked(xps, xprt); + xprt_switch_remove_xprt_locked(xps, xprt, offline); spin_unlock(&xps->xps_lock); xprt_put(xprt); } @@ -155,7 +156,7 @@ static void xprt_switch_free_entries(struct rpc_xprt_switch *xps) xprt = list_first_entry(&xps->xps_xprt_list, struct rpc_xprt, xprt_switch); - xprt_switch_remove_xprt_locked(xps, xprt); + xprt_switch_remove_xprt_locked(xps, xprt, true); spin_unlock(&xps->xps_lock); xprt_put(xprt); spin_lock(&xps->xps_lock); -- cgit v1.2.3 From 273d6aed9e5a1859dda15256f45561315c3d237a Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 25 Jul 2022 09:32:29 -0400 Subject: SUNRPC export xprt_iter_rewind function Make xprt_iter_rewind callable outside of xprtmultipath.c Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xprtmultipath.h | 2 ++ net/sunrpc/xprtmultipath.c | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xprtmultipath.h b/include/linux/sunrpc/xprtmultipath.h index 9fff0768d942..c0514c684b2c 100644 --- a/include/linux/sunrpc/xprtmultipath.h +++ b/include/linux/sunrpc/xprtmultipath.h @@ -68,6 +68,8 @@ extern void xprt_iter_init_listoffline(struct rpc_xprt_iter *xpi, extern void xprt_iter_destroy(struct rpc_xprt_iter *xpi); +extern void xprt_iter_rewind(struct rpc_xprt_iter *xpi); + extern struct rpc_xprt_switch *xprt_iter_xchg_switch( struct rpc_xprt_iter *xpi, struct rpc_xprt_switch *newswitch); diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c index 55da01730311..685db598acbe 100644 --- a/net/sunrpc/xprtmultipath.c +++ b/net/sunrpc/xprtmultipath.c @@ -481,7 +481,6 @@ struct rpc_xprt *xprt_iter_next_entry_offline(struct rpc_xprt_iter *xpi) * Resets xpi to ensure that it points to the first entry in the list * of transports. */ -static void xprt_iter_rewind(struct rpc_xprt_iter *xpi) { rcu_read_lock(); -- cgit v1.2.3 From 92cc04f60ab4ae199eee507e5cd4d5aa6c722e9c Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 25 Jul 2022 09:32:30 -0400 Subject: SUNRPC create a function that probes only offline transports For only offline transports, attempt to check connectivity via a NULL call and, if that succeeds, call a provided session trunking detection function. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/linux/sunrpc/clnt.h | 2 ++ net/sunrpc/clnt.c | 65 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 7a43fd514398..75eea5ebb179 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -235,6 +235,8 @@ int rpc_clnt_setup_test_and_add_xprt(struct rpc_clnt *, struct rpc_xprt *, void *); void rpc_clnt_manage_trunked_xprts(struct rpc_clnt *); +void rpc_clnt_probe_trunked_xprts(struct rpc_clnt *, + struct rpc_add_xprt_test *); const char *rpc_proc_name(const struct rpc_task *task); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 9c9712274ca8..bbfc47f03480 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -807,6 +807,13 @@ int rpc_clnt_xprt_iter_init(struct rpc_clnt *clnt, struct rpc_xprt_iter *xpi) return _rpc_clnt_xprt_iter_init(clnt, xpi, xprt_iter_init_listall); } +static +int rpc_clnt_xprt_iter_offline_init(struct rpc_clnt *clnt, + struct rpc_xprt_iter *xpi) +{ + return _rpc_clnt_xprt_iter_init(clnt, xpi, xprt_iter_init_listoffline); +} + /** * rpc_clnt_iterate_for_each_xprt - Apply a function to all transports * @clnt: pointer to client @@ -3018,6 +3025,64 @@ out_put_switch: } EXPORT_SYMBOL_GPL(rpc_clnt_add_xprt); +static int rpc_xprt_probe_trunked(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, + struct rpc_add_xprt_test *data) +{ + struct rpc_xprt_switch *xps; + struct rpc_xprt *main_xprt; + int status = 0; + + xprt_get(xprt); + + rcu_read_lock(); + main_xprt = xprt_get(rcu_dereference(clnt->cl_xprt)); + xps = xprt_switch_get(rcu_dereference(clnt->cl_xpi.xpi_xpswitch)); + status = rpc_cmp_addr_port((struct sockaddr *)&xprt->addr, + (struct sockaddr *)&main_xprt->addr); + rcu_read_unlock(); + xprt_put(main_xprt); + if (status || !test_bit(XPRT_OFFLINE, &xprt->state)) + goto out; + + status = rpc_clnt_add_xprt_helper(clnt, xprt, data); +out: + xprt_put(xprt); + xprt_switch_put(xps); + return status; +} + +/* rpc_clnt_probe_trunked_xprt -- probe offlined transport for session trunking + * @clnt rpc_clnt structure + * + * For each offlined transport found in the rpc_clnt structure call + * the function rpc_xprt_probe_trunked() which will determine if this + * transport still belongs to the trunking group. + */ +void rpc_clnt_probe_trunked_xprts(struct rpc_clnt *clnt, + struct rpc_add_xprt_test *data) +{ + struct rpc_xprt_iter xpi; + int ret; + + ret = rpc_clnt_xprt_iter_offline_init(clnt, &xpi); + if (ret) + return; + for (;;) { + struct rpc_xprt *xprt = xprt_iter_get_next(&xpi); + + if (!xprt) + break; + ret = rpc_xprt_probe_trunked(clnt, xprt, data); + xprt_put(xprt); + if (ret < 0) + break; + xprt_iter_rewind(&xpi); + } + xprt_iter_destroy(&xpi); +} +EXPORT_SYMBOL_GPL(rpc_clnt_probe_trunked_xprts); + static int rpc_xprt_offline(struct rpc_clnt *clnt, struct rpc_xprt *xprt, void *data) -- cgit v1.2.3 From 39ade048a32ea653b94dcfbf816b0b13a6be8a33 Mon Sep 17 00:00:00 2001 From: "Fabio M. De Francesco" Date: Wed, 6 Jul 2022 13:15:19 +0200 Subject: highmem: Make __kunmap_{local,atomic}() take const void pointer __kunmap_ {local,atomic}() currently take pointers to void. However, this is semantically incorrect, since these functions do not change the memory their arguments point to. Therefore, make this semantics explicit by modifying the __kunmap_{local,atomic}() prototypes to take pointers to const void. As a side effect, compilers may produce more efficient code. Acked-by: Andrew Morton Acked-by: Helge Deller # parisc Suggested-by: David Sterba Suggested-by: Ira Weiny Reviewed-by: Ira Weiny Signed-off-by: Fabio M. De Francesco Reviewed-by: David Sterba Signed-off-by: David Sterba --- arch/parisc/include/asm/cacheflush.h | 6 +++--- arch/parisc/kernel/cache.c | 2 +- include/linux/highmem-internal.h | 10 +++++----- mm/highmem.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index 8d03b3b26229..0bdee6724132 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h @@ -22,7 +22,7 @@ void flush_kernel_icache_range_asm(unsigned long, unsigned long); void flush_user_dcache_range_asm(unsigned long, unsigned long); void flush_kernel_dcache_range_asm(unsigned long, unsigned long); void purge_kernel_dcache_range_asm(unsigned long, unsigned long); -void flush_kernel_dcache_page_asm(void *); +void flush_kernel_dcache_page_asm(const void *addr); void flush_kernel_icache_page(void *); /* Cache flush operations */ @@ -31,7 +31,7 @@ void flush_cache_all_local(void); void flush_cache_all(void); void flush_cache_mm(struct mm_struct *mm); -void flush_kernel_dcache_page_addr(void *addr); +void flush_kernel_dcache_page_addr(const void *addr); #define flush_kernel_dcache_range(start,size) \ flush_kernel_dcache_range_asm((start), (start)+(size)); @@ -75,7 +75,7 @@ void flush_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr); void flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr); #define ARCH_HAS_FLUSH_ON_KUNMAP -static inline void kunmap_flush_on_unmap(void *addr) +static inline void kunmap_flush_on_unmap(const void *addr) { flush_kernel_dcache_page_addr(addr); } diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index a9bc578e4c52..993999a65e54 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -549,7 +549,7 @@ extern void purge_kernel_dcache_page_asm(unsigned long); extern void clear_user_page_asm(void *, unsigned long); extern void copy_user_page_asm(void *, void *, unsigned long); -void flush_kernel_dcache_page_addr(void *addr) +void flush_kernel_dcache_page_addr(const void *addr) { unsigned long flags; diff --git a/include/linux/highmem-internal.h b/include/linux/highmem-internal.h index cddb42ff0473..034b1106d022 100644 --- a/include/linux/highmem-internal.h +++ b/include/linux/highmem-internal.h @@ -8,7 +8,7 @@ #ifdef CONFIG_KMAP_LOCAL void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot); void *__kmap_local_page_prot(struct page *page, pgprot_t prot); -void kunmap_local_indexed(void *vaddr); +void kunmap_local_indexed(const void *vaddr); void kmap_local_fork(struct task_struct *tsk); void __kmap_local_sched_out(void); void __kmap_local_sched_in(void); @@ -89,7 +89,7 @@ static inline void *kmap_local_pfn(unsigned long pfn) return __kmap_local_pfn_prot(pfn, kmap_prot); } -static inline void __kunmap_local(void *vaddr) +static inline void __kunmap_local(const void *vaddr) { kunmap_local_indexed(vaddr); } @@ -121,7 +121,7 @@ static inline void *kmap_atomic_pfn(unsigned long pfn) return __kmap_local_pfn_prot(pfn, kmap_prot); } -static inline void __kunmap_atomic(void *addr) +static inline void __kunmap_atomic(const void *addr) { kunmap_local_indexed(addr); pagefault_enable(); @@ -197,7 +197,7 @@ static inline void *kmap_local_pfn(unsigned long pfn) return kmap_local_page(pfn_to_page(pfn)); } -static inline void __kunmap_local(void *addr) +static inline void __kunmap_local(const void *addr) { #ifdef ARCH_HAS_FLUSH_ON_KUNMAP kunmap_flush_on_unmap(addr); @@ -224,7 +224,7 @@ static inline void *kmap_atomic_pfn(unsigned long pfn) return kmap_atomic(pfn_to_page(pfn)); } -static inline void __kunmap_atomic(void *addr) +static inline void __kunmap_atomic(const void *addr) { #ifdef ARCH_HAS_FLUSH_ON_KUNMAP kunmap_flush_on_unmap(addr); diff --git a/mm/highmem.c b/mm/highmem.c index 1a692997fac4..e32083e4ce0d 100644 --- a/mm/highmem.c +++ b/mm/highmem.c @@ -561,7 +561,7 @@ void *__kmap_local_page_prot(struct page *page, pgprot_t prot) } EXPORT_SYMBOL(__kmap_local_page_prot); -void kunmap_local_indexed(void *vaddr) +void kunmap_local_indexed(const void *vaddr) { unsigned long addr = (unsigned long) vaddr & PAGE_MASK; pte_t *kmap_pte; -- cgit v1.2.3 From 65ea1b66482f415d51cd46515b02477257330339 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Sat, 9 Jul 2022 08:18:38 +0900 Subject: block: add bdev_max_segments() helper Add bdev_max_segments() like other queue parameters. Reviewed-by: Johannes Thumshirn Reviewed-by: Jens Axboe Reviewed-by: Christoph Hellwig Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- include/linux/blkdev.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2f7b43444c5f..62e3ff52ab03 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1206,6 +1206,11 @@ bdev_max_zone_append_sectors(struct block_device *bdev) return queue_max_zone_append_sectors(bdev_get_queue(bdev)); } +static inline unsigned int bdev_max_segments(struct block_device *bdev) +{ + return queue_max_segments(bdev_get_queue(bdev)); +} + static inline unsigned queue_logical_block_size(const struct request_queue *q) { int retval = 512; -- cgit v1.2.3 From 44abdd1646e1fbfb781972c0bffc90b4eb3e87b3 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Fri, 22 Jul 2022 19:02:51 -0700 Subject: vfio: Pass in starting IOVA to vfio_pin/unpin_pages API The vfio_pin/unpin_pages() so far accepted arrays of PFNs of user IOVA. Among all three callers, there was only one caller possibly passing in a non-contiguous PFN list, which is now ensured to have contiguous PFN inputs too. Pass in the starting address with "iova" alone to simplify things, so callers no longer need to maintain a PFN list or to pin/unpin one page at a time. This also allows VFIO to use more efficient implementations of pin/unpin_pages. For now, also update vfio_iommu_type1 to fit this new parameter too, while keeping its input intact (being user_iova) since we don't want to spend too much effort swapping its parameters and local variables at that level. Reviewed-by: Christoph Hellwig Reviewed-by: Kirti Wankhede Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Tony Krowiak Acked-by: Eric Farman Tested-by: Terrence Xu Tested-by: Eric Farman Signed-off-by: Nicolin Chen Link: https://lore.kernel.org/r/20220723020256.30081-6-nicolinc@nvidia.com Signed-off-by: Alex Williamson --- Documentation/driver-api/vfio-mediated-device.rst | 4 ++-- drivers/gpu/drm/i915/gvt/kvmgt.c | 18 +++++---------- drivers/s390/cio/vfio_ccw_cp.c | 4 ++-- drivers/s390/crypto/vfio_ap_ops.c | 9 ++++---- drivers/vfio/vfio.c | 27 ++++++++++------------- drivers/vfio/vfio.h | 4 ++-- drivers/vfio/vfio_iommu_type1.c | 15 ++++++------- include/linux/vfio.h | 5 ++--- 8 files changed, 37 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst index b0fdf76b339a..ea32a0f13ddb 100644 --- a/Documentation/driver-api/vfio-mediated-device.rst +++ b/Documentation/driver-api/vfio-mediated-device.rst @@ -262,10 +262,10 @@ Translation APIs for Mediated Devices The following APIs are provided for translating user pfn to host pfn in a VFIO driver:: - int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, + int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, int npage, int prot, unsigned long *phys_pfn); - void vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, + void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage); These functions call back into the back-end IOMMU module by using the pin_pages diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 2fee5695515a..8be75c282611 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -231,14 +231,8 @@ static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, unsigned long size) { - int total_pages = DIV_ROUND_UP(size, PAGE_SIZE); - int npage; - - for (npage = 0; npage < total_pages; npage++) { - unsigned long cur_gfn = gfn + npage; - - vfio_unpin_pages(&vgpu->vfio_device, &cur_gfn, 1); - } + vfio_unpin_pages(&vgpu->vfio_device, gfn << PAGE_SHIFT, + DIV_ROUND_UP(size, PAGE_SIZE)); } /* Pin a normal or compound guest page for dma. */ @@ -255,14 +249,14 @@ static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, * on stack to hold pfns. */ for (npage = 0; npage < total_pages; npage++) { - unsigned long cur_gfn = gfn + npage; + dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT; unsigned long pfn; - ret = vfio_pin_pages(&vgpu->vfio_device, &cur_gfn, 1, + ret = vfio_pin_pages(&vgpu->vfio_device, cur_iova, 1, IOMMU_READ | IOMMU_WRITE, &pfn); if (ret != 1) { - gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n", - cur_gfn, ret); + gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n", + &cur_iova, ret); goto err; } diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 3b94863ad24e..a739262f988d 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -114,7 +114,7 @@ static void pfn_array_unpin(struct pfn_array *pa, continue; } - vfio_unpin_pages(vdev, first, npage); + vfio_unpin_pages(vdev, *first << PAGE_SHIFT, npage); unpinned += npage; npage = 1; } @@ -146,7 +146,7 @@ static int pfn_array_pin(struct pfn_array *pa, struct vfio_device *vdev) continue; } - ret = vfio_pin_pages(vdev, first, npage, + ret = vfio_pin_pages(vdev, *first << PAGE_SHIFT, npage, IOMMU_READ | IOMMU_WRITE, &pa->pa_pfn[pinned]); if (ret < 0) { diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 5781059d3ed2..70f484668ca0 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -124,7 +124,7 @@ static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q) q->saved_isc = VFIO_AP_ISC_INVALID; } if (q->saved_pfn && !WARN_ON(!q->matrix_mdev)) { - vfio_unpin_pages(&q->matrix_mdev->vdev, &q->saved_pfn, 1); + vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_pfn << PAGE_SHIFT, 1); q->saved_pfn = 0; } } @@ -258,7 +258,7 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, return status; } - ret = vfio_pin_pages(&q->matrix_mdev->vdev, &g_pfn, 1, + ret = vfio_pin_pages(&q->matrix_mdev->vdev, g_pfn << PAGE_SHIFT, 1, IOMMU_READ | IOMMU_WRITE, &h_pfn); switch (ret) { case 1: @@ -301,7 +301,7 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, break; case AP_RESPONSE_OTHERWISE_CHANGED: /* We could not modify IRQ setings: clear new configuration */ - vfio_unpin_pages(&q->matrix_mdev->vdev, &g_pfn, 1); + vfio_unpin_pages(&q->matrix_mdev->vdev, g_pfn << PAGE_SHIFT, 1); kvm_s390_gisc_unregister(kvm, isc); break; default: @@ -1232,9 +1232,8 @@ static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova, { struct ap_matrix_mdev *matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev); - unsigned long g_pfn = iova >> PAGE_SHIFT; - vfio_unpin_pages(&matrix_mdev->vdev, &g_pfn, 1); + vfio_unpin_pages(&matrix_mdev->vdev, iova, 1); } /** diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 92b10aafae28..ffd1a492eea9 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1934,17 +1934,17 @@ int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); /* - * Pin a set of guest PFNs and return their associated host PFNs for local + * Pin contiguous user pages and return their associated host pages for local * domain only. * @device [in] : device - * @user_pfn [in]: array of user/guest PFNs to be pinned. - * @npage [in] : count of elements in user_pfn array. This count should not - * be greater VFIO_PIN_PAGES_MAX_ENTRIES. + * @iova [in] : starting IOVA of user pages to be pinned. + * @npage [in] : count of pages to be pinned. This count should not + * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. * @prot [in] : protection flags * @phys_pfn[out]: array of host PFNs * Return error or number of pages pinned. */ -int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, +int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, int npage, int prot, unsigned long *phys_pfn) { struct vfio_container *container; @@ -1952,8 +1952,7 @@ int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, struct vfio_iommu_driver *driver; int ret; - if (!user_pfn || !phys_pfn || !npage || - !vfio_assert_device_open(device)) + if (!phys_pfn || !npage || !vfio_assert_device_open(device)) return -EINVAL; if (npage > VFIO_PIN_PAGES_MAX_ENTRIES) @@ -1967,7 +1966,7 @@ int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, driver = container->iommu_driver; if (likely(driver && driver->ops->pin_pages)) ret = driver->ops->pin_pages(container->iommu_data, - group->iommu_group, user_pfn, + group->iommu_group, iova, npage, prot, phys_pfn); else ret = -ENOTTY; @@ -1977,15 +1976,13 @@ int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, EXPORT_SYMBOL(vfio_pin_pages); /* - * Unpin set of host PFNs for local domain only. + * Unpin contiguous host pages for local domain only. * @device [in] : device - * @user_pfn [in]: array of user/guest PFNs to be unpinned. Number of user/guest - * PFNs should not be greater than VFIO_PIN_PAGES_MAX_ENTRIES. - * @npage [in] : count of elements in user_pfn array. This count should not + * @iova [in] : starting address of user pages to be unpinned. + * @npage [in] : count of pages to be unpinned. This count should not * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. */ -void vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, - int npage) +void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) { struct vfio_container *container; struct vfio_iommu_driver *driver; @@ -2000,7 +1997,7 @@ void vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, container = device->group->container; driver = container->iommu_driver; - driver->ops->unpin_pages(container->iommu_data, user_pfn, npage); + driver->ops->unpin_pages(container->iommu_data, iova, npage); } EXPORT_SYMBOL(vfio_unpin_pages); diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index 6a8424b407c7..e9767e13f00f 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -50,11 +50,11 @@ struct vfio_iommu_driver_ops { struct iommu_group *group); int (*pin_pages)(void *iommu_data, struct iommu_group *group, - unsigned long *user_pfn, + dma_addr_t user_iova, int npage, int prot, unsigned long *phys_pfn); void (*unpin_pages)(void *iommu_data, - unsigned long *user_pfn, int npage); + dma_addr_t user_iova, int npage); void (*register_device)(void *iommu_data, struct vfio_device *vdev); void (*unregister_device)(void *iommu_data, diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e49fbe9968ef..e629e059118c 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -829,7 +829,7 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, static int vfio_iommu_type1_pin_pages(void *iommu_data, struct iommu_group *iommu_group, - unsigned long *user_pfn, + dma_addr_t user_iova, int npage, int prot, unsigned long *phys_pfn) { @@ -841,7 +841,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, bool do_accounting; dma_addr_t iova; - if (!iommu || !user_pfn || !phys_pfn) + if (!iommu || !phys_pfn) return -EINVAL; /* Supported for v2 version only */ @@ -857,7 +857,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, again: if (iommu->vaddr_invalid_count) { for (i = 0; i < npage; i++) { - iova = user_pfn[i] << PAGE_SHIFT; + iova = user_iova + PAGE_SIZE * i; ret = vfio_find_dma_valid(iommu, iova, PAGE_SIZE, &dma); if (ret < 0) goto pin_done; @@ -882,7 +882,7 @@ again: for (i = 0; i < npage; i++) { struct vfio_pfn *vpfn; - iova = user_pfn[i] << PAGE_SHIFT; + iova = user_iova + PAGE_SIZE * i; dma = vfio_find_dma(iommu, iova, PAGE_SIZE); if (!dma) { ret = -EINVAL; @@ -939,7 +939,7 @@ pin_unwind: for (j = 0; j < i; j++) { dma_addr_t iova; - iova = user_pfn[j] << PAGE_SHIFT; + iova = user_iova + PAGE_SIZE * j; dma = vfio_find_dma(iommu, iova, PAGE_SIZE); vfio_unpin_page_external(dma, iova, do_accounting); phys_pfn[j] = 0; @@ -950,7 +950,7 @@ pin_done: } static void vfio_iommu_type1_unpin_pages(void *iommu_data, - unsigned long *user_pfn, int npage) + dma_addr_t user_iova, int npage) { struct vfio_iommu *iommu = iommu_data; bool do_accounting; @@ -964,10 +964,9 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, do_accounting = list_empty(&iommu->domain_list); for (i = 0; i < npage; i++) { + dma_addr_t iova = user_iova + PAGE_SIZE * i; struct vfio_dma *dma; - dma_addr_t iova; - iova = user_pfn[i] << PAGE_SHIFT; dma = vfio_find_dma(iommu, iova, PAGE_SIZE); if (!dma) break; diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 9f7d74c24925..9e3b6abcf890 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -161,10 +161,9 @@ bool vfio_file_has_dev(struct file *file, struct vfio_device *device); #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) -int vfio_pin_pages(struct vfio_device *device, unsigned long *user_pfn, +int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, int npage, int prot, unsigned long *phys_pfn); -void vfio_unpin_pages(struct vfio_device *device, unsigned long *user_pfn, - int npage); +void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage); int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, void *data, size_t len, bool write); -- cgit v1.2.3 From 8561aa4fb7d72011c2352af2b1b9caf588942181 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Fri, 22 Jul 2022 19:02:54 -0700 Subject: vfio: Rename user_iova of vfio_dma_rw() Following the updated vfio_pin/unpin_pages(), use the simpler "iova". Reviewed-by: Christoph Hellwig Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Terrence Xu Tested-by: Eric Farman Signed-off-by: Nicolin Chen Link: https://lore.kernel.org/r/20220723020256.30081-9-nicolinc@nvidia.com Signed-off-by: Alex Williamson --- drivers/vfio/vfio.c | 6 +++--- include/linux/vfio.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index ffd1a492eea9..606a20b605ba 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -2012,13 +2012,13 @@ EXPORT_SYMBOL(vfio_unpin_pages); * not a real device DMA, it is not necessary to pin the user space memory. * * @device [in] : VFIO device - * @user_iova [in] : base IOVA of a user space buffer + * @iova [in] : base IOVA of a user space buffer * @data [in] : pointer to kernel buffer * @len [in] : kernel buffer length * @write : indicate read or write * Return error code on failure or 0 on success. */ -int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, void *data, +int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, size_t len, bool write) { struct vfio_container *container; @@ -2034,7 +2034,7 @@ int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, void *data, if (likely(driver && driver->ops->dma_rw)) ret = driver->ops->dma_rw(container->iommu_data, - user_iova, data, len, write); + iova, data, len, write); else ret = -ENOTTY; return ret; diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 9e3b6abcf890..acefd663e63b 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -164,7 +164,7 @@ bool vfio_file_has_dev(struct file *file, struct vfio_device *device); int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, int npage, int prot, unsigned long *phys_pfn); void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage); -int vfio_dma_rw(struct vfio_device *device, dma_addr_t user_iova, +int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, size_t len, bool write); /* -- cgit v1.2.3 From 34a255e67615995f729254307a0581c143e03752 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Fri, 22 Jul 2022 19:02:56 -0700 Subject: vfio: Replace phys_pfn with pages for vfio_pin_pages() Most of the callers of vfio_pin_pages() want "struct page *" and the low-level mm code to pin pages returns a list of "struct page *" too. So there's no gain in converting "struct page *" to PFN in between. Replace the output parameter "phys_pfn" list with a "pages" list, to simplify callers. This also allows us to replace the vfio_iommu_type1 implementation with a more efficient one. And drop the pfn_valid check in the gvt code, as there is no need to do such a check at a page-backed struct page pointer. For now, also update vfio_iommu_type1 to fit this new parameter too. Reviewed-by: Christoph Hellwig Reviewed-by: Kirti Wankhede Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Acked-by: Eric Farman Tested-by: Terrence Xu Tested-by: Eric Farman Signed-off-by: Nicolin Chen Link: https://lore.kernel.org/r/20220723020256.30081-11-nicolinc@nvidia.com Signed-off-by: Alex Williamson --- Documentation/driver-api/vfio-mediated-device.rst | 2 +- drivers/gpu/drm/i915/gvt/kvmgt.c | 19 ++++++------------- drivers/s390/cio/vfio_ccw_cp.c | 19 +++++++++---------- drivers/s390/crypto/vfio_ap_ops.c | 6 +++--- drivers/vfio/vfio.c | 8 ++++---- drivers/vfio/vfio.h | 2 +- drivers/vfio/vfio_iommu_type1.c | 19 +++++++++++-------- include/linux/vfio.h | 2 +- 8 files changed, 36 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst index ea32a0f13ddb..ba5fefcdae1a 100644 --- a/Documentation/driver-api/vfio-mediated-device.rst +++ b/Documentation/driver-api/vfio-mediated-device.rst @@ -263,7 +263,7 @@ The following APIs are provided for translating user pfn to host pfn in a VFIO driver:: int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, - int npage, int prot, unsigned long *phys_pfn); + int npage, int prot, struct page **pages); void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 8be75c282611..e3cd58946477 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -240,7 +240,7 @@ static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, unsigned long size, struct page **page) { int total_pages = DIV_ROUND_UP(size, PAGE_SIZE); - unsigned long base_pfn = 0; + struct page *base_page = NULL; int npage; int ret; @@ -250,26 +250,19 @@ static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, */ for (npage = 0; npage < total_pages; npage++) { dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT; - unsigned long pfn; + struct page *cur_page; ret = vfio_pin_pages(&vgpu->vfio_device, cur_iova, 1, - IOMMU_READ | IOMMU_WRITE, &pfn); + IOMMU_READ | IOMMU_WRITE, &cur_page); if (ret != 1) { gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n", &cur_iova, ret); goto err; } - if (!pfn_valid(pfn)) { - gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn); - npage++; - ret = -EFAULT; - goto err; - } - if (npage == 0) - base_pfn = pfn; - else if (base_pfn + npage != pfn) { + base_page = cur_page; + else if (base_page + npage != cur_page) { gvt_vgpu_err("The pages are not continuous\n"); ret = -EINVAL; npage++; @@ -277,7 +270,7 @@ static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, } } - *page = pfn_to_page(base_pfn); + *page = base_page; return 0; err: gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE); diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index cd4ec4f6d6ff..8963f452f963 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -22,8 +22,8 @@ struct page_array { /* Array that stores pages need to pin. */ dma_addr_t *pa_iova; - /* Array that receives PFNs of the pages pinned. */ - unsigned long *pa_pfn; + /* Array that receives the pinned pages. */ + struct page **pa_page; /* Number of pages pinned from @pa_iova. */ int pa_nr; }; @@ -68,19 +68,19 @@ static int page_array_alloc(struct page_array *pa, u64 iova, unsigned int len) return -EINVAL; pa->pa_iova = kcalloc(pa->pa_nr, - sizeof(*pa->pa_iova) + sizeof(*pa->pa_pfn), + sizeof(*pa->pa_iova) + sizeof(*pa->pa_page), GFP_KERNEL); if (unlikely(!pa->pa_iova)) { pa->pa_nr = 0; return -ENOMEM; } - pa->pa_pfn = (unsigned long *)&pa->pa_iova[pa->pa_nr]; + pa->pa_page = (struct page **)&pa->pa_iova[pa->pa_nr]; pa->pa_iova[0] = iova; - pa->pa_pfn[0] = -1ULL; + pa->pa_page[0] = NULL; for (i = 1; i < pa->pa_nr; i++) { pa->pa_iova[i] = pa->pa_iova[i - 1] + PAGE_SIZE; - pa->pa_pfn[i] = -1ULL; + pa->pa_page[i] = NULL; } return 0; @@ -144,7 +144,7 @@ static int page_array_pin(struct page_array *pa, struct vfio_device *vdev) ret = vfio_pin_pages(vdev, *first, npage, IOMMU_READ | IOMMU_WRITE, - &pa->pa_pfn[pinned]); + &pa->pa_page[pinned]); if (ret < 0) { goto err_out; } else if (ret > 0 && ret != npage) { @@ -195,7 +195,7 @@ static inline void page_array_idal_create_words(struct page_array *pa, */ for (i = 0; i < pa->pa_nr; i++) - idaws[i] = pa->pa_pfn[i] << PAGE_SHIFT; + idaws[i] = page_to_phys(pa->pa_page[i]); /* Adjust the first IDAW, since it may not start on a page boundary */ idaws[0] += pa->pa_iova[0] & (PAGE_SIZE - 1); @@ -246,8 +246,7 @@ static long copy_from_iova(struct vfio_device *vdev, void *to, u64 iova, l = n; for (i = 0; i < pa.pa_nr; i++) { - struct page *page = pfn_to_page(pa.pa_pfn[i]); - void *from = kmap_local_page(page); + void *from = kmap_local_page(pa.pa_page[i]); m = PAGE_SIZE; if (i == 0) { diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index d7c38c82f694..75cd92c291e3 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -234,9 +234,9 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, struct ap_qirq_ctrl aqic_gisa = {}; struct ap_queue_status status = {}; struct kvm_s390_gisa *gisa; + struct page *h_page; int nisc; struct kvm *kvm; - unsigned long h_pfn; phys_addr_t h_nib; dma_addr_t nib; int ret; @@ -251,7 +251,7 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, } ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1, - IOMMU_READ | IOMMU_WRITE, &h_pfn); + IOMMU_READ | IOMMU_WRITE, &h_page); switch (ret) { case 1: break; @@ -267,7 +267,7 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, kvm = q->matrix_mdev->kvm; gisa = kvm->arch.gisa_int.origin; - h_nib = (h_pfn << PAGE_SHIFT) | (nib & ~PAGE_MASK); + h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK); aqic_gisa.gisc = isc; nisc = kvm_s390_gisc_register(kvm, isc); diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 606a20b605ba..8e23ca59ceed 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1941,18 +1941,18 @@ EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); * @npage [in] : count of pages to be pinned. This count should not * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. * @prot [in] : protection flags - * @phys_pfn[out]: array of host PFNs + * @pages[out] : array of host pages * Return error or number of pages pinned. */ int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, - int npage, int prot, unsigned long *phys_pfn) + int npage, int prot, struct page **pages) { struct vfio_container *container; struct vfio_group *group = device->group; struct vfio_iommu_driver *driver; int ret; - if (!phys_pfn || !npage || !vfio_assert_device_open(device)) + if (!pages || !npage || !vfio_assert_device_open(device)) return -EINVAL; if (npage > VFIO_PIN_PAGES_MAX_ENTRIES) @@ -1967,7 +1967,7 @@ int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, if (likely(driver && driver->ops->pin_pages)) ret = driver->ops->pin_pages(container->iommu_data, group->iommu_group, iova, - npage, prot, phys_pfn); + npage, prot, pages); else ret = -ENOTTY; diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index e9767e13f00f..503bea6c843d 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -52,7 +52,7 @@ struct vfio_iommu_driver_ops { struct iommu_group *group, dma_addr_t user_iova, int npage, int prot, - unsigned long *phys_pfn); + struct page **pages); void (*unpin_pages)(void *iommu_data, dma_addr_t user_iova, int npage); void (*register_device)(void *iommu_data, diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e629e059118c..db516c90a977 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -831,7 +831,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, struct iommu_group *iommu_group, dma_addr_t user_iova, int npage, int prot, - unsigned long *phys_pfn) + struct page **pages) { struct vfio_iommu *iommu = iommu_data; struct vfio_iommu_group *group; @@ -841,7 +841,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, bool do_accounting; dma_addr_t iova; - if (!iommu || !phys_pfn) + if (!iommu || !pages) return -EINVAL; /* Supported for v2 version only */ @@ -880,6 +880,7 @@ again: do_accounting = list_empty(&iommu->domain_list); for (i = 0; i < npage; i++) { + unsigned long phys_pfn; struct vfio_pfn *vpfn; iova = user_iova + PAGE_SIZE * i; @@ -896,23 +897,25 @@ again: vpfn = vfio_iova_get_vfio_pfn(dma, iova); if (vpfn) { - phys_pfn[i] = vpfn->pfn; + pages[i] = pfn_to_page(vpfn->pfn); continue; } remote_vaddr = dma->vaddr + (iova - dma->iova); - ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn[i], + ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, do_accounting); if (ret) goto pin_unwind; - ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]); + ret = vfio_add_to_pfn_list(dma, iova, phys_pfn); if (ret) { - if (put_pfn(phys_pfn[i], dma->prot) && do_accounting) + if (put_pfn(phys_pfn, dma->prot) && do_accounting) vfio_lock_acct(dma, -1, true); goto pin_unwind; } + pages[i] = pfn_to_page(phys_pfn); + if (iommu->dirty_page_tracking) { unsigned long pgshift = __ffs(iommu->pgsize_bitmap); @@ -935,14 +938,14 @@ again: goto pin_done; pin_unwind: - phys_pfn[i] = 0; + pages[i] = NULL; for (j = 0; j < i; j++) { dma_addr_t iova; iova = user_iova + PAGE_SIZE * j; dma = vfio_find_dma(iommu, iova, PAGE_SIZE); vfio_unpin_page_external(dma, iova, do_accounting); - phys_pfn[j] = 0; + pages[j] = NULL; } pin_done: mutex_unlock(&iommu->lock); diff --git a/include/linux/vfio.h b/include/linux/vfio.h index acefd663e63b..e05ddc6fe6a5 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -162,7 +162,7 @@ bool vfio_file_has_dev(struct file *file, struct vfio_device *device); #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, - int npage, int prot, unsigned long *phys_pfn); + int npage, int prot, struct page **pages); void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage); int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, size_t len, bool write); -- cgit v1.2.3 From e948d32c54fa45cc1601c98956bdbbf5f17a3db2 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sat, 23 Jul 2022 19:57:17 +0200 Subject: video: fbdev: imxfb: Drop platform data support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No source file but the driver itself includes the header containing the platform data definition. The last user is gone since commit 8485adf17a15 ("ARM: imx: Remove imx device directory"). So we can safely drop platform data support. Signed-off-by: Uwe Kleine-König Signed-off-by: Helge Deller --- drivers/video/fbdev/imxfb.c | 99 +++++++++++-------------------- include/linux/platform_data/video-imxfb.h | 12 ---- 2 files changed, 34 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c index a2f644c97f28..85a5bf5639d9 100644 --- a/drivers/video/fbdev/imxfb.c +++ b/drivers/video/fbdev/imxfb.c @@ -656,7 +656,6 @@ static int imxfb_activate_var(struct fb_var_screeninfo *var, struct fb_info *inf static int imxfb_init_fbinfo(struct platform_device *pdev) { - struct imx_fb_platform_data *pdata = dev_get_platdata(&pdev->dev); struct fb_info *info = platform_get_drvdata(pdev); struct imxfb_info *fbi = info->par; struct device_node *np; @@ -690,25 +689,20 @@ static int imxfb_init_fbinfo(struct platform_device *pdev) info->fbops = &imxfb_ops; info->flags = FBINFO_FLAG_DEFAULT | FBINFO_READS_FAST; - if (pdata) { - fbi->lscr1 = pdata->lscr1; - fbi->dmacr = pdata->dmacr; - fbi->pwmr = pdata->pwmr; - } else { - np = pdev->dev.of_node; - info->var.grayscale = of_property_read_bool(np, - "cmap-greyscale"); - fbi->cmap_inverse = of_property_read_bool(np, "cmap-inverse"); - fbi->cmap_static = of_property_read_bool(np, "cmap-static"); - fbi->lscr1 = IMXFB_LSCR1_DEFAULT; + np = pdev->dev.of_node; + info->var.grayscale = of_property_read_bool(np, + "cmap-greyscale"); + fbi->cmap_inverse = of_property_read_bool(np, "cmap-inverse"); + fbi->cmap_static = of_property_read_bool(np, "cmap-static"); - of_property_read_u32(np, "fsl,lpccr", &fbi->pwmr); + fbi->lscr1 = IMXFB_LSCR1_DEFAULT; - of_property_read_u32(np, "fsl,lscr1", &fbi->lscr1); + of_property_read_u32(np, "fsl,lpccr", &fbi->pwmr); - of_property_read_u32(np, "fsl,dmacr", &fbi->dmacr); - } + of_property_read_u32(np, "fsl,lscr1", &fbi->lscr1); + + of_property_read_u32(np, "fsl,dmacr", &fbi->dmacr); return 0; } @@ -863,10 +857,10 @@ static int imxfb_probe(struct platform_device *pdev) struct imxfb_info *fbi; struct lcd_device *lcd; struct fb_info *info; - struct imx_fb_platform_data *pdata; struct resource *res; struct imx_fb_videomode *m; const struct of_device_id *of_id; + struct device_node *display_np; int ret, i; int bytes_per_pixel; @@ -884,8 +878,6 @@ static int imxfb_probe(struct platform_device *pdev) if (!res) return -ENODEV; - pdata = dev_get_platdata(&pdev->dev); - info = framebuffer_alloc(sizeof(struct imxfb_info), &pdev->dev); if (!info) return -ENOMEM; @@ -898,43 +890,34 @@ static int imxfb_probe(struct platform_device *pdev) if (ret < 0) goto failed_init; - if (pdata) { - if (!fb_mode) - fb_mode = pdata->mode[0].mode.name; - - fbi->mode = pdata->mode; - fbi->num_modes = pdata->num_modes; - } else { - struct device_node *display_np; - fb_mode = NULL; - - display_np = of_parse_phandle(pdev->dev.of_node, "display", 0); - if (!display_np) { - dev_err(&pdev->dev, "No display defined in devicetree\n"); - ret = -EINVAL; - goto failed_of_parse; - } + fb_mode = NULL; - /* - * imxfb does not support more modes, we choose only the native - * mode. - */ - fbi->num_modes = 1; - - fbi->mode = devm_kzalloc(&pdev->dev, - sizeof(struct imx_fb_videomode), GFP_KERNEL); - if (!fbi->mode) { - ret = -ENOMEM; - of_node_put(display_np); - goto failed_of_parse; - } + display_np = of_parse_phandle(pdev->dev.of_node, "display", 0); + if (!display_np) { + dev_err(&pdev->dev, "No display defined in devicetree\n"); + ret = -EINVAL; + goto failed_of_parse; + } - ret = imxfb_of_read_mode(&pdev->dev, display_np, fbi->mode); + /* + * imxfb does not support more modes, we choose only the native + * mode. + */ + fbi->num_modes = 1; + + fbi->mode = devm_kzalloc(&pdev->dev, + sizeof(struct imx_fb_videomode), GFP_KERNEL); + if (!fbi->mode) { + ret = -ENOMEM; of_node_put(display_np); - if (ret) - goto failed_of_parse; + goto failed_of_parse; } + ret = imxfb_of_read_mode(&pdev->dev, display_np, fbi->mode); + of_node_put(display_np); + if (ret) + goto failed_of_parse; + /* Calculate maximum bytes used per pixel. In most cases this should * be the same as m->bpp/8 */ m = &fbi->mode[0]; @@ -1001,13 +984,6 @@ static int imxfb_probe(struct platform_device *pdev) info->fix.smem_start = fbi->map_dma; - if (pdata && pdata->init) { - ret = pdata->init(fbi->pdev); - if (ret) - goto failed_platform_init; - } - - INIT_LIST_HEAD(&info->modelist); for (i = 0; i < fbi->num_modes; i++) fb_add_videomode(&fbi->mode[i].mode, &info->modelist); @@ -1059,9 +1035,6 @@ failed_lcd: failed_register: fb_dealloc_cmap(&info->cmap); failed_cmap: - if (pdata && pdata->exit) - pdata->exit(fbi->pdev); -failed_platform_init: dma_free_wc(&pdev->dev, fbi->map_size, info->screen_buffer, fbi->map_dma); failed_map: @@ -1079,7 +1052,6 @@ failed_init: static int imxfb_remove(struct platform_device *pdev) { - struct imx_fb_platform_data *pdata; struct fb_info *info = platform_get_drvdata(pdev); struct imxfb_info *fbi = info->par; struct resource *res; @@ -1092,9 +1064,6 @@ static int imxfb_remove(struct platform_device *pdev) unregister_framebuffer(info); fb_dealloc_cmap(&info->cmap); - pdata = dev_get_platdata(&pdev->dev); - if (pdata && pdata->exit) - pdata->exit(fbi->pdev); dma_free_wc(&pdev->dev, fbi->map_size, info->screen_buffer, fbi->map_dma); iounmap(fbi->regs); diff --git a/include/linux/platform_data/video-imxfb.h b/include/linux/platform_data/video-imxfb.h index 02812651af7d..b80a156a6617 100644 --- a/include/linux/platform_data/video-imxfb.h +++ b/include/linux/platform_data/video-imxfb.h @@ -55,16 +55,4 @@ struct imx_fb_videomode { unsigned char bpp; }; -struct imx_fb_platform_data { - struct imx_fb_videomode *mode; - int num_modes; - - u_int pwmr; - u_int lscr1; - u_int dmacr; - - int (*init)(struct platform_device *); - void (*exit)(struct platform_device *); -}; - #endif /* ifndef __MACH_IMXFB_H__ */ -- cgit v1.2.3 From e2279cc92919f37b4af985cb28ae350bb3e62e71 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sat, 23 Jul 2022 19:57:18 +0200 Subject: video: fbdev: imxfb: Drop unused symbols from header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only file that includes is the imxfb driver. Drop all symbols that are unused there. Signed-off-by: Uwe Kleine-König Signed-off-by: Helge Deller --- include/linux/platform_data/video-imxfb.h | 35 ------------------------------- 1 file changed, 35 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/video-imxfb.h b/include/linux/platform_data/video-imxfb.h index b80a156a6617..a16837c5e43c 100644 --- a/include/linux/platform_data/video-imxfb.h +++ b/include/linux/platform_data/video-imxfb.h @@ -8,45 +8,10 @@ #include #define PCR_TFT (1 << 31) -#define PCR_COLOR (1 << 30) -#define PCR_PBSIZ_1 (0 << 28) -#define PCR_PBSIZ_2 (1 << 28) -#define PCR_PBSIZ_4 (2 << 28) -#define PCR_PBSIZ_8 (3 << 28) -#define PCR_BPIX_1 (0 << 25) -#define PCR_BPIX_2 (1 << 25) -#define PCR_BPIX_4 (2 << 25) #define PCR_BPIX_8 (3 << 25) #define PCR_BPIX_12 (4 << 25) #define PCR_BPIX_16 (5 << 25) #define PCR_BPIX_18 (6 << 25) -#define PCR_PIXPOL (1 << 24) -#define PCR_FLMPOL (1 << 23) -#define PCR_LPPOL (1 << 22) -#define PCR_CLKPOL (1 << 21) -#define PCR_OEPOL (1 << 20) -#define PCR_SCLKIDLE (1 << 19) -#define PCR_END_SEL (1 << 18) -#define PCR_END_BYTE_SWAP (1 << 17) -#define PCR_REV_VS (1 << 16) -#define PCR_ACD_SEL (1 << 15) -#define PCR_ACD(x) (((x) & 0x7f) << 8) -#define PCR_SCLK_SEL (1 << 7) -#define PCR_SHARP (1 << 6) -#define PCR_PCD(x) ((x) & 0x3f) - -#define PWMR_CLS(x) (((x) & 0x1ff) << 16) -#define PWMR_LDMSK (1 << 15) -#define PWMR_SCR1 (1 << 10) -#define PWMR_SCR0 (1 << 9) -#define PWMR_CC_EN (1 << 8) -#define PWMR_PW(x) ((x) & 0xff) - -#define LSCR1_PS_RISE_DELAY(x) (((x) & 0x7f) << 26) -#define LSCR1_CLS_RISE_DELAY(x) (((x) & 0x3f) << 16) -#define LSCR1_REV_TOGGLE_DELAY(x) (((x) & 0xf) << 8) -#define LSCR1_GRAY2(x) (((x) & 0xf) << 4) -#define LSCR1_GRAY1(x) (((x) & 0xf)) struct imx_fb_videomode { struct fb_videomode mode; -- cgit v1.2.3 From ded77a74ee6bc3dea72ad41129823a812e4b64f3 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sat, 23 Jul 2022 19:57:19 +0200 Subject: video: fbdev: imxfb: Fold into only user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No source file but the driver itself includes the header containing the platform data definition. The last user is gone since commit 8485adf17a15 ("ARM: imx: Remove imx device directory"). Move the remaining symbols directly into the driver and remove the then unused header file. Signed-off-by: Uwe Kleine-König Signed-off-by: Helge Deller --- MAINTAINERS | 1 - drivers/video/fbdev/imxfb.c | 13 ++++++++++++- include/linux/platform_data/video-imxfb.h | 23 ----------------------- 3 files changed, 12 insertions(+), 25 deletions(-) delete mode 100644 include/linux/platform_data/video-imxfb.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 651616ed8ae2..75c4d7ebef09 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8011,7 +8011,6 @@ L: linux-fbdev@vger.kernel.org L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained F: drivers/video/fbdev/imxfb.c -F: include/linux/platform_data/video-imxfb.h FREESCALE IMX DDR PMU DRIVER M: Frank Li diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c index 85a5bf5639d9..fa6a19c1ae9b 100644 --- a/drivers/video/fbdev/imxfb.c +++ b/drivers/video/fbdev/imxfb.c @@ -41,7 +41,18 @@ #include