From 534acc057b5a08ec33fa57cdd2f5a09ef124e7f2 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Wed, 29 Jul 2009 15:04:18 -0700 Subject: lib: flexible array implementation Once a structure goes over PAGE_SIZE*2, we see occasional allocation failures. Some people have chosen to switch over to things like vmalloc() that will let them keep array-like access to such a large structures. But, vmalloc() has plenty of downsides. Here's an alternative. I think it's what Andrew was suggesting here: http://lkml.org/lkml/2009/7/2/518 I call it a flexible array. It does all of its work in PAGE_SIZE bits, so never does an order>0 allocation. The base level has PAGE_SIZE-2*sizeof(int) bytes of storage for pointers to the second level. So, with a 32-bit arch, you get about 4MB (4183112 bytes) of total storage when the objects pack nicely into a page. It is half that on 64-bit because the pointers are twice the size. There's a table detailing this in the code. There are kerneldocs for the functions, but here's an overview: flex_array_alloc() - dynamically allocate a base structure flex_array_free() - free the array and all of the second-level pages flex_array_free_parts() - free the second-level pages, but not the base (for static bases) flex_array_put() - copy into the array at the given index flex_array_get() - copy out of the array at the given index flex_array_prealloc() - preallocate the second-level pages between the given indexes to guarantee no allocs will occur at put() time. We could also potentially just pass the "element_size" into each of the API functions instead of storing it internally. That would get us one more base pointer on 32-bit. I've been testing this by running it in userspace. The header and patch that I've been using are here, as well as the little script I'm using to generate the size table which goes in the kerneldocs. http://sr71.net/~dave/linux/flexarray/ [akpm@linux-foundation.org: coding-style fixes] Signed-off-by: Dave Hansen Reviewed-by: KAMEZAWA Hiroyuki Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 include/linux/flex_array.h (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h new file mode 100644 index 000000000000..23c1ec79a31b --- /dev/null +++ b/include/linux/flex_array.h @@ -0,0 +1,47 @@ +#ifndef _FLEX_ARRAY_H +#define _FLEX_ARRAY_H + +#include +#include + +#define FLEX_ARRAY_PART_SIZE PAGE_SIZE +#define FLEX_ARRAY_BASE_SIZE PAGE_SIZE + +struct flex_array_part; + +/* + * This is meant to replace cases where an array-like + * structure has gotten too big to fit into kmalloc() + * and the developer is getting tempted to use + * vmalloc(). + */ + +struct flex_array { + union { + struct { + int element_size; + int total_nr_elements; + struct flex_array_part *parts[0]; + }; + /* + * This little trick makes sure that + * sizeof(flex_array) == PAGE_SIZE + */ + char padding[FLEX_ARRAY_BASE_SIZE]; + }; +}; + +#define FLEX_ARRAY_INIT(size, total) { { {\ + .element_size = (size), \ + .total_nr_elements = (total), \ +} } } + +struct flex_array *flex_array_alloc(int element_size, int total, gfp_t flags); +int flex_array_prealloc(struct flex_array *fa, int start, int end, gfp_t flags); +void flex_array_free(struct flex_array *fa); +void flex_array_free_parts(struct flex_array *fa); +int flex_array_put(struct flex_array *fa, int element_nr, void *src, + gfp_t flags); +void *flex_array_get(struct flex_array *fa, int element_nr); + +#endif /* _FLEX_ARRAY_H */ -- cgit v1.2.3 From 8e7ee27095aee87b5db1b0061e2ceea5878a1bbd Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Wed, 26 Aug 2009 14:29:21 -0700 Subject: flex_array: declare parts member to have incomplete type The `parts' member of struct flex_array should evaluate to an incomplete type so that sizeof() cannot be used and C99 does not require the zero-length specification. Signed-off-by: David Rientjes Acked-by: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 23c1ec79a31b..603160db7c98 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -21,7 +21,7 @@ struct flex_array { struct { int element_size; int total_nr_elements; - struct flex_array_part *parts[0]; + struct flex_array_part *parts[]; }; /* * This little trick makes sure that -- cgit v1.2.3 From b62e408c05228f40e69bb38a48db8961cac6cd23 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Wed, 26 Aug 2009 14:29:22 -0700 Subject: flex_array: convert element_nr formals to unsigned It's problematic to allow signed element_nr's or total's to be passed as part of the flex array API. flex_array_alloc() allows total_nr_elements to be set to a negative quantity, which is obviously erroneous. flex_array_get() and flex_array_put() allows negative array indices in dereferencing an array part, which could address memory mapped before struct flex_array. The fix is to convert all existing element_nr formals to be qualified as unsigned. Existing checks to compare it to total_nr_elements or the max array size based on element_size need not be changed. Signed-off-by: David Rientjes Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 10 ++++++---- lib/flex_array.c | 24 +++++++++++++----------- 2 files changed, 19 insertions(+), 15 deletions(-) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 603160db7c98..45ff18491514 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -36,12 +36,14 @@ struct flex_array { .total_nr_elements = (total), \ } } } -struct flex_array *flex_array_alloc(int element_size, int total, gfp_t flags); -int flex_array_prealloc(struct flex_array *fa, int start, int end, gfp_t flags); +struct flex_array *flex_array_alloc(int element_size, unsigned int total, + gfp_t flags); +int flex_array_prealloc(struct flex_array *fa, unsigned int start, + unsigned int end, gfp_t flags); void flex_array_free(struct flex_array *fa); void flex_array_free_parts(struct flex_array *fa); -int flex_array_put(struct flex_array *fa, int element_nr, void *src, +int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, gfp_t flags); -void *flex_array_get(struct flex_array *fa, int element_nr); +void *flex_array_get(struct flex_array *fa, unsigned int element_nr); #endif /* _FLEX_ARRAY_H */ diff --git a/lib/flex_array.c b/lib/flex_array.c index cf4e372cae90..7baed2fc3bc8 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -99,7 +99,8 @@ static inline int elements_fit_in_base(struct flex_array *fa) * capacity in the base structure. Also note that no effort is made * to efficiently pack objects across page boundaries. */ -struct flex_array *flex_array_alloc(int element_size, int total, gfp_t flags) +struct flex_array *flex_array_alloc(int element_size, unsigned int total, + gfp_t flags) { struct flex_array *ret; int max_size = nr_base_part_ptrs() * __elements_per_part(element_size); @@ -115,7 +116,8 @@ struct flex_array *flex_array_alloc(int element_size, int total, gfp_t flags) return ret; } -static int fa_element_to_part_nr(struct flex_array *fa, int element_nr) +static int fa_element_to_part_nr(struct flex_array *fa, + unsigned int element_nr) { return element_nr / __elements_per_part(fa->element_size); } @@ -143,14 +145,12 @@ void flex_array_free(struct flex_array *fa) kfree(fa); } -static int fa_index_inside_part(struct flex_array *fa, int element_nr) +static unsigned int index_inside_part(struct flex_array *fa, + unsigned int element_nr) { - return element_nr % __elements_per_part(fa->element_size); -} + unsigned int part_offset; -static int index_inside_part(struct flex_array *fa, int element_nr) -{ - int part_offset = fa_index_inside_part(fa, element_nr); + part_offset = element_nr % __elements_per_part(fa->element_size); return part_offset * fa->element_size; } @@ -185,7 +185,8 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) * * Locking must be provided by the caller. */ -int flex_array_put(struct flex_array *fa, int element_nr, void *src, gfp_t flags) +int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, + gfp_t flags) { int part_nr = fa_element_to_part_nr(fa, element_nr); struct flex_array_part *part; @@ -217,7 +218,8 @@ int flex_array_put(struct flex_array *fa, int element_nr, void *src, gfp_t flags * * Locking must be provided by the caller. */ -int flex_array_prealloc(struct flex_array *fa, int start, int end, gfp_t flags) +int flex_array_prealloc(struct flex_array *fa, unsigned int start, + unsigned int end, gfp_t flags) { int start_part; int end_part; @@ -248,7 +250,7 @@ int flex_array_prealloc(struct flex_array *fa, int start, int end, gfp_t flags) * * Locking must be provided by the caller. */ -void *flex_array_get(struct flex_array *fa, int element_nr) +void *flex_array_get(struct flex_array *fa, unsigned int element_nr) { int part_nr = fa_element_to_part_nr(fa, element_nr); struct flex_array_part *part; -- cgit v1.2.3 From e6de3988aa52debb25a427d085061f3bf1181d54 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:30 -0700 Subject: flex_array: add flex_array_clear function Add a new function to the flex_array API: int flex_array_clear(struct flex_array *fa, unsigned int element_nr) This function will zero the element at element_nr in the flex_array. Although this is equivalent to using flex_array_put() and passing a pointer to zero'd memory, flex_array_clear() does not require such a pointer to memory that would most likely need to be allocated on the caller's stack which could be significantly large depending on element_size. Signed-off-by: David Rientjes Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 1 + lib/flex_array.c | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 45ff18491514..3887b21f883f 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -44,6 +44,7 @@ void flex_array_free(struct flex_array *fa); void flex_array_free_parts(struct flex_array *fa); int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, gfp_t flags); +int flex_array_clear(struct flex_array *fa, unsigned int element_nr); void *flex_array_get(struct flex_array *fa, unsigned int element_nr); #endif /* _FLEX_ARRAY_H */ diff --git a/lib/flex_array.c b/lib/flex_array.c index 7baed2fc3bc8..b68f99be4080 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -206,6 +206,32 @@ int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, return 0; } +/** + * flex_array_clear - clear element in array at @element_nr + * @element_nr: index of the position to clear. + * + * Locking must be provided by the caller. + */ +int flex_array_clear(struct flex_array *fa, unsigned int element_nr) +{ + int part_nr = fa_element_to_part_nr(fa, element_nr); + struct flex_array_part *part; + void *dst; + + if (element_nr >= fa->total_nr_elements) + return -ENOSPC; + if (elements_fit_in_base(fa)) + part = (struct flex_array_part *)&fa->parts[0]; + else { + part = fa->parts[part_nr]; + if (!part) + return -EINVAL; + } + dst = &part->elements[index_inside_part(fa, element_nr)]; + memset(dst, 0, fa->element_size); + return 0; +} + /** * flex_array_prealloc - guarantee that array space exists * @start: index of first array element for which space is allocated -- cgit v1.2.3 From 4af5a2f770cc8575840ccb1514ec76ecb592985c Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:31 -0700 Subject: flex_array: add flex_array_shrink function Add a new function to the flex_array API: int flex_array_shrink(struct flex_array *fa) This function will free all unused second-level pages. Since elements are now poisoned if they are not allocated with __GFP_ZERO, it's possible to identify parts that consist solely of unused elements. flex_array_shrink() returns the number of pages freed. Signed-off-by: David Rientjes Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 1 + lib/flex_array.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 3887b21f883f..f12401e485fe 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -46,5 +46,6 @@ int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, gfp_t flags); int flex_array_clear(struct flex_array *fa, unsigned int element_nr); void *flex_array_get(struct flex_array *fa, unsigned int element_nr); +int flex_array_shrink(struct flex_array *fa); #endif /* _FLEX_ARRAY_H */ diff --git a/lib/flex_array.c b/lib/flex_array.c index e22d0e9776aa..1b03bb553410 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -291,3 +291,43 @@ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) } return &part->elements[index_inside_part(fa, element_nr)]; } + +static int part_is_free(struct flex_array_part *part) +{ + int i; + + for (i = 0; i < sizeof(struct flex_array_part); i++) + if (part->elements[i] != FLEX_ARRAY_FREE) + return 0; + return 1; +} + +/** + * flex_array_shrink - free unused second-level pages + * + * Frees all second-level pages that consist solely of unused + * elements. Returns the number of pages freed. + * + * Locking must be provided by the caller. + */ +int flex_array_shrink(struct flex_array *fa) +{ + struct flex_array_part *part; + int max_part = nr_base_part_ptrs(); + int part_nr; + int ret = 0; + + if (elements_fit_in_base(fa)) + return ret; + for (part_nr = 0; part_nr < max_part; part_nr++) { + part = fa->parts[part_nr]; + if (!part) + continue; + if (part_is_free(part)) { + fa->parts[part_nr] = NULL; + kfree(part); + ret++; + } + } + return ret; +} -- cgit v1.2.3 From 45b588d6e5cc172704bac0c998ce54873b149b22 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:33 -0700 Subject: flex_array: introduce DEFINE_FLEX_ARRAY FLEX_ARRAY_INIT(element_size, total_nr_elements) cannot determine if either parameter is valid, so flex arrays which are statically allocated with this interface can easily become corrupted or reference beyond its allocated memory. This removes FLEX_ARRAY_INIT() as a struct flex_array initializer since no initializer may perform the required checking. Instead, the array is now defined with a new interface: DEFINE_FLEX_ARRAY(name, element_size, total_nr_elements) This may be prefixed with `static' for file scope. This interface includes compile-time checking of the parameters to ensure they are valid. Since the validity of both element_size and total_nr_elements depend on FLEX_ARRAY_BASE_SIZE and FLEX_ARRAY_PART_SIZE, the kernel build will fail if either of these predefined values changes such that the array parameters are no longer valid. Since BUILD_BUG_ON() requires compile time constants, several of the static inline functions that were once local to lib/flex_array.c had to be moved to include/linux/flex_array.h. Signed-off-by: David Rientjes Acked-by: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 30 ++++++++++++++++++++++++++---- lib/flex_array.c | 36 ++++++++++-------------------------- 2 files changed, 36 insertions(+), 30 deletions(-) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index f12401e485fe..1d747f72298b 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -31,10 +31,32 @@ struct flex_array { }; }; -#define FLEX_ARRAY_INIT(size, total) { { {\ - .element_size = (size), \ - .total_nr_elements = (total), \ -} } } +/* Number of bytes left in base struct flex_array, excluding metadata */ +#define FLEX_ARRAY_BASE_BYTES_LEFT \ + (FLEX_ARRAY_BASE_SIZE - offsetof(struct flex_array, parts)) + +/* Number of pointers in base to struct flex_array_part pages */ +#define FLEX_ARRAY_NR_BASE_PTRS \ + (FLEX_ARRAY_BASE_BYTES_LEFT / sizeof(struct flex_array_part *)) + +/* Number of elements of size that fit in struct flex_array_part */ +#define FLEX_ARRAY_ELEMENTS_PER_PART(size) \ + (FLEX_ARRAY_PART_SIZE / size) + +/* + * Defines a statically allocated flex array and ensures its parameters are + * valid. + */ +#define DEFINE_FLEX_ARRAY(__arrayname, __element_size, __total) \ + struct flex_array __arrayname = { { { \ + .element_size = (__element_size), \ + .total_nr_elements = (__total), \ + } } }; \ + static inline void __arrayname##_invalid_parameter(void) \ + { \ + BUILD_BUG_ON((__total) > FLEX_ARRAY_NR_BASE_PTRS * \ + FLEX_ARRAY_ELEMENTS_PER_PART(__element_size)); \ + } struct flex_array *flex_array_alloc(int element_size, unsigned int total, gfp_t flags); diff --git a/lib/flex_array.c b/lib/flex_array.c index 1b03bb553410..b62ce6cffd0a 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -28,23 +28,6 @@ struct flex_array_part { char elements[FLEX_ARRAY_PART_SIZE]; }; -static inline int __elements_per_part(int element_size) -{ - return FLEX_ARRAY_PART_SIZE / element_size; -} - -static inline int bytes_left_in_base(void) -{ - int element_offset = offsetof(struct flex_array, parts); - int bytes_left = FLEX_ARRAY_BASE_SIZE - element_offset; - return bytes_left; -} - -static inline int nr_base_part_ptrs(void) -{ - return bytes_left_in_base() / sizeof(struct flex_array_part *); -} - /* * If a user requests an allocation which is small * enough, we may simply use the space in the @@ -54,7 +37,7 @@ static inline int nr_base_part_ptrs(void) static inline int elements_fit_in_base(struct flex_array *fa) { int data_size = fa->element_size * fa->total_nr_elements; - if (data_size <= bytes_left_in_base()) + if (data_size <= FLEX_ARRAY_BASE_BYTES_LEFT) return 1; return 0; } @@ -103,7 +86,8 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, gfp_t flags) { struct flex_array *ret; - int max_size = nr_base_part_ptrs() * __elements_per_part(element_size); + int max_size = FLEX_ARRAY_NR_BASE_PTRS * + FLEX_ARRAY_ELEMENTS_PER_PART(element_size); /* max_size will end up 0 if element_size > PAGE_SIZE */ if (total > max_size) @@ -114,14 +98,15 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, ret->element_size = element_size; ret->total_nr_elements = total; if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) - memset(ret->parts[0], FLEX_ARRAY_FREE, bytes_left_in_base()); + memset(ret->parts[0], FLEX_ARRAY_FREE, + FLEX_ARRAY_BASE_BYTES_LEFT); return ret; } static int fa_element_to_part_nr(struct flex_array *fa, unsigned int element_nr) { - return element_nr / __elements_per_part(fa->element_size); + return element_nr / FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); } /** @@ -133,11 +118,10 @@ static int fa_element_to_part_nr(struct flex_array *fa, void flex_array_free_parts(struct flex_array *fa) { int part_nr; - int max_part = nr_base_part_ptrs(); if (elements_fit_in_base(fa)) return; - for (part_nr = 0; part_nr < max_part; part_nr++) + for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) kfree(fa->parts[part_nr]); } @@ -152,7 +136,8 @@ static unsigned int index_inside_part(struct flex_array *fa, { unsigned int part_offset; - part_offset = element_nr % __elements_per_part(fa->element_size); + part_offset = element_nr % + FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); return part_offset * fa->element_size; } @@ -313,13 +298,12 @@ static int part_is_free(struct flex_array_part *part) int flex_array_shrink(struct flex_array *fa) { struct flex_array_part *part; - int max_part = nr_base_part_ptrs(); int part_nr; int ret = 0; if (elements_fit_in_base(fa)) return ret; - for (part_nr = 0; part_nr < max_part; part_nr++) { + for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) { part = fa->parts[part_nr]; if (!part) continue; -- cgit v1.2.3 From ea98eed9bcb62d1319db8b1210712c6a110a886c Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 9 Aug 2010 17:20:56 -0700 Subject: flex_array: add helpers to get and put to make pointers easy to use Getting and putting arrays of pointers with flex arrays is a PITA. You have to remember to pass &ptr to the _put and you have to do weird and wacky casting to get the ptr back from the _get. Add two functions flex_array_get_ptr() and flex_array_put_ptr() to handle all of the magic. [akpm@linux-foundation.org: simplification suggested by Joe] Signed-off-by: Eric Paris Cc: David Rientjes Cc: Dave Hansen Cc: Joe Perches Cc: James Morris Cc: Joe Perches Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 5 +++++ lib/flex_array.c | 25 ++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 1d747f72298b..631b77f2ac70 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -70,4 +70,9 @@ int flex_array_clear(struct flex_array *fa, unsigned int element_nr); void *flex_array_get(struct flex_array *fa, unsigned int element_nr); int flex_array_shrink(struct flex_array *fa); +#define flex_array_put_ptr(fa, nr, src, gfp) \ + flex_array_put(fa, nr, &(void *)(src), gfp) + +void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr); + #endif /* _FLEX_ARRAY_H */ diff --git a/lib/flex_array.c b/lib/flex_array.c index 41b1804fa728..77a6fea7481e 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -171,6 +171,8 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) * Note that this *copies* the contents of @src into * the array. If you are trying to store an array of * pointers, make sure to pass in &ptr instead of ptr. + * You may instead wish to use the flex_array_put_ptr() + * helper function. * * Locking must be provided by the caller. */ @@ -265,7 +267,8 @@ int flex_array_prealloc(struct flex_array *fa, unsigned int start, * * Returns a pointer to the data at index @element_nr. Note * that this is a copy of the data that was passed in. If you - * are using this to store pointers, you'll get back &ptr. + * are using this to store pointers, you'll get back &ptr. You + * may instead wish to use the flex_array_get_ptr helper. * * Locking must be provided by the caller. */ @@ -286,6 +289,26 @@ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) return &part->elements[index_inside_part(fa, element_nr)]; } +/** + * flex_array_get_ptr - pull a ptr back out of the array + * @fa: the flex array from which to extract data + * @element_nr: index of the element to fetch from the array + * + * Returns the pointer placed in the flex array at element_nr using + * flex_array_put_ptr(). This function should not be called if the + * element in question was not set using the _put_ptr() helper. + */ +void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr) +{ + void **tmp; + + tmp = flex_array_get(fa, element_nr); + if (!tmp) + return NULL; + + return *tmp; +} + static int part_is_free(struct flex_array_part *part) { int i; -- cgit v1.2.3 From c41ab6a1b9028de33e74101cb0aae13098a56fdb Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 29 Nov 2010 15:47:09 -0500 Subject: flex_array: fix flex_array_put_ptr macro to be valid C MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using flex_array_put_ptr() results in a compile error "error: lvalue required as unary ‘&’ operand" fix the casting order to fix this. Signed-off-by: Eric Paris --- include/linux/flex_array.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 631b77f2ac70..70e4efabe0fb 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -71,7 +71,7 @@ void *flex_array_get(struct flex_array *fa, unsigned int element_nr); int flex_array_shrink(struct flex_array *fa); #define flex_array_put_ptr(fa, nr, src, gfp) \ - flex_array_put(fa, nr, &(void *)(src), gfp) + flex_array_put(fa, nr, (void *)&(src), gfp) void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr); -- cgit v1.2.3 From 5d30b10bd68df007e7ae21e77d1e0ce184b53040 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Thu, 28 Apr 2011 15:55:52 -0400 Subject: flex_array: flex_array_prealloc takes a number of elements, not an end Change flex_array_prealloc to take the number of elements for which space should be allocated instead of the last (inclusive) element. Users and documentation are updated accordingly. flex_arrays got introduced before they had users. When folks started using it, they ended up needing a different API than was coded up originally. This swaps over to the API that folks apparently need. Based-on-patch-by: Steffen Klassert Signed-off-by: Eric Paris Tested-by: Chris Richards Acked-by: Dave Hansen Cc: stable@kernel.org [2.6.38+] --- Documentation/flexible-arrays.txt | 4 ++-- include/linux/flex_array.h | 2 +- lib/flex_array.c | 13 ++++++++----- security/selinux/ss/policydb.c | 6 +++--- 4 files changed, 14 insertions(+), 11 deletions(-) (limited to 'include/linux/flex_array.h') diff --git a/Documentation/flexible-arrays.txt b/Documentation/flexible-arrays.txt index cb8a3a00cc92..df904aec9904 100644 --- a/Documentation/flexible-arrays.txt +++ b/Documentation/flexible-arrays.txt @@ -66,10 +66,10 @@ trick is to ensure that any needed memory allocations are done before entering atomic context, using: int flex_array_prealloc(struct flex_array *array, unsigned int start, - unsigned int end, gfp_t flags); + unsigned int nr_elements, gfp_t flags); This function will ensure that memory for the elements indexed in the range -defined by start and end has been allocated. Thereafter, a +defined by start and nr_elements has been allocated. Thereafter, a flex_array_put() call on an element in that range is guaranteed not to block. diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 70e4efabe0fb..ebeb2f3ad068 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -61,7 +61,7 @@ struct flex_array { struct flex_array *flex_array_alloc(int element_size, unsigned int total, gfp_t flags); int flex_array_prealloc(struct flex_array *fa, unsigned int start, - unsigned int end, gfp_t flags); + unsigned int nr_elements, gfp_t flags); void flex_array_free(struct flex_array *fa); void flex_array_free_parts(struct flex_array *fa); int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, diff --git a/lib/flex_array.c b/lib/flex_array.c index c0ea40ba2082..0c33b24498ba 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -232,10 +232,10 @@ EXPORT_SYMBOL(flex_array_clear); /** * flex_array_prealloc - guarantee that array space exists - * @fa: the flex array for which to preallocate parts - * @start: index of first array element for which space is allocated - * @end: index of last (inclusive) element for which space is allocated - * @flags: page allocation flags + * @fa: the flex array for which to preallocate parts + * @start: index of first array element for which space is allocated + * @nr_elements: number of elements for which space is allocated + * @flags: page allocation flags * * This will guarantee that no future calls to flex_array_put() * will allocate memory. It can be used if you are expecting to @@ -245,13 +245,16 @@ EXPORT_SYMBOL(flex_array_clear); * Locking must be provided by the caller. */ int flex_array_prealloc(struct flex_array *fa, unsigned int start, - unsigned int end, gfp_t flags) + unsigned int nr_elements, gfp_t flags) { int start_part; int end_part; int part_nr; + unsigned int end; struct flex_array_part *part; + end = start + nr_elements - 1; + if (start >= fa->total_nr_elements || end >= fa->total_nr_elements) return -ENOSPC; if (elements_fit_in_base(fa)) diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index e7b850ad57ee..e6e7ce0d3d55 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -502,7 +502,7 @@ static int policydb_index(struct policydb *p) goto out; rc = flex_array_prealloc(p->type_val_to_struct_array, 0, - p->p_types.nprim - 1, GFP_KERNEL | __GFP_ZERO); + p->p_types.nprim, GFP_KERNEL | __GFP_ZERO); if (rc) goto out; @@ -519,7 +519,7 @@ static int policydb_index(struct policydb *p) goto out; rc = flex_array_prealloc(p->sym_val_to_name[i], - 0, p->symtab[i].nprim - 1, + 0, p->symtab[i].nprim, GFP_KERNEL | __GFP_ZERO); if (rc) goto out; @@ -2375,7 +2375,7 @@ int policydb_read(struct policydb *p, void *fp) goto bad; /* preallocate so we don't have to worry about the put ever failing */ - rc = flex_array_prealloc(p->type_attr_map_array, 0, p->p_types.nprim - 1, + rc = flex_array_prealloc(p->type_attr_map_array, 0, p->p_types.nprim, GFP_KERNEL | __GFP_ZERO); if (rc) goto bad; -- cgit v1.2.3 From 704f15ddb5fc2a7f25a12eb0913302d8ad9ffab3 Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Thu, 26 May 2011 16:25:02 -0700 Subject: flex_array: avoid divisions when accessing elements On most architectures division is an expensive operation and accessing an element currently requires four of them. This performance penalty effectively precludes flex arrays from being used on any kind of fast path. However, two of these divisions can be handled at creation time and the others can be replaced by a reciprocal divide, completely avoiding real divisions on access. [eparis@redhat.com: rebase on top of changes to support 0 len elements] [eparis@redhat.com: initialize part_nr when array fits entirely in base] Signed-off-by: Jesse Gross Signed-off-by: Eric Paris Cc: Dave Hansen Cc: David Rientjes Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 2 ++ lib/flex_array.c | 51 ++++++++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 22 deletions(-) (limited to 'include/linux/flex_array.h') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index ebeb2f3ad068..6843cf193a44 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -21,6 +21,8 @@ struct flex_array { struct { int element_size; int total_nr_elements; + int elems_per_part; + u32 reciprocal_elems; struct flex_array_part *parts[]; }; /* diff --git a/lib/flex_array.c b/lib/flex_array.c index cab7621f98aa..9b8b89458c4c 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -24,6 +24,7 @@ #include #include #include +#include struct flex_array_part { char elements[FLEX_ARRAY_PART_SIZE]; @@ -70,15 +71,15 @@ static inline int elements_fit_in_base(struct flex_array *fa) * Element size | Objects | Objects | * PAGE_SIZE=4k | 32-bit | 64-bit | * ---------------------------------| - * 1 bytes | 4186112 | 2093056 | - * 2 bytes | 2093056 | 1046528 | - * 3 bytes | 1395030 | 697515 | - * 4 bytes | 1046528 | 523264 | - * 32 bytes | 130816 | 65408 | - * 33 bytes | 126728 | 63364 | - * 2048 bytes | 2044 | 1022 | - * 2049 bytes | 1022 | 511 | - * void * | 1046528 | 261632 | + * 1 bytes | 4177920 | 2088960 | + * 2 bytes | 2088960 | 1044480 | + * 3 bytes | 1392300 | 696150 | + * 4 bytes | 1044480 | 522240 | + * 32 bytes | 130560 | 65408 | + * 33 bytes | 126480 | 63240 | + * 2048 bytes | 2040 | 1020 | + * 2049 bytes | 1020 | 510 | + * void * | 1044480 | 261120 | * * Since 64-bit pointers are twice the size, we lose half the * capacity in the base structure. Also note that no effort is made @@ -88,11 +89,15 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, gfp_t flags) { struct flex_array *ret; + int elems_per_part = 0; + int reciprocal_elems = 0; int max_size = 0; - if (element_size) - max_size = FLEX_ARRAY_NR_BASE_PTRS * - FLEX_ARRAY_ELEMENTS_PER_PART(element_size); + if (element_size) { + elems_per_part = FLEX_ARRAY_ELEMENTS_PER_PART(element_size); + reciprocal_elems = reciprocal_value(elems_per_part); + max_size = FLEX_ARRAY_NR_BASE_PTRS * elems_per_part; + } /* max_size will end up 0 if element_size > PAGE_SIZE */ if (total > max_size) @@ -102,6 +107,8 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, return NULL; ret->element_size = element_size; ret->total_nr_elements = total; + ret->elems_per_part = elems_per_part; + ret->reciprocal_elems = reciprocal_elems; if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) memset(&ret->parts[0], FLEX_ARRAY_FREE, FLEX_ARRAY_BASE_BYTES_LEFT); @@ -112,7 +119,7 @@ EXPORT_SYMBOL(flex_array_alloc); static int fa_element_to_part_nr(struct flex_array *fa, unsigned int element_nr) { - return element_nr / FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); + return reciprocal_divide(element_nr, fa->reciprocal_elems); } /** @@ -141,12 +148,12 @@ void flex_array_free(struct flex_array *fa) EXPORT_SYMBOL(flex_array_free); static unsigned int index_inside_part(struct flex_array *fa, - unsigned int element_nr) + unsigned int element_nr, + unsigned int part_nr) { unsigned int part_offset; - part_offset = element_nr % - FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); + part_offset = element_nr - part_nr * fa->elems_per_part; return part_offset * fa->element_size; } @@ -186,7 +193,7 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, gfp_t flags) { - int part_nr; + int part_nr = 0; struct flex_array_part *part; void *dst; @@ -202,7 +209,7 @@ int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, if (!part) return -ENOMEM; } - dst = &part->elements[index_inside_part(fa, element_nr)]; + dst = &part->elements[index_inside_part(fa, element_nr, part_nr)]; memcpy(dst, src, fa->element_size); return 0; } @@ -217,7 +224,7 @@ EXPORT_SYMBOL(flex_array_put); */ int flex_array_clear(struct flex_array *fa, unsigned int element_nr) { - int part_nr; + int part_nr = 0; struct flex_array_part *part; void *dst; @@ -233,7 +240,7 @@ int flex_array_clear(struct flex_array *fa, unsigned int element_nr) if (!part) return -EINVAL; } - dst = &part->elements[index_inside_part(fa, element_nr)]; + dst = &part->elements[index_inside_part(fa, element_nr, part_nr)]; memset(dst, FLEX_ARRAY_FREE, fa->element_size); return 0; } @@ -302,7 +309,7 @@ EXPORT_SYMBOL(flex_array_prealloc); */ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) { - int part_nr; + int part_nr = 0; struct flex_array_part *part; if (!fa->element_size) @@ -317,7 +324,7 @@ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) if (!part) return NULL; } - return &part->elements[index_inside_part(fa, element_nr)]; + return &part->elements[index_inside_part(fa, element_nr, part_nr)]; } EXPORT_SYMBOL(flex_array_get); -- cgit v1.2.3 From 809fa972fd90ff27225294b17a027e908b2d7b7a Mon Sep 17 00:00:00 2001 From: Hannes Frederic Sowa Date: Wed, 22 Jan 2014 02:29:41 +0100 Subject: reciprocal_divide: update/correction of the algorithm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jakub Zawadzki noticed that some divisions by reciprocal_divide() were not correct [1][2], which he could also show with BPF code after divisions are transformed into reciprocal_value() for runtime invariance which can be passed to reciprocal_divide() later on; reverse in BPF dump ended up with a different, off-by-one K in some situations. This has been fixed by Eric Dumazet in commit aee636c4809fa5 ("bpf: do not use reciprocal divide"). This follow-up patch improves reciprocal_value() and reciprocal_divide() to work in all cases by using Granlund and Montgomery method, so that also future use is safe and without any non-obvious side-effects. Known problems with the old implementation were that division by 1 always returned 0 and some off-by-ones when the dividend and divisor where very large. This seemed to not be problematic with its current users, as far as we can tell. Eric Dumazet checked for the slab usage, we cannot surely say so in the case of flex_array. Still, in order to fix that, we propose an extension from the original implementation from commit 6a2d7a955d8d resp. [3][4], by using the algorithm proposed in "Division by Invariant Integers Using Multiplication" [5], Torbjörn Granlund and Peter L. Montgomery, that is, pseudocode for q = n/d where q, n, d is in u32 universe: 1) Initialization: int l = ceil(log_2 d) uword m' = floor((1<<32)*((1<>32 q = (t+((n-t)>>sh_1))>>sh_2 The assembler implementation from Agner Fog [6] also helped a lot while implementing. We have tested the implementation on x86_64, ppc64, i686, s390x; on x86_64/haswell we're still half the latency compared to normal divide. Joint work with Daniel Borkmann. [1] http://www.wireshark.org/~darkjames/reciprocal-buggy.c [2] http://www.wireshark.org/~darkjames/set-and-dump-filter-k-bug.c [3] https://gmplib.org/~tege/division-paper.pdf [4] http://homepage.cs.uiowa.edu/~jones/bcd/divide.html [5] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.1.2556 [6] http://www.agner.org/optimize/asmlib.zip Reported-by: Jakub Zawadzki Cc: Eric Dumazet Cc: Austin S Hemmelgarn Cc: linux-kernel@vger.kernel.org Cc: Jesse Gross Cc: Jamal Hadi Salim Cc: Stephen Hemminger Cc: Matt Mackall Cc: Pekka Enberg Cc: Christoph Lameter Cc: Andy Gospodarek Cc: Veaceslav Falico Cc: Jay Vosburgh Cc: Jakub Zawadzki Signed-off-by: Daniel Borkmann Signed-off-by: Hannes Frederic Sowa Signed-off-by: David S. Miller --- drivers/net/bonding/bond_main.c | 24 ++++++++++++++++------- drivers/net/bonding/bond_netlink.c | 4 ---- drivers/net/bonding/bond_options.c | 15 ++++++++++----- drivers/net/bonding/bond_sysfs.c | 5 ----- drivers/net/bonding/bonding.h | 3 +++ include/linux/flex_array.h | 3 ++- include/linux/reciprocal_div.h | 39 ++++++++++++++++++++------------------ include/linux/slab_def.h | 4 +++- include/net/red.h | 3 ++- lib/flex_array.c | 7 ++++++- lib/reciprocal_div.c | 24 +++++++++++++++++++---- net/sched/sch_netem.c | 6 ++++-- 12 files changed, 88 insertions(+), 49 deletions(-) (limited to 'include/linux/flex_array.h') diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 3220b488dd1e..f100bd958b88 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -79,7 +79,6 @@ #include #include #include -#include #include "bonding.h" #include "bond_3ad.h" #include "bond_alb.h" @@ -3596,8 +3595,9 @@ static void bond_xmit_slave_id(struct bonding *bond, struct sk_buff *skb, int sl */ static u32 bond_rr_gen_slave_id(struct bonding *bond) { - int packets_per_slave = bond->params.packets_per_slave; u32 slave_id; + struct reciprocal_value reciprocal_packets_per_slave; + int packets_per_slave = bond->params.packets_per_slave; switch (packets_per_slave) { case 0: @@ -3607,8 +3607,10 @@ static u32 bond_rr_gen_slave_id(struct bonding *bond) slave_id = bond->rr_tx_counter; break; default: + reciprocal_packets_per_slave = + bond->params.reciprocal_packets_per_slave; slave_id = reciprocal_divide(bond->rr_tx_counter, - packets_per_slave); + reciprocal_packets_per_slave); break; } bond->rr_tx_counter++; @@ -4343,10 +4345,18 @@ static int bond_check_params(struct bond_params *params) params->resend_igmp = resend_igmp; params->min_links = min_links; params->lp_interval = lp_interval; - if (packets_per_slave > 1) - params->packets_per_slave = reciprocal_value(packets_per_slave); - else - params->packets_per_slave = packets_per_slave; + params->packets_per_slave = packets_per_slave; + if (packets_per_slave > 0) { + params->reciprocal_packets_per_slave = + reciprocal_value(packets_per_slave); + } else { + /* reciprocal_packets_per_slave is unused if + * packets_per_slave is 0 or 1, just initialize it + */ + params->reciprocal_packets_per_slave = + (struct reciprocal_value) { 0 }; + } + if (primary) { strncpy(params->primary, primary, IFNAMSIZ); params->primary[IFNAMSIZ - 1] = 0; diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 21c648854a8c..e8526552790c 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -19,7 +19,6 @@ #include #include #include -#include #include "bonding.h" int bond_get_slave(struct net_device *slave_dev, struct sk_buff *skb) @@ -452,9 +451,6 @@ static int bond_fill_info(struct sk_buff *skb, goto nla_put_failure; packets_per_slave = bond->params.packets_per_slave; - if (packets_per_slave > 1) - packets_per_slave = reciprocal_value(packets_per_slave); - if (nla_put_u32(skb, IFLA_BOND_PACKETS_PER_SLAVE, packets_per_slave)) goto nla_put_failure; diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index 945a6668da83..85e434886f2e 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -16,7 +16,6 @@ #include #include #include -#include #include "bonding.h" int bond_option_mode_set(struct bonding *bond, int mode) @@ -671,11 +670,17 @@ int bond_option_packets_per_slave_set(struct bonding *bond, pr_warn("%s: Warning: packets_per_slave has effect only in balance-rr mode\n", bond->dev->name); - if (packets_per_slave > 1) - bond->params.packets_per_slave = + bond->params.packets_per_slave = packets_per_slave; + if (packets_per_slave > 0) { + bond->params.reciprocal_packets_per_slave = reciprocal_value(packets_per_slave); - else - bond->params.packets_per_slave = packets_per_slave; + } else { + /* reciprocal_packets_per_slave is unused if + * packets_per_slave is 0 or 1, just initialize it + */ + bond->params.reciprocal_packets_per_slave = + (struct reciprocal_value) { 0 }; + } return 0; } diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c index 011f163c2c67..c083e9a66ece 100644 --- a/drivers/net/bonding/bond_sysfs.c +++ b/drivers/net/bonding/bond_sysfs.c @@ -39,7 +39,6 @@ #include #include #include -#include #include "bonding.h" @@ -1374,10 +1373,6 @@ static ssize_t bonding_show_packets_per_slave(struct device *d, { struct bonding *bond = to_bond(d); unsigned int packets_per_slave = bond->params.packets_per_slave; - - if (packets_per_slave > 1) - packets_per_slave = reciprocal_value(packets_per_slave); - return sprintf(buf, "%u\n", packets_per_slave); } diff --git a/drivers/net/bonding/bonding.h b/drivers/net/bonding/bonding.h index 8a935f8f2b3c..0a616c41dc94 100644 --- a/drivers/net/bonding/bonding.h +++ b/drivers/net/bonding/bonding.h @@ -23,6 +23,8 @@ #include #include #include +#include + #include "bond_3ad.h" #include "bond_alb.h" @@ -171,6 +173,7 @@ struct bond_params { int resend_igmp; int lp_interval; int packets_per_slave; + struct reciprocal_value reciprocal_packets_per_slave; }; struct bond_parm_tbl { diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index 6843cf193a44..b6efb0c64408 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -2,6 +2,7 @@ #define _FLEX_ARRAY_H #include +#include #include #define FLEX_ARRAY_PART_SIZE PAGE_SIZE @@ -22,7 +23,7 @@ struct flex_array { int element_size; int total_nr_elements; int elems_per_part; - u32 reciprocal_elems; + struct reciprocal_value reciprocal_elems; struct flex_array_part *parts[]; }; /* diff --git a/include/linux/reciprocal_div.h b/include/linux/reciprocal_div.h index f9c90b33285b..8c5a3fb6c6c5 100644 --- a/include/linux/reciprocal_div.h +++ b/include/linux/reciprocal_div.h @@ -4,29 +4,32 @@ #include /* - * This file describes reciprocical division. + * This algorithm is based on the paper "Division by Invariant + * Integers Using Multiplication" by Torbjörn Granlund and Peter + * L. Montgomery. * - * This optimizes the (A/B) problem, when A and B are two u32 - * and B is a known value (but not known at compile time) + * The assembler implementation from Agner Fog, which this code is + * based on, can be found here: + * http://www.agner.org/optimize/asmlib.zip * - * The math principle used is : - * Let RECIPROCAL_VALUE(B) be (((1LL << 32) + (B - 1))/ B) - * Then A / B = (u32)(((u64)(A) * (R)) >> 32) - * - * This replaces a divide by a multiply (and a shift), and - * is generally less expensive in CPU cycles. + * This optimization for A/B is helpful if the divisor B is mostly + * runtime invariant. The reciprocal of B is calculated in the + * slow-path with reciprocal_value(). The fast-path can then just use + * a much faster multiplication operation with a variable dividend A + * to calculate the division A/B. */ -/* - * Computes the reciprocal value (R) for the value B of the divisor. - * Should not be called before each reciprocal_divide(), - * or else the performance is slower than a normal divide. - */ -extern u32 reciprocal_value(u32 B); +struct reciprocal_value { + u32 m; + u8 sh1, sh2; +}; +struct reciprocal_value reciprocal_value(u32 d); -static inline u32 reciprocal_divide(u32 A, u32 R) +static inline u32 reciprocal_divide(u32 a, struct reciprocal_value R) { - return (u32)(((u64)A * R) >> 32); + u32 t = (u32)(((u64)a * R.m) >> 32); + return (t + ((a - t) >> R.sh1)) >> R.sh2; } -#endif + +#endif /* _LINUX_RECIPROCAL_DIV_H */ diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 09bfffb08a56..96e8abae19a9 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -1,6 +1,8 @@ #ifndef _LINUX_SLAB_DEF_H #define _LINUX_SLAB_DEF_H +#include + /* * Definitions unique to the original Linux SLAB allocator. */ @@ -12,7 +14,7 @@ struct kmem_cache { unsigned int shared; unsigned int size; - u32 reciprocal_buffer_size; + struct reciprocal_value reciprocal_buffer_size; /* 2) touched by every alloc & free from the backend */ unsigned int flags; /* constant flags */ diff --git a/include/net/red.h b/include/net/red.h index 168bb2f495f2..76e0b5f922c6 100644 --- a/include/net/red.h +++ b/include/net/red.h @@ -130,7 +130,8 @@ struct red_parms { u32 qth_max; /* Max avg length threshold: Wlog scaled */ u32 Scell_max; u32 max_P; /* probability, [0 .. 1.0] 32 scaled */ - u32 max_P_reciprocal; /* reciprocal_value(max_P / qth_delta) */ + /* reciprocal_value(max_P / qth_delta) */ + struct reciprocal_value max_P_reciprocal; u32 qth_delta; /* max_th - min_th */ u32 target_min; /* min_th + 0.4*(max_th - min_th) */ u32 target_max; /* min_th + 0.6*(max_th - min_th) */ diff --git a/lib/flex_array.c b/lib/flex_array.c index 6948a6692fc4..2eed22fa507c 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -90,8 +90,8 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, { struct flex_array *ret; int elems_per_part = 0; - int reciprocal_elems = 0; int max_size = 0; + struct reciprocal_value reciprocal_elems = { 0 }; if (element_size) { elems_per_part = FLEX_ARRAY_ELEMENTS_PER_PART(element_size); @@ -119,6 +119,11 @@ EXPORT_SYMBOL(flex_array_alloc); static int fa_element_to_part_nr(struct flex_array *fa, unsigned int element_nr) { + /* + * if element_size == 0 we don't get here, so we never touch + * the zeroed fa->reciprocal_elems, which would yield invalid + * results + */ return reciprocal_divide(element_nr, fa->reciprocal_elems); } diff --git a/lib/reciprocal_div.c b/lib/reciprocal_div.c index 75510e94f7d0..464152410c51 100644 --- a/lib/reciprocal_div.c +++ b/lib/reciprocal_div.c @@ -1,11 +1,27 @@ +#include #include #include #include -u32 reciprocal_value(u32 k) +/* + * For a description of the algorithm please have a look at + * include/linux/reciprocal_div.h + */ + +struct reciprocal_value reciprocal_value(u32 d) { - u64 val = (1LL << 32) + (k - 1); - do_div(val, k); - return (u32)val; + struct reciprocal_value R; + u64 m; + int l; + + l = fls(d - 1); + m = ((1ULL << 32) * ((1ULL << l) - d)); + do_div(m, d); + ++m; + R.m = (u32)m; + R.sh1 = min(l, 1); + R.sh2 = max(l - 1, 0); + + return R; } EXPORT_SYMBOL(reciprocal_value); diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index a2bfc371b44a..de1059af6da1 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -91,7 +91,7 @@ struct netem_sched_data { u64 rate; s32 packet_overhead; u32 cell_size; - u32 cell_size_reciprocal; + struct reciprocal_value cell_size_reciprocal; s32 cell_overhead; struct crndstate { @@ -725,9 +725,11 @@ static void get_rate(struct Qdisc *sch, const struct nlattr *attr) q->rate = r->rate; q->packet_overhead = r->packet_overhead; q->cell_size = r->cell_size; + q->cell_overhead = r->cell_overhead; if (q->cell_size) q->cell_size_reciprocal = reciprocal_value(q->cell_size); - q->cell_overhead = r->cell_overhead; + else + q->cell_size_reciprocal = (struct reciprocal_value) { 0 }; } static int get_loss_clg(struct Qdisc *sch, const struct nlattr *attr) -- cgit v1.2.3