summaryrefslogtreecommitdiff
path: root/include/linux/pageblock-flags.h
diff options
context:
space:
mode:
authorMel Gorman <mgorman@suse.de>2014-06-05 03:10:16 +0400
committerJiri Slaby <jslaby@suse.cz>2014-07-02 14:06:09 +0400
commit4d6dd826592bc985202d47307f5a0560412f8613 (patch)
tree3eb235f6cc722ee7452af3671ffb56d0be6ea7b4 /include/linux/pageblock-flags.h
parentc4c34dd67a3588a00713407f797468c02454ca5b (diff)
downloadlinux-4d6dd826592bc985202d47307f5a0560412f8613.tar.xz
mm: page_alloc: use word-based accesses for get/set pageblock bitmaps
commit e58469bafd0524e848c3733bc3918d854595e20f upstream. The test_bit operations in get/set pageblock flags are expensive. This patch reads the bitmap on a word basis and use shifts and masks to isolate the bits of interest. Similarly masks are used to set a local copy of the bitmap and then use cmpxchg to update the bitmap if there have been no other changes made in parallel. In a test running dd onto tmpfs the overhead of the pageblock-related functions went from 1.27% in profiles to 0.5%. In addition to the performance benefits, this patch closes races that are possible between: a) get_ and set_pageblock_migratetype(), where get_pageblock_migratetype() reads part of the bits before and other part of the bits after set_pageblock_migratetype() has updated them. b) set_pageblock_migratetype() and set_pageblock_skip(), where the non-atomic read-modify-update set bit operation in set_pageblock_skip() will cause lost updates to some bits changed in the set_pageblock_migratetype(). Joonsoo Kim first reported the case a) via code inspection. Vlastimil Babka's testing with a debug patch showed that either a) or b) occurs roughly once per mmtests' stress-highalloc benchmark (although not necessarily in the same pageblock). Furthermore during development of unrelated compaction patches, it was observed that frequent calls to {start,undo}_isolate_page_range() the race occurs several thousands of times and has resulted in NULL pointer dereferences in move_freepages() and free_one_page() in places where free_list[migratetype] is manipulated by e.g. list_move(). Further debugging confirmed that migratetype had invalid value of 6, causing out of bounds access to the free_list array. That confirmed that the race exist, although it may be extremely rare, and currently only fatal where page isolation is performed due to memory hot remove. Races on pageblocks being updated by set_pageblock_migratetype(), where both old and new migratetype are lower MIGRATE_RESERVE, currently cannot result in an invalid value being observed, although theoretically they may still lead to unexpected creation or destruction of MIGRATE_RESERVE pageblocks. Furthermore, things could get suddenly worse when memory isolation is used more, or when new migratetypes are added. After this patch, the race has no longer been observed in testing. Signed-off-by: Mel Gorman <mgorman@suse.de> Acked-by: Vlastimil Babka <vbabka@suse.cz> Reported-by: Joonsoo Kim <iamjoonsoo.kim@lge.com> Reported-and-tested-by: Vlastimil Babka <vbabka@suse.cz> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Jan Kara <jack@suse.cz> Cc: Michal Hocko <mhocko@suse.cz> Cc: Hugh Dickins <hughd@google.com> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Theodore Ts'o <tytso@mit.edu> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Rik van Riel <riel@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Jiri Slaby <jslaby@suse.cz>
Diffstat (limited to 'include/linux/pageblock-flags.h')
-rw-r--r--include/linux/pageblock-flags.h37
1 files changed, 31 insertions, 6 deletions
diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
index 2ee8cd2466b5..c08730c10c7a 100644
--- a/include/linux/pageblock-flags.h
+++ b/include/linux/pageblock-flags.h
@@ -30,9 +30,12 @@ enum pageblock_bits {
PB_migrate,
PB_migrate_end = PB_migrate + 3 - 1,
/* 3 bits required for migrate types */
-#ifdef CONFIG_COMPACTION
PB_migrate_skip,/* If set the block is skipped by compaction */
-#endif /* CONFIG_COMPACTION */
+
+ /*
+ * Assume the bits will always align on a word. If this assumption
+ * changes then get/set pageblock needs updating.
+ */
NR_PAGEBLOCK_BITS
};
@@ -62,11 +65,33 @@ extern int pageblock_order;
/* Forward declaration */
struct page;
+unsigned long get_pageblock_flags_mask(struct page *page,
+ unsigned long end_bitidx,
+ unsigned long mask);
+void set_pageblock_flags_mask(struct page *page,
+ unsigned long flags,
+ unsigned long end_bitidx,
+ unsigned long mask);
+
/* Declarations for getting and setting flags. See mm/page_alloc.c */
-unsigned long get_pageblock_flags_group(struct page *page,
- int start_bitidx, int end_bitidx);
-void set_pageblock_flags_group(struct page *page, unsigned long flags,
- int start_bitidx, int end_bitidx);
+static inline unsigned long get_pageblock_flags_group(struct page *page,
+ int start_bitidx, int end_bitidx)
+{
+ unsigned long nr_flag_bits = end_bitidx - start_bitidx + 1;
+ unsigned long mask = (1 << nr_flag_bits) - 1;
+
+ return get_pageblock_flags_mask(page, end_bitidx, mask);
+}
+
+static inline void set_pageblock_flags_group(struct page *page,
+ unsigned long flags,
+ int start_bitidx, int end_bitidx)
+{
+ unsigned long nr_flag_bits = end_bitidx - start_bitidx + 1;
+ unsigned long mask = (1 << nr_flag_bits) - 1;
+
+ set_pageblock_flags_mask(page, flags, end_bitidx, mask);
+}
#ifdef CONFIG_COMPACTION
#define get_pageblock_skip(page) \