diff options
| -rw-r--r-- | arch/x86/include/asm/arch_hweight.h | 13 | ||||
| -rw-r--r-- | include/linux/bitmap.h | 2 | ||||
| -rw-r--r-- | include/linux/bitops.h | 6 | ||||
| -rw-r--r-- | include/linux/jiffies.h | 4 |
4 files changed, 14 insertions, 11 deletions
diff --git a/arch/x86/include/asm/arch_hweight.h b/arch/x86/include/asm/arch_hweight.h index 9686c3d9ff73..259a7c1ef709 100644 --- a/arch/x86/include/asm/arch_hweight.h +++ b/arch/x86/include/asm/arch_hweight.h @@ -21,7 +21,7 @@ * ARCH_HWEIGHT_CFLAGS in <arch/x86/Kconfig> for the respective * compiler switches. */ -static inline unsigned int __arch_hweight32(unsigned int w) +static __always_inline unsigned int __arch_hweight32(unsigned int w) { unsigned int res = 0; @@ -42,20 +42,23 @@ static inline unsigned int __arch_hweight8(unsigned int w) return __arch_hweight32(w & 0xff); } +#ifdef CONFIG_X86_32 static inline unsigned long __arch_hweight64(__u64 w) { - unsigned long res = 0; - -#ifdef CONFIG_X86_32 return __arch_hweight32((u32)w) + __arch_hweight32((u32)(w >> 32)); +} #else +static __always_inline unsigned long __arch_hweight64(__u64 w) +{ + unsigned long res = 0; + asm (ALTERNATIVE("call __sw_hweight64", POPCNT64, X86_FEATURE_POPCNT) : "="REG_OUT (res) : REG_IN (w)); -#endif /* CONFIG_X86_32 */ return res; } +#endif /* CONFIG_X86_32 */ #endif diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index ea17cca9e685..9653fdb76a42 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -295,7 +295,7 @@ static inline int bitmap_full(const unsigned long *src, unsigned int nbits) return find_first_zero_bit(src, nbits) == nbits; } -static inline int bitmap_weight(const unsigned long *src, unsigned int nbits) +static __always_inline int bitmap_weight(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 297f5bda4fdf..e63553386ae7 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -57,7 +57,7 @@ extern unsigned long __sw_hweight64(__u64 w); (bit) < (size); \ (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) -static __inline__ int get_bitmask_order(unsigned int count) +static inline int get_bitmask_order(unsigned int count) { int order; @@ -65,7 +65,7 @@ static __inline__ int get_bitmask_order(unsigned int count) return order; /* We could be slightly more clever with -1 here... */ } -static __inline__ int get_count_order(unsigned int count) +static inline int get_count_order(unsigned int count) { int order; @@ -75,7 +75,7 @@ static __inline__ int get_count_order(unsigned int count) return order; } -static inline unsigned long hweight_long(unsigned long w) +static __always_inline unsigned long hweight_long(unsigned long w) { return sizeof(w) == 4 ? hweight32(w) : hweight64(w); } diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index 535fd3bb1ba8..1ba48a18c1d7 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -351,7 +351,7 @@ static inline unsigned long _msecs_to_jiffies(const unsigned int m) * directly here and from __msecs_to_jiffies() in the case where * constant folding is not possible. */ -static inline unsigned long msecs_to_jiffies(const unsigned int m) +static __always_inline unsigned long msecs_to_jiffies(const unsigned int m) { if (__builtin_constant_p(m)) { if ((int)m < 0) @@ -405,7 +405,7 @@ static inline unsigned long _usecs_to_jiffies(const unsigned int u) * directly here and from __msecs_to_jiffies() in the case where * constant folding is not possible. */ -static inline unsigned long usecs_to_jiffies(const unsigned int u) +static __always_inline unsigned long usecs_to_jiffies(const unsigned int u) { if (__builtin_constant_p(u)) { if (u > jiffies_to_usecs(MAX_JIFFY_OFFSET)) |