1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
// SPDX-License-Identifier: GPL-2.0-only
/*
* String functions optimized for hardware which doesn't
* handle unaligned memory accesses efficiently.
*
* Copyright (C) 2021 Matteo Croce
*/
#define __NO_FORTIFY
#include <linux/types.h>
#include <linux/module.h>
/* Minimum size for a word copy to be convenient */
#define BYTES_LONG sizeof(long)
#define WORD_MASK (BYTES_LONG - 1)
#define MIN_THRESHOLD (BYTES_LONG * 2)
/* convenience union to avoid cast between different pointer types */
union types {
u8 *as_u8;
unsigned long *as_ulong;
uintptr_t as_uptr;
};
union const_types {
const u8 *as_u8;
unsigned long *as_ulong;
uintptr_t as_uptr;
};
void *__memcpy(void *dest, const void *src, size_t count)
{
union const_types s = { .as_u8 = src };
union types d = { .as_u8 = dest };
int distance = 0;
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
if (count < MIN_THRESHOLD)
goto copy_remainder;
/* Copy a byte at time until destination is aligned. */
for (; d.as_uptr & WORD_MASK; count--)
*d.as_u8++ = *s.as_u8++;
distance = s.as_uptr & WORD_MASK;
}
if (distance) {
unsigned long last, next;
/*
* s is distance bytes ahead of d, and d just reached
* the alignment boundary. Move s backward to word align it
* and shift data to compensate for distance, in order to do
* word-by-word copy.
*/
s.as_u8 -= distance;
next = s.as_ulong[0];
for (; count >= BYTES_LONG; count -= BYTES_LONG) {
last = next;
next = s.as_ulong[1];
d.as_ulong[0] = last >> (distance * 8) |
next << ((BYTES_LONG - distance) * 8);
d.as_ulong++;
s.as_ulong++;
}
/* Restore s with the original offset. */
s.as_u8 += distance;
} else {
/*
* If the source and dest lower bits are the same, do a simple
* 32/64 bit wide copy.
*/
for (; count >= BYTES_LONG; count -= BYTES_LONG)
*d.as_ulong++ = *s.as_ulong++;
}
copy_remainder:
while (count--)
*d.as_u8++ = *s.as_u8++;
return dest;
}
EXPORT_SYMBOL(__memcpy);
void *memcpy(void *dest, const void *src, size_t count) __weak __alias(__memcpy);
EXPORT_SYMBOL(memcpy);
/*
* Simply check if the buffer overlaps an call memcpy() in case,
* otherwise do a simple one byte at time backward copy.
*/
void *__memmove(void *dest, const void *src, size_t count)
{
if (dest < src || src + count <= dest)
return __memcpy(dest, src, count);
if (dest > src) {
const char *s = src + count;
char *tmp = dest + count;
while (count--)
*--tmp = *--s;
}
return dest;
}
EXPORT_SYMBOL(__memmove);
void *memmove(void *dest, const void *src, size_t count) __weak __alias(__memmove);
EXPORT_SYMBOL(memmove);
void *__memset(void *s, int c, size_t count)
{
union types dest = { .as_u8 = s };
if (count >= MIN_THRESHOLD) {
unsigned long cu = (unsigned long)c;
/* Compose an ulong with 'c' repeated 4/8 times */
#ifdef CONFIG_ARCH_HAS_FAST_MULTIPLIER
cu *= 0x0101010101010101UL;
#else
cu |= cu << 8;
cu |= cu << 16;
/* Suppress warning on 32 bit machines */
cu |= (cu << 16) << 16;
#endif
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
/*
* Fill the buffer one byte at time until
* the destination is word aligned.
*/
for (; count && dest.as_uptr & WORD_MASK; count--)
*dest.as_u8++ = c;
}
/* Copy using the largest size allowed */
for (; count >= BYTES_LONG; count -= BYTES_LONG)
*dest.as_ulong++ = cu;
}
/* copy the remainder */
while (count--)
*dest.as_u8++ = c;
return s;
}
EXPORT_SYMBOL(__memset);
void *memset(void *s, int c, size_t count) __weak __alias(__memset);
EXPORT_SYMBOL(memset);
|
