1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
// SPDX-License-Identifier: GPL-2.0-only
/* ----------------------------------------------------------------------- *
*
* Copyright 2014 Intel Corporation; author: H. Peter Anvin
*
* ----------------------------------------------------------------------- */
/*
* The IRET instruction, when returning to a 16-bit segment, only
* restores the bottom 16 bits of the user space stack pointer. This
* causes some 16-bit software to break, but it also leaks kernel state
* to user space.
*
* This works around this by creating percpu "ministacks", each of which
* is mapped 2^16 times 64K apart. When we detect that the return SS is
* on the LDT, we copy the IRET frame to the ministack and use the
* relevant alias to return to userspace. The ministacks are mapped
* readonly, so if the IRET fault we promote #GP to #DF which is an IST
* vector and thus has its own stack; we then do the fixup in the #DF
* handler.
*
* This file sets up the ministacks and the related page tables. The
* actual ministack invocation is in entry_64.S.
*/
#include <linux/init.h>
#include <linux/init_task.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/random.h>
#include <linux/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/setup.h>
#include <asm/espfix.h>
/*
* Note: we only need 6*8 = 48 bytes for the espfix stack, but round
* it up to a cache line to avoid unnecessary sharing.
*/
#define ESPFIX_STACK_SIZE (8*8UL)
#define ESPFIX_STACKS_PER_PAGE (PAGE_SIZE/ESPFIX_STACK_SIZE)
/* There is address space for how many espfix pages? */
#define ESPFIX_PAGE_SPACE (1UL << (P4D_SHIFT-PAGE_SHIFT-16))
#define ESPFIX_MAX_CPUS (ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE)
#if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS
# error "Need more virtual address space for the ESPFIX hack"
#endif
#define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO)
/* This contains the *bottom* address of the espfix stack */
DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack);
DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr);
/* Initialization mutex - should this be a spinlock? */
static DEFINE_MUTEX(espfix_init_mutex);
/* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */
#define ESPFIX_MAX_PAGES DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
static void *espfix_pages[ESPFIX_MAX_PAGES];
static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
__aligned(PAGE_SIZE);
static unsigned int page_random, slot_random;
/*
* This returns the bottom address of the espfix stack for a specific CPU.
* The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case
* we have to account for some amount of padding at the end of each page.
*/
static inline unsigned long espfix_base_addr(unsigned int cpu)
{
unsigned long page, slot;
unsigned long addr;
page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random;
slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE;
addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE);
addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16);
addr += ESPFIX_BASE_ADDR;
return addr;
}
#define PTE_STRIDE (65536/PAGE_SIZE)
#define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE)
#define ESPFIX_PMD_CLONES PTRS_PER_PMD
#define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES))
#define PGTABLE_PROT ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX)
static void init_espfix_random(void)
{
unsigned long rand;
/*
* This is run before the entropy pools are initialized,
* but this is hopefully better than nothing.
*/
if (!arch_get_random_long(&rand)) {
/* The constant is an arbitrary large prime */
rand = rdtsc();
rand *= 0xc345c6b72fd16123UL;
}
slot_random = rand % ESPFIX_STACKS_PER_PAGE;
page_random = (rand / ESPFIX_STACKS_PER_PAGE)
& (ESPFIX_PAGE_SPACE - 1);
}
void __init init_espfix_bsp(void)
{
pgd_t *pgd;
p4d_t *p4d;
/* Install the espfix pud into the kernel page directory */
pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)];
p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR);
p4d_populate(&init_mm, p4d, espfix_pud_page);
/* Randomize the locations */
init_espfix_random();
/* The rest is the same as for any other processor */
init_espfix_ap(0);
}
void init_espfix_ap(int cpu)
{
unsigned int page;
unsigned long addr;
pud_t pud, *pud_p;
pmd_t pmd, *pmd_p;
pte_t pte, *pte_p;
int n, node;
void *stack_page;
pteval_t ptemask;
/* We only have to do this once... */
if (likely(per_cpu(espfix_stack, cpu)))
return; /* Already initialized */
addr = espfix_base_addr(cpu);
page = cpu/ESPFIX_STACKS_PER_PAGE;
/* Did another CPU already set this up? */
stack_page = READ_ONCE(espfix_pages[page]);
if (likely(stack_page))
goto done;
mutex_lock(&espfix_init_mutex);
/* Did we race on the lock? */
stack_page = READ_ONCE(espfix_pages[page]);
if (stack_page)
goto unlock_done;
node = cpu_to_node(cpu);
ptemask = __supported_pte_mask;
pud_p = &espfix_pud_page[pud_index(addr)];
pud = *pud_p;
if (!pud_present(pud)) {
struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
pmd_p = (pmd_t *)page_address(page);
pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask));
paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT);
for (n = 0; n < ESPFIX_PUD_CLONES; n++)
set_pud(&pud_p[n], pud);
}
pmd_p = pmd_offset(&pud, addr);
pmd = *pmd_p;
if (!pmd_present(pmd)) {
struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
pte_p = (pte_t *)page_address(page);
pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask));
paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT);
for (n = 0; n < ESPFIX_PMD_CLONES; n++)
set_pmd(&pmd_p[n], pmd);
}
pte_p = pte_offset_kernel(&pmd, addr);
stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0));
/*
* __PAGE_KERNEL_* includes _PAGE_GLOBAL, which we want since
* this is mapped to userspace.
*/
pte = __pte(__pa(stack_page) | ((__PAGE_KERNEL_RO | _PAGE_ENC) & ptemask));
for (n = 0; n < ESPFIX_PTE_CLONES; n++)
set_pte(&pte_p[n*PTE_STRIDE], pte);
/* Job is done for this CPU and any CPU which shares this page */
WRITE_ONCE(espfix_pages[page], stack_page);
unlock_done:
mutex_unlock(&espfix_init_mutex);
done:
per_cpu(espfix_stack, cpu) = addr;
per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page
+ (addr & ~PAGE_MASK);
}
|
