1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/init.h>
35#include <linux/init_task.h>
36#include <linux/kernel.h>
37#include <linux/percpu.h>
38#include <linux/gfp.h>
39#include <linux/random.h>
40#include <asm/pgtable.h>
41#include <asm/pgalloc.h>
42#include <asm/setup.h>
43#include <asm/espfix.h>
44
45
46
47
48
49#define ESPFIX_STACK_SIZE (8*8UL)
50#define ESPFIX_STACKS_PER_PAGE (PAGE_SIZE/ESPFIX_STACK_SIZE)
51
52
53#define ESPFIX_PAGE_SPACE (1UL << (PGDIR_SHIFT-PAGE_SHIFT-16))
54
55#define ESPFIX_MAX_CPUS (ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE)
56#if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS
57# error "Need more than one PGD for the ESPFIX hack"
58#endif
59
60#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO)
61
62
63DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack);
64DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr);
65
66
67static DEFINE_MUTEX(espfix_init_mutex);
68
69
70#define ESPFIX_MAX_PAGES DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
71static void *espfix_pages[ESPFIX_MAX_PAGES];
72
73static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
74 __aligned(PAGE_SIZE);
75
76static unsigned int page_random, slot_random;
77
78
79
80
81
82
83static inline unsigned long espfix_base_addr(unsigned int cpu)
84{
85 unsigned long page, slot;
86 unsigned long addr;
87
88 page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random;
89 slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE;
90 addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE);
91 addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16);
92 addr += ESPFIX_BASE_ADDR;
93 return addr;
94}
95
96#define PTE_STRIDE (65536/PAGE_SIZE)
97#define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE)
98#define ESPFIX_PMD_CLONES PTRS_PER_PMD
99#define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES))
100
101#define PGTABLE_PROT ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX)
102
103static void init_espfix_random(void)
104{
105 unsigned long rand;
106
107
108
109
110
111 if (!arch_get_random_long(&rand)) {
112
113 rand = rdtsc();
114 rand *= 0xc345c6b72fd16123UL;
115 }
116
117 slot_random = rand % ESPFIX_STACKS_PER_PAGE;
118 page_random = (rand / ESPFIX_STACKS_PER_PAGE)
119 & (ESPFIX_PAGE_SPACE - 1);
120}
121
122void __init init_espfix_bsp(void)
123{
124 pgd_t *pgd_p;
125
126
127 pgd_p = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)];
128 pgd_populate(&init_mm, pgd_p, (pud_t *)espfix_pud_page);
129
130
131 init_espfix_random();
132
133
134 init_espfix_ap(0);
135}
136
137void init_espfix_ap(int cpu)
138{
139 unsigned int page;
140 unsigned long addr;
141 pud_t pud, *pud_p;
142 pmd_t pmd, *pmd_p;
143 pte_t pte, *pte_p;
144 int n, node;
145 void *stack_page;
146 pteval_t ptemask;
147
148
149 if (likely(per_cpu(espfix_stack, cpu)))
150 return;
151
152 addr = espfix_base_addr(cpu);
153 page = cpu/ESPFIX_STACKS_PER_PAGE;
154
155
156 stack_page = ACCESS_ONCE(espfix_pages[page]);
157 if (likely(stack_page))
158 goto done;
159
160 mutex_lock(&espfix_init_mutex);
161
162
163 stack_page = ACCESS_ONCE(espfix_pages[page]);
164 if (stack_page)
165 goto unlock_done;
166
167 node = cpu_to_node(cpu);
168 ptemask = __supported_pte_mask;
169
170 pud_p = &espfix_pud_page[pud_index(addr)];
171 pud = *pud_p;
172 if (!pud_present(pud)) {
173 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
174
175 pmd_p = (pmd_t *)page_address(page);
176 pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask));
177 paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT);
178 for (n = 0; n < ESPFIX_PUD_CLONES; n++)
179 set_pud(&pud_p[n], pud);
180 }
181
182 pmd_p = pmd_offset(&pud, addr);
183 pmd = *pmd_p;
184 if (!pmd_present(pmd)) {
185 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
186
187 pte_p = (pte_t *)page_address(page);
188 pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask));
189 paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT);
190 for (n = 0; n < ESPFIX_PMD_CLONES; n++)
191 set_pmd(&pmd_p[n], pmd);
192 }
193
194 pte_p = pte_offset_kernel(&pmd, addr);
195 stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0));
196 pte = __pte(__pa(stack_page) | (__PAGE_KERNEL_RO & ptemask));
197 for (n = 0; n < ESPFIX_PTE_CLONES; n++)
198 set_pte(&pte_p[n*PTE_STRIDE], pte);
199
200
201 ACCESS_ONCE(espfix_pages[page]) = stack_page;
202
203unlock_done:
204 mutex_unlock(&espfix_init_mutex);
205done:
206 per_cpu(espfix_stack, cpu) = addr;
207 per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page
208 + (addr & ~PAGE_MASK);
209}
210