1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/linkage.h>
14#include <linux/threads.h>
15#include <linux/init.h>
16#include <asm/segment.h>
17#include <asm/pgtable.h>
18#include <asm/page.h>
19#include <asm/msr.h>
20#include <asm/cache.h>
21#include <asm/processor-flags.h>
22#include <asm/percpu.h>
23#include <asm/nops.h>
24#include "../entry/calling.h"
25#include <asm/export.h>
26#include <asm/nospec-branch.h>
27
28#ifdef CONFIG_PARAVIRT
29#include <asm/asm-offsets.h>
30#include <asm/paravirt.h>
31#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg
32#else
33#define GET_CR2_INTO(reg) movq %cr2, reg
34#define INTERRUPT_RETURN iretq
35#endif
36
37
38
39
40
41
42#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
43
44
45PGD_PAGE_OFFSET = pgd_index(__PAGE_OFFSET_BASE)
46PGD_START_KERNEL = pgd_index(__START_KERNEL_map)
47#endif
48L3_START_KERNEL = pud_index(__START_KERNEL_map)
49
50 .text
51 __HEAD
52 .code64
53 .globl startup_64
54startup_64:
55 UNWIND_HINT_EMPTY
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75 leaq (__end_init_task - SIZEOF_PTREGS)(%rip), %rsp
76
77
78 call verify_cpu
79
80
81
82
83
84
85
86 leaq _text(%rip), %rdi
87 pushq %rsi
88 call __startup_64
89 popq %rsi
90
91
92 addq $(early_top_pgt - __START_KERNEL_map), %rax
93 jmp 1f
94ENTRY(secondary_startup_64)
95 UNWIND_HINT_EMPTY
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111 call verify_cpu
112
113
114
115
116
117 pushq %rsi
118 call __startup_secondary_64
119 popq %rsi
120
121
122 addq $(init_top_pgt - __START_KERNEL_map), %rax
1231:
124
125
126 movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
127#ifdef CONFIG_X86_5LEVEL
128 orl $X86_CR4_LA57, %ecx
129#endif
130 movq %rcx, %cr4
131
132
133 addq phys_base(%rip), %rax
134 movq %rax, %cr3
135
136
137 movq $1f, %rax
138 ANNOTATE_RETPOLINE_SAFE
139 jmp *%rax
1401:
141 UNWIND_HINT_EMPTY
142
143
144 movl $0x80000001, %eax
145 cpuid
146 movl %edx,%edi
147
148
149 movl $MSR_EFER, %ecx
150 rdmsr
151 btsl $_EFER_SCE, %eax
152 btl $20,%edi
153 jnc 1f
154 btsl $_EFER_NX, %eax
155 btsq $_PAGE_BIT_NX,early_pmd_flags(%rip)
1561: wrmsr
157
158
159 movl $CR0_STATE, %eax
160
161 movq %rax, %cr0
162
163
164 movq initial_stack(%rip), %rsp
165
166
167 pushq $0
168 popfq
169
170
171
172
173
174
175
176 lgdt early_gdt_descr(%rip)
177
178
179 xorl %eax,%eax
180 movl %eax,%ds
181 movl %eax,%ss
182 movl %eax,%es
183
184
185
186
187
188
189 movl %eax,%fs
190 movl %eax,%gs
191
192
193
194
195
196
197
198
199 movl $MSR_GS_BASE,%ecx
200 movl initial_gs(%rip),%eax
201 movl initial_gs+4(%rip),%edx
202 wrmsr
203
204
205
206 movq %rsi, %rdi
207
208.Ljump_to_C_code:
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234 pushq $.Lafter_lret
235 xorq %rbp, %rbp
236 movq initial_code(%rip), %rax
237 pushq $__KERNEL_CS
238 pushq %rax
239 lretq
240.Lafter_lret:
241END(secondary_startup_64)
242
243#include "verify_cpu.S"
244
245#ifdef CONFIG_HOTPLUG_CPU
246
247
248
249
250
251ENTRY(start_cpu0)
252 movq initial_stack(%rip), %rsp
253 UNWIND_HINT_EMPTY
254 jmp .Ljump_to_C_code
255ENDPROC(start_cpu0)
256#endif
257
258
259 __REFDATA
260 .balign 8
261 GLOBAL(initial_code)
262 .quad x86_64_start_kernel
263 GLOBAL(initial_gs)
264 .quad INIT_PER_CPU_VAR(irq_stack_union)
265 GLOBAL(initial_stack)
266
267
268
269
270 .quad init_thread_union + THREAD_SIZE - SIZEOF_PTREGS
271 __FINITDATA
272
273 __INIT
274ENTRY(early_idt_handler_array)
275 i = 0
276 .rept NUM_EXCEPTION_VECTORS
277 .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
278 UNWIND_HINT_IRET_REGS
279 pushq $0
280 .else
281 UNWIND_HINT_IRET_REGS offset=8
282 .endif
283 pushq $i
284 jmp early_idt_handler_common
285 UNWIND_HINT_IRET_REGS
286 i = i + 1
287 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
288 .endr
289 UNWIND_HINT_IRET_REGS offset=16
290END(early_idt_handler_array)
291
292early_idt_handler_common:
293
294
295
296
297 cld
298
299 incl early_recursion_flag(%rip)
300
301
302 pushq %rsi
303 movq 8(%rsp), %rsi
304 movq %rdi, 8(%rsp)
305 pushq %rdx
306 pushq %rcx
307 pushq %rax
308 pushq %r8
309 pushq %r9
310 pushq %r10
311 pushq %r11
312 pushq %rbx
313 pushq %rbp
314 pushq %r12
315 pushq %r13
316 pushq %r14
317 pushq %r15
318 UNWIND_HINT_REGS
319
320 cmpq $14,%rsi
321 jnz 10f
322 GET_CR2_INTO(%rdi)
323 call early_make_pgtable
324 andl %eax,%eax
325 jz 20f
326
32710:
328 movq %rsp,%rdi
329 call early_fixup_exception
330
33120:
332 decl early_recursion_flag(%rip)
333 jmp restore_regs_and_return_to_kernel
334END(early_idt_handler_common)
335
336 __INITDATA
337
338 .balign 4
339GLOBAL(early_recursion_flag)
340 .long 0
341
342#define NEXT_PAGE(name) \
343 .balign PAGE_SIZE; \
344GLOBAL(name)
345
346#ifdef CONFIG_PAGE_TABLE_ISOLATION
347
348
349
350
351
352
353
354
355
356
357#define PTI_USER_PGD_FILL 512
358
359#define NEXT_PGD_PAGE(name) \
360 .balign 2 * PAGE_SIZE; \
361GLOBAL(name)
362#else
363#define NEXT_PGD_PAGE(name) NEXT_PAGE(name)
364#define PTI_USER_PGD_FILL 0
365#endif
366
367
368#define PMDS(START, PERM, COUNT) \
369 i = 0 ; \
370 .rept (COUNT) ; \
371 .quad (START) + (i << PMD_SHIFT) + (PERM) ; \
372 i = i + 1 ; \
373 .endr
374
375 __INITDATA
376NEXT_PGD_PAGE(early_top_pgt)
377 .fill 511,8,0
378#ifdef CONFIG_X86_5LEVEL
379 .quad level4_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
380#else
381 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
382#endif
383 .fill PTI_USER_PGD_FILL,8,0
384
385NEXT_PAGE(early_dynamic_pgts)
386 .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
387
388 .data
389
390
391NEXT_PGD_PAGE(init_top_pgt)
392 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
393 .org init_top_pgt + PGD_PAGE_OFFSET*8, 0
394 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
395 .org init_top_pgt + PGD_START_KERNEL*8, 0
396
397 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
398 .fill PTI_USER_PGD_FILL,8,0
399
400NEXT_PAGE(level3_ident_pgt)
401 .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
402 .fill 511, 8, 0
403NEXT_PAGE(level2_ident_pgt)
404
405
406
407 PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
408#else
409NEXT_PGD_PAGE(init_top_pgt)
410 .fill 512,8,0
411 .fill PTI_USER_PGD_FILL,8,0
412#endif
413
414#ifdef CONFIG_X86_5LEVEL
415NEXT_PAGE(level4_kernel_pgt)
416 .fill 511,8,0
417 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
418#endif
419
420NEXT_PAGE(level3_kernel_pgt)
421 .fill L3_START_KERNEL,8,0
422
423 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
424 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
425
426NEXT_PAGE(level2_kernel_pgt)
427
428
429
430
431
432
433
434
435
436
437 PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
438 KERNEL_IMAGE_SIZE/PMD_SIZE)
439
440NEXT_PAGE(level2_fixmap_pgt)
441 .fill 506,8,0
442 .quad level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
443
444 .fill 5,8,0
445
446NEXT_PAGE(level1_fixmap_pgt)
447 .fill 512,8,0
448
449#undef PMDS
450
451 .data
452 .align 16
453 .globl early_gdt_descr
454early_gdt_descr:
455 .word GDT_ENTRIES*8-1
456early_gdt_descr_base:
457 .quad INIT_PER_CPU_VAR(gdt_page)
458
459ENTRY(phys_base)
460
461 .quad 0x0000000000000000
462EXPORT_SYMBOL(phys_base)
463
464#include "../../x86/xen/xen-head.S"
465
466 __PAGE_ALIGNED_BSS
467NEXT_PAGE(empty_zero_page)
468 .skip PAGE_SIZE
469EXPORT_SYMBOL(empty_zero_page)
470
471