1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/linkage.h>
14#include <linux/threads.h>
15#include <linux/init.h>
16#include <asm/segment.h>
17#include <asm/pgtable.h>
18#include <asm/page.h>
19#include <asm/msr.h>
20#include <asm/cache.h>
21#include <asm/processor-flags.h>
22#include <asm/percpu.h>
23#include <asm/nops.h>
24#include "../entry/calling.h"
25#include <asm/export.h>
26#include <asm/nospec-branch.h>
27#include <asm/fixmap.h>
28
29#ifdef CONFIG_PARAVIRT_XXL
30#include <asm/asm-offsets.h>
31#include <asm/paravirt.h>
32#else
33#define INTERRUPT_RETURN iretq
34#endif
35
36
37
38
39
40
41#define l4_index(x) (((x) >> 39) & 511)
42#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
43
44L4_PAGE_OFFSET = l4_index(__PAGE_OFFSET_BASE_L4)
45L4_START_KERNEL = l4_index(__START_KERNEL_map)
46
47L3_START_KERNEL = pud_index(__START_KERNEL_map)
48
49 .text
50 __HEAD
51 .code64
52 .globl startup_64
53startup_64:
54 UNWIND_HINT_EMPTY
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74 leaq (__end_init_task - SIZEOF_PTREGS)(%rip), %rsp
75
76
77 call verify_cpu
78
79
80
81
82
83
84
85 leaq _text(%rip), %rdi
86 pushq %rsi
87 call __startup_64
88 popq %rsi
89
90
91 addq $(early_top_pgt - __START_KERNEL_map), %rax
92 jmp 1f
93ENTRY(secondary_startup_64)
94 UNWIND_HINT_EMPTY
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110 call verify_cpu
111
112
113
114
115
116 pushq %rsi
117 call __startup_secondary_64
118 popq %rsi
119
120
121 addq $(init_top_pgt - __START_KERNEL_map), %rax
1221:
123
124
125 movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
126#ifdef CONFIG_X86_5LEVEL
127 testl $1, __pgtable_l5_enabled(%rip)
128 jz 1f
129 orl $X86_CR4_LA57, %ecx
1301:
131#endif
132 movq %rcx, %cr4
133
134
135 addq phys_base(%rip), %rax
136 movq %rax, %cr3
137
138
139 movq $1f, %rax
140 ANNOTATE_RETPOLINE_SAFE
141 jmp *%rax
1421:
143 UNWIND_HINT_EMPTY
144
145
146 movl $0x80000001, %eax
147 cpuid
148 movl %edx,%edi
149
150
151 movl $MSR_EFER, %ecx
152 rdmsr
153 btsl $_EFER_SCE, %eax
154 btl $20,%edi
155 jnc 1f
156 btsl $_EFER_NX, %eax
157 btsq $_PAGE_BIT_NX,early_pmd_flags(%rip)
1581: wrmsr
159
160
161 movl $CR0_STATE, %eax
162
163 movq %rax, %cr0
164
165
166 movq initial_stack(%rip), %rsp
167
168
169 pushq $0
170 popfq
171
172
173
174
175
176
177
178 lgdt early_gdt_descr(%rip)
179
180
181 xorl %eax,%eax
182 movl %eax,%ds
183 movl %eax,%ss
184 movl %eax,%es
185
186
187
188
189
190
191 movl %eax,%fs
192 movl %eax,%gs
193
194
195
196
197
198
199
200
201 movl $MSR_GS_BASE,%ecx
202 movl initial_gs(%rip),%eax
203 movl initial_gs+4(%rip),%edx
204 wrmsr
205
206
207
208 movq %rsi, %rdi
209
210.Ljump_to_C_code:
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236 pushq $.Lafter_lret
237 xorl %ebp, %ebp
238 movq initial_code(%rip), %rax
239 pushq $__KERNEL_CS
240 pushq %rax
241 lretq
242.Lafter_lret:
243END(secondary_startup_64)
244
245#include "verify_cpu.S"
246
247#ifdef CONFIG_HOTPLUG_CPU
248
249
250
251
252
253ENTRY(start_cpu0)
254 UNWIND_HINT_EMPTY
255 movq initial_stack(%rip), %rsp
256 jmp .Ljump_to_C_code
257END(start_cpu0)
258#endif
259
260
261 __REFDATA
262 .balign 8
263 GLOBAL(initial_code)
264 .quad x86_64_start_kernel
265 GLOBAL(initial_gs)
266 .quad INIT_PER_CPU_VAR(fixed_percpu_data)
267 GLOBAL(initial_stack)
268
269
270
271
272 .quad init_thread_union + THREAD_SIZE - SIZEOF_PTREGS
273 __FINITDATA
274
275 __INIT
276ENTRY(early_idt_handler_array)
277 i = 0
278 .rept NUM_EXCEPTION_VECTORS
279 .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
280 UNWIND_HINT_IRET_REGS
281 pushq $0
282 .else
283 UNWIND_HINT_IRET_REGS offset=8
284 .endif
285 pushq $i
286 jmp early_idt_handler_common
287 UNWIND_HINT_IRET_REGS
288 i = i + 1
289 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
290 .endr
291 UNWIND_HINT_IRET_REGS offset=16
292END(early_idt_handler_array)
293
294early_idt_handler_common:
295
296
297
298
299 cld
300
301 incl early_recursion_flag(%rip)
302
303
304 pushq %rsi
305 movq 8(%rsp), %rsi
306 movq %rdi, 8(%rsp)
307 pushq %rdx
308 pushq %rcx
309 pushq %rax
310 pushq %r8
311 pushq %r9
312 pushq %r10
313 pushq %r11
314 pushq %rbx
315 pushq %rbp
316 pushq %r12
317 pushq %r13
318 pushq %r14
319 pushq %r15
320 UNWIND_HINT_REGS
321
322 cmpq $14,%rsi
323 jnz 10f
324 GET_CR2_INTO(%rdi)
325 call early_make_pgtable
326 andl %eax,%eax
327 jz 20f
328
32910:
330 movq %rsp,%rdi
331 call early_fixup_exception
332
33320:
334 decl early_recursion_flag(%rip)
335 jmp restore_regs_and_return_to_kernel
336END(early_idt_handler_common)
337
338 __INITDATA
339
340 .balign 4
341GLOBAL(early_recursion_flag)
342 .long 0
343
344#define NEXT_PAGE(name) \
345 .balign PAGE_SIZE; \
346GLOBAL(name)
347
348#ifdef CONFIG_PAGE_TABLE_ISOLATION
349
350
351
352
353
354
355
356
357
358
359#define PTI_USER_PGD_FILL 512
360
361#define NEXT_PGD_PAGE(name) \
362 .balign 2 * PAGE_SIZE; \
363GLOBAL(name)
364#else
365#define NEXT_PGD_PAGE(name) NEXT_PAGE(name)
366#define PTI_USER_PGD_FILL 0
367#endif
368
369
370#define PMDS(START, PERM, COUNT) \
371 i = 0 ; \
372 .rept (COUNT) ; \
373 .quad (START) + (i << PMD_SHIFT) + (PERM) ; \
374 i = i + 1 ; \
375 .endr
376
377 __INITDATA
378NEXT_PGD_PAGE(early_top_pgt)
379 .fill 512,8,0
380 .fill PTI_USER_PGD_FILL,8,0
381
382NEXT_PAGE(early_dynamic_pgts)
383 .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
384
385 .data
386
387
388NEXT_PGD_PAGE(init_top_pgt)
389 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
390 .org init_top_pgt + L4_PAGE_OFFSET*8, 0
391 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
392 .org init_top_pgt + L4_START_KERNEL*8, 0
393
394 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
395 .fill PTI_USER_PGD_FILL,8,0
396
397NEXT_PAGE(level3_ident_pgt)
398 .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
399 .fill 511, 8, 0
400NEXT_PAGE(level2_ident_pgt)
401
402
403
404
405
406
407
408
409 PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
410#else
411NEXT_PGD_PAGE(init_top_pgt)
412 .fill 512,8,0
413 .fill PTI_USER_PGD_FILL,8,0
414#endif
415
416#ifdef CONFIG_X86_5LEVEL
417NEXT_PAGE(level4_kernel_pgt)
418 .fill 511,8,0
419 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
420#endif
421
422NEXT_PAGE(level3_kernel_pgt)
423 .fill L3_START_KERNEL,8,0
424
425 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
426 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
427
428NEXT_PAGE(level2_kernel_pgt)
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443 PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
444 KERNEL_IMAGE_SIZE/PMD_SIZE)
445
446NEXT_PAGE(level2_fixmap_pgt)
447 .fill (512 - 4 - FIXMAP_PMD_NUM),8,0
448 pgtno = 0
449 .rept (FIXMAP_PMD_NUM)
450 .quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
451 + _PAGE_TABLE_NOENC;
452 pgtno = pgtno + 1
453 .endr
454
455 .fill 4,8,0
456
457NEXT_PAGE(level1_fixmap_pgt)
458 .rept (FIXMAP_PMD_NUM)
459 .fill 512,8,0
460 .endr
461
462#undef PMDS
463
464 .data
465 .align 16
466 .globl early_gdt_descr
467early_gdt_descr:
468 .word GDT_ENTRIES*8-1
469early_gdt_descr_base:
470 .quad INIT_PER_CPU_VAR(gdt_page)
471
472ENTRY(phys_base)
473
474 .quad 0x0000000000000000
475EXPORT_SYMBOL(phys_base)
476
477#include "../../x86/xen/xen-head.S"
478
479 __PAGE_ALIGNED_BSS
480NEXT_PAGE(empty_zero_page)
481 .skip PAGE_SIZE
482EXPORT_SYMBOL(empty_zero_page)
483
484