1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/linkage.h>
14#include <linux/threads.h>
15#include <linux/init.h>
16#include <asm/segment.h>
17#include <asm/pgtable.h>
18#include <asm/page.h>
19#include <asm/msr.h>
20#include <asm/cache.h>
21#include <asm/processor-flags.h>
22#include <asm/percpu.h>
23#include <asm/nops.h>
24#include "../entry/calling.h"
25#include <asm/export.h>
26#include <asm/nospec-branch.h>
27#include <asm/fixmap.h>
28
29#ifdef CONFIG_PARAVIRT_XXL
30#include <asm/asm-offsets.h>
31#include <asm/paravirt.h>
32#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg
33#else
34#define GET_CR2_INTO(reg) movq %cr2, reg
35#define INTERRUPT_RETURN iretq
36#endif
37
38
39
40
41
42
43#define l4_index(x) (((x) >> 39) & 511)
44#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
45
46L4_PAGE_OFFSET = l4_index(__PAGE_OFFSET_BASE_L4)
47L4_START_KERNEL = l4_index(__START_KERNEL_map)
48
49L3_START_KERNEL = pud_index(__START_KERNEL_map)
50
51 .text
52 __HEAD
53 .code64
54 .globl startup_64
55startup_64:
56 UNWIND_HINT_EMPTY
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76 leaq (__end_init_task - SIZEOF_PTREGS)(%rip), %rsp
77
78
79 call verify_cpu
80
81
82
83
84
85
86
87 leaq _text(%rip), %rdi
88 pushq %rsi
89 call __startup_64
90 popq %rsi
91
92
93 addq $(early_top_pgt - __START_KERNEL_map), %rax
94 jmp 1f
95ENTRY(secondary_startup_64)
96 UNWIND_HINT_EMPTY
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112 call verify_cpu
113
114
115
116
117
118 pushq %rsi
119 call __startup_secondary_64
120 popq %rsi
121
122
123 addq $(init_top_pgt - __START_KERNEL_map), %rax
1241:
125
126
127 movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
128#ifdef CONFIG_X86_5LEVEL
129 testl $1, __pgtable_l5_enabled(%rip)
130 jz 1f
131 orl $X86_CR4_LA57, %ecx
1321:
133#endif
134 movq %rcx, %cr4
135
136
137 addq phys_base(%rip), %rax
138 movq %rax, %cr3
139
140
141 movq $1f, %rax
142 ANNOTATE_RETPOLINE_SAFE
143 jmp *%rax
1441:
145 UNWIND_HINT_EMPTY
146
147
148 movl $0x80000001, %eax
149 cpuid
150 movl %edx,%edi
151
152
153 movl $MSR_EFER, %ecx
154 rdmsr
155 btsl $_EFER_SCE, %eax
156 btl $20,%edi
157 jnc 1f
158 btsl $_EFER_NX, %eax
159 btsq $_PAGE_BIT_NX,early_pmd_flags(%rip)
1601: wrmsr
161
162
163 movl $CR0_STATE, %eax
164
165 movq %rax, %cr0
166
167
168 movq initial_stack(%rip), %rsp
169
170
171 pushq $0
172 popfq
173
174
175
176
177
178
179
180 lgdt early_gdt_descr(%rip)
181
182
183 xorl %eax,%eax
184 movl %eax,%ds
185 movl %eax,%ss
186 movl %eax,%es
187
188
189
190
191
192
193 movl %eax,%fs
194 movl %eax,%gs
195
196
197
198
199
200
201
202
203 movl $MSR_GS_BASE,%ecx
204 movl initial_gs(%rip),%eax
205 movl initial_gs+4(%rip),%edx
206 wrmsr
207
208
209
210 movq %rsi, %rdi
211
212.Ljump_to_C_code:
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238 pushq $.Lafter_lret
239 xorl %ebp, %ebp
240 movq initial_code(%rip), %rax
241 pushq $__KERNEL_CS
242 pushq %rax
243 lretq
244.Lafter_lret:
245END(secondary_startup_64)
246
247#include "verify_cpu.S"
248
249#ifdef CONFIG_HOTPLUG_CPU
250
251
252
253
254
255ENTRY(start_cpu0)
256 movq initial_stack(%rip), %rsp
257 UNWIND_HINT_EMPTY
258 jmp .Ljump_to_C_code
259ENDPROC(start_cpu0)
260#endif
261
262
263 __REFDATA
264 .balign 8
265 GLOBAL(initial_code)
266 .quad x86_64_start_kernel
267 GLOBAL(initial_gs)
268 .quad INIT_PER_CPU_VAR(fixed_percpu_data)
269 GLOBAL(initial_stack)
270
271
272
273
274 .quad init_thread_union + THREAD_SIZE - SIZEOF_PTREGS
275 __FINITDATA
276
277 __INIT
278ENTRY(early_idt_handler_array)
279 i = 0
280 .rept NUM_EXCEPTION_VECTORS
281 .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
282 UNWIND_HINT_IRET_REGS
283 pushq $0
284 .else
285 UNWIND_HINT_IRET_REGS offset=8
286 .endif
287 pushq $i
288 jmp early_idt_handler_common
289 UNWIND_HINT_IRET_REGS
290 i = i + 1
291 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
292 .endr
293 UNWIND_HINT_IRET_REGS offset=16
294END(early_idt_handler_array)
295
296early_idt_handler_common:
297
298
299
300
301 cld
302
303 incl early_recursion_flag(%rip)
304
305
306 pushq %rsi
307 movq 8(%rsp), %rsi
308 movq %rdi, 8(%rsp)
309 pushq %rdx
310 pushq %rcx
311 pushq %rax
312 pushq %r8
313 pushq %r9
314 pushq %r10
315 pushq %r11
316 pushq %rbx
317 pushq %rbp
318 pushq %r12
319 pushq %r13
320 pushq %r14
321 pushq %r15
322 UNWIND_HINT_REGS
323
324 cmpq $14,%rsi
325 jnz 10f
326 GET_CR2_INTO(%rdi)
327 call early_make_pgtable
328 andl %eax,%eax
329 jz 20f
330
33110:
332 movq %rsp,%rdi
333 call early_fixup_exception
334
33520:
336 decl early_recursion_flag(%rip)
337 jmp restore_regs_and_return_to_kernel
338END(early_idt_handler_common)
339
340 __INITDATA
341
342 .balign 4
343GLOBAL(early_recursion_flag)
344 .long 0
345
346#define NEXT_PAGE(name) \
347 .balign PAGE_SIZE; \
348GLOBAL(name)
349
350#ifdef CONFIG_PAGE_TABLE_ISOLATION
351
352
353
354
355
356
357
358
359
360
361#define PTI_USER_PGD_FILL 512
362
363#define NEXT_PGD_PAGE(name) \
364 .balign 2 * PAGE_SIZE; \
365GLOBAL(name)
366#else
367#define NEXT_PGD_PAGE(name) NEXT_PAGE(name)
368#define PTI_USER_PGD_FILL 0
369#endif
370
371
372#define PMDS(START, PERM, COUNT) \
373 i = 0 ; \
374 .rept (COUNT) ; \
375 .quad (START) + (i << PMD_SHIFT) + (PERM) ; \
376 i = i + 1 ; \
377 .endr
378
379 __INITDATA
380NEXT_PGD_PAGE(early_top_pgt)
381 .fill 512,8,0
382 .fill PTI_USER_PGD_FILL,8,0
383
384NEXT_PAGE(early_dynamic_pgts)
385 .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
386
387 .data
388
389
390NEXT_PGD_PAGE(init_top_pgt)
391 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
392 .org init_top_pgt + L4_PAGE_OFFSET*8, 0
393 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
394 .org init_top_pgt + L4_START_KERNEL*8, 0
395
396 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
397 .fill PTI_USER_PGD_FILL,8,0
398
399NEXT_PAGE(level3_ident_pgt)
400 .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
401 .fill 511, 8, 0
402NEXT_PAGE(level2_ident_pgt)
403
404
405
406
407
408
409
410
411 PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
412#else
413NEXT_PGD_PAGE(init_top_pgt)
414 .fill 512,8,0
415 .fill PTI_USER_PGD_FILL,8,0
416#endif
417
418#ifdef CONFIG_X86_5LEVEL
419NEXT_PAGE(level4_kernel_pgt)
420 .fill 511,8,0
421 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
422#endif
423
424NEXT_PAGE(level3_kernel_pgt)
425 .fill L3_START_KERNEL,8,0
426
427 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
428 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
429
430NEXT_PAGE(level2_kernel_pgt)
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445 PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
446 KERNEL_IMAGE_SIZE/PMD_SIZE)
447
448NEXT_PAGE(level2_fixmap_pgt)
449 .fill (512 - 4 - FIXMAP_PMD_NUM),8,0
450 pgtno = 0
451 .rept (FIXMAP_PMD_NUM)
452 .quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
453 + _PAGE_TABLE_NOENC;
454 pgtno = pgtno + 1
455 .endr
456
457 .fill 4,8,0
458
459NEXT_PAGE(level1_fixmap_pgt)
460 .rept (FIXMAP_PMD_NUM)
461 .fill 512,8,0
462 .endr
463
464#undef PMDS
465
466 .data
467 .align 16
468 .globl early_gdt_descr
469early_gdt_descr:
470 .word GDT_ENTRIES*8-1
471early_gdt_descr_base:
472 .quad INIT_PER_CPU_VAR(gdt_page)
473
474ENTRY(phys_base)
475
476 .quad 0x0000000000000000
477EXPORT_SYMBOL(phys_base)
478
479#include "../../x86/xen/xen-head.S"
480
481 __PAGE_ALIGNED_BSS
482NEXT_PAGE(empty_zero_page)
483 .skip PAGE_SIZE
484EXPORT_SYMBOL(empty_zero_page)
485
486