1
2
3
4
5
6
7#define pr_fmt(fmt) "kexec: " fmt
8
9#include <linux/mm.h>
10#include <linux/kexec.h>
11#include <linux/string.h>
12#include <linux/gfp.h>
13#include <linux/reboot.h>
14#include <linux/numa.h>
15#include <linux/ftrace.h>
16#include <linux/io.h>
17#include <linux/suspend.h>
18#include <linux/vmalloc.h>
19
20#include <asm/init.h>
21#include <asm/pgtable.h>
22#include <asm/tlbflush.h>
23#include <asm/mmu_context.h>
24#include <asm/io_apic.h>
25#include <asm/debugreg.h>
26#include <asm/kexec-bzimage64.h>
27#include <asm/setup.h>
28#include <asm/set_memory.h>
29
30#ifdef CONFIG_KEXEC_FILE
31const struct kexec_file_ops * const kexec_file_loaders[] = {
32 &kexec_bzImage64_ops,
33 NULL
34};
35#endif
36
37static void free_transition_pgtable(struct kimage *image)
38{
39 free_page((unsigned long)image->arch.p4d);
40 image->arch.p4d = NULL;
41 free_page((unsigned long)image->arch.pud);
42 image->arch.pud = NULL;
43 free_page((unsigned long)image->arch.pmd);
44 image->arch.pmd = NULL;
45 free_page((unsigned long)image->arch.pte);
46 image->arch.pte = NULL;
47}
48
49static int init_transition_pgtable(struct kimage *image, pgd_t *pgd)
50{
51 p4d_t *p4d;
52 pud_t *pud;
53 pmd_t *pmd;
54 pte_t *pte;
55 unsigned long vaddr, paddr;
56 int result = -ENOMEM;
57
58 vaddr = (unsigned long)relocate_kernel;
59 paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE);
60 pgd += pgd_index(vaddr);
61 if (!pgd_present(*pgd)) {
62 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
63 if (!p4d)
64 goto err;
65 image->arch.p4d = p4d;
66 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
67 }
68 p4d = p4d_offset(pgd, vaddr);
69 if (!p4d_present(*p4d)) {
70 pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
71 if (!pud)
72 goto err;
73 image->arch.pud = pud;
74 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
75 }
76 pud = pud_offset(p4d, vaddr);
77 if (!pud_present(*pud)) {
78 pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
79 if (!pmd)
80 goto err;
81 image->arch.pmd = pmd;
82 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
83 }
84 pmd = pmd_offset(pud, vaddr);
85 if (!pmd_present(*pmd)) {
86 pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
87 if (!pte)
88 goto err;
89 image->arch.pte = pte;
90 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
91 }
92 pte = pte_offset_kernel(pmd, vaddr);
93 set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC_NOENC));
94 return 0;
95err:
96 return result;
97}
98
99static void *alloc_pgt_page(void *data)
100{
101 struct kimage *image = (struct kimage *)data;
102 struct page *page;
103 void *p = NULL;
104
105 page = kimage_alloc_control_pages(image, 0);
106 if (page) {
107 p = page_address(page);
108 clear_page(p);
109 }
110
111 return p;
112}
113
114static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
115{
116 struct x86_mapping_info info = {
117 .alloc_pgt_page = alloc_pgt_page,
118 .context = image,
119 .page_flag = __PAGE_KERNEL_LARGE_EXEC,
120 .kernpg_flag = _KERNPG_TABLE_NOENC,
121 };
122 unsigned long mstart, mend;
123 pgd_t *level4p;
124 int result;
125 int i;
126
127 level4p = (pgd_t *)__va(start_pgtable);
128 clear_page(level4p);
129
130 if (direct_gbpages)
131 info.direct_gbpages = true;
132
133 for (i = 0; i < nr_pfn_mapped; i++) {
134 mstart = pfn_mapped[i].start << PAGE_SHIFT;
135 mend = pfn_mapped[i].end << PAGE_SHIFT;
136
137 result = kernel_ident_mapping_init(&info,
138 level4p, mstart, mend);
139 if (result)
140 return result;
141 }
142
143
144
145
146
147
148
149 for (i = 0; i < image->nr_segments; i++) {
150 mstart = image->segment[i].mem;
151 mend = mstart + image->segment[i].memsz;
152
153 result = kernel_ident_mapping_init(&info,
154 level4p, mstart, mend);
155
156 if (result)
157 return result;
158 }
159
160 return init_transition_pgtable(image, level4p);
161}
162
163static void set_idt(void *newidt, u16 limit)
164{
165 struct desc_ptr curidt;
166
167
168 curidt.size = limit;
169 curidt.address = (unsigned long)newidt;
170
171 __asm__ __volatile__ (
172 "lidtq %0\n"
173 : : "m" (curidt)
174 );
175};
176
177
178static void set_gdt(void *newgdt, u16 limit)
179{
180 struct desc_ptr curgdt;
181
182
183 curgdt.size = limit;
184 curgdt.address = (unsigned long)newgdt;
185
186 __asm__ __volatile__ (
187 "lgdtq %0\n"
188 : : "m" (curgdt)
189 );
190};
191
192static void load_segments(void)
193{
194 __asm__ __volatile__ (
195 "\tmovl %0,%%ds\n"
196 "\tmovl %0,%%es\n"
197 "\tmovl %0,%%ss\n"
198 "\tmovl %0,%%fs\n"
199 "\tmovl %0,%%gs\n"
200 : : "a" (__KERNEL_DS) : "memory"
201 );
202}
203
204#ifdef CONFIG_KEXEC_FILE
205
206static int arch_update_purgatory(struct kimage *image)
207{
208 int ret = 0;
209
210 if (!image->file_mode)
211 return 0;
212
213
214 if (image->type == KEXEC_TYPE_CRASH) {
215 ret = kexec_purgatory_get_set_symbol(image,
216 "purgatory_backup_dest",
217 &image->arch.backup_load_addr,
218 sizeof(image->arch.backup_load_addr), 0);
219 if (ret)
220 return ret;
221
222 ret = kexec_purgatory_get_set_symbol(image,
223 "purgatory_backup_src",
224 &image->arch.backup_src_start,
225 sizeof(image->arch.backup_src_start), 0);
226 if (ret)
227 return ret;
228
229 ret = kexec_purgatory_get_set_symbol(image,
230 "purgatory_backup_sz",
231 &image->arch.backup_src_sz,
232 sizeof(image->arch.backup_src_sz), 0);
233 if (ret)
234 return ret;
235 }
236
237 return ret;
238}
239#else
240static inline int arch_update_purgatory(struct kimage *image)
241{
242 return 0;
243}
244#endif
245
246int machine_kexec_prepare(struct kimage *image)
247{
248 unsigned long start_pgtable;
249 int result;
250
251
252 start_pgtable = page_to_pfn(image->control_code_page) << PAGE_SHIFT;
253
254
255 result = init_pgtable(image, start_pgtable);
256 if (result)
257 return result;
258
259
260 result = arch_update_purgatory(image);
261 if (result)
262 return result;
263
264 return 0;
265}
266
267void machine_kexec_cleanup(struct kimage *image)
268{
269 free_transition_pgtable(image);
270}
271
272
273
274
275
276void machine_kexec(struct kimage *image)
277{
278 unsigned long page_list[PAGES_NR];
279 void *control_page;
280 int save_ftrace_enabled;
281
282#ifdef CONFIG_KEXEC_JUMP
283 if (image->preserve_context)
284 save_processor_state();
285#endif
286
287 save_ftrace_enabled = __ftrace_enabled_save();
288
289
290 local_irq_disable();
291 hw_breakpoint_disable();
292
293 if (image->preserve_context) {
294#ifdef CONFIG_X86_IO_APIC
295
296
297
298
299
300
301 clear_IO_APIC();
302 restore_boot_irq_mode();
303#endif
304 }
305
306 control_page = page_address(image->control_code_page) + PAGE_SIZE;
307 memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
308
309 page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page);
310 page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
311 page_list[PA_TABLE_PAGE] =
312 (unsigned long)__pa(page_address(image->control_code_page));
313
314 if (image->type == KEXEC_TYPE_DEFAULT)
315 page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
316 << PAGE_SHIFT);
317
318
319
320
321
322
323
324
325
326
327
328 load_segments();
329
330
331
332
333 set_gdt(phys_to_virt(0), 0);
334 set_idt(phys_to_virt(0), 0);
335
336
337 image->start = relocate_kernel((unsigned long)image->head,
338 (unsigned long)page_list,
339 image->start,
340 image->preserve_context,
341 sme_active());
342
343#ifdef CONFIG_KEXEC_JUMP
344 if (image->preserve_context)
345 restore_processor_state();
346#endif
347
348 __ftrace_enabled_restore(save_ftrace_enabled);
349}
350
351void arch_crash_save_vmcoreinfo(void)
352{
353 u64 sme_mask = sme_me_mask;
354
355 VMCOREINFO_NUMBER(phys_base);
356 VMCOREINFO_SYMBOL(init_top_pgt);
357 vmcoreinfo_append_str("NUMBER(pgtable_l5_enabled)=%d\n",
358 pgtable_l5_enabled());
359
360#ifdef CONFIG_NUMA
361 VMCOREINFO_SYMBOL(node_data);
362 VMCOREINFO_LENGTH(node_data, MAX_NUMNODES);
363#endif
364 vmcoreinfo_append_str("KERNELOFFSET=%lx\n",
365 kaslr_offset());
366 VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE);
367 VMCOREINFO_NUMBER(sme_mask);
368}
369
370
371
372#ifdef CONFIG_KEXEC_FILE
373void *arch_kexec_kernel_image_load(struct kimage *image)
374{
375 vfree(image->arch.elf_headers);
376 image->arch.elf_headers = NULL;
377
378 if (!image->fops || !image->fops->load)
379 return ERR_PTR(-ENOEXEC);
380
381 return image->fops->load(image, image->kernel_buf,
382 image->kernel_buf_len, image->initrd_buf,
383 image->initrd_buf_len, image->cmdline_buf,
384 image->cmdline_buf_len);
385}
386
387
388
389
390
391
392
393
394
395
396
397int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
398 Elf_Shdr *section, const Elf_Shdr *relsec,
399 const Elf_Shdr *symtabsec)
400{
401 unsigned int i;
402 Elf64_Rela *rel;
403 Elf64_Sym *sym;
404 void *location;
405 unsigned long address, sec_base, value;
406 const char *strtab, *name, *shstrtab;
407 const Elf_Shdr *sechdrs;
408
409
410 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
411 strtab = (char *)pi->ehdr + sechdrs[symtabsec->sh_link].sh_offset;
412 shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
413
414 rel = (void *)pi->ehdr + relsec->sh_offset;
415
416 pr_debug("Applying relocate section %s to %u\n",
417 shstrtab + relsec->sh_name, relsec->sh_info);
418
419 for (i = 0; i < relsec->sh_size / sizeof(*rel); i++) {
420
421
422
423
424
425
426
427
428
429
430
431 location = pi->purgatory_buf;
432 location += section->sh_offset;
433 location += rel[i].r_offset;
434
435
436 address = section->sh_addr + rel[i].r_offset;
437
438
439
440
441
442
443
444 sym = (void *)pi->ehdr + symtabsec->sh_offset;
445 sym += ELF64_R_SYM(rel[i].r_info);
446
447 if (sym->st_name)
448 name = strtab + sym->st_name;
449 else
450 name = shstrtab + sechdrs[sym->st_shndx].sh_name;
451
452 pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
453 name, sym->st_info, sym->st_shndx, sym->st_value,
454 sym->st_size);
455
456 if (sym->st_shndx == SHN_UNDEF) {
457 pr_err("Undefined symbol: %s\n", name);
458 return -ENOEXEC;
459 }
460
461 if (sym->st_shndx == SHN_COMMON) {
462 pr_err("symbol '%s' in common section\n", name);
463 return -ENOEXEC;
464 }
465
466 if (sym->st_shndx == SHN_ABS)
467 sec_base = 0;
468 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
469 pr_err("Invalid section %d for symbol %s\n",
470 sym->st_shndx, name);
471 return -ENOEXEC;
472 } else
473 sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
474
475 value = sym->st_value;
476 value += sec_base;
477 value += rel[i].r_addend;
478
479 switch (ELF64_R_TYPE(rel[i].r_info)) {
480 case R_X86_64_NONE:
481 break;
482 case R_X86_64_64:
483 *(u64 *)location = value;
484 break;
485 case R_X86_64_32:
486 *(u32 *)location = value;
487 if (value != *(u32 *)location)
488 goto overflow;
489 break;
490 case R_X86_64_32S:
491 *(s32 *)location = value;
492 if ((s64)value != *(s32 *)location)
493 goto overflow;
494 break;
495 case R_X86_64_PC32:
496 case R_X86_64_PLT32:
497 value -= (u64)address;
498 *(u32 *)location = value;
499 break;
500 default:
501 pr_err("Unknown rela relocation: %llu\n",
502 ELF64_R_TYPE(rel[i].r_info));
503 return -ENOEXEC;
504 }
505 }
506 return 0;
507
508overflow:
509 pr_err("Overflow in relocation type %d value 0x%lx\n",
510 (int)ELF64_R_TYPE(rel[i].r_info), value);
511 return -ENOEXEC;
512}
513#endif
514
515static int
516kexec_mark_range(unsigned long start, unsigned long end, bool protect)
517{
518 struct page *page;
519 unsigned int nr_pages;
520
521
522
523
524
525 if (!end || start > end)
526 return 0;
527
528 page = pfn_to_page(start >> PAGE_SHIFT);
529 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
530 if (protect)
531 return set_pages_ro(page, nr_pages);
532 else
533 return set_pages_rw(page, nr_pages);
534}
535
536static void kexec_mark_crashkres(bool protect)
537{
538 unsigned long control;
539
540 kexec_mark_range(crashk_low_res.start, crashk_low_res.end, protect);
541
542
543 control = PFN_PHYS(page_to_pfn(kexec_crash_image->control_code_page));
544
545 kexec_mark_range(crashk_res.start, control + PAGE_SIZE - 1, protect);
546 control += KEXEC_CONTROL_PAGE_SIZE;
547 kexec_mark_range(control, crashk_res.end, protect);
548}
549
550void arch_kexec_protect_crashkres(void)
551{
552 kexec_mark_crashkres(true);
553}
554
555void arch_kexec_unprotect_crashkres(void)
556{
557 kexec_mark_crashkres(false);
558}
559
560int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp)
561{
562
563
564
565
566
567 return set_memory_decrypted((unsigned long)vaddr, pages);
568}
569
570void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages)
571{
572
573
574
575
576 set_memory_encrypted((unsigned long)vaddr, pages);
577}
578