1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) "kexec: " fmt
14
15#include <linux/types.h>
16#include <linux/kernel.h>
17#include <linux/smp.h>
18#include <linux/reboot.h>
19#include <linux/kexec.h>
20#include <linux/delay.h>
21#include <linux/elf.h>
22#include <linux/elfcore.h>
23#include <linux/export.h>
24#include <linux/slab.h>
25#include <linux/vmalloc.h>
26
27#include <asm/processor.h>
28#include <asm/hardirq.h>
29#include <asm/nmi.h>
30#include <asm/hw_irq.h>
31#include <asm/apic.h>
32#include <asm/e820/types.h>
33#include <asm/io_apic.h>
34#include <asm/hpet.h>
35#include <linux/kdebug.h>
36#include <asm/cpu.h>
37#include <asm/reboot.h>
38#include <asm/virtext.h>
39#include <asm/intel_pt.h>
40
41
42#define ELF_CORE_HEADER_ALIGN 4096
43
44
45#define CRASH_MAX_RANGES 16
46
47struct crash_mem_range {
48 u64 start, end;
49};
50
51struct crash_mem {
52 unsigned int nr_ranges;
53 struct crash_mem_range ranges[CRASH_MAX_RANGES];
54};
55
56
57struct crash_elf_data {
58 struct kimage *image;
59
60
61
62
63 unsigned int max_nr_ranges;
64
65
66 void *ehdr;
67
68 void *bufp;
69 struct crash_mem mem;
70};
71
72
73struct crash_memmap_data {
74 struct boot_params *params;
75
76 unsigned int type;
77};
78
79
80
81
82
83
84
85
86crash_vmclear_fn __rcu *crash_vmclear_loaded_vmcss = NULL;
87EXPORT_SYMBOL_GPL(crash_vmclear_loaded_vmcss);
88unsigned long crash_zero_bytes;
89
90static inline void cpu_crash_vmclear_loaded_vmcss(void)
91{
92 crash_vmclear_fn *do_vmclear_operation = NULL;
93
94 rcu_read_lock();
95 do_vmclear_operation = rcu_dereference(crash_vmclear_loaded_vmcss);
96 if (do_vmclear_operation)
97 do_vmclear_operation();
98 rcu_read_unlock();
99}
100
101#if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC)
102
103static void kdump_nmi_callback(int cpu, struct pt_regs *regs)
104{
105#ifdef CONFIG_X86_32
106 struct pt_regs fixed_regs;
107
108 if (!user_mode(regs)) {
109 crash_fixup_ss_esp(&fixed_regs, regs);
110 regs = &fixed_regs;
111 }
112#endif
113 crash_save_cpu(regs, cpu);
114
115
116
117
118 cpu_crash_vmclear_loaded_vmcss();
119
120
121
122
123
124
125
126 cpu_emergency_vmxoff();
127 cpu_emergency_svm_disable();
128
129
130
131
132 cpu_emergency_stop_pt();
133
134 disable_local_APIC();
135}
136
137void kdump_nmi_shootdown_cpus(void)
138{
139 nmi_shootdown_cpus(kdump_nmi_callback);
140
141 disable_local_APIC();
142}
143
144
145void crash_smp_send_stop(void)
146{
147 static int cpus_stopped;
148
149 if (cpus_stopped)
150 return;
151
152 if (smp_ops.crash_stop_other_cpus)
153 smp_ops.crash_stop_other_cpus();
154 else
155 smp_send_stop();
156
157 cpus_stopped = 1;
158}
159
160#else
161void crash_smp_send_stop(void)
162{
163
164}
165#endif
166
167void native_machine_crash_shutdown(struct pt_regs *regs)
168{
169
170
171
172
173
174
175
176
177
178 local_irq_disable();
179
180 crash_smp_send_stop();
181
182
183
184
185 cpu_crash_vmclear_loaded_vmcss();
186
187
188
189
190
191 cpu_emergency_vmxoff();
192 cpu_emergency_svm_disable();
193
194
195
196
197 cpu_emergency_stop_pt();
198
199#ifdef CONFIG_X86_IO_APIC
200
201 ioapic_zap_locks();
202 disable_IO_APIC();
203#endif
204 lapic_shutdown();
205#ifdef CONFIG_HPET_TIMER
206 hpet_disable();
207#endif
208 crash_save_cpu(regs, safe_smp_processor_id());
209}
210
211#ifdef CONFIG_KEXEC_FILE
212static int get_nr_ram_ranges_callback(u64 start, u64 end, void *arg)
213{
214 unsigned int *nr_ranges = arg;
215
216 (*nr_ranges)++;
217 return 0;
218}
219
220
221
222static void fill_up_crash_elf_data(struct crash_elf_data *ced,
223 struct kimage *image)
224{
225 unsigned int nr_ranges = 0;
226
227 ced->image = image;
228
229 walk_system_ram_res(0, -1, &nr_ranges,
230 get_nr_ram_ranges_callback);
231
232 ced->max_nr_ranges = nr_ranges;
233
234
235 ced->max_nr_ranges++;
236
237
238 if (crashk_low_res.end)
239 ced->max_nr_ranges++;
240}
241
242static int exclude_mem_range(struct crash_mem *mem,
243 unsigned long long mstart, unsigned long long mend)
244{
245 int i, j;
246 unsigned long long start, end;
247 struct crash_mem_range temp_range = {0, 0};
248
249 for (i = 0; i < mem->nr_ranges; i++) {
250 start = mem->ranges[i].start;
251 end = mem->ranges[i].end;
252
253 if (mstart > end || mend < start)
254 continue;
255
256
257 if (mstart < start)
258 mstart = start;
259 if (mend > end)
260 mend = end;
261
262
263 if (mstart == start && mend == end) {
264 mem->ranges[i].start = 0;
265 mem->ranges[i].end = 0;
266 if (i < mem->nr_ranges - 1) {
267
268 for (j = i; j < mem->nr_ranges - 1; j++) {
269 mem->ranges[j].start =
270 mem->ranges[j+1].start;
271 mem->ranges[j].end =
272 mem->ranges[j+1].end;
273 }
274 }
275 mem->nr_ranges--;
276 return 0;
277 }
278
279 if (mstart > start && mend < end) {
280
281 mem->ranges[i].end = mstart - 1;
282 temp_range.start = mend + 1;
283 temp_range.end = end;
284 } else if (mstart != start)
285 mem->ranges[i].end = mstart - 1;
286 else
287 mem->ranges[i].start = mend + 1;
288 break;
289 }
290
291
292 if (!temp_range.end)
293 return 0;
294
295
296 if (i == CRASH_MAX_RANGES - 1) {
297 pr_err("Too many crash ranges after split\n");
298 return -ENOMEM;
299 }
300
301
302 j = i + 1;
303 if (j < mem->nr_ranges) {
304
305 for (i = mem->nr_ranges - 1; i >= j; i--)
306 mem->ranges[i + 1] = mem->ranges[i];
307 }
308
309 mem->ranges[j].start = temp_range.start;
310 mem->ranges[j].end = temp_range.end;
311 mem->nr_ranges++;
312 return 0;
313}
314
315
316
317
318
319static int elf_header_exclude_ranges(struct crash_elf_data *ced,
320 unsigned long long mstart, unsigned long long mend)
321{
322 struct crash_mem *cmem = &ced->mem;
323 int ret = 0;
324
325 memset(cmem->ranges, 0, sizeof(cmem->ranges));
326
327 cmem->ranges[0].start = mstart;
328 cmem->ranges[0].end = mend;
329 cmem->nr_ranges = 1;
330
331
332 ret = exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
333 if (ret)
334 return ret;
335
336 if (crashk_low_res.end) {
337 ret = exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end);
338 if (ret)
339 return ret;
340 }
341
342 return ret;
343}
344
345static int prepare_elf64_ram_headers_callback(u64 start, u64 end, void *arg)
346{
347 struct crash_elf_data *ced = arg;
348 Elf64_Ehdr *ehdr;
349 Elf64_Phdr *phdr;
350 unsigned long mstart, mend;
351 struct kimage *image = ced->image;
352 struct crash_mem *cmem;
353 int ret, i;
354
355 ehdr = ced->ehdr;
356
357
358 ret = elf_header_exclude_ranges(ced, start, end);
359 if (ret)
360 return ret;
361
362
363 cmem = &ced->mem;
364
365 for (i = 0; i < cmem->nr_ranges; i++) {
366 mstart = cmem->ranges[i].start;
367 mend = cmem->ranges[i].end;
368
369 phdr = ced->bufp;
370 ced->bufp += sizeof(Elf64_Phdr);
371
372 phdr->p_type = PT_LOAD;
373 phdr->p_flags = PF_R|PF_W|PF_X;
374 phdr->p_offset = mstart;
375
376
377
378
379
380 if (mstart == image->arch.backup_src_start &&
381 (mend - mstart + 1) == image->arch.backup_src_sz)
382 phdr->p_offset = image->arch.backup_load_addr;
383
384 phdr->p_paddr = mstart;
385 phdr->p_vaddr = (unsigned long long) __va(mstart);
386 phdr->p_filesz = phdr->p_memsz = mend - mstart + 1;
387 phdr->p_align = 0;
388 ehdr->e_phnum++;
389 pr_debug("Crash PT_LOAD elf header. phdr=%p vaddr=0x%llx, paddr=0x%llx, sz=0x%llx e_phnum=%d p_offset=0x%llx\n",
390 phdr, phdr->p_vaddr, phdr->p_paddr, phdr->p_filesz,
391 ehdr->e_phnum, phdr->p_offset);
392 }
393
394 return ret;
395}
396
397static int prepare_elf64_headers(struct crash_elf_data *ced,
398 void **addr, unsigned long *sz)
399{
400 Elf64_Ehdr *ehdr;
401 Elf64_Phdr *phdr;
402 unsigned long nr_cpus = num_possible_cpus(), nr_phdr, elf_sz;
403 unsigned char *buf, *bufp;
404 unsigned int cpu;
405 unsigned long long notes_addr;
406 int ret;
407
408
409 nr_phdr = nr_cpus + 1;
410 nr_phdr += ced->max_nr_ranges;
411
412
413
414
415
416
417
418
419
420 nr_phdr++;
421 elf_sz = sizeof(Elf64_Ehdr) + nr_phdr * sizeof(Elf64_Phdr);
422 elf_sz = ALIGN(elf_sz, ELF_CORE_HEADER_ALIGN);
423
424 buf = vzalloc(elf_sz);
425 if (!buf)
426 return -ENOMEM;
427
428 bufp = buf;
429 ehdr = (Elf64_Ehdr *)bufp;
430 bufp += sizeof(Elf64_Ehdr);
431 memcpy(ehdr->e_ident, ELFMAG, SELFMAG);
432 ehdr->e_ident[EI_CLASS] = ELFCLASS64;
433 ehdr->e_ident[EI_DATA] = ELFDATA2LSB;
434 ehdr->e_ident[EI_VERSION] = EV_CURRENT;
435 ehdr->e_ident[EI_OSABI] = ELF_OSABI;
436 memset(ehdr->e_ident + EI_PAD, 0, EI_NIDENT - EI_PAD);
437 ehdr->e_type = ET_CORE;
438 ehdr->e_machine = ELF_ARCH;
439 ehdr->e_version = EV_CURRENT;
440 ehdr->e_phoff = sizeof(Elf64_Ehdr);
441 ehdr->e_ehsize = sizeof(Elf64_Ehdr);
442 ehdr->e_phentsize = sizeof(Elf64_Phdr);
443
444
445 for_each_present_cpu(cpu) {
446 phdr = (Elf64_Phdr *)bufp;
447 bufp += sizeof(Elf64_Phdr);
448 phdr->p_type = PT_NOTE;
449 notes_addr = per_cpu_ptr_to_phys(per_cpu_ptr(crash_notes, cpu));
450 phdr->p_offset = phdr->p_paddr = notes_addr;
451 phdr->p_filesz = phdr->p_memsz = sizeof(note_buf_t);
452 (ehdr->e_phnum)++;
453 }
454
455
456 phdr = (Elf64_Phdr *)bufp;
457 bufp += sizeof(Elf64_Phdr);
458 phdr->p_type = PT_NOTE;
459 phdr->p_offset = phdr->p_paddr = paddr_vmcoreinfo_note();
460 phdr->p_filesz = phdr->p_memsz = VMCOREINFO_NOTE_SIZE;
461 (ehdr->e_phnum)++;
462
463#ifdef CONFIG_X86_64
464
465 phdr = (Elf64_Phdr *)bufp;
466 bufp += sizeof(Elf64_Phdr);
467 phdr->p_type = PT_LOAD;
468 phdr->p_flags = PF_R|PF_W|PF_X;
469 phdr->p_vaddr = (Elf64_Addr)_text;
470 phdr->p_filesz = phdr->p_memsz = _end - _text;
471 phdr->p_offset = phdr->p_paddr = __pa_symbol(_text);
472 (ehdr->e_phnum)++;
473#endif
474
475
476 ced->ehdr = ehdr;
477 ced->bufp = bufp;
478 ret = walk_system_ram_res(0, -1, ced,
479 prepare_elf64_ram_headers_callback);
480 if (ret < 0)
481 return ret;
482
483 *addr = buf;
484 *sz = elf_sz;
485 return 0;
486}
487
488
489static int prepare_elf_headers(struct kimage *image, void **addr,
490 unsigned long *sz)
491{
492 struct crash_elf_data *ced;
493 int ret;
494
495 ced = kzalloc(sizeof(*ced), GFP_KERNEL);
496 if (!ced)
497 return -ENOMEM;
498
499 fill_up_crash_elf_data(ced, image);
500
501
502 ret = prepare_elf64_headers(ced, addr, sz);
503 kfree(ced);
504 return ret;
505}
506
507static int add_e820_entry(struct boot_params *params, struct e820_entry *entry)
508{
509 unsigned int nr_e820_entries;
510
511 nr_e820_entries = params->e820_entries;
512 if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE)
513 return 1;
514
515 memcpy(¶ms->e820_table[nr_e820_entries], entry,
516 sizeof(struct e820_entry));
517 params->e820_entries++;
518 return 0;
519}
520
521static int memmap_entry_callback(u64 start, u64 end, void *arg)
522{
523 struct crash_memmap_data *cmd = arg;
524 struct boot_params *params = cmd->params;
525 struct e820_entry ei;
526
527 ei.addr = start;
528 ei.size = end - start + 1;
529 ei.type = cmd->type;
530 add_e820_entry(params, &ei);
531
532 return 0;
533}
534
535static int memmap_exclude_ranges(struct kimage *image, struct crash_mem *cmem,
536 unsigned long long mstart,
537 unsigned long long mend)
538{
539 unsigned long start, end;
540 int ret = 0;
541
542 cmem->ranges[0].start = mstart;
543 cmem->ranges[0].end = mend;
544 cmem->nr_ranges = 1;
545
546
547 start = image->arch.backup_load_addr;
548 end = start + image->arch.backup_src_sz - 1;
549 ret = exclude_mem_range(cmem, start, end);
550 if (ret)
551 return ret;
552
553
554 start = image->arch.elf_load_addr;
555 end = start + image->arch.elf_headers_sz - 1;
556 return exclude_mem_range(cmem, start, end);
557}
558
559
560int crash_setup_memmap_entries(struct kimage *image, struct boot_params *params)
561{
562 int i, ret = 0;
563 unsigned long flags;
564 struct e820_entry ei;
565 struct crash_memmap_data cmd;
566 struct crash_mem *cmem;
567
568 cmem = vzalloc(sizeof(struct crash_mem));
569 if (!cmem)
570 return -ENOMEM;
571
572 memset(&cmd, 0, sizeof(struct crash_memmap_data));
573 cmd.params = params;
574
575
576 ei.addr = image->arch.backup_src_start;
577 ei.size = image->arch.backup_src_sz;
578 ei.type = E820_TYPE_RAM;
579 add_e820_entry(params, &ei);
580
581
582 cmd.type = E820_TYPE_ACPI;
583 flags = IORESOURCE_MEM | IORESOURCE_BUSY;
584 walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1, &cmd,
585 memmap_entry_callback);
586
587
588 cmd.type = E820_TYPE_NVS;
589 walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1, &cmd,
590 memmap_entry_callback);
591
592
593 if (crashk_low_res.end) {
594 ei.addr = crashk_low_res.start;
595 ei.size = crashk_low_res.end - crashk_low_res.start + 1;
596 ei.type = E820_TYPE_RAM;
597 add_e820_entry(params, &ei);
598 }
599
600
601 ret = memmap_exclude_ranges(image, cmem, crashk_res.start,
602 crashk_res.end);
603 if (ret)
604 goto out;
605
606 for (i = 0; i < cmem->nr_ranges; i++) {
607 ei.size = cmem->ranges[i].end - cmem->ranges[i].start + 1;
608
609
610 if (ei.size < PAGE_SIZE)
611 continue;
612 ei.addr = cmem->ranges[i].start;
613 ei.type = E820_TYPE_RAM;
614 add_e820_entry(params, &ei);
615 }
616
617out:
618 vfree(cmem);
619 return ret;
620}
621
622static int determine_backup_region(u64 start, u64 end, void *arg)
623{
624 struct kimage *image = arg;
625
626 image->arch.backup_src_start = start;
627 image->arch.backup_src_sz = end - start + 1;
628
629
630 return 1;
631}
632
633int crash_load_segments(struct kimage *image)
634{
635 int ret;
636 struct kexec_buf kbuf = { .image = image, .buf_min = 0,
637 .buf_max = ULONG_MAX, .top_down = false };
638
639
640
641
642
643
644 ret = walk_system_ram_res(KEXEC_BACKUP_SRC_START, KEXEC_BACKUP_SRC_END,
645 image, determine_backup_region);
646
647
648 if (ret < 0)
649 return ret;
650
651
652 if (image->arch.backup_src_sz) {
653 kbuf.buffer = &crash_zero_bytes;
654 kbuf.bufsz = sizeof(crash_zero_bytes);
655 kbuf.memsz = image->arch.backup_src_sz;
656 kbuf.buf_align = PAGE_SIZE;
657
658
659
660
661
662 ret = kexec_add_buffer(&kbuf);
663 if (ret)
664 return ret;
665 image->arch.backup_load_addr = kbuf.mem;
666 pr_debug("Loaded backup region at 0x%lx backup_start=0x%lx memsz=0x%lx\n",
667 image->arch.backup_load_addr,
668 image->arch.backup_src_start, kbuf.memsz);
669 }
670
671
672 ret = prepare_elf_headers(image, &kbuf.buffer, &kbuf.bufsz);
673 if (ret)
674 return ret;
675
676 image->arch.elf_headers = kbuf.buffer;
677 image->arch.elf_headers_sz = kbuf.bufsz;
678
679 kbuf.memsz = kbuf.bufsz;
680 kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
681 ret = kexec_add_buffer(&kbuf);
682 if (ret) {
683 vfree((void *)image->arch.elf_headers);
684 return ret;
685 }
686 image->arch.elf_load_addr = kbuf.mem;
687 pr_debug("Loaded ELF headers at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
688 image->arch.elf_load_addr, kbuf.bufsz, kbuf.bufsz);
689
690 return ret;
691}
692#endif
693