1
2
3
4
5
6
7
8
9#include <linux/mm.h>
10#include <linux/kexec.h>
11#include <linux/delay.h>
12#include <linux/numa.h>
13#include <linux/ftrace.h>
14#include <linux/suspend.h>
15#include <linux/gfp.h>
16#include <linux/io.h>
17
18#include <asm/pgtable.h>
19#include <asm/pgalloc.h>
20#include <asm/tlbflush.h>
21#include <asm/mmu_context.h>
22#include <asm/apic.h>
23#include <asm/io_apic.h>
24#include <asm/cpufeature.h>
25#include <asm/desc.h>
26#include <asm/set_memory.h>
27#include <asm/debugreg.h>
28
29static void set_gdt(void *newgdt, __u16 limit)
30{
31 struct desc_ptr curgdt;
32
33
34 curgdt.size = limit;
35 curgdt.address = (unsigned long)newgdt;
36
37 load_gdt(&curgdt);
38}
39
40static void load_segments(void)
41{
42#define __STR(X) #X
43#define STR(X) __STR(X)
44
45 __asm__ __volatile__ (
46 "\tljmp $"STR(__KERNEL_CS)",$1f\n"
47 "\t1:\n"
48 "\tmovl $"STR(__KERNEL_DS)",%%eax\n"
49 "\tmovl %%eax,%%ds\n"
50 "\tmovl %%eax,%%es\n"
51 "\tmovl %%eax,%%ss\n"
52 : : : "eax", "memory");
53#undef STR
54#undef __STR
55}
56
57static void machine_kexec_free_page_tables(struct kimage *image)
58{
59 free_pages((unsigned long)image->arch.pgd, PGD_ALLOCATION_ORDER);
60 image->arch.pgd = NULL;
61#ifdef CONFIG_X86_PAE
62 free_page((unsigned long)image->arch.pmd0);
63 image->arch.pmd0 = NULL;
64 free_page((unsigned long)image->arch.pmd1);
65 image->arch.pmd1 = NULL;
66#endif
67 free_page((unsigned long)image->arch.pte0);
68 image->arch.pte0 = NULL;
69 free_page((unsigned long)image->arch.pte1);
70 image->arch.pte1 = NULL;
71}
72
73static int machine_kexec_alloc_page_tables(struct kimage *image)
74{
75 image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
76 PGD_ALLOCATION_ORDER);
77#ifdef CONFIG_X86_PAE
78 image->arch.pmd0 = (pmd_t *)get_zeroed_page(GFP_KERNEL);
79 image->arch.pmd1 = (pmd_t *)get_zeroed_page(GFP_KERNEL);
80#endif
81 image->arch.pte0 = (pte_t *)get_zeroed_page(GFP_KERNEL);
82 image->arch.pte1 = (pte_t *)get_zeroed_page(GFP_KERNEL);
83 if (!image->arch.pgd ||
84#ifdef CONFIG_X86_PAE
85 !image->arch.pmd0 || !image->arch.pmd1 ||
86#endif
87 !image->arch.pte0 || !image->arch.pte1) {
88 return -ENOMEM;
89 }
90 return 0;
91}
92
93static void machine_kexec_page_table_set_one(
94 pgd_t *pgd, pmd_t *pmd, pte_t *pte,
95 unsigned long vaddr, unsigned long paddr)
96{
97 p4d_t *p4d;
98 pud_t *pud;
99
100 pgd += pgd_index(vaddr);
101#ifdef CONFIG_X86_PAE
102 if (!(pgd_val(*pgd) & _PAGE_PRESENT))
103 set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT));
104#endif
105 p4d = p4d_offset(pgd, vaddr);
106 pud = pud_offset(p4d, vaddr);
107 pmd = pmd_offset(pud, vaddr);
108 if (!(pmd_val(*pmd) & _PAGE_PRESENT))
109 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
110 pte = pte_offset_kernel(pmd, vaddr);
111 set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
112}
113
114static void machine_kexec_prepare_page_tables(struct kimage *image)
115{
116 void *control_page;
117 pmd_t *pmd = NULL;
118
119 control_page = page_address(image->control_code_page);
120#ifdef CONFIG_X86_PAE
121 pmd = image->arch.pmd0;
122#endif
123 machine_kexec_page_table_set_one(
124 image->arch.pgd, pmd, image->arch.pte0,
125 (unsigned long)control_page, __pa(control_page));
126#ifdef CONFIG_X86_PAE
127 pmd = image->arch.pmd1;
128#endif
129 machine_kexec_page_table_set_one(
130 image->arch.pgd, pmd, image->arch.pte1,
131 __pa(control_page), __pa(control_page));
132}
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149int machine_kexec_prepare(struct kimage *image)
150{
151 int error;
152
153 set_pages_x(image->control_code_page, 1);
154 error = machine_kexec_alloc_page_tables(image);
155 if (error)
156 return error;
157 machine_kexec_prepare_page_tables(image);
158 return 0;
159}
160
161
162
163
164
165void machine_kexec_cleanup(struct kimage *image)
166{
167 set_pages_nx(image->control_code_page, 1);
168 machine_kexec_free_page_tables(image);
169}
170
171
172
173
174
175void machine_kexec(struct kimage *image)
176{
177 unsigned long page_list[PAGES_NR];
178 void *control_page;
179 int save_ftrace_enabled;
180 asmlinkage unsigned long
181 (*relocate_kernel_ptr)(unsigned long indirection_page,
182 unsigned long control_page,
183 unsigned long start_address,
184 unsigned int has_pae,
185 unsigned int preserve_context);
186
187#ifdef CONFIG_KEXEC_JUMP
188 if (image->preserve_context)
189 save_processor_state();
190#endif
191
192 save_ftrace_enabled = __ftrace_enabled_save();
193
194
195 local_irq_disable();
196 hw_breakpoint_disable();
197
198 if (image->preserve_context) {
199#ifdef CONFIG_X86_IO_APIC
200
201
202
203
204
205
206 clear_IO_APIC();
207 restore_boot_irq_mode();
208#endif
209 }
210
211 control_page = page_address(image->control_code_page);
212 memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
213
214 relocate_kernel_ptr = control_page;
215 page_list[PA_CONTROL_PAGE] = __pa(control_page);
216 page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
217 page_list[PA_PGD] = __pa(image->arch.pgd);
218
219 if (image->type == KEXEC_TYPE_DEFAULT)
220 page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
221 << PAGE_SHIFT);
222
223
224
225
226
227
228
229
230
231
232
233 load_segments();
234
235
236
237
238 idt_invalidate(phys_to_virt(0));
239 set_gdt(phys_to_virt(0), 0);
240
241
242 image->start = relocate_kernel_ptr((unsigned long)image->head,
243 (unsigned long)page_list,
244 image->start,
245 boot_cpu_has(X86_FEATURE_PAE),
246 image->preserve_context);
247
248#ifdef CONFIG_KEXEC_JUMP
249 if (image->preserve_context)
250 restore_processor_state();
251#endif
252
253 __ftrace_enabled_restore(save_ftrace_enabled);
254}
255
256void arch_crash_save_vmcoreinfo(void)
257{
258#ifdef CONFIG_NUMA
259 VMCOREINFO_SYMBOL(node_data);
260 VMCOREINFO_LENGTH(node_data, MAX_NUMNODES);
261#endif
262#ifdef CONFIG_X86_PAE
263 VMCOREINFO_CONFIG(X86_PAE);
264#endif
265}
266
267