1
2
3
4
5
6
7
8
9
10#include <linux/suspend.h>
11#include <linux/export.h>
12#include <linux/smp.h>
13#include <linux/perf_event.h>
14#include <linux/tboot.h>
15#include <linux/dmi.h>
16#include <linux/pgtable.h>
17
18#include <asm/proto.h>
19#include <asm/mtrr.h>
20#include <asm/page.h>
21#include <asm/mce.h>
22#include <asm/suspend.h>
23#include <asm/fpu/internal.h>
24#include <asm/debugreg.h>
25#include <asm/cpu.h>
26#include <asm/mmu_context.h>
27#include <asm/cpu_device_id.h>
28
29#ifdef CONFIG_X86_32
30__visible unsigned long saved_context_ebx;
31__visible unsigned long saved_context_esp, saved_context_ebp;
32__visible unsigned long saved_context_esi, saved_context_edi;
33__visible unsigned long saved_context_eflags;
34#endif
35struct saved_context saved_context;
36
37static void msr_save_context(struct saved_context *ctxt)
38{
39 struct saved_msr *msr = ctxt->saved_msrs.array;
40 struct saved_msr *end = msr + ctxt->saved_msrs.num;
41
42 while (msr < end) {
43 msr->valid = !rdmsrl_safe(msr->info.msr_no, &msr->info.reg.q);
44 msr++;
45 }
46}
47
48static void msr_restore_context(struct saved_context *ctxt)
49{
50 struct saved_msr *msr = ctxt->saved_msrs.array;
51 struct saved_msr *end = msr + ctxt->saved_msrs.num;
52
53 while (msr < end) {
54 if (msr->valid)
55 wrmsrl(msr->info.msr_no, msr->info.reg.q);
56 msr++;
57 }
58}
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76static void __save_processor_state(struct saved_context *ctxt)
77{
78#ifdef CONFIG_X86_32
79 mtrr_save_fixed_ranges(NULL);
80#endif
81 kernel_fpu_begin();
82
83
84
85
86 store_idt(&ctxt->idt);
87
88
89
90
91
92
93
94 ctxt->gdt_desc.size = GDT_SIZE - 1;
95 ctxt->gdt_desc.address = (unsigned long)get_cpu_gdt_rw(smp_processor_id());
96
97 store_tr(ctxt->tr);
98
99
100
101
102
103 savesegment(gs, ctxt->gs);
104#ifdef CONFIG_X86_64
105 savesegment(fs, ctxt->fs);
106 savesegment(ds, ctxt->ds);
107 savesegment(es, ctxt->es);
108
109 rdmsrl(MSR_FS_BASE, ctxt->fs_base);
110 rdmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base);
111 rdmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base);
112 mtrr_save_fixed_ranges(NULL);
113
114 rdmsrl(MSR_EFER, ctxt->efer);
115#endif
116
117
118
119
120 ctxt->cr0 = read_cr0();
121 ctxt->cr2 = read_cr2();
122 ctxt->cr3 = __read_cr3();
123 ctxt->cr4 = __read_cr4();
124 ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE,
125 &ctxt->misc_enable);
126 msr_save_context(ctxt);
127}
128
129
130void save_processor_state(void)
131{
132 __save_processor_state(&saved_context);
133 x86_platform.save_sched_clock_state();
134}
135#ifdef CONFIG_X86_32
136EXPORT_SYMBOL(save_processor_state);
137#endif
138
139static void do_fpu_end(void)
140{
141
142
143
144 kernel_fpu_end();
145}
146
147static void fix_processor_context(void)
148{
149 int cpu = smp_processor_id();
150#ifdef CONFIG_X86_64
151 struct desc_struct *desc = get_cpu_gdt_rw(cpu);
152 tss_desc tss;
153#endif
154
155
156
157
158
159
160
161
162 set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
163
164#ifdef CONFIG_X86_64
165 memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
166 tss.type = 0x9;
167 write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
168
169 syscall_init();
170#else
171 if (boot_cpu_has(X86_FEATURE_SEP))
172 enable_sep_cpu();
173#endif
174 load_TR_desc();
175 load_mm_ldt(current->active_mm);
176 initialize_tlbstate_and_flush();
177
178 fpu__resume_cpu();
179
180
181 load_fixmap_gdt(cpu);
182}
183
184
185
186
187
188
189
190
191
192static void notrace __restore_processor_state(struct saved_context *ctxt)
193{
194 struct cpuinfo_x86 *c;
195
196 if (ctxt->misc_enable_saved)
197 wrmsrl(MSR_IA32_MISC_ENABLE, ctxt->misc_enable);
198
199
200
201
202#ifdef CONFIG_X86_32
203 if (ctxt->cr4)
204 __write_cr4(ctxt->cr4);
205#else
206
207 wrmsrl(MSR_EFER, ctxt->efer);
208 __write_cr4(ctxt->cr4);
209#endif
210 write_cr3(ctxt->cr3);
211 write_cr2(ctxt->cr2);
212 write_cr0(ctxt->cr0);
213
214
215 load_idt(&ctxt->idt);
216
217
218
219
220
221 loadsegment(ss, __KERNEL_DS);
222 loadsegment(ds, __USER_DS);
223 loadsegment(es, __USER_DS);
224
225
226
227
228
229#ifdef CONFIG_X86_64
230 wrmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base);
231#else
232 loadsegment(fs, __KERNEL_PERCPU);
233#endif
234
235
236 fix_processor_context();
237
238
239
240
241
242#ifdef CONFIG_X86_64
243 loadsegment(ds, ctxt->es);
244 loadsegment(es, ctxt->es);
245 loadsegment(fs, ctxt->fs);
246 load_gs_index(ctxt->gs);
247
248
249
250
251
252
253 wrmsrl(MSR_FS_BASE, ctxt->fs_base);
254 wrmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base);
255#else
256 loadsegment(gs, ctxt->gs);
257#endif
258
259 do_fpu_end();
260 tsc_verify_tsc_adjust(true);
261 x86_platform.restore_sched_clock_state();
262 mtrr_bp_restore();
263 perf_restore_debug_store();
264 msr_restore_context(ctxt);
265
266 c = &cpu_data(smp_processor_id());
267 if (cpu_has(c, X86_FEATURE_MSR_IA32_FEAT_CTL))
268 init_ia32_feat_ctl(c);
269}
270
271
272void notrace restore_processor_state(void)
273{
274 __restore_processor_state(&saved_context);
275}
276#ifdef CONFIG_X86_32
277EXPORT_SYMBOL(restore_processor_state);
278#endif
279
280#if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU)
281static void resume_play_dead(void)
282{
283 play_dead_common();
284 tboot_shutdown(TB_SHUTDOWN_WFS);
285 hlt_play_dead();
286}
287
288int hibernate_resume_nonboot_cpu_disable(void)
289{
290 void (*play_dead)(void) = smp_ops.play_dead;
291 int ret;
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309 ret = cpuhp_smt_enable();
310 if (ret)
311 return ret;
312 smp_ops.play_dead = resume_play_dead;
313 ret = freeze_secondary_cpus(0);
314 smp_ops.play_dead = play_dead;
315 return ret;
316}
317#endif
318
319
320
321
322
323
324static int bsp_check(void)
325{
326 if (cpumask_first(cpu_online_mask) != 0) {
327 pr_warn("CPU0 is offline.\n");
328 return -ENODEV;
329 }
330
331 return 0;
332}
333
334static int bsp_pm_callback(struct notifier_block *nb, unsigned long action,
335 void *ptr)
336{
337 int ret = 0;
338
339 switch (action) {
340 case PM_SUSPEND_PREPARE:
341 case PM_HIBERNATION_PREPARE:
342 ret = bsp_check();
343 break;
344#ifdef CONFIG_DEBUG_HOTPLUG_CPU0
345 case PM_RESTORE_PREPARE:
346
347
348
349
350
351 if (!cpu_online(0))
352 _debug_hotplug_cpu(0, 1);
353 break;
354 case PM_POST_RESTORE:
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378 _debug_hotplug_cpu(0, 0);
379 break;
380#endif
381 default:
382 break;
383 }
384 return notifier_from_errno(ret);
385}
386
387static int __init bsp_pm_check_init(void)
388{
389
390
391
392
393
394 pm_notifier(bsp_pm_callback, -INT_MAX);
395 return 0;
396}
397
398core_initcall(bsp_pm_check_init);
399
400static int msr_build_context(const u32 *msr_id, const int num)
401{
402 struct saved_msrs *saved_msrs = &saved_context.saved_msrs;
403 struct saved_msr *msr_array;
404 int total_num;
405 int i, j;
406
407 total_num = saved_msrs->num + num;
408
409 msr_array = kmalloc_array(total_num, sizeof(struct saved_msr), GFP_KERNEL);
410 if (!msr_array) {
411 pr_err("x86/pm: Can not allocate memory to save/restore MSRs during suspend.\n");
412 return -ENOMEM;
413 }
414
415 if (saved_msrs->array) {
416
417
418
419
420 memcpy(msr_array, saved_msrs->array,
421 sizeof(struct saved_msr) * saved_msrs->num);
422
423 kfree(saved_msrs->array);
424 }
425
426 for (i = saved_msrs->num, j = 0; i < total_num; i++, j++) {
427 msr_array[i].info.msr_no = msr_id[j];
428 msr_array[i].valid = false;
429 msr_array[i].info.reg.q = 0;
430 }
431 saved_msrs->num = total_num;
432 saved_msrs->array = msr_array;
433
434 return 0;
435}
436
437
438
439
440
441
442
443
444
445
446
447static int msr_initialize_bdw(const struct dmi_system_id *d)
448{
449
450 u32 bdw_msr_id[] = { MSR_IA32_THERM_CONTROL };
451
452 pr_info("x86/pm: %s detected, MSR saving is needed during suspending.\n", d->ident);
453 return msr_build_context(bdw_msr_id, ARRAY_SIZE(bdw_msr_id));
454}
455
456static const struct dmi_system_id msr_save_dmi_table[] = {
457 {
458 .callback = msr_initialize_bdw,
459 .ident = "BROADWELL BDX_EP",
460 .matches = {
461 DMI_MATCH(DMI_PRODUCT_NAME, "GRANTLEY"),
462 DMI_MATCH(DMI_PRODUCT_VERSION, "E63448-400"),
463 },
464 },
465 {}
466};
467
468static int msr_save_cpuid_features(const struct x86_cpu_id *c)
469{
470 u32 cpuid_msr_id[] = {
471 MSR_AMD64_CPUID_FN_1,
472 };
473
474 pr_info("x86/pm: family %#hx cpu detected, MSR saving is needed during suspending.\n",
475 c->family);
476
477 return msr_build_context(cpuid_msr_id, ARRAY_SIZE(cpuid_msr_id));
478}
479
480static const struct x86_cpu_id msr_save_cpu_table[] = {
481 X86_MATCH_VENDOR_FAM(AMD, 0x15, &msr_save_cpuid_features),
482 X86_MATCH_VENDOR_FAM(AMD, 0x16, &msr_save_cpuid_features),
483 {}
484};
485
486typedef int (*pm_cpu_match_t)(const struct x86_cpu_id *);
487static int pm_cpu_check(const struct x86_cpu_id *c)
488{
489 const struct x86_cpu_id *m;
490 int ret = 0;
491
492 m = x86_match_cpu(msr_save_cpu_table);
493 if (m) {
494 pm_cpu_match_t fn;
495
496 fn = (pm_cpu_match_t)m->driver_data;
497 ret = fn(m);
498 }
499
500 return ret;
501}
502
503static int pm_check_save_msr(void)
504{
505 dmi_check_system(msr_save_dmi_table);
506 pm_cpu_check(msr_save_cpu_table);
507
508 return 0;
509}
510
511device_initcall(pm_check_save_msr);
512