1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#define pr_fmt(x) "hibernate: " x
17#include <linux/cpu.h>
18#include <linux/kvm_host.h>
19#include <linux/pm.h>
20#include <linux/sched.h>
21#include <linux/suspend.h>
22#include <linux/utsname.h>
23
24#include <asm/barrier.h>
25#include <asm/cacheflush.h>
26#include <asm/cputype.h>
27#include <asm/daifflags.h>
28#include <asm/irqflags.h>
29#include <asm/kexec.h>
30#include <asm/memory.h>
31#include <asm/mmu_context.h>
32#include <asm/mte.h>
33#include <asm/sections.h>
34#include <asm/smp.h>
35#include <asm/smp_plat.h>
36#include <asm/suspend.h>
37#include <asm/sysreg.h>
38#include <asm/trans_pgd.h>
39#include <asm/virt.h>
40
41
42
43
44
45
46
47
48
49extern int in_suspend;
50
51
52#define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode())
53
54
55extern char hibernate_el2_vectors[];
56
57
58extern char __hyp_stub_vectors[];
59
60
61
62
63
64static int sleep_cpu = -EINVAL;
65
66
67
68
69
70
71struct arch_hibernate_hdr_invariants {
72 char uts_version[__NEW_UTS_LEN + 1];
73};
74
75
76static struct arch_hibernate_hdr {
77 struct arch_hibernate_hdr_invariants invariants;
78
79
80 phys_addr_t ttbr1_el1;
81 void (*reenter_kernel)(void);
82
83
84
85
86
87 phys_addr_t __hyp_stub_vectors;
88
89 u64 sleep_cpu_mpidr;
90} resume_hdr;
91
92static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
93{
94 memset(i, 0, sizeof(*i));
95 memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
96}
97
98int pfn_is_nosave(unsigned long pfn)
99{
100 unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
101 unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
102
103 return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) ||
104 crash_is_nosave(pfn);
105}
106
107void notrace save_processor_state(void)
108{
109 WARN_ON(num_online_cpus() != 1);
110}
111
112void notrace restore_processor_state(void)
113{
114}
115
116int arch_hibernation_header_save(void *addr, unsigned int max_size)
117{
118 struct arch_hibernate_hdr *hdr = addr;
119
120 if (max_size < sizeof(*hdr))
121 return -EOVERFLOW;
122
123 arch_hdr_invariants(&hdr->invariants);
124 hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir);
125 hdr->reenter_kernel = _cpu_resume;
126
127
128 if (el2_reset_needed())
129 hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors);
130 else
131 hdr->__hyp_stub_vectors = 0;
132
133
134 if (sleep_cpu < 0) {
135 pr_err("Failing to hibernate on an unknown CPU.\n");
136 return -ENODEV;
137 }
138 hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu);
139 pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu,
140 hdr->sleep_cpu_mpidr);
141
142 return 0;
143}
144EXPORT_SYMBOL(arch_hibernation_header_save);
145
146int arch_hibernation_header_restore(void *addr)
147{
148 int ret;
149 struct arch_hibernate_hdr_invariants invariants;
150 struct arch_hibernate_hdr *hdr = addr;
151
152 arch_hdr_invariants(&invariants);
153 if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) {
154 pr_crit("Hibernate image not generated by this kernel!\n");
155 return -EINVAL;
156 }
157
158 sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr);
159 pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu,
160 hdr->sleep_cpu_mpidr);
161 if (sleep_cpu < 0) {
162 pr_crit("Hibernated on a CPU not known to this kernel!\n");
163 sleep_cpu = -EINVAL;
164 return -EINVAL;
165 }
166
167 ret = bringup_hibernate_cpu(sleep_cpu);
168 if (ret) {
169 sleep_cpu = -EINVAL;
170 return ret;
171 }
172
173 resume_hdr = *hdr;
174
175 return 0;
176}
177EXPORT_SYMBOL(arch_hibernation_header_restore);
178
179static void *hibernate_page_alloc(void *arg)
180{
181 return (void *)get_safe_page((__force gfp_t)(unsigned long)arg);
182}
183
184
185
186
187
188
189
190
191
192
193
194
195
196static int create_safe_exec_page(void *src_start, size_t length,
197 phys_addr_t *phys_dst_addr)
198{
199 struct trans_pgd_info trans_info = {
200 .trans_alloc_page = hibernate_page_alloc,
201 .trans_alloc_arg = (__force void *)GFP_ATOMIC,
202 };
203
204 void *page = (void *)get_safe_page(GFP_ATOMIC);
205 phys_addr_t trans_ttbr0;
206 unsigned long t0sz;
207 int rc;
208
209 if (!page)
210 return -ENOMEM;
211
212 memcpy(page, src_start, length);
213 caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length);
214 rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page);
215 if (rc)
216 return rc;
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232 cpu_set_reserved_ttbr0();
233 local_flush_tlb_all();
234 __cpu_set_tcr_t0sz(t0sz);
235 write_sysreg(trans_ttbr0, ttbr0_el1);
236 isb();
237
238 *phys_dst_addr = virt_to_phys(page);
239
240 return 0;
241}
242
243#ifdef CONFIG_ARM64_MTE
244
245static DEFINE_XARRAY(mte_pages);
246
247static int save_tags(struct page *page, unsigned long pfn)
248{
249 void *tag_storage, *ret;
250
251 tag_storage = mte_allocate_tag_storage();
252 if (!tag_storage)
253 return -ENOMEM;
254
255 mte_save_page_tags(page_address(page), tag_storage);
256
257 ret = xa_store(&mte_pages, pfn, tag_storage, GFP_KERNEL);
258 if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
259 mte_free_tag_storage(tag_storage);
260 return xa_err(ret);
261 } else if (WARN(ret, "swsusp: %s: Duplicate entry", __func__)) {
262 mte_free_tag_storage(ret);
263 }
264
265 return 0;
266}
267
268static void swsusp_mte_free_storage(void)
269{
270 XA_STATE(xa_state, &mte_pages, 0);
271 void *tags;
272
273 xa_lock(&mte_pages);
274 xas_for_each(&xa_state, tags, ULONG_MAX) {
275 mte_free_tag_storage(tags);
276 }
277 xa_unlock(&mte_pages);
278
279 xa_destroy(&mte_pages);
280}
281
282static int swsusp_mte_save_tags(void)
283{
284 struct zone *zone;
285 unsigned long pfn, max_zone_pfn;
286 int ret = 0;
287 int n = 0;
288
289 if (!system_supports_mte())
290 return 0;
291
292 for_each_populated_zone(zone) {
293 max_zone_pfn = zone_end_pfn(zone);
294 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
295 struct page *page = pfn_to_online_page(pfn);
296
297 if (!page)
298 continue;
299
300 if (!test_bit(PG_mte_tagged, &page->flags))
301 continue;
302
303 ret = save_tags(page, pfn);
304 if (ret) {
305 swsusp_mte_free_storage();
306 goto out;
307 }
308
309 n++;
310 }
311 }
312 pr_info("Saved %d MTE pages\n", n);
313
314out:
315 return ret;
316}
317
318static void swsusp_mte_restore_tags(void)
319{
320 XA_STATE(xa_state, &mte_pages, 0);
321 int n = 0;
322 void *tags;
323
324 xa_lock(&mte_pages);
325 xas_for_each(&xa_state, tags, ULONG_MAX) {
326 unsigned long pfn = xa_state.xa_index;
327 struct page *page = pfn_to_online_page(pfn);
328
329
330
331
332
333
334 mte_restore_page_tags(page_address(page), tags);
335
336 mte_free_tag_storage(tags);
337 n++;
338 }
339 xa_unlock(&mte_pages);
340
341 pr_info("Restored %d MTE pages\n", n);
342
343 xa_destroy(&mte_pages);
344}
345
346#else
347
348static int swsusp_mte_save_tags(void)
349{
350 return 0;
351}
352
353static void swsusp_mte_restore_tags(void)
354{
355}
356
357#endif
358
359int swsusp_arch_suspend(void)
360{
361 int ret = 0;
362 unsigned long flags;
363 struct sleep_stack_data state;
364
365 if (cpus_are_stuck_in_kernel()) {
366 pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n");
367 return -EBUSY;
368 }
369
370 flags = local_daif_save();
371
372 if (__cpu_suspend_enter(&state)) {
373
374 crash_prepare_suspend();
375
376 ret = swsusp_mte_save_tags();
377 if (ret)
378 return ret;
379
380 sleep_cpu = smp_processor_id();
381 ret = swsusp_save();
382 } else {
383
384 dcache_clean_inval_poc((unsigned long)__mmuoff_data_start,
385 (unsigned long)__mmuoff_data_end);
386 dcache_clean_inval_poc((unsigned long)__idmap_text_start,
387 (unsigned long)__idmap_text_end);
388
389
390 if (el2_reset_needed()) {
391 dcache_clean_inval_poc(
392 (unsigned long)__hyp_idmap_text_start,
393 (unsigned long)__hyp_idmap_text_end);
394 dcache_clean_inval_poc((unsigned long)__hyp_text_start,
395 (unsigned long)__hyp_text_end);
396 }
397
398 swsusp_mte_restore_tags();
399
400
401 crash_post_resume();
402
403
404
405
406
407 in_suspend = 0;
408
409 sleep_cpu = -EINVAL;
410 __cpu_suspend_exit();
411
412
413
414
415
416
417 spectre_v4_enable_mitigation(NULL);
418 }
419
420 local_daif_restore(flags);
421
422 return ret;
423}
424
425
426
427
428
429
430
431int swsusp_arch_resume(void)
432{
433 int rc;
434 void *zero_page;
435 size_t exit_size;
436 pgd_t *tmp_pg_dir;
437 void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *,
438 void *, phys_addr_t, phys_addr_t);
439 struct trans_pgd_info trans_info = {
440 .trans_alloc_page = hibernate_page_alloc,
441 .trans_alloc_arg = (void *)GFP_ATOMIC,
442 };
443
444
445
446
447
448
449 rc = trans_pgd_create_copy(&trans_info, &tmp_pg_dir, PAGE_OFFSET,
450 PAGE_END);
451 if (rc)
452 return rc;
453
454
455
456
457
458 zero_page = (void *)get_safe_page(GFP_ATOMIC);
459 if (!zero_page) {
460 pr_err("Failed to allocate zero page.\n");
461 return -ENOMEM;
462 }
463
464 exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start;
465
466
467
468
469 rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size,
470 (phys_addr_t *)&hibernate_exit);
471 if (rc) {
472 pr_err("Failed to create safe executable page for hibernate_exit code.\n");
473 return rc;
474 }
475
476
477
478
479
480 dcache_clean_inval_poc((unsigned long)hibernate_exit,
481 (unsigned long)hibernate_exit + exit_size);
482
483
484
485
486
487
488
489 if (el2_reset_needed()) {
490 phys_addr_t el2_vectors = (phys_addr_t)hibernate_exit;
491 el2_vectors += hibernate_el2_vectors -
492 __hibernate_exit_text_start;
493
494 __hyp_set_vectors(el2_vectors);
495 }
496
497 hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1,
498 resume_hdr.reenter_kernel, restore_pblist,
499 resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page));
500
501 return 0;
502}
503
504int hibernate_resume_nonboot_cpu_disable(void)
505{
506 if (sleep_cpu < 0) {
507 pr_err("Failing to resume from hibernate on an unknown CPU.\n");
508 return -ENODEV;
509 }
510
511 return freeze_secondary_cpus(sleep_cpu);
512}
513