1
2
3
4
5
6
7
8#include <linux/types.h>
9#include <linux/mm.h>
10#include <linux/slab.h>
11#include <linux/of.h>
12#include <linux/device.h>
13#include <linux/cpu.h>
14
15#include <asm/asm-prototypes.h>
16#include <asm/firmware.h>
17#include <asm/interrupt.h>
18#include <asm/machdep.h>
19#include <asm/opal.h>
20#include <asm/cputhreads.h>
21#include <asm/cpuidle.h>
22#include <asm/code-patching.h>
23#include <asm/smp.h>
24#include <asm/runlatch.h>
25#include <asm/dbell.h>
26
27#include "powernv.h"
28#include "subcore.h"
29
30
31#define MAX_STOP_STATE 0xF
32
33#define P9_STOP_SPR_MSR 2000
34#define P9_STOP_SPR_PSSCR 855
35
36static u32 supported_cpuidle_states;
37struct pnv_idle_states_t *pnv_idle_states;
38int nr_pnv_idle_states;
39
40
41
42
43
44static u64 pnv_default_stop_val;
45static u64 pnv_default_stop_mask;
46static bool default_stop_found;
47
48
49
50
51static u64 pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
52static u64 deep_spr_loss_state = MAX_STOP_STATE + 1;
53
54
55
56
57
58static u64 pnv_deepest_stop_psscr_val;
59static u64 pnv_deepest_stop_psscr_mask;
60static u64 pnv_deepest_stop_flag;
61static bool deepest_stop_found;
62
63static unsigned long power7_offline_type;
64
65static int pnv_save_sprs_for_deep_states(void)
66{
67 int cpu;
68 int rc;
69
70
71
72
73
74
75 uint64_t lpcr_val = mfspr(SPRN_LPCR);
76 uint64_t hid0_val = mfspr(SPRN_HID0);
77 uint64_t hmeer_val = mfspr(SPRN_HMEER);
78 uint64_t msr_val = MSR_IDLE;
79 uint64_t psscr_val = pnv_deepest_stop_psscr_val;
80
81 for_each_present_cpu(cpu) {
82 uint64_t pir = get_hard_smp_processor_id(cpu);
83 uint64_t hsprg0_val = (uint64_t)paca_ptrs[cpu];
84
85 rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val);
86 if (rc != 0)
87 return rc;
88
89 rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
90 if (rc != 0)
91 return rc;
92
93 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
94 rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val);
95 if (rc)
96 return rc;
97
98 rc = opal_slw_set_reg(pir,
99 P9_STOP_SPR_PSSCR, psscr_val);
100
101 if (rc)
102 return rc;
103 }
104
105
106 if (cpu_thread_in_core(cpu) == 0) {
107
108 rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val);
109 if (rc != 0)
110 return rc;
111
112 rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val);
113 if (rc != 0)
114 return rc;
115
116
117 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
118 uint64_t hid1_val = mfspr(SPRN_HID1);
119 uint64_t hid4_val = mfspr(SPRN_HID4);
120 uint64_t hid5_val = mfspr(SPRN_HID5);
121
122 rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val);
123 if (rc != 0)
124 return rc;
125
126 rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val);
127 if (rc != 0)
128 return rc;
129
130 rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val);
131 if (rc != 0)
132 return rc;
133 }
134 }
135 }
136
137 return 0;
138}
139
140u32 pnv_get_supported_cpuidle_states(void)
141{
142 return supported_cpuidle_states;
143}
144EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states);
145
146static void pnv_fastsleep_workaround_apply(void *info)
147
148{
149 int rc;
150 int *err = info;
151
152 rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP,
153 OPAL_CONFIG_IDLE_APPLY);
154 if (rc)
155 *err = 1;
156}
157
158static bool power7_fastsleep_workaround_entry = true;
159static bool power7_fastsleep_workaround_exit = true;
160
161
162
163
164
165
166static u8 fastsleep_workaround_applyonce;
167
168static ssize_t show_fastsleep_workaround_applyonce(struct device *dev,
169 struct device_attribute *attr, char *buf)
170{
171 return sprintf(buf, "%u\n", fastsleep_workaround_applyonce);
172}
173
174static ssize_t store_fastsleep_workaround_applyonce(struct device *dev,
175 struct device_attribute *attr, const char *buf,
176 size_t count)
177{
178 cpumask_t primary_thread_mask;
179 int err;
180 u8 val;
181
182 if (kstrtou8(buf, 0, &val) || val != 1)
183 return -EINVAL;
184
185 if (fastsleep_workaround_applyonce == 1)
186 return count;
187
188
189
190
191
192
193
194
195
196
197
198
199
200 power7_fastsleep_workaround_exit = false;
201
202 cpus_read_lock();
203 primary_thread_mask = cpu_online_cores_map();
204 on_each_cpu_mask(&primary_thread_mask,
205 pnv_fastsleep_workaround_apply,
206 &err, 1);
207 cpus_read_unlock();
208 if (err) {
209 pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply");
210 goto fail;
211 }
212
213 power7_fastsleep_workaround_entry = false;
214
215 fastsleep_workaround_applyonce = 1;
216
217 return count;
218fail:
219 return -EIO;
220}
221
222static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600,
223 show_fastsleep_workaround_applyonce,
224 store_fastsleep_workaround_applyonce);
225
226static inline void atomic_start_thread_idle(void)
227{
228 int cpu = raw_smp_processor_id();
229 int first = cpu_first_thread_sibling(cpu);
230 int thread_nr = cpu_thread_in_core(cpu);
231 unsigned long *state = &paca_ptrs[first]->idle_state;
232
233 clear_bit(thread_nr, state);
234}
235
236static inline void atomic_stop_thread_idle(void)
237{
238 int cpu = raw_smp_processor_id();
239 int first = cpu_first_thread_sibling(cpu);
240 int thread_nr = cpu_thread_in_core(cpu);
241 unsigned long *state = &paca_ptrs[first]->idle_state;
242
243 set_bit(thread_nr, state);
244}
245
246static inline void atomic_lock_thread_idle(void)
247{
248 int cpu = raw_smp_processor_id();
249 int first = cpu_first_thread_sibling(cpu);
250 unsigned long *state = &paca_ptrs[first]->idle_state;
251
252 while (unlikely(test_and_set_bit_lock(NR_PNV_CORE_IDLE_LOCK_BIT, state)))
253 barrier();
254}
255
256static inline void atomic_unlock_and_stop_thread_idle(void)
257{
258 int cpu = raw_smp_processor_id();
259 int first = cpu_first_thread_sibling(cpu);
260 unsigned long thread = 1UL << cpu_thread_in_core(cpu);
261 unsigned long *state = &paca_ptrs[first]->idle_state;
262 u64 s = READ_ONCE(*state);
263 u64 new, tmp;
264
265 BUG_ON(!(s & PNV_CORE_IDLE_LOCK_BIT));
266 BUG_ON(s & thread);
267
268again:
269 new = (s | thread) & ~PNV_CORE_IDLE_LOCK_BIT;
270 tmp = cmpxchg(state, s, new);
271 if (unlikely(tmp != s)) {
272 s = tmp;
273 goto again;
274 }
275}
276
277static inline void atomic_unlock_thread_idle(void)
278{
279 int cpu = raw_smp_processor_id();
280 int first = cpu_first_thread_sibling(cpu);
281 unsigned long *state = &paca_ptrs[first]->idle_state;
282
283 BUG_ON(!test_bit(NR_PNV_CORE_IDLE_LOCK_BIT, state));
284 clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state);
285}
286
287
288struct p7_sprs {
289
290 u64 tscr;
291 u64 worc;
292
293
294 u64 sdr1;
295 u64 rpr;
296
297
298 u64 lpcr;
299 u64 hfscr;
300 u64 fscr;
301 u64 purr;
302 u64 spurr;
303 u64 dscr;
304 u64 wort;
305
306
307 u64 amr;
308 u64 iamr;
309 u64 amor;
310 u64 uamor;
311};
312
313static unsigned long power7_idle_insn(unsigned long type)
314{
315 int cpu = raw_smp_processor_id();
316 int first = cpu_first_thread_sibling(cpu);
317 unsigned long *state = &paca_ptrs[first]->idle_state;
318 unsigned long thread = 1UL << cpu_thread_in_core(cpu);
319 unsigned long core_thread_mask = (1UL << threads_per_core) - 1;
320 unsigned long srr1;
321 bool full_winkle;
322 struct p7_sprs sprs = {};
323 bool sprs_saved = false;
324 int rc;
325
326 if (unlikely(type != PNV_THREAD_NAP)) {
327 atomic_lock_thread_idle();
328
329 BUG_ON(!(*state & thread));
330 *state &= ~thread;
331
332 if (power7_fastsleep_workaround_entry) {
333 if ((*state & core_thread_mask) == 0) {
334 rc = opal_config_cpu_idle_state(
335 OPAL_CONFIG_IDLE_FASTSLEEP,
336 OPAL_CONFIG_IDLE_APPLY);
337 BUG_ON(rc);
338 }
339 }
340
341 if (type == PNV_THREAD_WINKLE) {
342 sprs.tscr = mfspr(SPRN_TSCR);
343 sprs.worc = mfspr(SPRN_WORC);
344
345 sprs.sdr1 = mfspr(SPRN_SDR1);
346 sprs.rpr = mfspr(SPRN_RPR);
347
348 sprs.lpcr = mfspr(SPRN_LPCR);
349 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
350 sprs.hfscr = mfspr(SPRN_HFSCR);
351 sprs.fscr = mfspr(SPRN_FSCR);
352 }
353 sprs.purr = mfspr(SPRN_PURR);
354 sprs.spurr = mfspr(SPRN_SPURR);
355 sprs.dscr = mfspr(SPRN_DSCR);
356 sprs.wort = mfspr(SPRN_WORT);
357
358 sprs_saved = true;
359
360
361
362
363
364
365
366
367 *state += 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT;
368 if ((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS)
369 >> PNV_CORE_IDLE_WINKLE_COUNT_SHIFT
370 == threads_per_core)
371 *state |= PNV_CORE_IDLE_THREAD_WINKLE_BITS;
372 WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0);
373 }
374
375 atomic_unlock_thread_idle();
376 }
377
378 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
379 sprs.amr = mfspr(SPRN_AMR);
380 sprs.iamr = mfspr(SPRN_IAMR);
381 sprs.amor = mfspr(SPRN_AMOR);
382 sprs.uamor = mfspr(SPRN_UAMOR);
383 }
384
385 local_paca->thread_idle_state = type;
386 srr1 = isa206_idle_insn_mayloss(type);
387 local_paca->thread_idle_state = PNV_THREAD_RUNNING;
388
389 WARN_ON_ONCE(!srr1);
390 WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
391
392 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
393 if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) {
394
395
396
397
398 mtspr(SPRN_AMR, sprs.amr);
399 mtspr(SPRN_IAMR, sprs.iamr);
400 mtspr(SPRN_AMOR, sprs.amor);
401 mtspr(SPRN_UAMOR, sprs.uamor);
402 }
403 }
404
405 if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI))
406 hmi_exception_realmode(NULL);
407
408 if (likely((srr1 & SRR1_WAKESTATE) != SRR1_WS_HVLOSS)) {
409 if (unlikely(type != PNV_THREAD_NAP)) {
410 atomic_lock_thread_idle();
411 if (type == PNV_THREAD_WINKLE) {
412 WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0);
413 *state -= 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT;
414 *state &= ~(thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT);
415 }
416 atomic_unlock_and_stop_thread_idle();
417 }
418 return srr1;
419 }
420
421
422 BUG_ON(type == PNV_THREAD_NAP);
423
424 atomic_lock_thread_idle();
425
426 full_winkle = false;
427 if (type == PNV_THREAD_WINKLE) {
428 WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0);
429 *state -= 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT;
430 if (*state & (thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT)) {
431 *state &= ~(thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT);
432 full_winkle = true;
433 BUG_ON(!sprs_saved);
434 }
435 }
436
437 WARN_ON(*state & thread);
438
439 if ((*state & core_thread_mask) != 0)
440 goto core_woken;
441
442
443 if (full_winkle) {
444 mtspr(SPRN_TSCR, sprs.tscr);
445 mtspr(SPRN_WORC, sprs.worc);
446 }
447
448 if (power7_fastsleep_workaround_exit) {
449 rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP,
450 OPAL_CONFIG_IDLE_UNDO);
451 BUG_ON(rc);
452 }
453
454
455 if (opal_resync_timebase() != OPAL_SUCCESS)
456 BUG();
457
458core_woken:
459 if (!full_winkle)
460 goto subcore_woken;
461
462 if ((*state & local_paca->subcore_sibling_mask) != 0)
463 goto subcore_woken;
464
465
466 mtspr(SPRN_SDR1, sprs.sdr1);
467 mtspr(SPRN_RPR, sprs.rpr);
468
469subcore_woken:
470
471
472
473
474
475 isync();
476 atomic_unlock_and_stop_thread_idle();
477
478
479 if (!full_winkle)
480 return srr1;
481
482
483 mtspr(SPRN_LPCR, sprs.lpcr);
484 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
485 mtspr(SPRN_HFSCR, sprs.hfscr);
486 mtspr(SPRN_FSCR, sprs.fscr);
487 }
488 mtspr(SPRN_PURR, sprs.purr);
489 mtspr(SPRN_SPURR, sprs.spurr);
490 mtspr(SPRN_DSCR, sprs.dscr);
491 mtspr(SPRN_WORT, sprs.wort);
492
493 mtspr(SPRN_SPRG3, local_paca->sprg_vdso);
494
495
496
497
498
499
500 __slb_restore_bolted_realmode();
501
502 return srr1;
503}
504
505extern unsigned long idle_kvm_start_guest(unsigned long srr1);
506
507#ifdef CONFIG_HOTPLUG_CPU
508static unsigned long power7_offline(void)
509{
510 unsigned long srr1;
511
512 mtmsr(MSR_IDLE);
513
514#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531 local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE;
532#endif
533
534 __ppc64_runlatch_off();
535 srr1 = power7_idle_insn(power7_offline_type);
536 __ppc64_runlatch_on();
537
538#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
539 local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL;
540
541 smp_mb();
542 if (local_paca->kvm_hstate.hwthread_req)
543 srr1 = idle_kvm_start_guest(srr1);
544#endif
545
546 mtmsr(MSR_KERNEL);
547
548 return srr1;
549}
550#endif
551
552void power7_idle_type(unsigned long type)
553{
554 unsigned long srr1;
555
556 if (!prep_irq_for_idle_irqsoff())
557 return;
558
559 mtmsr(MSR_IDLE);
560 __ppc64_runlatch_off();
561 srr1 = power7_idle_insn(type);
562 __ppc64_runlatch_on();
563 mtmsr(MSR_KERNEL);
564
565 fini_irq_for_idle_irqsoff();
566 irq_set_pending_from_srr1(srr1);
567}
568
569static void power7_idle(void)
570{
571 if (!powersave_nap)
572 return;
573
574 power7_idle_type(PNV_THREAD_NAP);
575}
576
577struct p9_sprs {
578
579 u64 ptcr;
580 u64 rpr;
581 u64 tscr;
582 u64 ldbar;
583
584
585 u64 lpcr;
586 u64 hfscr;
587 u64 fscr;
588 u64 pid;
589 u64 purr;
590 u64 spurr;
591 u64 dscr;
592 u64 wort;
593 u64 ciabr;
594
595 u64 mmcra;
596 u32 mmcr0;
597 u32 mmcr1;
598 u64 mmcr2;
599
600
601 u64 amr;
602 u64 iamr;
603 u64 amor;
604 u64 uamor;
605};
606
607static unsigned long power9_idle_stop(unsigned long psscr)
608{
609 int cpu = raw_smp_processor_id();
610 int first = cpu_first_thread_sibling(cpu);
611 unsigned long *state = &paca_ptrs[first]->idle_state;
612 unsigned long core_thread_mask = (1UL << threads_per_core) - 1;
613 unsigned long srr1;
614 unsigned long pls;
615 unsigned long mmcr0 = 0;
616 unsigned long mmcra = 0;
617 struct p9_sprs sprs = {};
618 bool sprs_saved = false;
619
620 if (!(psscr & (PSSCR_EC|PSSCR_ESL))) {
621
622
623
624
625
626
627 srr1 = isa300_idle_stop_noloss(psscr);
628 if (likely(!srr1))
629 return 0;
630
631
632
633
634
635 BUG_ON((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS);
636
637 goto out;
638 }
639
640
641#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
642 if (cpu_has_feature(CPU_FTR_P9_TM_XER_SO_BUG)) {
643 local_paca->requested_psscr = psscr;
644
645 smp_mb();
646 if (atomic_read(&local_paca->dont_stop)) {
647 local_paca->requested_psscr = 0;
648 return 0;
649 }
650 }
651#endif
652
653 if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
654
655
656
657
658
659 mmcr0 = mfspr(SPRN_MMCR0);
660 }
661
662 if ((psscr & PSSCR_RL_MASK) >= deep_spr_loss_state) {
663 sprs.lpcr = mfspr(SPRN_LPCR);
664 sprs.hfscr = mfspr(SPRN_HFSCR);
665 sprs.fscr = mfspr(SPRN_FSCR);
666 sprs.pid = mfspr(SPRN_PID);
667 sprs.purr = mfspr(SPRN_PURR);
668 sprs.spurr = mfspr(SPRN_SPURR);
669 sprs.dscr = mfspr(SPRN_DSCR);
670 sprs.ciabr = mfspr(SPRN_CIABR);
671
672 sprs.mmcra = mfspr(SPRN_MMCRA);
673 sprs.mmcr0 = mfspr(SPRN_MMCR0);
674 sprs.mmcr1 = mfspr(SPRN_MMCR1);
675 sprs.mmcr2 = mfspr(SPRN_MMCR2);
676
677 sprs.ptcr = mfspr(SPRN_PTCR);
678 sprs.rpr = mfspr(SPRN_RPR);
679 sprs.tscr = mfspr(SPRN_TSCR);
680 if (!firmware_has_feature(FW_FEATURE_ULTRAVISOR))
681 sprs.ldbar = mfspr(SPRN_LDBAR);
682
683 sprs_saved = true;
684
685 atomic_start_thread_idle();
686 }
687
688 sprs.amr = mfspr(SPRN_AMR);
689 sprs.iamr = mfspr(SPRN_IAMR);
690 sprs.amor = mfspr(SPRN_AMOR);
691 sprs.uamor = mfspr(SPRN_UAMOR);
692
693 srr1 = isa300_idle_stop_mayloss(psscr);
694
695#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
696 local_paca->requested_psscr = 0;
697#endif
698
699 psscr = mfspr(SPRN_PSSCR);
700
701 WARN_ON_ONCE(!srr1);
702 WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
703
704 if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) {
705
706
707
708
709 mtspr(SPRN_AMR, sprs.amr);
710 mtspr(SPRN_IAMR, sprs.iamr);
711 mtspr(SPRN_AMOR, sprs.amor);
712 mtspr(SPRN_UAMOR, sprs.uamor);
713
714
715
716
717
718
719 if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
720 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT);
721 mtspr(SPRN_MMCR0, mmcr0);
722 }
723
724
725
726
727
728 mmcra = mfspr(SPRN_MMCRA);
729 mmcra |= PPC_BIT(60);
730 mtspr(SPRN_MMCRA, mmcra);
731 mmcra &= ~PPC_BIT(60);
732 mtspr(SPRN_MMCRA, mmcra);
733 }
734
735 if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI))
736 hmi_exception_realmode(NULL);
737
738
739
740
741
742
743 pls = (psscr & PSSCR_PLS) >> PSSCR_PLS_SHIFT;
744 if (likely(pls < deep_spr_loss_state)) {
745 if (sprs_saved)
746 atomic_stop_thread_idle();
747 goto out;
748 }
749
750
751 BUG_ON(!sprs_saved);
752
753 atomic_lock_thread_idle();
754
755 if ((*state & core_thread_mask) != 0)
756 goto core_woken;
757
758
759 mtspr(SPRN_PTCR, sprs.ptcr);
760 mtspr(SPRN_RPR, sprs.rpr);
761 mtspr(SPRN_TSCR, sprs.tscr);
762
763 if (pls >= pnv_first_tb_loss_level) {
764
765 if (opal_resync_timebase() != OPAL_SUCCESS)
766 BUG();
767 }
768
769
770
771
772
773
774 isync();
775
776core_woken:
777 atomic_unlock_and_stop_thread_idle();
778
779
780 mtspr(SPRN_LPCR, sprs.lpcr);
781 mtspr(SPRN_HFSCR, sprs.hfscr);
782 mtspr(SPRN_FSCR, sprs.fscr);
783 mtspr(SPRN_PID, sprs.pid);
784 mtspr(SPRN_PURR, sprs.purr);
785 mtspr(SPRN_SPURR, sprs.spurr);
786 mtspr(SPRN_DSCR, sprs.dscr);
787 mtspr(SPRN_CIABR, sprs.ciabr);
788
789 mtspr(SPRN_MMCRA, sprs.mmcra);
790 mtspr(SPRN_MMCR0, sprs.mmcr0);
791 mtspr(SPRN_MMCR1, sprs.mmcr1);
792 mtspr(SPRN_MMCR2, sprs.mmcr2);
793 if (!firmware_has_feature(FW_FEATURE_ULTRAVISOR))
794 mtspr(SPRN_LDBAR, sprs.ldbar);
795
796 mtspr(SPRN_SPRG3, local_paca->sprg_vdso);
797
798 if (!radix_enabled())
799 __slb_restore_bolted_realmode();
800
801out:
802 mtmsr(MSR_KERNEL);
803
804 return srr1;
805}
806
807#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
808
809
810
811
812
813
814
815
816
817void pnv_power9_force_smt4_catch(void)
818{
819 int cpu, cpu0, thr;
820 int awake_threads = 1;
821 int poke_threads = 0;
822 int need_awake = threads_per_core;
823
824 cpu = smp_processor_id();
825 cpu0 = cpu & ~(threads_per_core - 1);
826 for (thr = 0; thr < threads_per_core; ++thr) {
827 if (cpu != cpu0 + thr)
828 atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop);
829 }
830
831 smp_mb();
832 for (thr = 0; thr < threads_per_core; ++thr) {
833 if (!paca_ptrs[cpu0+thr]->requested_psscr)
834 ++awake_threads;
835 else
836 poke_threads |= (1 << thr);
837 }
838
839
840 if (awake_threads < need_awake) {
841
842 for (thr = 0; thr < threads_per_core; ++thr) {
843 if (poke_threads & (1 << thr)) {
844 ppc_msgsnd_sync();
845 ppc_msgsnd(PPC_DBELL_MSGTYPE, 0,
846 paca_ptrs[cpu0+thr]->hw_cpu_id);
847 }
848 }
849
850 do {
851 for (thr = 0; thr < threads_per_core; ++thr) {
852 if ((poke_threads & (1 << thr)) &&
853 !paca_ptrs[cpu0+thr]->requested_psscr) {
854 ++awake_threads;
855 poke_threads &= ~(1 << thr);
856 }
857 }
858 } while (awake_threads < need_awake);
859 }
860}
861EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_catch);
862
863void pnv_power9_force_smt4_release(void)
864{
865 int cpu, cpu0, thr;
866
867 cpu = smp_processor_id();
868 cpu0 = cpu & ~(threads_per_core - 1);
869
870
871 for (thr = 0; thr < threads_per_core; ++thr) {
872 if (cpu != cpu0 + thr)
873 atomic_dec(&paca_ptrs[cpu0+thr]->dont_stop);
874 }
875}
876EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_release);
877#endif
878
879struct p10_sprs {
880
881
882
883
884
885
886
887
888
889
890
891};
892
893static unsigned long power10_idle_stop(unsigned long psscr)
894{
895 int cpu = raw_smp_processor_id();
896 int first = cpu_first_thread_sibling(cpu);
897 unsigned long *state = &paca_ptrs[first]->idle_state;
898 unsigned long core_thread_mask = (1UL << threads_per_core) - 1;
899 unsigned long srr1;
900 unsigned long pls;
901
902 bool sprs_saved = false;
903
904 if (!(psscr & (PSSCR_EC|PSSCR_ESL))) {
905
906
907
908
909
910
911 srr1 = isa300_idle_stop_noloss(psscr);
912 if (likely(!srr1))
913 return 0;
914
915
916
917
918
919 BUG_ON((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS);
920
921 goto out;
922 }
923
924
925 if ((psscr & PSSCR_RL_MASK) >= deep_spr_loss_state) {
926
927
928 sprs_saved = true;
929
930 atomic_start_thread_idle();
931 }
932
933 srr1 = isa300_idle_stop_mayloss(psscr);
934
935 psscr = mfspr(SPRN_PSSCR);
936
937 WARN_ON_ONCE(!srr1);
938 WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
939
940 if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI))
941 hmi_exception_realmode(NULL);
942
943
944
945
946
947
948 pls = (psscr & PSSCR_PLS) >> PSSCR_PLS_SHIFT;
949 if (likely(pls < deep_spr_loss_state)) {
950 if (sprs_saved)
951 atomic_stop_thread_idle();
952 goto out;
953 }
954
955
956 BUG_ON(!sprs_saved);
957
958 atomic_lock_thread_idle();
959
960 if ((*state & core_thread_mask) != 0)
961 goto core_woken;
962
963
964
965 if (pls >= pnv_first_tb_loss_level) {
966
967 if (opal_resync_timebase() != OPAL_SUCCESS)
968 BUG();
969 }
970
971
972
973
974
975
976 isync();
977
978core_woken:
979 atomic_unlock_and_stop_thread_idle();
980
981
982
983 if (!radix_enabled())
984 __slb_restore_bolted_realmode();
985
986out:
987 mtmsr(MSR_KERNEL);
988
989 return srr1;
990}
991
992#ifdef CONFIG_HOTPLUG_CPU
993static unsigned long arch300_offline_stop(unsigned long psscr)
994{
995 unsigned long srr1;
996
997 if (cpu_has_feature(CPU_FTR_ARCH_31))
998 srr1 = power10_idle_stop(psscr);
999 else
1000 srr1 = power9_idle_stop(psscr);
1001
1002 return srr1;
1003}
1004#endif
1005
1006void arch300_idle_type(unsigned long stop_psscr_val,
1007 unsigned long stop_psscr_mask)
1008{
1009 unsigned long psscr;
1010 unsigned long srr1;
1011
1012 if (!prep_irq_for_idle_irqsoff())
1013 return;
1014
1015 psscr = mfspr(SPRN_PSSCR);
1016 psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val;
1017
1018 __ppc64_runlatch_off();
1019 if (cpu_has_feature(CPU_FTR_ARCH_31))
1020 srr1 = power10_idle_stop(psscr);
1021 else
1022 srr1 = power9_idle_stop(psscr);
1023 __ppc64_runlatch_on();
1024
1025 fini_irq_for_idle_irqsoff();
1026
1027 irq_set_pending_from_srr1(srr1);
1028}
1029
1030
1031
1032
1033static void arch300_idle(void)
1034{
1035 arch300_idle_type(pnv_default_stop_val, pnv_default_stop_mask);
1036}
1037
1038#ifdef CONFIG_HOTPLUG_CPU
1039
1040void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val)
1041{
1042 u64 pir = get_hard_smp_processor_id(cpu);
1043
1044 mtspr(SPRN_LPCR, lpcr_val);
1045
1046
1047
1048
1049
1050 if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT)
1051 opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
1052}
1053
1054
1055
1056
1057
1058
1059unsigned long pnv_cpu_offline(unsigned int cpu)
1060{
1061 unsigned long srr1;
1062
1063 __ppc64_runlatch_off();
1064
1065 if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) {
1066 unsigned long psscr;
1067
1068 psscr = mfspr(SPRN_PSSCR);
1069 psscr = (psscr & ~pnv_deepest_stop_psscr_mask) |
1070 pnv_deepest_stop_psscr_val;
1071 srr1 = arch300_offline_stop(psscr);
1072 } else if (cpu_has_feature(CPU_FTR_ARCH_206) && power7_offline_type) {
1073 srr1 = power7_offline();
1074 } else {
1075
1076 while (!generic_check_cpu_restart(cpu)) {
1077 HMT_low();
1078 HMT_very_low();
1079 }
1080 srr1 = 0;
1081 HMT_medium();
1082 }
1083
1084 __ppc64_runlatch_on();
1085
1086 return srr1;
1087}
1088#endif
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags)
1128{
1129 int err = 0;
1130
1131
1132
1133
1134
1135
1136 if (*psscr_mask == 0xf) {
1137 *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL;
1138 *psscr_mask = PSSCR_HV_DEFAULT_MASK;
1139 return err;
1140 }
1141
1142
1143
1144
1145
1146
1147
1148
1149 if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) {
1150 err = ERR_EC_ESL_MISMATCH;
1151 } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
1152 GET_PSSCR_ESL(*psscr_val) == 0) {
1153 err = ERR_DEEP_STATE_ESL_MISMATCH;
1154 }
1155
1156 return err;
1157}
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169static void __init pnv_arch300_idle_init(void)
1170{
1171 u64 max_residency_ns = 0;
1172 int i;
1173
1174
1175 if (!pvr_version_is(PVR_POWER10) && !pvr_version_is(PVR_POWER9))
1176 return;
1177
1178
1179
1180
1181
1182
1183
1184
1185 pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
1186 deep_spr_loss_state = MAX_STOP_STATE + 1;
1187 for (i = 0; i < nr_pnv_idle_states; i++) {
1188 int err;
1189 struct pnv_idle_states_t *state = &pnv_idle_states[i];
1190 u64 psscr_rl = state->psscr_val & PSSCR_RL_MASK;
1191
1192
1193 if (pvr_version_is(PVR_POWER10) &&
1194 state->flags & (OPAL_PM_TIMEBASE_STOP|OPAL_PM_LOSE_FULL_CONTEXT))
1195 continue;
1196
1197 if ((state->flags & OPAL_PM_TIMEBASE_STOP) &&
1198 (pnv_first_tb_loss_level > psscr_rl))
1199 pnv_first_tb_loss_level = psscr_rl;
1200
1201 if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
1202 (deep_spr_loss_state > psscr_rl))
1203 deep_spr_loss_state = psscr_rl;
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213 if ((state->flags & OPAL_PM_TIMEBASE_STOP) &&
1214 (deep_spr_loss_state > psscr_rl))
1215 deep_spr_loss_state = psscr_rl;
1216
1217 err = validate_psscr_val_mask(&state->psscr_val,
1218 &state->psscr_mask,
1219 state->flags);
1220 if (err) {
1221 report_invalid_psscr_val(state->psscr_val, err);
1222 continue;
1223 }
1224
1225 state->valid = true;
1226
1227 if (max_residency_ns < state->residency_ns) {
1228 max_residency_ns = state->residency_ns;
1229 pnv_deepest_stop_psscr_val = state->psscr_val;
1230 pnv_deepest_stop_psscr_mask = state->psscr_mask;
1231 pnv_deepest_stop_flag = state->flags;
1232 deepest_stop_found = true;
1233 }
1234
1235 if (!default_stop_found &&
1236 (state->flags & OPAL_PM_STOP_INST_FAST)) {
1237 pnv_default_stop_val = state->psscr_val;
1238 pnv_default_stop_mask = state->psscr_mask;
1239 default_stop_found = true;
1240 WARN_ON(state->flags & OPAL_PM_LOSE_FULL_CONTEXT);
1241 }
1242 }
1243
1244 if (unlikely(!default_stop_found)) {
1245 pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n");
1246 } else {
1247 ppc_md.power_save = arch300_idle;
1248 pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n",
1249 pnv_default_stop_val, pnv_default_stop_mask);
1250 }
1251
1252 if (unlikely(!deepest_stop_found)) {
1253 pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait");
1254 } else {
1255 pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n",
1256 pnv_deepest_stop_psscr_val,
1257 pnv_deepest_stop_psscr_mask);
1258 }
1259
1260 pr_info("cpuidle-powernv: First stop level that may lose SPRs = 0x%llx\n",
1261 deep_spr_loss_state);
1262
1263 pr_info("cpuidle-powernv: First stop level that may lose timebase = 0x%llx\n",
1264 pnv_first_tb_loss_level);
1265}
1266
1267static void __init pnv_disable_deep_states(void)
1268{
1269
1270
1271
1272
1273
1274 supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
1275 pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
1276 pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
1277
1278 if (cpu_has_feature(CPU_FTR_ARCH_300) &&
1279 (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
1280
1281
1282
1283
1284 if (default_stop_found) {
1285 pnv_deepest_stop_psscr_val = pnv_default_stop_val;
1286 pnv_deepest_stop_psscr_mask = pnv_default_stop_mask;
1287 pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
1288 pnv_deepest_stop_psscr_val);
1289 } else {
1290 deepest_stop_found = false;
1291 pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
1292 }
1293 }
1294}
1295
1296
1297
1298
1299static void __init pnv_probe_idle_states(void)
1300{
1301 int i;
1302
1303 if (nr_pnv_idle_states < 0) {
1304 pr_warn("cpuidle-powernv: no idle states found in the DT\n");
1305 return;
1306 }
1307
1308 if (cpu_has_feature(CPU_FTR_ARCH_300))
1309 pnv_arch300_idle_init();
1310
1311 for (i = 0; i < nr_pnv_idle_states; i++)
1312 supported_cpuidle_states |= pnv_idle_states[i].flags;
1313}
1314
1315
1316
1317
1318
1319
1320
1321static int pnv_parse_cpuidle_dt(void)
1322{
1323 struct device_node *np;
1324 int nr_idle_states, i;
1325 int rc = 0;
1326 u32 *temp_u32;
1327 u64 *temp_u64;
1328 const char **temp_string;
1329
1330 np = of_find_node_by_path("/ibm,opal/power-mgt");
1331 if (!np) {
1332 pr_warn("opal: PowerMgmt Node not found\n");
1333 return -ENODEV;
1334 }
1335 nr_idle_states = of_property_count_u32_elems(np,
1336 "ibm,cpu-idle-state-flags");
1337
1338 pnv_idle_states = kcalloc(nr_idle_states, sizeof(*pnv_idle_states),
1339 GFP_KERNEL);
1340 temp_u32 = kcalloc(nr_idle_states, sizeof(u32), GFP_KERNEL);
1341 temp_u64 = kcalloc(nr_idle_states, sizeof(u64), GFP_KERNEL);
1342 temp_string = kcalloc(nr_idle_states, sizeof(char *), GFP_KERNEL);
1343
1344 if (!(pnv_idle_states && temp_u32 && temp_u64 && temp_string)) {
1345 pr_err("Could not allocate memory for dt parsing\n");
1346 rc = -ENOMEM;
1347 goto out;
1348 }
1349
1350
1351 if (of_property_read_u32_array(np, "ibm,cpu-idle-state-flags",
1352 temp_u32, nr_idle_states)) {
1353 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n");
1354 rc = -EINVAL;
1355 goto out;
1356 }
1357 for (i = 0; i < nr_idle_states; i++)
1358 pnv_idle_states[i].flags = temp_u32[i];
1359
1360
1361 if (of_property_read_u32_array(np, "ibm,cpu-idle-state-latencies-ns",
1362 temp_u32, nr_idle_states)) {
1363 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
1364 rc = -EINVAL;
1365 goto out;
1366 }
1367 for (i = 0; i < nr_idle_states; i++)
1368 pnv_idle_states[i].latency_ns = temp_u32[i];
1369
1370
1371 if (of_property_read_u32_array(np, "ibm,cpu-idle-state-residency-ns",
1372 temp_u32, nr_idle_states)) {
1373 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n");
1374 rc = -EINVAL;
1375 goto out;
1376 }
1377 for (i = 0; i < nr_idle_states; i++)
1378 pnv_idle_states[i].residency_ns = temp_u32[i];
1379
1380
1381 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1382
1383 if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr",
1384 temp_u64, nr_idle_states)) {
1385 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n");
1386 rc = -EINVAL;
1387 goto out;
1388 }
1389 for (i = 0; i < nr_idle_states; i++)
1390 pnv_idle_states[i].psscr_val = temp_u64[i];
1391
1392
1393 if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr-mask",
1394 temp_u64, nr_idle_states)) {
1395 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n");
1396 rc = -EINVAL;
1397 goto out;
1398 }
1399 for (i = 0; i < nr_idle_states; i++)
1400 pnv_idle_states[i].psscr_mask = temp_u64[i];
1401 }
1402
1403
1404
1405
1406
1407
1408
1409 if (of_property_read_string_array(np, "ibm,cpu-idle-state-names",
1410 temp_string, nr_idle_states) < 0) {
1411 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n");
1412 rc = -EINVAL;
1413 goto out;
1414 }
1415 for (i = 0; i < nr_idle_states; i++)
1416 strlcpy(pnv_idle_states[i].name, temp_string[i],
1417 PNV_IDLE_NAME_LEN);
1418 nr_pnv_idle_states = nr_idle_states;
1419 rc = 0;
1420out:
1421 kfree(temp_u32);
1422 kfree(temp_u64);
1423 kfree(temp_string);
1424 return rc;
1425}
1426
1427static int __init pnv_init_idle_states(void)
1428{
1429 int cpu;
1430 int rc = 0;
1431
1432
1433 for_each_present_cpu(cpu) {
1434 struct paca_struct *p = paca_ptrs[cpu];
1435
1436 p->idle_state = 0;
1437 if (cpu == cpu_first_thread_sibling(cpu))
1438 p->idle_state = (1 << threads_per_core) - 1;
1439
1440 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
1441
1442 p->thread_idle_state = PNV_THREAD_RUNNING;
1443 } else if (pvr_version_is(PVR_POWER9)) {
1444
1445#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1446 p->requested_psscr = 0;
1447 atomic_set(&p->dont_stop, 0);
1448#endif
1449 }
1450 }
1451
1452
1453 nr_pnv_idle_states = 0;
1454 supported_cpuidle_states = 0;
1455
1456 if (cpuidle_disable != IDLE_NO_OVERRIDE)
1457 goto out;
1458 rc = pnv_parse_cpuidle_dt();
1459 if (rc)
1460 return rc;
1461 pnv_probe_idle_states();
1462
1463 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
1464 if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
1465 power7_fastsleep_workaround_entry = false;
1466 power7_fastsleep_workaround_exit = false;
1467 } else {
1468
1469
1470
1471
1472
1473
1474 device_create_file(cpu_subsys.dev_root,
1475 &dev_attr_fastsleep_workaround_applyonce);
1476 }
1477
1478 update_subcore_sibling_mask();
1479
1480 if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) {
1481 ppc_md.power_save = power7_idle;
1482 power7_offline_type = PNV_THREAD_NAP;
1483 }
1484
1485 if ((supported_cpuidle_states & OPAL_PM_WINKLE_ENABLED) &&
1486 (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT))
1487 power7_offline_type = PNV_THREAD_WINKLE;
1488 else if ((supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED) ||
1489 (supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1))
1490 power7_offline_type = PNV_THREAD_SLEEP;
1491 }
1492
1493 if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
1494 if (pnv_save_sprs_for_deep_states())
1495 pnv_disable_deep_states();
1496 }
1497
1498out:
1499 return 0;
1500}
1501machine_subsys_initcall(powernv, pnv_init_idle_states);
1502