1
2
3
4
5
6
7
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/init.h>
11#include <linux/moduleparam.h>
12#include <linux/cpuidle.h>
13#include <linux/cpu.h>
14#include <linux/notifier.h>
15
16#include <asm/paca.h>
17#include <asm/reg.h>
18#include <asm/machdep.h>
19#include <asm/firmware.h>
20#include <asm/runlatch.h>
21
22#include "plpar_wrappers.h"
23#include "pseries.h"
24
25struct cpuidle_driver pseries_idle_driver = {
26 .name = "pseries_idle",
27 .owner = THIS_MODULE,
28};
29
30#define MAX_IDLE_STATE_COUNT 2
31
32static int max_idle_state = MAX_IDLE_STATE_COUNT - 1;
33static struct cpuidle_device __percpu *pseries_cpuidle_devices;
34static struct cpuidle_state *cpuidle_state_table;
35
36static inline void idle_loop_prolog(unsigned long *in_purr)
37{
38 *in_purr = mfspr(SPRN_PURR);
39
40
41
42
43 get_lppaca()->idle = 1;
44}
45
46static inline void idle_loop_epilog(unsigned long in_purr)
47{
48 get_lppaca()->wait_state_cycles += mfspr(SPRN_PURR) - in_purr;
49 get_lppaca()->idle = 0;
50}
51
52static int snooze_loop(struct cpuidle_device *dev,
53 struct cpuidle_driver *drv,
54 int index)
55{
56 unsigned long in_purr;
57 int cpu = dev->cpu;
58
59 idle_loop_prolog(&in_purr);
60 local_irq_enable();
61 set_thread_flag(TIF_POLLING_NRFLAG);
62
63 while ((!need_resched()) && cpu_online(cpu)) {
64 ppc64_runlatch_off();
65 HMT_low();
66 HMT_very_low();
67 }
68
69 HMT_medium();
70 clear_thread_flag(TIF_POLLING_NRFLAG);
71 smp_mb();
72
73 idle_loop_epilog(in_purr);
74
75 return index;
76}
77
78static void check_and_cede_processor(void)
79{
80
81
82
83
84
85 if (prep_irq_for_idle()) {
86 cede_processor();
87#ifdef CONFIG_TRACE_IRQFLAGS
88
89 if (WARN_ON(!(mfmsr() & MSR_EE)))
90 __hard_irq_enable();
91#endif
92 }
93}
94
95static int dedicated_cede_loop(struct cpuidle_device *dev,
96 struct cpuidle_driver *drv,
97 int index)
98{
99 unsigned long in_purr;
100
101 idle_loop_prolog(&in_purr);
102 get_lppaca()->donate_dedicated_cpu = 1;
103
104 ppc64_runlatch_off();
105 HMT_medium();
106 check_and_cede_processor();
107
108 get_lppaca()->donate_dedicated_cpu = 0;
109
110 idle_loop_epilog(in_purr);
111
112 return index;
113}
114
115static int shared_cede_loop(struct cpuidle_device *dev,
116 struct cpuidle_driver *drv,
117 int index)
118{
119 unsigned long in_purr;
120
121 idle_loop_prolog(&in_purr);
122
123
124
125
126
127
128
129
130 check_and_cede_processor();
131
132 idle_loop_epilog(in_purr);
133
134 return index;
135}
136
137
138
139
140static struct cpuidle_state dedicated_states[MAX_IDLE_STATE_COUNT] = {
141 {
142 .name = "snooze",
143 .desc = "snooze",
144 .flags = CPUIDLE_FLAG_TIME_VALID,
145 .exit_latency = 0,
146 .target_residency = 0,
147 .enter = &snooze_loop },
148 {
149 .name = "CEDE",
150 .desc = "CEDE",
151 .flags = CPUIDLE_FLAG_TIME_VALID,
152 .exit_latency = 10,
153 .target_residency = 100,
154 .enter = &dedicated_cede_loop },
155};
156
157
158
159
160static struct cpuidle_state shared_states[MAX_IDLE_STATE_COUNT] = {
161 {
162 .name = "Shared Cede",
163 .desc = "Shared Cede",
164 .flags = CPUIDLE_FLAG_TIME_VALID,
165 .exit_latency = 0,
166 .target_residency = 0,
167 .enter = &shared_cede_loop },
168};
169
170void update_smt_snooze_delay(int cpu, int residency)
171{
172 struct cpuidle_driver *drv = cpuidle_get_driver();
173 struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
174
175 if (cpuidle_state_table != dedicated_states)
176 return;
177
178 if (residency < 0) {
179
180 if (dev)
181 dev->states_usage[1].disable = 1;
182 } else
183 if (drv)
184 drv->states[1].target_residency = residency;
185}
186
187static int pseries_cpuidle_add_cpu_notifier(struct notifier_block *n,
188 unsigned long action, void *hcpu)
189{
190 int hotcpu = (unsigned long)hcpu;
191 struct cpuidle_device *dev =
192 per_cpu_ptr(pseries_cpuidle_devices, hotcpu);
193
194 if (dev && cpuidle_get_driver()) {
195 switch (action) {
196 case CPU_ONLINE:
197 case CPU_ONLINE_FROZEN:
198 cpuidle_pause_and_lock();
199 cpuidle_enable_device(dev);
200 cpuidle_resume_and_unlock();
201 break;
202
203 case CPU_DEAD:
204 case CPU_DEAD_FROZEN:
205 cpuidle_pause_and_lock();
206 cpuidle_disable_device(dev);
207 cpuidle_resume_and_unlock();
208 break;
209
210 default:
211 return NOTIFY_DONE;
212 }
213 }
214 return NOTIFY_OK;
215}
216
217static struct notifier_block setup_hotplug_notifier = {
218 .notifier_call = pseries_cpuidle_add_cpu_notifier,
219};
220
221
222
223
224static int pseries_cpuidle_driver_init(void)
225{
226 int idle_state;
227 struct cpuidle_driver *drv = &pseries_idle_driver;
228
229 drv->state_count = 0;
230
231 for (idle_state = 0; idle_state < MAX_IDLE_STATE_COUNT; ++idle_state) {
232
233 if (idle_state > max_idle_state)
234 break;
235
236
237 if (cpuidle_state_table[idle_state].enter == NULL)
238 continue;
239
240 drv->states[drv->state_count] =
241 cpuidle_state_table[idle_state];
242
243 drv->state_count += 1;
244 }
245
246 return 0;
247}
248
249
250
251
252static void pseries_idle_devices_uninit(void)
253{
254 int i;
255 struct cpuidle_device *dev;
256
257 for_each_possible_cpu(i) {
258 dev = per_cpu_ptr(pseries_cpuidle_devices, i);
259 cpuidle_unregister_device(dev);
260 }
261
262 free_percpu(pseries_cpuidle_devices);
263 return;
264}
265
266
267
268
269static int pseries_idle_devices_init(void)
270{
271 int i;
272 struct cpuidle_driver *drv = &pseries_idle_driver;
273 struct cpuidle_device *dev;
274
275 pseries_cpuidle_devices = alloc_percpu(struct cpuidle_device);
276 if (pseries_cpuidle_devices == NULL)
277 return -ENOMEM;
278
279 for_each_possible_cpu(i) {
280 dev = per_cpu_ptr(pseries_cpuidle_devices, i);
281 dev->state_count = drv->state_count;
282 dev->cpu = i;
283 if (cpuidle_register_device(dev)) {
284 printk(KERN_DEBUG \
285 "cpuidle_register_device %d failed!\n", i);
286 return -EIO;
287 }
288 }
289
290 return 0;
291}
292
293
294
295
296
297static int pseries_idle_probe(void)
298{
299
300 if (!firmware_has_feature(FW_FEATURE_SPLPAR))
301 return -ENODEV;
302
303 if (cpuidle_disable != IDLE_NO_OVERRIDE)
304 return -ENODEV;
305
306 if (max_idle_state == 0) {
307 printk(KERN_DEBUG "pseries processor idle disabled.\n");
308 return -EPERM;
309 }
310
311 if (get_lppaca()->shared_proc)
312 cpuidle_state_table = shared_states;
313 else
314 cpuidle_state_table = dedicated_states;
315
316 return 0;
317}
318
319static int __init pseries_processor_idle_init(void)
320{
321 int retval;
322
323 retval = pseries_idle_probe();
324 if (retval)
325 return retval;
326
327 pseries_cpuidle_driver_init();
328 retval = cpuidle_register_driver(&pseries_idle_driver);
329 if (retval) {
330 printk(KERN_DEBUG "Registration of pseries driver failed.\n");
331 return retval;
332 }
333
334 retval = pseries_idle_devices_init();
335 if (retval) {
336 pseries_idle_devices_uninit();
337 cpuidle_unregister_driver(&pseries_idle_driver);
338 return retval;
339 }
340
341 register_cpu_notifier(&setup_hotplug_notifier);
342 printk(KERN_DEBUG "pseries_idle_driver registered\n");
343
344 return 0;
345}
346
347static void __exit pseries_processor_idle_exit(void)
348{
349
350 unregister_cpu_notifier(&setup_hotplug_notifier);
351 pseries_idle_devices_uninit();
352 cpuidle_unregister_driver(&pseries_idle_driver);
353
354 return;
355}
356
357module_init(pseries_processor_idle_init);
358module_exit(pseries_processor_idle_exit);
359
360MODULE_AUTHOR("Deepthi Dharwar <deepthi@linux.vnet.ibm.com>");
361MODULE_DESCRIPTION("Cpuidle driver for POWER");
362MODULE_LICENSE("GPL");
363