1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#define DEBUG
41
42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44#include <linux/kernel.h>
45#include <linux/cpuidle.h>
46#include <linux/tick.h>
47#include <trace/events/power.h>
48#include <linux/sched.h>
49#include <linux/notifier.h>
50#include <linux/cpu.h>
51#include <linux/moduleparam.h>
52#include <asm/cpu_device_id.h>
53#include <asm/intel-family.h>
54#include <asm/mwait.h>
55#include <asm/msr.h>
56
57#define INTEL_IDLE_VERSION "0.4.1"
58
59static struct cpuidle_driver intel_idle_driver = {
60 .name = "intel_idle",
61 .owner = THIS_MODULE,
62};
63
64static int max_cstate = CPUIDLE_STATE_MAX - 1;
65
66static unsigned int mwait_substates;
67
68#define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
69
70static unsigned int lapic_timer_reliable_states = (1 << 1);
71
72struct idle_cpu {
73 struct cpuidle_state *state_table;
74
75
76
77
78
79 unsigned long auto_demotion_disable_flags;
80 bool byt_auto_demotion_disable_flag;
81 bool disable_promotion_to_c1e;
82};
83
84static const struct idle_cpu *icpu;
85static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
86static int intel_idle(struct cpuidle_device *dev,
87 struct cpuidle_driver *drv, int index);
88static void intel_idle_s2idle(struct cpuidle_device *dev,
89 struct cpuidle_driver *drv, int index);
90static struct cpuidle_state *cpuidle_state_table;
91
92
93
94
95
96
97
98#define CPUIDLE_FLAG_TLB_FLUSHED 0x10000
99
100
101
102
103
104
105
106
107#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
108#define MWAIT2flg(eax) ((eax & 0xFF) << 24)
109
110
111
112
113
114
115static struct cpuidle_state nehalem_cstates[] = {
116 {
117 .name = "C1",
118 .desc = "MWAIT 0x00",
119 .flags = MWAIT2flg(0x00),
120 .exit_latency = 3,
121 .target_residency = 6,
122 .enter = &intel_idle,
123 .enter_s2idle = intel_idle_s2idle, },
124 {
125 .name = "C1E",
126 .desc = "MWAIT 0x01",
127 .flags = MWAIT2flg(0x01),
128 .exit_latency = 10,
129 .target_residency = 20,
130 .enter = &intel_idle,
131 .enter_s2idle = intel_idle_s2idle, },
132 {
133 .name = "C3",
134 .desc = "MWAIT 0x10",
135 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
136 .exit_latency = 20,
137 .target_residency = 80,
138 .enter = &intel_idle,
139 .enter_s2idle = intel_idle_s2idle, },
140 {
141 .name = "C6",
142 .desc = "MWAIT 0x20",
143 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
144 .exit_latency = 200,
145 .target_residency = 800,
146 .enter = &intel_idle,
147 .enter_s2idle = intel_idle_s2idle, },
148 {
149 .enter = NULL }
150};
151
152static struct cpuidle_state snb_cstates[] = {
153 {
154 .name = "C1",
155 .desc = "MWAIT 0x00",
156 .flags = MWAIT2flg(0x00),
157 .exit_latency = 2,
158 .target_residency = 2,
159 .enter = &intel_idle,
160 .enter_s2idle = intel_idle_s2idle, },
161 {
162 .name = "C1E",
163 .desc = "MWAIT 0x01",
164 .flags = MWAIT2flg(0x01),
165 .exit_latency = 10,
166 .target_residency = 20,
167 .enter = &intel_idle,
168 .enter_s2idle = intel_idle_s2idle, },
169 {
170 .name = "C3",
171 .desc = "MWAIT 0x10",
172 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
173 .exit_latency = 80,
174 .target_residency = 211,
175 .enter = &intel_idle,
176 .enter_s2idle = intel_idle_s2idle, },
177 {
178 .name = "C6",
179 .desc = "MWAIT 0x20",
180 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
181 .exit_latency = 104,
182 .target_residency = 345,
183 .enter = &intel_idle,
184 .enter_s2idle = intel_idle_s2idle, },
185 {
186 .name = "C7",
187 .desc = "MWAIT 0x30",
188 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
189 .exit_latency = 109,
190 .target_residency = 345,
191 .enter = &intel_idle,
192 .enter_s2idle = intel_idle_s2idle, },
193 {
194 .enter = NULL }
195};
196
197static struct cpuidle_state byt_cstates[] = {
198 {
199 .name = "C1",
200 .desc = "MWAIT 0x00",
201 .flags = MWAIT2flg(0x00),
202 .exit_latency = 1,
203 .target_residency = 1,
204 .enter = &intel_idle,
205 .enter_s2idle = intel_idle_s2idle, },
206 {
207 .name = "C6N",
208 .desc = "MWAIT 0x58",
209 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
210 .exit_latency = 300,
211 .target_residency = 275,
212 .enter = &intel_idle,
213 .enter_s2idle = intel_idle_s2idle, },
214 {
215 .name = "C6S",
216 .desc = "MWAIT 0x52",
217 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
218 .exit_latency = 500,
219 .target_residency = 560,
220 .enter = &intel_idle,
221 .enter_s2idle = intel_idle_s2idle, },
222 {
223 .name = "C7",
224 .desc = "MWAIT 0x60",
225 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
226 .exit_latency = 1200,
227 .target_residency = 4000,
228 .enter = &intel_idle,
229 .enter_s2idle = intel_idle_s2idle, },
230 {
231 .name = "C7S",
232 .desc = "MWAIT 0x64",
233 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
234 .exit_latency = 10000,
235 .target_residency = 20000,
236 .enter = &intel_idle,
237 .enter_s2idle = intel_idle_s2idle, },
238 {
239 .enter = NULL }
240};
241
242static struct cpuidle_state cht_cstates[] = {
243 {
244 .name = "C1",
245 .desc = "MWAIT 0x00",
246 .flags = MWAIT2flg(0x00),
247 .exit_latency = 1,
248 .target_residency = 1,
249 .enter = &intel_idle,
250 .enter_s2idle = intel_idle_s2idle, },
251 {
252 .name = "C6N",
253 .desc = "MWAIT 0x58",
254 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
255 .exit_latency = 80,
256 .target_residency = 275,
257 .enter = &intel_idle,
258 .enter_s2idle = intel_idle_s2idle, },
259 {
260 .name = "C6S",
261 .desc = "MWAIT 0x52",
262 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
263 .exit_latency = 200,
264 .target_residency = 560,
265 .enter = &intel_idle,
266 .enter_s2idle = intel_idle_s2idle, },
267 {
268 .name = "C7",
269 .desc = "MWAIT 0x60",
270 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
271 .exit_latency = 1200,
272 .target_residency = 4000,
273 .enter = &intel_idle,
274 .enter_s2idle = intel_idle_s2idle, },
275 {
276 .name = "C7S",
277 .desc = "MWAIT 0x64",
278 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
279 .exit_latency = 10000,
280 .target_residency = 20000,
281 .enter = &intel_idle,
282 .enter_s2idle = intel_idle_s2idle, },
283 {
284 .enter = NULL }
285};
286
287static struct cpuidle_state ivb_cstates[] = {
288 {
289 .name = "C1",
290 .desc = "MWAIT 0x00",
291 .flags = MWAIT2flg(0x00),
292 .exit_latency = 1,
293 .target_residency = 1,
294 .enter = &intel_idle,
295 .enter_s2idle = intel_idle_s2idle, },
296 {
297 .name = "C1E",
298 .desc = "MWAIT 0x01",
299 .flags = MWAIT2flg(0x01),
300 .exit_latency = 10,
301 .target_residency = 20,
302 .enter = &intel_idle,
303 .enter_s2idle = intel_idle_s2idle, },
304 {
305 .name = "C3",
306 .desc = "MWAIT 0x10",
307 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
308 .exit_latency = 59,
309 .target_residency = 156,
310 .enter = &intel_idle,
311 .enter_s2idle = intel_idle_s2idle, },
312 {
313 .name = "C6",
314 .desc = "MWAIT 0x20",
315 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
316 .exit_latency = 80,
317 .target_residency = 300,
318 .enter = &intel_idle,
319 .enter_s2idle = intel_idle_s2idle, },
320 {
321 .name = "C7",
322 .desc = "MWAIT 0x30",
323 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
324 .exit_latency = 87,
325 .target_residency = 300,
326 .enter = &intel_idle,
327 .enter_s2idle = intel_idle_s2idle, },
328 {
329 .enter = NULL }
330};
331
332static struct cpuidle_state ivt_cstates[] = {
333 {
334 .name = "C1",
335 .desc = "MWAIT 0x00",
336 .flags = MWAIT2flg(0x00),
337 .exit_latency = 1,
338 .target_residency = 1,
339 .enter = &intel_idle,
340 .enter_s2idle = intel_idle_s2idle, },
341 {
342 .name = "C1E",
343 .desc = "MWAIT 0x01",
344 .flags = MWAIT2flg(0x01),
345 .exit_latency = 10,
346 .target_residency = 80,
347 .enter = &intel_idle,
348 .enter_s2idle = intel_idle_s2idle, },
349 {
350 .name = "C3",
351 .desc = "MWAIT 0x10",
352 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
353 .exit_latency = 59,
354 .target_residency = 156,
355 .enter = &intel_idle,
356 .enter_s2idle = intel_idle_s2idle, },
357 {
358 .name = "C6",
359 .desc = "MWAIT 0x20",
360 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
361 .exit_latency = 82,
362 .target_residency = 300,
363 .enter = &intel_idle,
364 .enter_s2idle = intel_idle_s2idle, },
365 {
366 .enter = NULL }
367};
368
369static struct cpuidle_state ivt_cstates_4s[] = {
370 {
371 .name = "C1",
372 .desc = "MWAIT 0x00",
373 .flags = MWAIT2flg(0x00),
374 .exit_latency = 1,
375 .target_residency = 1,
376 .enter = &intel_idle,
377 .enter_s2idle = intel_idle_s2idle, },
378 {
379 .name = "C1E",
380 .desc = "MWAIT 0x01",
381 .flags = MWAIT2flg(0x01),
382 .exit_latency = 10,
383 .target_residency = 250,
384 .enter = &intel_idle,
385 .enter_s2idle = intel_idle_s2idle, },
386 {
387 .name = "C3",
388 .desc = "MWAIT 0x10",
389 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
390 .exit_latency = 59,
391 .target_residency = 300,
392 .enter = &intel_idle,
393 .enter_s2idle = intel_idle_s2idle, },
394 {
395 .name = "C6",
396 .desc = "MWAIT 0x20",
397 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
398 .exit_latency = 84,
399 .target_residency = 400,
400 .enter = &intel_idle,
401 .enter_s2idle = intel_idle_s2idle, },
402 {
403 .enter = NULL }
404};
405
406static struct cpuidle_state ivt_cstates_8s[] = {
407 {
408 .name = "C1",
409 .desc = "MWAIT 0x00",
410 .flags = MWAIT2flg(0x00),
411 .exit_latency = 1,
412 .target_residency = 1,
413 .enter = &intel_idle,
414 .enter_s2idle = intel_idle_s2idle, },
415 {
416 .name = "C1E",
417 .desc = "MWAIT 0x01",
418 .flags = MWAIT2flg(0x01),
419 .exit_latency = 10,
420 .target_residency = 500,
421 .enter = &intel_idle,
422 .enter_s2idle = intel_idle_s2idle, },
423 {
424 .name = "C3",
425 .desc = "MWAIT 0x10",
426 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
427 .exit_latency = 59,
428 .target_residency = 600,
429 .enter = &intel_idle,
430 .enter_s2idle = intel_idle_s2idle, },
431 {
432 .name = "C6",
433 .desc = "MWAIT 0x20",
434 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
435 .exit_latency = 88,
436 .target_residency = 700,
437 .enter = &intel_idle,
438 .enter_s2idle = intel_idle_s2idle, },
439 {
440 .enter = NULL }
441};
442
443static struct cpuidle_state hsw_cstates[] = {
444 {
445 .name = "C1",
446 .desc = "MWAIT 0x00",
447 .flags = MWAIT2flg(0x00),
448 .exit_latency = 2,
449 .target_residency = 2,
450 .enter = &intel_idle,
451 .enter_s2idle = intel_idle_s2idle, },
452 {
453 .name = "C1E",
454 .desc = "MWAIT 0x01",
455 .flags = MWAIT2flg(0x01),
456 .exit_latency = 10,
457 .target_residency = 20,
458 .enter = &intel_idle,
459 .enter_s2idle = intel_idle_s2idle, },
460 {
461 .name = "C3",
462 .desc = "MWAIT 0x10",
463 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
464 .exit_latency = 33,
465 .target_residency = 100,
466 .enter = &intel_idle,
467 .enter_s2idle = intel_idle_s2idle, },
468 {
469 .name = "C6",
470 .desc = "MWAIT 0x20",
471 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
472 .exit_latency = 133,
473 .target_residency = 400,
474 .enter = &intel_idle,
475 .enter_s2idle = intel_idle_s2idle, },
476 {
477 .name = "C7s",
478 .desc = "MWAIT 0x32",
479 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
480 .exit_latency = 166,
481 .target_residency = 500,
482 .enter = &intel_idle,
483 .enter_s2idle = intel_idle_s2idle, },
484 {
485 .name = "C8",
486 .desc = "MWAIT 0x40",
487 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
488 .exit_latency = 300,
489 .target_residency = 900,
490 .enter = &intel_idle,
491 .enter_s2idle = intel_idle_s2idle, },
492 {
493 .name = "C9",
494 .desc = "MWAIT 0x50",
495 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
496 .exit_latency = 600,
497 .target_residency = 1800,
498 .enter = &intel_idle,
499 .enter_s2idle = intel_idle_s2idle, },
500 {
501 .name = "C10",
502 .desc = "MWAIT 0x60",
503 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
504 .exit_latency = 2600,
505 .target_residency = 7700,
506 .enter = &intel_idle,
507 .enter_s2idle = intel_idle_s2idle, },
508 {
509 .enter = NULL }
510};
511static struct cpuidle_state bdw_cstates[] = {
512 {
513 .name = "C1",
514 .desc = "MWAIT 0x00",
515 .flags = MWAIT2flg(0x00),
516 .exit_latency = 2,
517 .target_residency = 2,
518 .enter = &intel_idle,
519 .enter_s2idle = intel_idle_s2idle, },
520 {
521 .name = "C1E",
522 .desc = "MWAIT 0x01",
523 .flags = MWAIT2flg(0x01),
524 .exit_latency = 10,
525 .target_residency = 20,
526 .enter = &intel_idle,
527 .enter_s2idle = intel_idle_s2idle, },
528 {
529 .name = "C3",
530 .desc = "MWAIT 0x10",
531 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
532 .exit_latency = 40,
533 .target_residency = 100,
534 .enter = &intel_idle,
535 .enter_s2idle = intel_idle_s2idle, },
536 {
537 .name = "C6",
538 .desc = "MWAIT 0x20",
539 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
540 .exit_latency = 133,
541 .target_residency = 400,
542 .enter = &intel_idle,
543 .enter_s2idle = intel_idle_s2idle, },
544 {
545 .name = "C7s",
546 .desc = "MWAIT 0x32",
547 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
548 .exit_latency = 166,
549 .target_residency = 500,
550 .enter = &intel_idle,
551 .enter_s2idle = intel_idle_s2idle, },
552 {
553 .name = "C8",
554 .desc = "MWAIT 0x40",
555 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
556 .exit_latency = 300,
557 .target_residency = 900,
558 .enter = &intel_idle,
559 .enter_s2idle = intel_idle_s2idle, },
560 {
561 .name = "C9",
562 .desc = "MWAIT 0x50",
563 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
564 .exit_latency = 600,
565 .target_residency = 1800,
566 .enter = &intel_idle,
567 .enter_s2idle = intel_idle_s2idle, },
568 {
569 .name = "C10",
570 .desc = "MWAIT 0x60",
571 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
572 .exit_latency = 2600,
573 .target_residency = 7700,
574 .enter = &intel_idle,
575 .enter_s2idle = intel_idle_s2idle, },
576 {
577 .enter = NULL }
578};
579
580static struct cpuidle_state skl_cstates[] = {
581 {
582 .name = "C1",
583 .desc = "MWAIT 0x00",
584 .flags = MWAIT2flg(0x00),
585 .exit_latency = 2,
586 .target_residency = 2,
587 .enter = &intel_idle,
588 .enter_s2idle = intel_idle_s2idle, },
589 {
590 .name = "C1E",
591 .desc = "MWAIT 0x01",
592 .flags = MWAIT2flg(0x01),
593 .exit_latency = 10,
594 .target_residency = 20,
595 .enter = &intel_idle,
596 .enter_s2idle = intel_idle_s2idle, },
597 {
598 .name = "C3",
599 .desc = "MWAIT 0x10",
600 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
601 .exit_latency = 70,
602 .target_residency = 100,
603 .enter = &intel_idle,
604 .enter_s2idle = intel_idle_s2idle, },
605 {
606 .name = "C6",
607 .desc = "MWAIT 0x20",
608 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
609 .exit_latency = 85,
610 .target_residency = 200,
611 .enter = &intel_idle,
612 .enter_s2idle = intel_idle_s2idle, },
613 {
614 .name = "C7s",
615 .desc = "MWAIT 0x33",
616 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
617 .exit_latency = 124,
618 .target_residency = 800,
619 .enter = &intel_idle,
620 .enter_s2idle = intel_idle_s2idle, },
621 {
622 .name = "C8",
623 .desc = "MWAIT 0x40",
624 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
625 .exit_latency = 200,
626 .target_residency = 800,
627 .enter = &intel_idle,
628 .enter_s2idle = intel_idle_s2idle, },
629 {
630 .name = "C9",
631 .desc = "MWAIT 0x50",
632 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
633 .exit_latency = 480,
634 .target_residency = 5000,
635 .enter = &intel_idle,
636 .enter_s2idle = intel_idle_s2idle, },
637 {
638 .name = "C10",
639 .desc = "MWAIT 0x60",
640 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
641 .exit_latency = 890,
642 .target_residency = 5000,
643 .enter = &intel_idle,
644 .enter_s2idle = intel_idle_s2idle, },
645 {
646 .enter = NULL }
647};
648
649static struct cpuidle_state skx_cstates[] = {
650 {
651 .name = "C1",
652 .desc = "MWAIT 0x00",
653 .flags = MWAIT2flg(0x00),
654 .exit_latency = 2,
655 .target_residency = 2,
656 .enter = &intel_idle,
657 .enter_s2idle = intel_idle_s2idle, },
658 {
659 .name = "C1E",
660 .desc = "MWAIT 0x01",
661 .flags = MWAIT2flg(0x01),
662 .exit_latency = 10,
663 .target_residency = 20,
664 .enter = &intel_idle,
665 .enter_s2idle = intel_idle_s2idle, },
666 {
667 .name = "C6",
668 .desc = "MWAIT 0x20",
669 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
670 .exit_latency = 133,
671 .target_residency = 600,
672 .enter = &intel_idle,
673 .enter_s2idle = intel_idle_s2idle, },
674 {
675 .enter = NULL }
676};
677
678static struct cpuidle_state atom_cstates[] = {
679 {
680 .name = "C1E",
681 .desc = "MWAIT 0x00",
682 .flags = MWAIT2flg(0x00),
683 .exit_latency = 10,
684 .target_residency = 20,
685 .enter = &intel_idle,
686 .enter_s2idle = intel_idle_s2idle, },
687 {
688 .name = "C2",
689 .desc = "MWAIT 0x10",
690 .flags = MWAIT2flg(0x10),
691 .exit_latency = 20,
692 .target_residency = 80,
693 .enter = &intel_idle,
694 .enter_s2idle = intel_idle_s2idle, },
695 {
696 .name = "C4",
697 .desc = "MWAIT 0x30",
698 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
699 .exit_latency = 100,
700 .target_residency = 400,
701 .enter = &intel_idle,
702 .enter_s2idle = intel_idle_s2idle, },
703 {
704 .name = "C6",
705 .desc = "MWAIT 0x52",
706 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
707 .exit_latency = 140,
708 .target_residency = 560,
709 .enter = &intel_idle,
710 .enter_s2idle = intel_idle_s2idle, },
711 {
712 .enter = NULL }
713};
714static struct cpuidle_state tangier_cstates[] = {
715 {
716 .name = "C1",
717 .desc = "MWAIT 0x00",
718 .flags = MWAIT2flg(0x00),
719 .exit_latency = 1,
720 .target_residency = 4,
721 .enter = &intel_idle,
722 .enter_s2idle = intel_idle_s2idle, },
723 {
724 .name = "C4",
725 .desc = "MWAIT 0x30",
726 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
727 .exit_latency = 100,
728 .target_residency = 400,
729 .enter = &intel_idle,
730 .enter_s2idle = intel_idle_s2idle, },
731 {
732 .name = "C6",
733 .desc = "MWAIT 0x52",
734 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
735 .exit_latency = 140,
736 .target_residency = 560,
737 .enter = &intel_idle,
738 .enter_s2idle = intel_idle_s2idle, },
739 {
740 .name = "C7",
741 .desc = "MWAIT 0x60",
742 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
743 .exit_latency = 1200,
744 .target_residency = 4000,
745 .enter = &intel_idle,
746 .enter_s2idle = intel_idle_s2idle, },
747 {
748 .name = "C9",
749 .desc = "MWAIT 0x64",
750 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
751 .exit_latency = 10000,
752 .target_residency = 20000,
753 .enter = &intel_idle,
754 .enter_s2idle = intel_idle_s2idle, },
755 {
756 .enter = NULL }
757};
758static struct cpuidle_state avn_cstates[] = {
759 {
760 .name = "C1",
761 .desc = "MWAIT 0x00",
762 .flags = MWAIT2flg(0x00),
763 .exit_latency = 2,
764 .target_residency = 2,
765 .enter = &intel_idle,
766 .enter_s2idle = intel_idle_s2idle, },
767 {
768 .name = "C6",
769 .desc = "MWAIT 0x51",
770 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
771 .exit_latency = 15,
772 .target_residency = 45,
773 .enter = &intel_idle,
774 .enter_s2idle = intel_idle_s2idle, },
775 {
776 .enter = NULL }
777};
778static struct cpuidle_state knl_cstates[] = {
779 {
780 .name = "C1",
781 .desc = "MWAIT 0x00",
782 .flags = MWAIT2flg(0x00),
783 .exit_latency = 1,
784 .target_residency = 2,
785 .enter = &intel_idle,
786 .enter_s2idle = intel_idle_s2idle },
787 {
788 .name = "C6",
789 .desc = "MWAIT 0x10",
790 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
791 .exit_latency = 120,
792 .target_residency = 500,
793 .enter = &intel_idle,
794 .enter_s2idle = intel_idle_s2idle },
795 {
796 .enter = NULL }
797};
798
799static struct cpuidle_state bxt_cstates[] = {
800 {
801 .name = "C1",
802 .desc = "MWAIT 0x00",
803 .flags = MWAIT2flg(0x00),
804 .exit_latency = 2,
805 .target_residency = 2,
806 .enter = &intel_idle,
807 .enter_s2idle = intel_idle_s2idle, },
808 {
809 .name = "C1E",
810 .desc = "MWAIT 0x01",
811 .flags = MWAIT2flg(0x01),
812 .exit_latency = 10,
813 .target_residency = 20,
814 .enter = &intel_idle,
815 .enter_s2idle = intel_idle_s2idle, },
816 {
817 .name = "C6",
818 .desc = "MWAIT 0x20",
819 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
820 .exit_latency = 133,
821 .target_residency = 133,
822 .enter = &intel_idle,
823 .enter_s2idle = intel_idle_s2idle, },
824 {
825 .name = "C7s",
826 .desc = "MWAIT 0x31",
827 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
828 .exit_latency = 155,
829 .target_residency = 155,
830 .enter = &intel_idle,
831 .enter_s2idle = intel_idle_s2idle, },
832 {
833 .name = "C8",
834 .desc = "MWAIT 0x40",
835 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
836 .exit_latency = 1000,
837 .target_residency = 1000,
838 .enter = &intel_idle,
839 .enter_s2idle = intel_idle_s2idle, },
840 {
841 .name = "C9",
842 .desc = "MWAIT 0x50",
843 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
844 .exit_latency = 2000,
845 .target_residency = 2000,
846 .enter = &intel_idle,
847 .enter_s2idle = intel_idle_s2idle, },
848 {
849 .name = "C10",
850 .desc = "MWAIT 0x60",
851 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
852 .exit_latency = 10000,
853 .target_residency = 10000,
854 .enter = &intel_idle,
855 .enter_s2idle = intel_idle_s2idle, },
856 {
857 .enter = NULL }
858};
859
860static struct cpuidle_state dnv_cstates[] = {
861 {
862 .name = "C1",
863 .desc = "MWAIT 0x00",
864 .flags = MWAIT2flg(0x00),
865 .exit_latency = 2,
866 .target_residency = 2,
867 .enter = &intel_idle,
868 .enter_s2idle = intel_idle_s2idle, },
869 {
870 .name = "C1E",
871 .desc = "MWAIT 0x01",
872 .flags = MWAIT2flg(0x01),
873 .exit_latency = 10,
874 .target_residency = 20,
875 .enter = &intel_idle,
876 .enter_s2idle = intel_idle_s2idle, },
877 {
878 .name = "C6",
879 .desc = "MWAIT 0x20",
880 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
881 .exit_latency = 50,
882 .target_residency = 500,
883 .enter = &intel_idle,
884 .enter_s2idle = intel_idle_s2idle, },
885 {
886 .enter = NULL }
887};
888
889
890
891
892
893
894
895
896
897static __cpuidle int intel_idle(struct cpuidle_device *dev,
898 struct cpuidle_driver *drv, int index)
899{
900 unsigned long ecx = 1;
901 struct cpuidle_state *state = &drv->states[index];
902 unsigned long eax = flg2MWAIT(state->flags);
903 unsigned int cstate;
904 bool uninitialized_var(tick);
905 int cpu = smp_processor_id();
906
907
908
909
910
911 if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
912 leave_mm(cpu);
913
914 if (!static_cpu_has(X86_FEATURE_ARAT)) {
915 cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) &
916 MWAIT_CSTATE_MASK) + 1;
917 tick = false;
918 if (!(lapic_timer_reliable_states & (1 << (cstate)))) {
919 tick = true;
920 tick_broadcast_enter();
921 }
922 }
923
924 mwait_idle_with_hints(eax, ecx);
925
926 if (!static_cpu_has(X86_FEATURE_ARAT) && tick)
927 tick_broadcast_exit();
928
929 return index;
930}
931
932
933
934
935
936
937
938static void intel_idle_s2idle(struct cpuidle_device *dev,
939 struct cpuidle_driver *drv, int index)
940{
941 unsigned long ecx = 1;
942 unsigned long eax = flg2MWAIT(drv->states[index].flags);
943
944 mwait_idle_with_hints(eax, ecx);
945}
946
947static void __setup_broadcast_timer(bool on)
948{
949 if (on)
950 tick_broadcast_enable();
951 else
952 tick_broadcast_disable();
953}
954
955static void auto_demotion_disable(void)
956{
957 unsigned long long msr_bits;
958
959 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
960 msr_bits &= ~(icpu->auto_demotion_disable_flags);
961 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
962}
963static void c1e_promotion_disable(void)
964{
965 unsigned long long msr_bits;
966
967 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
968 msr_bits &= ~0x2;
969 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
970}
971
972static const struct idle_cpu idle_cpu_nehalem = {
973 .state_table = nehalem_cstates,
974 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
975 .disable_promotion_to_c1e = true,
976};
977
978static const struct idle_cpu idle_cpu_atom = {
979 .state_table = atom_cstates,
980};
981
982static const struct idle_cpu idle_cpu_tangier = {
983 .state_table = tangier_cstates,
984};
985
986static const struct idle_cpu idle_cpu_lincroft = {
987 .state_table = atom_cstates,
988 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
989};
990
991static const struct idle_cpu idle_cpu_snb = {
992 .state_table = snb_cstates,
993 .disable_promotion_to_c1e = true,
994};
995
996static const struct idle_cpu idle_cpu_byt = {
997 .state_table = byt_cstates,
998 .disable_promotion_to_c1e = true,
999 .byt_auto_demotion_disable_flag = true,
1000};
1001
1002static const struct idle_cpu idle_cpu_cht = {
1003 .state_table = cht_cstates,
1004 .disable_promotion_to_c1e = true,
1005 .byt_auto_demotion_disable_flag = true,
1006};
1007
1008static const struct idle_cpu idle_cpu_ivb = {
1009 .state_table = ivb_cstates,
1010 .disable_promotion_to_c1e = true,
1011};
1012
1013static const struct idle_cpu idle_cpu_ivt = {
1014 .state_table = ivt_cstates,
1015 .disable_promotion_to_c1e = true,
1016};
1017
1018static const struct idle_cpu idle_cpu_hsw = {
1019 .state_table = hsw_cstates,
1020 .disable_promotion_to_c1e = true,
1021};
1022
1023static const struct idle_cpu idle_cpu_bdw = {
1024 .state_table = bdw_cstates,
1025 .disable_promotion_to_c1e = true,
1026};
1027
1028static const struct idle_cpu idle_cpu_skl = {
1029 .state_table = skl_cstates,
1030 .disable_promotion_to_c1e = true,
1031};
1032
1033static const struct idle_cpu idle_cpu_skx = {
1034 .state_table = skx_cstates,
1035 .disable_promotion_to_c1e = true,
1036};
1037
1038static const struct idle_cpu idle_cpu_avn = {
1039 .state_table = avn_cstates,
1040 .disable_promotion_to_c1e = true,
1041};
1042
1043static const struct idle_cpu idle_cpu_knl = {
1044 .state_table = knl_cstates,
1045};
1046
1047static const struct idle_cpu idle_cpu_bxt = {
1048 .state_table = bxt_cstates,
1049 .disable_promotion_to_c1e = true,
1050};
1051
1052static const struct idle_cpu idle_cpu_dnv = {
1053 .state_table = dnv_cstates,
1054 .disable_promotion_to_c1e = true,
1055};
1056
1057static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1058 INTEL_CPU_FAM6(NEHALEM_EP, idle_cpu_nehalem),
1059 INTEL_CPU_FAM6(NEHALEM, idle_cpu_nehalem),
1060 INTEL_CPU_FAM6(NEHALEM_G, idle_cpu_nehalem),
1061 INTEL_CPU_FAM6(WESTMERE, idle_cpu_nehalem),
1062 INTEL_CPU_FAM6(WESTMERE_EP, idle_cpu_nehalem),
1063 INTEL_CPU_FAM6(NEHALEM_EX, idle_cpu_nehalem),
1064 INTEL_CPU_FAM6(ATOM_BONNELL, idle_cpu_atom),
1065 INTEL_CPU_FAM6(ATOM_BONNELL_MID, idle_cpu_lincroft),
1066 INTEL_CPU_FAM6(WESTMERE_EX, idle_cpu_nehalem),
1067 INTEL_CPU_FAM6(SANDYBRIDGE, idle_cpu_snb),
1068 INTEL_CPU_FAM6(SANDYBRIDGE_X, idle_cpu_snb),
1069 INTEL_CPU_FAM6(ATOM_SALTWELL, idle_cpu_atom),
1070 INTEL_CPU_FAM6(ATOM_SILVERMONT, idle_cpu_byt),
1071 INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, idle_cpu_tangier),
1072 INTEL_CPU_FAM6(ATOM_AIRMONT, idle_cpu_cht),
1073 INTEL_CPU_FAM6(IVYBRIDGE, idle_cpu_ivb),
1074 INTEL_CPU_FAM6(IVYBRIDGE_X, idle_cpu_ivt),
1075 INTEL_CPU_FAM6(HASWELL_CORE, idle_cpu_hsw),
1076 INTEL_CPU_FAM6(HASWELL_X, idle_cpu_hsw),
1077 INTEL_CPU_FAM6(HASWELL_ULT, idle_cpu_hsw),
1078 INTEL_CPU_FAM6(HASWELL_GT3E, idle_cpu_hsw),
1079 INTEL_CPU_FAM6(ATOM_SILVERMONT_X, idle_cpu_avn),
1080 INTEL_CPU_FAM6(BROADWELL_CORE, idle_cpu_bdw),
1081 INTEL_CPU_FAM6(BROADWELL_GT3E, idle_cpu_bdw),
1082 INTEL_CPU_FAM6(BROADWELL_X, idle_cpu_bdw),
1083 INTEL_CPU_FAM6(BROADWELL_XEON_D, idle_cpu_bdw),
1084 INTEL_CPU_FAM6(SKYLAKE_MOBILE, idle_cpu_skl),
1085 INTEL_CPU_FAM6(SKYLAKE_DESKTOP, idle_cpu_skl),
1086 INTEL_CPU_FAM6(KABYLAKE_MOBILE, idle_cpu_skl),
1087 INTEL_CPU_FAM6(KABYLAKE_DESKTOP, idle_cpu_skl),
1088 INTEL_CPU_FAM6(SKYLAKE_X, idle_cpu_skx),
1089 INTEL_CPU_FAM6(XEON_PHI_KNL, idle_cpu_knl),
1090 INTEL_CPU_FAM6(XEON_PHI_KNM, idle_cpu_knl),
1091 INTEL_CPU_FAM6(ATOM_GOLDMONT, idle_cpu_bxt),
1092 INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, idle_cpu_bxt),
1093 INTEL_CPU_FAM6(ATOM_GOLDMONT_X, idle_cpu_dnv),
1094 INTEL_CPU_FAM6(ATOM_TREMONT_X, idle_cpu_dnv),
1095 {}
1096};
1097
1098
1099
1100
1101static int __init intel_idle_probe(void)
1102{
1103 unsigned int eax, ebx, ecx;
1104 const struct x86_cpu_id *id;
1105
1106 if (max_cstate == 0) {
1107 pr_debug("disabled\n");
1108 return -EPERM;
1109 }
1110
1111 id = x86_match_cpu(intel_idle_ids);
1112 if (!id) {
1113 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
1114 boot_cpu_data.x86 == 6)
1115 pr_debug("does not run on family %d model %d\n",
1116 boot_cpu_data.x86, boot_cpu_data.x86_model);
1117 return -ENODEV;
1118 }
1119
1120 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
1121 pr_debug("Please enable MWAIT in BIOS SETUP\n");
1122 return -ENODEV;
1123 }
1124
1125 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
1126 return -ENODEV;
1127
1128 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
1129
1130 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
1131 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
1132 !mwait_substates)
1133 return -ENODEV;
1134
1135 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
1136
1137 icpu = (const struct idle_cpu *)id->driver_data;
1138 cpuidle_state_table = icpu->state_table;
1139
1140 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
1141 boot_cpu_data.x86_model);
1142
1143 return 0;
1144}
1145
1146
1147
1148
1149
1150static void intel_idle_cpuidle_devices_uninit(void)
1151{
1152 int i;
1153 struct cpuidle_device *dev;
1154
1155 for_each_online_cpu(i) {
1156 dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
1157 cpuidle_unregister_device(dev);
1158 }
1159}
1160
1161
1162
1163
1164
1165
1166
1167static void ivt_idle_state_table_update(void)
1168{
1169
1170 int cpu, package_num, num_sockets = 1;
1171
1172 for_each_online_cpu(cpu) {
1173 package_num = topology_physical_package_id(cpu);
1174 if (package_num + 1 > num_sockets) {
1175 num_sockets = package_num + 1;
1176
1177 if (num_sockets > 4) {
1178 cpuidle_state_table = ivt_cstates_8s;
1179 return;
1180 }
1181 }
1182 }
1183
1184 if (num_sockets > 2)
1185 cpuidle_state_table = ivt_cstates_4s;
1186
1187
1188}
1189
1190
1191
1192
1193
1194static unsigned int irtl_ns_units[] = {
1195 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
1196
1197static unsigned long long irtl_2_usec(unsigned long long irtl)
1198{
1199 unsigned long long ns;
1200
1201 if (!irtl)
1202 return 0;
1203
1204 ns = irtl_ns_units[(irtl >> 10) & 0x7];
1205
1206 return div64_u64((irtl & 0x3FF) * ns, 1000);
1207}
1208
1209
1210
1211
1212
1213
1214static void bxt_idle_state_table_update(void)
1215{
1216 unsigned long long msr;
1217 unsigned int usec;
1218
1219 rdmsrl(MSR_PKGC6_IRTL, msr);
1220 usec = irtl_2_usec(msr);
1221 if (usec) {
1222 bxt_cstates[2].exit_latency = usec;
1223 bxt_cstates[2].target_residency = usec;
1224 }
1225
1226 rdmsrl(MSR_PKGC7_IRTL, msr);
1227 usec = irtl_2_usec(msr);
1228 if (usec) {
1229 bxt_cstates[3].exit_latency = usec;
1230 bxt_cstates[3].target_residency = usec;
1231 }
1232
1233 rdmsrl(MSR_PKGC8_IRTL, msr);
1234 usec = irtl_2_usec(msr);
1235 if (usec) {
1236 bxt_cstates[4].exit_latency = usec;
1237 bxt_cstates[4].target_residency = usec;
1238 }
1239
1240 rdmsrl(MSR_PKGC9_IRTL, msr);
1241 usec = irtl_2_usec(msr);
1242 if (usec) {
1243 bxt_cstates[5].exit_latency = usec;
1244 bxt_cstates[5].target_residency = usec;
1245 }
1246
1247 rdmsrl(MSR_PKGC10_IRTL, msr);
1248 usec = irtl_2_usec(msr);
1249 if (usec) {
1250 bxt_cstates[6].exit_latency = usec;
1251 bxt_cstates[6].target_residency = usec;
1252 }
1253
1254}
1255
1256
1257
1258
1259
1260
1261static void sklh_idle_state_table_update(void)
1262{
1263 unsigned long long msr;
1264 unsigned int eax, ebx, ecx, edx;
1265
1266
1267
1268 if (max_cstate <= 7)
1269 return;
1270
1271
1272 if ((mwait_substates & (0xF << 28)) == 0)
1273 return;
1274
1275 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1276
1277
1278 if ((msr & 0xF) != 8)
1279 return;
1280
1281 ecx = 0;
1282 cpuid(7, &eax, &ebx, &ecx, &edx);
1283
1284
1285 if (ebx & (1 << 2)) {
1286
1287 rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
1288
1289
1290 if (msr & (1 << 18))
1291 return;
1292 }
1293
1294 skl_cstates[5].disabled = 1;
1295 skl_cstates[6].disabled = 1;
1296}
1297
1298
1299
1300
1301
1302
1303static void intel_idle_state_table_update(void)
1304{
1305 switch (boot_cpu_data.x86_model) {
1306
1307 case INTEL_FAM6_IVYBRIDGE_X:
1308 ivt_idle_state_table_update();
1309 break;
1310 case INTEL_FAM6_ATOM_GOLDMONT:
1311 case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1312 bxt_idle_state_table_update();
1313 break;
1314 case INTEL_FAM6_SKYLAKE_DESKTOP:
1315 sklh_idle_state_table_update();
1316 break;
1317 }
1318}
1319
1320
1321
1322
1323
1324static void __init intel_idle_cpuidle_driver_init(void)
1325{
1326 int cstate;
1327 struct cpuidle_driver *drv = &intel_idle_driver;
1328
1329 intel_idle_state_table_update();
1330
1331 cpuidle_poll_state_init(drv);
1332 drv->state_count = 1;
1333
1334 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
1335 int num_substates, mwait_hint, mwait_cstate;
1336
1337 if ((cpuidle_state_table[cstate].enter == NULL) &&
1338 (cpuidle_state_table[cstate].enter_s2idle == NULL))
1339 break;
1340
1341 if (cstate + 1 > max_cstate) {
1342 pr_info("max_cstate %d reached\n", max_cstate);
1343 break;
1344 }
1345
1346 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
1347 mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint);
1348
1349
1350 num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4))
1351 & MWAIT_SUBSTATE_MASK;
1352
1353
1354 if (num_substates == 0)
1355 continue;
1356
1357
1358 if (cpuidle_state_table[cstate].disabled != 0) {
1359 pr_debug("state %s is disabled\n",
1360 cpuidle_state_table[cstate].name);
1361 continue;
1362 }
1363
1364
1365 if (((mwait_cstate + 1) > 2) &&
1366 !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1367 mark_tsc_unstable("TSC halts in idle"
1368 " states deeper than C2");
1369
1370 drv->states[drv->state_count] =
1371 cpuidle_state_table[cstate];
1372
1373 drv->state_count += 1;
1374 }
1375
1376 if (icpu->byt_auto_demotion_disable_flag) {
1377 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
1378 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
1379 }
1380}
1381
1382
1383
1384
1385
1386
1387
1388static int intel_idle_cpu_init(unsigned int cpu)
1389{
1390 struct cpuidle_device *dev;
1391
1392 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1393 dev->cpu = cpu;
1394
1395 if (cpuidle_register_device(dev)) {
1396 pr_debug("cpuidle_register_device %d failed!\n", cpu);
1397 return -EIO;
1398 }
1399
1400 if (icpu->auto_demotion_disable_flags)
1401 auto_demotion_disable();
1402
1403 if (icpu->disable_promotion_to_c1e)
1404 c1e_promotion_disable();
1405
1406 return 0;
1407}
1408
1409static int intel_idle_cpu_online(unsigned int cpu)
1410{
1411 struct cpuidle_device *dev;
1412
1413 if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
1414 __setup_broadcast_timer(true);
1415
1416
1417
1418
1419
1420
1421 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1422 if (!dev->registered)
1423 return intel_idle_cpu_init(cpu);
1424
1425 return 0;
1426}
1427
1428static int __init intel_idle_init(void)
1429{
1430 int retval;
1431
1432
1433 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
1434 return -ENODEV;
1435
1436 retval = intel_idle_probe();
1437 if (retval)
1438 return retval;
1439
1440 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
1441 if (intel_idle_cpuidle_devices == NULL)
1442 return -ENOMEM;
1443
1444 intel_idle_cpuidle_driver_init();
1445 retval = cpuidle_register_driver(&intel_idle_driver);
1446 if (retval) {
1447 struct cpuidle_driver *drv = cpuidle_get_driver();
1448 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
1449 drv ? drv->name : "none");
1450 goto init_driver_fail;
1451 }
1452
1453 if (boot_cpu_has(X86_FEATURE_ARAT))
1454 lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
1455
1456 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
1457 intel_idle_cpu_online, NULL);
1458 if (retval < 0)
1459 goto hp_setup_fail;
1460
1461 pr_debug("lapic_timer_reliable_states 0x%x\n",
1462 lapic_timer_reliable_states);
1463
1464 return 0;
1465
1466hp_setup_fail:
1467 intel_idle_cpuidle_devices_uninit();
1468 cpuidle_unregister_driver(&intel_idle_driver);
1469init_driver_fail:
1470 free_percpu(intel_idle_cpuidle_devices);
1471 return retval;
1472
1473}
1474device_initcall(intel_idle_init);
1475
1476
1477
1478
1479
1480
1481
1482module_param(max_cstate, int, 0444);
1483
