1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14
15#include <linux/kernel.h>
16#include <linux/kernel_stat.h>
17#include <linux/module.h>
18#include <linux/ktime.h>
19#include <linux/hrtimer.h>
20#include <linux/tick.h>
21#include <linux/slab.h>
22#include <linux/sched.h>
23#include <linux/list.h>
24#include <linux/cpu.h>
25#include <linux/cpufreq.h>
26#include <linux/sysfs.h>
27#include <linux/types.h>
28#include <linux/fs.h>
29#include <linux/debugfs.h>
30#include <linux/acpi.h>
31#include <linux/vmalloc.h>
32#include <trace/events/power.h>
33
34#include <asm/div64.h>
35#include <asm/msr.h>
36#include <asm/cpu_device_id.h>
37#include <asm/cpufeature.h>
38#include <asm/intel-family.h>
39
40#define ATOM_RATIOS 0x66a
41#define ATOM_VIDS 0x66b
42#define ATOM_TURBO_RATIOS 0x66c
43#define ATOM_TURBO_VIDS 0x66d
44
45#ifdef CONFIG_ACPI
46#include <acpi/processor.h>
47#endif
48
49#define FRAC_BITS 8
50#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
51#define fp_toint(X) ((X) >> FRAC_BITS)
52
53#define EXT_BITS 6
54#define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
55
56static inline int32_t mul_fp(int32_t x, int32_t y)
57{
58 return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
59}
60
61static inline int32_t div_fp(s64 x, s64 y)
62{
63 return div64_s64((int64_t)x << FRAC_BITS, y);
64}
65
66static inline int ceiling_fp(int32_t x)
67{
68 int mask, ret;
69
70 ret = fp_toint(x);
71 mask = (1 << FRAC_BITS) - 1;
72 if (x & mask)
73 ret += 1;
74 return ret;
75}
76
77static inline u64 mul_ext_fp(u64 x, u64 y)
78{
79 return (x * y) >> EXT_FRAC_BITS;
80}
81
82static inline u64 div_ext_fp(u64 x, u64 y)
83{
84 return div64_u64(x << EXT_FRAC_BITS, y);
85}
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105struct sample {
106 int32_t core_avg_perf;
107 int32_t busy_scaled;
108 u64 aperf;
109 u64 mperf;
110 u64 tsc;
111 u64 time;
112};
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128struct pstate_data {
129 int current_pstate;
130 int min_pstate;
131 int max_pstate;
132 int max_pstate_physical;
133 int scaling;
134 int turbo_pstate;
135};
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150struct vid_data {
151 int min;
152 int max;
153 int turbo;
154 int32_t ratio;
155};
156
157
158
159
160
161
162
163
164
165
166
167
168
169struct _pid {
170 int setpoint;
171 int32_t integral;
172 int32_t p_gain;
173 int32_t i_gain;
174 int32_t d_gain;
175 int deadband;
176 int32_t last_err;
177};
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202struct cpudata {
203 int cpu;
204
205 unsigned int policy;
206 struct update_util_data update_util;
207 bool update_util_set;
208
209 struct pstate_data pstate;
210 struct vid_data vid;
211 struct _pid pid;
212
213 u64 last_update;
214 u64 last_sample_time;
215 u64 prev_aperf;
216 u64 prev_mperf;
217 u64 prev_tsc;
218 u64 prev_cummulative_iowait;
219 struct sample sample;
220#ifdef CONFIG_ACPI
221 struct acpi_processor_performance acpi_perf_data;
222 bool valid_pss_table;
223#endif
224 unsigned int iowait_boost;
225};
226
227static struct cpudata **all_cpu_data;
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242struct pstate_adjust_policy {
243 int sample_rate_ms;
244 s64 sample_rate_ns;
245 int deadband;
246 int setpoint;
247 int p_gain_pct;
248 int d_gain_pct;
249 int i_gain_pct;
250 bool boost_iowait;
251};
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267struct pstate_funcs {
268 int (*get_max)(void);
269 int (*get_max_physical)(void);
270 int (*get_min)(void);
271 int (*get_turbo)(void);
272 int (*get_scaling)(void);
273 u64 (*get_val)(struct cpudata*, int pstate);
274 void (*get_vid)(struct cpudata *);
275 int32_t (*get_target_pstate)(struct cpudata *);
276};
277
278
279
280
281
282
283struct cpu_defaults {
284 struct pstate_adjust_policy pid_policy;
285 struct pstate_funcs funcs;
286};
287
288static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu);
289static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
290
291static struct pstate_adjust_policy pid_params __read_mostly;
292static struct pstate_funcs pstate_funcs __read_mostly;
293static int hwp_active __read_mostly;
294
295#ifdef CONFIG_ACPI
296static bool acpi_ppc;
297#endif
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326struct perf_limits {
327 int no_turbo;
328 int turbo_disabled;
329 int max_perf_pct;
330 int min_perf_pct;
331 int32_t max_perf;
332 int32_t min_perf;
333 int max_policy_pct;
334 int max_sysfs_pct;
335 int min_policy_pct;
336 int min_sysfs_pct;
337};
338
339static struct perf_limits performance_limits = {
340 .no_turbo = 0,
341 .turbo_disabled = 0,
342 .max_perf_pct = 100,
343 .max_perf = int_tofp(1),
344 .min_perf_pct = 100,
345 .min_perf = int_tofp(1),
346 .max_policy_pct = 100,
347 .max_sysfs_pct = 100,
348 .min_policy_pct = 0,
349 .min_sysfs_pct = 0,
350};
351
352static struct perf_limits powersave_limits = {
353 .no_turbo = 0,
354 .turbo_disabled = 0,
355 .max_perf_pct = 100,
356 .max_perf = int_tofp(1),
357 .min_perf_pct = 0,
358 .min_perf = 0,
359 .max_policy_pct = 100,
360 .max_sysfs_pct = 100,
361 .min_policy_pct = 0,
362 .min_sysfs_pct = 0,
363};
364
365#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
366static struct perf_limits *limits = &performance_limits;
367#else
368static struct perf_limits *limits = &powersave_limits;
369#endif
370
371#ifdef CONFIG_ACPI
372
373static bool intel_pstate_get_ppc_enable_status(void)
374{
375 if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
376 acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
377 return true;
378
379 return acpi_ppc;
380}
381
382static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
383{
384 struct cpudata *cpu;
385 int ret;
386 int i;
387
388 if (hwp_active)
389 return;
390
391 if (!intel_pstate_get_ppc_enable_status())
392 return;
393
394 cpu = all_cpu_data[policy->cpu];
395
396 ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
397 policy->cpu);
398 if (ret)
399 return;
400
401
402
403
404
405
406 if (cpu->acpi_perf_data.control_register.space_id !=
407 ACPI_ADR_SPACE_FIXED_HARDWARE)
408 goto err;
409
410
411
412
413
414 if (cpu->acpi_perf_data.state_count < 2)
415 goto err;
416
417 pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
418 for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
419 pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n",
420 (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
421 (u32) cpu->acpi_perf_data.states[i].core_frequency,
422 (u32) cpu->acpi_perf_data.states[i].power,
423 (u32) cpu->acpi_perf_data.states[i].control);
424 }
425
426
427
428
429
430
431
432
433
434
435
436
437 if (!limits->turbo_disabled)
438 cpu->acpi_perf_data.states[0].core_frequency =
439 policy->cpuinfo.max_freq / 1000;
440 cpu->valid_pss_table = true;
441 pr_debug("_PPC limits will be enforced\n");
442
443 return;
444
445 err:
446 cpu->valid_pss_table = false;
447 acpi_processor_unregister_performance(policy->cpu);
448}
449
450static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
451{
452 struct cpudata *cpu;
453
454 cpu = all_cpu_data[policy->cpu];
455 if (!cpu->valid_pss_table)
456 return;
457
458 acpi_processor_unregister_performance(policy->cpu);
459}
460
461#else
462static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
463{
464}
465
466static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
467{
468}
469#endif
470
471static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
472 int deadband, int integral) {
473 pid->setpoint = int_tofp(setpoint);
474 pid->deadband = int_tofp(deadband);
475 pid->integral = int_tofp(integral);
476 pid->last_err = int_tofp(setpoint) - int_tofp(busy);
477}
478
479static inline void pid_p_gain_set(struct _pid *pid, int percent)
480{
481 pid->p_gain = div_fp(percent, 100);
482}
483
484static inline void pid_i_gain_set(struct _pid *pid, int percent)
485{
486 pid->i_gain = div_fp(percent, 100);
487}
488
489static inline void pid_d_gain_set(struct _pid *pid, int percent)
490{
491 pid->d_gain = div_fp(percent, 100);
492}
493
494static signed int pid_calc(struct _pid *pid, int32_t busy)
495{
496 signed int result;
497 int32_t pterm, dterm, fp_error;
498 int32_t integral_limit;
499
500 fp_error = pid->setpoint - busy;
501
502 if (abs(fp_error) <= pid->deadband)
503 return 0;
504
505 pterm = mul_fp(pid->p_gain, fp_error);
506
507 pid->integral += fp_error;
508
509
510
511
512
513
514
515
516
517 integral_limit = int_tofp(30);
518 if (pid->integral > integral_limit)
519 pid->integral = integral_limit;
520 if (pid->integral < -integral_limit)
521 pid->integral = -integral_limit;
522
523 dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
524 pid->last_err = fp_error;
525
526 result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
527 result = result + (1 << (FRAC_BITS-1));
528 return (signed int)fp_toint(result);
529}
530
531static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
532{
533 pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
534 pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
535 pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
536
537 pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
538}
539
540static inline void intel_pstate_reset_all_pid(void)
541{
542 unsigned int cpu;
543
544 for_each_online_cpu(cpu) {
545 if (all_cpu_data[cpu])
546 intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
547 }
548}
549
550static inline void update_turbo_state(void)
551{
552 u64 misc_en;
553 struct cpudata *cpu;
554
555 cpu = all_cpu_data[0];
556 rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
557 limits->turbo_disabled =
558 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
559 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
560}
561
562static void intel_pstate_hwp_set(const struct cpumask *cpumask)
563{
564 int min, hw_min, max, hw_max, cpu, range, adj_range;
565 u64 value, cap;
566
567 for_each_cpu(cpu, cpumask) {
568 rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
569 hw_min = HWP_LOWEST_PERF(cap);
570 hw_max = HWP_HIGHEST_PERF(cap);
571 range = hw_max - hw_min;
572
573 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
574 adj_range = limits->min_perf_pct * range / 100;
575 min = hw_min + adj_range;
576 value &= ~HWP_MIN_PERF(~0L);
577 value |= HWP_MIN_PERF(min);
578
579 adj_range = limits->max_perf_pct * range / 100;
580 max = hw_min + adj_range;
581 if (limits->no_turbo) {
582 hw_max = HWP_GUARANTEED_PERF(cap);
583 if (hw_max < max)
584 max = hw_max;
585 }
586
587 value &= ~HWP_MAX_PERF(~0L);
588 value |= HWP_MAX_PERF(max);
589 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
590 }
591}
592
593static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy)
594{
595 if (hwp_active)
596 intel_pstate_hwp_set(policy->cpus);
597
598 return 0;
599}
600
601static void intel_pstate_hwp_set_online_cpus(void)
602{
603 get_online_cpus();
604 intel_pstate_hwp_set(cpu_online_mask);
605 put_online_cpus();
606}
607
608
609static int pid_param_set(void *data, u64 val)
610{
611 *(u32 *)data = val;
612 intel_pstate_reset_all_pid();
613 return 0;
614}
615
616static int pid_param_get(void *data, u64 *val)
617{
618 *val = *(u32 *)data;
619 return 0;
620}
621DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
622
623struct pid_param {
624 char *name;
625 void *value;
626};
627
628static struct pid_param pid_files[] = {
629 {"sample_rate_ms", &pid_params.sample_rate_ms},
630 {"d_gain_pct", &pid_params.d_gain_pct},
631 {"i_gain_pct", &pid_params.i_gain_pct},
632 {"deadband", &pid_params.deadband},
633 {"setpoint", &pid_params.setpoint},
634 {"p_gain_pct", &pid_params.p_gain_pct},
635 {NULL, NULL}
636};
637
638static void __init intel_pstate_debug_expose_params(void)
639{
640 struct dentry *debugfs_parent;
641 int i = 0;
642
643 if (hwp_active)
644 return;
645 debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
646 if (IS_ERR_OR_NULL(debugfs_parent))
647 return;
648 while (pid_files[i].name) {
649 debugfs_create_file(pid_files[i].name, 0660,
650 debugfs_parent, pid_files[i].value,
651 &fops_pid_param);
652 i++;
653 }
654}
655
656
657
658
659#define show_one(file_name, object) \
660 static ssize_t show_##file_name \
661 (struct kobject *kobj, struct attribute *attr, char *buf) \
662 { \
663 return sprintf(buf, "%u\n", limits->object); \
664 }
665
666static ssize_t show_turbo_pct(struct kobject *kobj,
667 struct attribute *attr, char *buf)
668{
669 struct cpudata *cpu;
670 int total, no_turbo, turbo_pct;
671 uint32_t turbo_fp;
672
673 cpu = all_cpu_data[0];
674
675 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
676 no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
677 turbo_fp = div_fp(no_turbo, total);
678 turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
679 return sprintf(buf, "%u\n", turbo_pct);
680}
681
682static ssize_t show_num_pstates(struct kobject *kobj,
683 struct attribute *attr, char *buf)
684{
685 struct cpudata *cpu;
686 int total;
687
688 cpu = all_cpu_data[0];
689 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
690 return sprintf(buf, "%u\n", total);
691}
692
693static ssize_t show_no_turbo(struct kobject *kobj,
694 struct attribute *attr, char *buf)
695{
696 ssize_t ret;
697
698 update_turbo_state();
699 if (limits->turbo_disabled)
700 ret = sprintf(buf, "%u\n", limits->turbo_disabled);
701 else
702 ret = sprintf(buf, "%u\n", limits->no_turbo);
703
704 return ret;
705}
706
707static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
708 const char *buf, size_t count)
709{
710 unsigned int input;
711 int ret;
712
713 ret = sscanf(buf, "%u", &input);
714 if (ret != 1)
715 return -EINVAL;
716
717 update_turbo_state();
718 if (limits->turbo_disabled) {
719 pr_warn("Turbo disabled by BIOS or unavailable on processor\n");
720 return -EPERM;
721 }
722
723 limits->no_turbo = clamp_t(int, input, 0, 1);
724
725 if (hwp_active)
726 intel_pstate_hwp_set_online_cpus();
727
728 return count;
729}
730
731static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
732 const char *buf, size_t count)
733{
734 unsigned int input;
735 int ret;
736
737 ret = sscanf(buf, "%u", &input);
738 if (ret != 1)
739 return -EINVAL;
740
741 limits->max_sysfs_pct = clamp_t(int, input, 0 , 100);
742 limits->max_perf_pct = min(limits->max_policy_pct,
743 limits->max_sysfs_pct);
744 limits->max_perf_pct = max(limits->min_policy_pct,
745 limits->max_perf_pct);
746 limits->max_perf_pct = max(limits->min_perf_pct,
747 limits->max_perf_pct);
748 limits->max_perf = div_fp(limits->max_perf_pct, 100);
749
750 if (hwp_active)
751 intel_pstate_hwp_set_online_cpus();
752 return count;
753}
754
755static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
756 const char *buf, size_t count)
757{
758 unsigned int input;
759 int ret;
760
761 ret = sscanf(buf, "%u", &input);
762 if (ret != 1)
763 return -EINVAL;
764
765 limits->min_sysfs_pct = clamp_t(int, input, 0 , 100);
766 limits->min_perf_pct = max(limits->min_policy_pct,
767 limits->min_sysfs_pct);
768 limits->min_perf_pct = min(limits->max_policy_pct,
769 limits->min_perf_pct);
770 limits->min_perf_pct = min(limits->max_perf_pct,
771 limits->min_perf_pct);
772 limits->min_perf = div_fp(limits->min_perf_pct, 100);
773
774 if (hwp_active)
775 intel_pstate_hwp_set_online_cpus();
776 return count;
777}
778
779show_one(max_perf_pct, max_perf_pct);
780show_one(min_perf_pct, min_perf_pct);
781
782define_one_global_rw(no_turbo);
783define_one_global_rw(max_perf_pct);
784define_one_global_rw(min_perf_pct);
785define_one_global_ro(turbo_pct);
786define_one_global_ro(num_pstates);
787
788static struct attribute *intel_pstate_attributes[] = {
789 &no_turbo.attr,
790 &max_perf_pct.attr,
791 &min_perf_pct.attr,
792 &turbo_pct.attr,
793 &num_pstates.attr,
794 NULL
795};
796
797static struct attribute_group intel_pstate_attr_group = {
798 .attrs = intel_pstate_attributes,
799};
800
801static void __init intel_pstate_sysfs_expose_params(void)
802{
803 struct kobject *intel_pstate_kobject;
804 int rc;
805
806 intel_pstate_kobject = kobject_create_and_add("intel_pstate",
807 &cpu_subsys.dev_root->kobj);
808 BUG_ON(!intel_pstate_kobject);
809 rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
810 BUG_ON(rc);
811}
812
813
814static void intel_pstate_hwp_enable(struct cpudata *cpudata)
815{
816
817 if (static_cpu_has(X86_FEATURE_HWP_NOTIFY))
818 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
819
820 wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
821}
822
823static int atom_get_min_pstate(void)
824{
825 u64 value;
826
827 rdmsrl(ATOM_RATIOS, value);
828 return (value >> 8) & 0x7F;
829}
830
831static int atom_get_max_pstate(void)
832{
833 u64 value;
834
835 rdmsrl(ATOM_RATIOS, value);
836 return (value >> 16) & 0x7F;
837}
838
839static int atom_get_turbo_pstate(void)
840{
841 u64 value;
842
843 rdmsrl(ATOM_TURBO_RATIOS, value);
844 return value & 0x7F;
845}
846
847static u64 atom_get_val(struct cpudata *cpudata, int pstate)
848{
849 u64 val;
850 int32_t vid_fp;
851 u32 vid;
852
853 val = (u64)pstate << 8;
854 if (limits->no_turbo && !limits->turbo_disabled)
855 val |= (u64)1 << 32;
856
857 vid_fp = cpudata->vid.min + mul_fp(
858 int_tofp(pstate - cpudata->pstate.min_pstate),
859 cpudata->vid.ratio);
860
861 vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
862 vid = ceiling_fp(vid_fp);
863
864 if (pstate > cpudata->pstate.max_pstate)
865 vid = cpudata->vid.turbo;
866
867 return val | vid;
868}
869
870static int silvermont_get_scaling(void)
871{
872 u64 value;
873 int i;
874
875 static int silvermont_freq_table[] = {
876 83300, 100000, 133300, 116700, 80000};
877
878 rdmsrl(MSR_FSB_FREQ, value);
879 i = value & 0x7;
880 WARN_ON(i > 4);
881
882 return silvermont_freq_table[i];
883}
884
885static int airmont_get_scaling(void)
886{
887 u64 value;
888 int i;
889
890 static int airmont_freq_table[] = {
891 83300, 100000, 133300, 116700, 80000,
892 93300, 90000, 88900, 87500};
893
894 rdmsrl(MSR_FSB_FREQ, value);
895 i = value & 0xF;
896 WARN_ON(i > 8);
897
898 return airmont_freq_table[i];
899}
900
901static void atom_get_vid(struct cpudata *cpudata)
902{
903 u64 value;
904
905 rdmsrl(ATOM_VIDS, value);
906 cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
907 cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
908 cpudata->vid.ratio = div_fp(
909 cpudata->vid.max - cpudata->vid.min,
910 int_tofp(cpudata->pstate.max_pstate -
911 cpudata->pstate.min_pstate));
912
913 rdmsrl(ATOM_TURBO_VIDS, value);
914 cpudata->vid.turbo = value & 0x7f;
915}
916
917static int core_get_min_pstate(void)
918{
919 u64 value;
920
921 rdmsrl(MSR_PLATFORM_INFO, value);
922 return (value >> 40) & 0xFF;
923}
924
925static int core_get_max_pstate_physical(void)
926{
927 u64 value;
928
929 rdmsrl(MSR_PLATFORM_INFO, value);
930 return (value >> 8) & 0xFF;
931}
932
933static int core_get_max_pstate(void)
934{
935 u64 tar;
936 u64 plat_info;
937 int max_pstate;
938 int err;
939
940 rdmsrl(MSR_PLATFORM_INFO, plat_info);
941 max_pstate = (plat_info >> 8) & 0xFF;
942
943 err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
944 if (!err) {
945
946 if (plat_info & 0x600000000) {
947 u64 tdp_ctrl;
948 u64 tdp_ratio;
949 int tdp_msr;
950
951 err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
952 if (err)
953 goto skip_tar;
954
955 tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x3);
956 err = rdmsrl_safe(tdp_msr, &tdp_ratio);
957 if (err)
958 goto skip_tar;
959
960
961 if (tdp_ctrl)
962 tdp_ratio >>= 16;
963
964 tdp_ratio &= 0xff;
965 if (tdp_ratio - 1 == tar) {
966 max_pstate = tar;
967 pr_debug("max_pstate=TAC %x\n", max_pstate);
968 } else {
969 goto skip_tar;
970 }
971 }
972 }
973
974skip_tar:
975 return max_pstate;
976}
977
978static int core_get_turbo_pstate(void)
979{
980 u64 value;
981 int nont, ret;
982
983 rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
984 nont = core_get_max_pstate();
985 ret = (value) & 255;
986 if (ret <= nont)
987 ret = nont;
988 return ret;
989}
990
991static inline int core_get_scaling(void)
992{
993 return 100000;
994}
995
996static u64 core_get_val(struct cpudata *cpudata, int pstate)
997{
998 u64 val;
999
1000 val = (u64)pstate << 8;
1001 if (limits->no_turbo && !limits->turbo_disabled)
1002 val |= (u64)1 << 32;
1003
1004 return val;
1005}
1006
1007static int knl_get_turbo_pstate(void)
1008{
1009 u64 value;
1010 int nont, ret;
1011
1012 rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
1013 nont = core_get_max_pstate();
1014 ret = (((value) >> 8) & 0xFF);
1015 if (ret <= nont)
1016 ret = nont;
1017 return ret;
1018}
1019
1020static struct cpu_defaults core_params = {
1021 .pid_policy = {
1022 .sample_rate_ms = 10,
1023 .deadband = 0,
1024 .setpoint = 97,
1025 .p_gain_pct = 20,
1026 .d_gain_pct = 0,
1027 .i_gain_pct = 0,
1028 },
1029 .funcs = {
1030 .get_max = core_get_max_pstate,
1031 .get_max_physical = core_get_max_pstate_physical,
1032 .get_min = core_get_min_pstate,
1033 .get_turbo = core_get_turbo_pstate,
1034 .get_scaling = core_get_scaling,
1035 .get_val = core_get_val,
1036 .get_target_pstate = get_target_pstate_use_performance,
1037 },
1038};
1039
1040static const struct cpu_defaults silvermont_params = {
1041 .pid_policy = {
1042 .sample_rate_ms = 10,
1043 .deadband = 0,
1044 .setpoint = 60,
1045 .p_gain_pct = 14,
1046 .d_gain_pct = 0,
1047 .i_gain_pct = 4,
1048 .boost_iowait = true,
1049 },
1050 .funcs = {
1051 .get_max = atom_get_max_pstate,
1052 .get_max_physical = atom_get_max_pstate,
1053 .get_min = atom_get_min_pstate,
1054 .get_turbo = atom_get_turbo_pstate,
1055 .get_val = atom_get_val,
1056 .get_scaling = silvermont_get_scaling,
1057 .get_vid = atom_get_vid,
1058 .get_target_pstate = get_target_pstate_use_cpu_load,
1059 },
1060};
1061
1062static const struct cpu_defaults airmont_params = {
1063 .pid_policy = {
1064 .sample_rate_ms = 10,
1065 .deadband = 0,
1066 .setpoint = 60,
1067 .p_gain_pct = 14,
1068 .d_gain_pct = 0,
1069 .i_gain_pct = 4,
1070 .boost_iowait = true,
1071 },
1072 .funcs = {
1073 .get_max = atom_get_max_pstate,
1074 .get_max_physical = atom_get_max_pstate,
1075 .get_min = atom_get_min_pstate,
1076 .get_turbo = atom_get_turbo_pstate,
1077 .get_val = atom_get_val,
1078 .get_scaling = airmont_get_scaling,
1079 .get_vid = atom_get_vid,
1080 .get_target_pstate = get_target_pstate_use_cpu_load,
1081 },
1082};
1083
1084static const struct cpu_defaults knl_params = {
1085 .pid_policy = {
1086 .sample_rate_ms = 10,
1087 .deadband = 0,
1088 .setpoint = 97,
1089 .p_gain_pct = 20,
1090 .d_gain_pct = 0,
1091 .i_gain_pct = 0,
1092 },
1093 .funcs = {
1094 .get_max = core_get_max_pstate,
1095 .get_max_physical = core_get_max_pstate_physical,
1096 .get_min = core_get_min_pstate,
1097 .get_turbo = knl_get_turbo_pstate,
1098 .get_scaling = core_get_scaling,
1099 .get_val = core_get_val,
1100 .get_target_pstate = get_target_pstate_use_performance,
1101 },
1102};
1103
1104static const struct cpu_defaults bxt_params = {
1105 .pid_policy = {
1106 .sample_rate_ms = 10,
1107 .deadband = 0,
1108 .setpoint = 60,
1109 .p_gain_pct = 14,
1110 .d_gain_pct = 0,
1111 .i_gain_pct = 4,
1112 .boost_iowait = true,
1113 },
1114 .funcs = {
1115 .get_max = core_get_max_pstate,
1116 .get_max_physical = core_get_max_pstate_physical,
1117 .get_min = core_get_min_pstate,
1118 .get_turbo = core_get_turbo_pstate,
1119 .get_scaling = core_get_scaling,
1120 .get_val = core_get_val,
1121 .get_target_pstate = get_target_pstate_use_cpu_load,
1122 },
1123};
1124
1125static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
1126{
1127 int max_perf = cpu->pstate.turbo_pstate;
1128 int max_perf_adj;
1129 int min_perf;
1130
1131 if (limits->no_turbo || limits->turbo_disabled)
1132 max_perf = cpu->pstate.max_pstate;
1133
1134
1135
1136
1137
1138
1139 max_perf_adj = fp_toint(max_perf * limits->max_perf);
1140 *max = clamp_t(int, max_perf_adj,
1141 cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
1142
1143 min_perf = fp_toint(max_perf * limits->min_perf);
1144 *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
1145}
1146
1147static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
1148{
1149 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1150 cpu->pstate.current_pstate = pstate;
1151
1152
1153
1154
1155
1156 wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
1157 pstate_funcs.get_val(cpu, pstate));
1158}
1159
1160static void intel_pstate_set_min_pstate(struct cpudata *cpu)
1161{
1162 intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
1163}
1164
1165static void intel_pstate_max_within_limits(struct cpudata *cpu)
1166{
1167 int min_pstate, max_pstate;
1168
1169 update_turbo_state();
1170 intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate);
1171 intel_pstate_set_pstate(cpu, max_pstate);
1172}
1173
1174static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
1175{
1176 cpu->pstate.min_pstate = pstate_funcs.get_min();
1177 cpu->pstate.max_pstate = pstate_funcs.get_max();
1178 cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
1179 cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
1180 cpu->pstate.scaling = pstate_funcs.get_scaling();
1181
1182 if (pstate_funcs.get_vid)
1183 pstate_funcs.get_vid(cpu);
1184
1185 intel_pstate_set_min_pstate(cpu);
1186}
1187
1188static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
1189{
1190 struct sample *sample = &cpu->sample;
1191
1192 sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
1193}
1194
1195static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
1196{
1197 u64 aperf, mperf;
1198 unsigned long flags;
1199 u64 tsc;
1200
1201 local_irq_save(flags);
1202 rdmsrl(MSR_IA32_APERF, aperf);
1203 rdmsrl(MSR_IA32_MPERF, mperf);
1204 tsc = rdtsc();
1205 if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
1206 local_irq_restore(flags);
1207 return false;
1208 }
1209 local_irq_restore(flags);
1210
1211 cpu->last_sample_time = cpu->sample.time;
1212 cpu->sample.time = time;
1213 cpu->sample.aperf = aperf;
1214 cpu->sample.mperf = mperf;
1215 cpu->sample.tsc = tsc;
1216 cpu->sample.aperf -= cpu->prev_aperf;
1217 cpu->sample.mperf -= cpu->prev_mperf;
1218 cpu->sample.tsc -= cpu->prev_tsc;
1219
1220 cpu->prev_aperf = aperf;
1221 cpu->prev_mperf = mperf;
1222 cpu->prev_tsc = tsc;
1223
1224
1225
1226
1227
1228
1229
1230 return !!cpu->last_sample_time;
1231}
1232
1233static inline int32_t get_avg_frequency(struct cpudata *cpu)
1234{
1235 return mul_ext_fp(cpu->sample.core_avg_perf,
1236 cpu->pstate.max_pstate_physical * cpu->pstate.scaling);
1237}
1238
1239static inline int32_t get_avg_pstate(struct cpudata *cpu)
1240{
1241 return mul_ext_fp(cpu->pstate.max_pstate_physical,
1242 cpu->sample.core_avg_perf);
1243}
1244
1245static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
1246{
1247 struct sample *sample = &cpu->sample;
1248 int32_t busy_frac, boost;
1249 int target, avg_pstate;
1250
1251 busy_frac = div_fp(sample->mperf, sample->tsc);
1252
1253 boost = cpu->iowait_boost;
1254 cpu->iowait_boost >>= 1;
1255
1256 if (busy_frac < boost)
1257 busy_frac = boost;
1258
1259 sample->busy_scaled = busy_frac * 100;
1260
1261 target = limits->no_turbo || limits->turbo_disabled ?
1262 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1263 target += target >> 2;
1264 target = mul_fp(target, busy_frac);
1265 if (target < cpu->pstate.min_pstate)
1266 target = cpu->pstate.min_pstate;
1267
1268
1269
1270
1271
1272
1273
1274
1275 avg_pstate = get_avg_pstate(cpu);
1276 if (avg_pstate > target)
1277 target += (avg_pstate - target) >> 1;
1278
1279 return target;
1280}
1281
1282static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
1283{
1284 int32_t perf_scaled, max_pstate, current_pstate, sample_ratio;
1285 u64 duration_ns;
1286
1287
1288
1289
1290
1291
1292
1293
1294 max_pstate = cpu->pstate.max_pstate_physical;
1295 current_pstate = cpu->pstate.current_pstate;
1296 perf_scaled = mul_ext_fp(cpu->sample.core_avg_perf,
1297 div_fp(100 * max_pstate, current_pstate));
1298
1299
1300
1301
1302
1303
1304
1305 duration_ns = cpu->sample.time - cpu->last_sample_time;
1306 if ((s64)duration_ns > pid_params.sample_rate_ns * 3) {
1307 sample_ratio = div_fp(pid_params.sample_rate_ns, duration_ns);
1308 perf_scaled = mul_fp(perf_scaled, sample_ratio);
1309 } else {
1310 sample_ratio = div_fp(100 * cpu->sample.mperf, cpu->sample.tsc);
1311 if (sample_ratio < int_tofp(1))
1312 perf_scaled = 0;
1313 }
1314
1315 cpu->sample.busy_scaled = perf_scaled;
1316 return cpu->pstate.current_pstate - pid_calc(&cpu->pid, perf_scaled);
1317}
1318
1319static inline void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
1320{
1321 int max_perf, min_perf;
1322
1323 update_turbo_state();
1324
1325 intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
1326 pstate = clamp_t(int, pstate, min_perf, max_perf);
1327 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1328 if (pstate == cpu->pstate.current_pstate)
1329 return;
1330
1331 cpu->pstate.current_pstate = pstate;
1332 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
1333}
1334
1335static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
1336{
1337 int from, target_pstate;
1338 struct sample *sample;
1339
1340 from = cpu->pstate.current_pstate;
1341
1342 target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
1343 cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
1344
1345 intel_pstate_update_pstate(cpu, target_pstate);
1346
1347 sample = &cpu->sample;
1348 trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
1349 fp_toint(sample->busy_scaled),
1350 from,
1351 cpu->pstate.current_pstate,
1352 sample->mperf,
1353 sample->aperf,
1354 sample->tsc,
1355 get_avg_frequency(cpu),
1356 fp_toint(cpu->iowait_boost * 100));
1357}
1358
1359static void intel_pstate_update_util(struct update_util_data *data, u64 time,
1360 unsigned int flags)
1361{
1362 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1363 u64 delta_ns;
1364
1365 if (pid_params.boost_iowait) {
1366 if (flags & SCHED_CPUFREQ_IOWAIT) {
1367 cpu->iowait_boost = int_tofp(1);
1368 } else if (cpu->iowait_boost) {
1369
1370 delta_ns = time - cpu->last_update;
1371 if (delta_ns > TICK_NSEC)
1372 cpu->iowait_boost = 0;
1373 }
1374 cpu->last_update = time;
1375 }
1376
1377 delta_ns = time - cpu->sample.time;
1378 if ((s64)delta_ns >= pid_params.sample_rate_ns) {
1379 bool sample_taken = intel_pstate_sample(cpu, time);
1380
1381 if (sample_taken) {
1382 intel_pstate_calc_avg_perf(cpu);
1383 if (!hwp_active)
1384 intel_pstate_adjust_busy_pstate(cpu);
1385 }
1386 }
1387}
1388
1389#define ICPU(model, policy) \
1390 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1391 (unsigned long)&policy }
1392
1393static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1394 ICPU(INTEL_FAM6_SANDYBRIDGE, core_params),
1395 ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_params),
1396 ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_params),
1397 ICPU(INTEL_FAM6_IVYBRIDGE, core_params),
1398 ICPU(INTEL_FAM6_HASWELL_CORE, core_params),
1399 ICPU(INTEL_FAM6_BROADWELL_CORE, core_params),
1400 ICPU(INTEL_FAM6_IVYBRIDGE_X, core_params),
1401 ICPU(INTEL_FAM6_HASWELL_X, core_params),
1402 ICPU(INTEL_FAM6_HASWELL_ULT, core_params),
1403 ICPU(INTEL_FAM6_HASWELL_GT3E, core_params),
1404 ICPU(INTEL_FAM6_BROADWELL_GT3E, core_params),
1405 ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_params),
1406 ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_params),
1407 ICPU(INTEL_FAM6_BROADWELL_X, core_params),
1408 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params),
1409 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params),
1410 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params),
1411 ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params),
1412 {}
1413};
1414MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1415
1416static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1417 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params),
1418 ICPU(INTEL_FAM6_BROADWELL_X, core_params),
1419 ICPU(INTEL_FAM6_SKYLAKE_X, core_params),
1420 {}
1421};
1422
1423static int intel_pstate_init_cpu(unsigned int cpunum)
1424{
1425 struct cpudata *cpu;
1426
1427 if (!all_cpu_data[cpunum])
1428 all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata),
1429 GFP_KERNEL);
1430 if (!all_cpu_data[cpunum])
1431 return -ENOMEM;
1432
1433 cpu = all_cpu_data[cpunum];
1434
1435 cpu->cpu = cpunum;
1436
1437 if (hwp_active) {
1438 intel_pstate_hwp_enable(cpu);
1439 pid_params.sample_rate_ms = 50;
1440 pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC;
1441 }
1442
1443 intel_pstate_get_cpu_pstates(cpu);
1444
1445 intel_pstate_busy_pid_reset(cpu);
1446
1447 pr_debug("controlling: cpu %d\n", cpunum);
1448
1449 return 0;
1450}
1451
1452static unsigned int intel_pstate_get(unsigned int cpu_num)
1453{
1454 struct cpudata *cpu = all_cpu_data[cpu_num];
1455
1456 return cpu ? get_avg_frequency(cpu) : 0;
1457}
1458
1459static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
1460{
1461 struct cpudata *cpu = all_cpu_data[cpu_num];
1462
1463 if (cpu->update_util_set)
1464 return;
1465
1466
1467 cpu->sample.time = 0;
1468 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
1469 intel_pstate_update_util);
1470 cpu->update_util_set = true;
1471}
1472
1473static void intel_pstate_clear_update_util_hook(unsigned int cpu)
1474{
1475 struct cpudata *cpu_data = all_cpu_data[cpu];
1476
1477 if (!cpu_data->update_util_set)
1478 return;
1479
1480 cpufreq_remove_update_util_hook(cpu);
1481 cpu_data->update_util_set = false;
1482 synchronize_sched();
1483}
1484
1485static void intel_pstate_set_performance_limits(struct perf_limits *limits)
1486{
1487 limits->no_turbo = 0;
1488 limits->turbo_disabled = 0;
1489 limits->max_perf_pct = 100;
1490 limits->max_perf = int_tofp(1);
1491 limits->min_perf_pct = 100;
1492 limits->min_perf = int_tofp(1);
1493 limits->max_policy_pct = 100;
1494 limits->max_sysfs_pct = 100;
1495 limits->min_policy_pct = 0;
1496 limits->min_sysfs_pct = 0;
1497}
1498
1499static int intel_pstate_set_policy(struct cpufreq_policy *policy)
1500{
1501 struct cpudata *cpu;
1502
1503 if (!policy->cpuinfo.max_freq)
1504 return -ENODEV;
1505
1506 pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
1507 policy->cpuinfo.max_freq, policy->max);
1508
1509 cpu = all_cpu_data[policy->cpu];
1510 cpu->policy = policy->policy;
1511
1512 if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
1513 policy->max < policy->cpuinfo.max_freq &&
1514 policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) {
1515 pr_debug("policy->max > max non turbo frequency\n");
1516 policy->max = policy->cpuinfo.max_freq;
1517 }
1518
1519 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
1520 limits = &performance_limits;
1521 if (policy->max >= policy->cpuinfo.max_freq) {
1522 pr_debug("set performance\n");
1523 intel_pstate_set_performance_limits(limits);
1524 goto out;
1525 }
1526 } else {
1527 pr_debug("set powersave\n");
1528 limits = &powersave_limits;
1529 }
1530
1531 limits->min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
1532 limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, 0 , 100);
1533 limits->max_policy_pct = DIV_ROUND_UP(policy->max * 100,
1534 policy->cpuinfo.max_freq);
1535 limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0 , 100);
1536
1537
1538 limits->min_perf_pct = max(limits->min_policy_pct,
1539 limits->min_sysfs_pct);
1540 limits->min_perf_pct = min(limits->max_policy_pct,
1541 limits->min_perf_pct);
1542 limits->max_perf_pct = min(limits->max_policy_pct,
1543 limits->max_sysfs_pct);
1544 limits->max_perf_pct = max(limits->min_policy_pct,
1545 limits->max_perf_pct);
1546
1547
1548 limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct);
1549
1550 limits->min_perf = div_fp(limits->min_perf_pct, 100);
1551 limits->max_perf = div_fp(limits->max_perf_pct, 100);
1552 limits->max_perf = round_up(limits->max_perf, FRAC_BITS);
1553
1554 out:
1555 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
1556
1557
1558
1559
1560 intel_pstate_clear_update_util_hook(policy->cpu);
1561 intel_pstate_max_within_limits(cpu);
1562 }
1563
1564 intel_pstate_set_update_util_hook(policy->cpu);
1565
1566 intel_pstate_hwp_set_policy(policy);
1567
1568 return 0;
1569}
1570
1571static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
1572{
1573 cpufreq_verify_within_cpu_limits(policy);
1574
1575 if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
1576 policy->policy != CPUFREQ_POLICY_PERFORMANCE)
1577 return -EINVAL;
1578
1579 return 0;
1580}
1581
1582static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
1583{
1584 int cpu_num = policy->cpu;
1585 struct cpudata *cpu = all_cpu_data[cpu_num];
1586
1587 pr_debug("CPU %d exiting\n", cpu_num);
1588
1589 intel_pstate_clear_update_util_hook(cpu_num);
1590
1591 if (hwp_active)
1592 return;
1593
1594 intel_pstate_set_min_pstate(cpu);
1595}
1596
1597static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
1598{
1599 struct cpudata *cpu;
1600 int rc;
1601
1602 rc = intel_pstate_init_cpu(policy->cpu);
1603 if (rc)
1604 return rc;
1605
1606 cpu = all_cpu_data[policy->cpu];
1607
1608 if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100)
1609 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
1610 else
1611 policy->policy = CPUFREQ_POLICY_POWERSAVE;
1612
1613 policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
1614 policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1615
1616
1617 policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
1618 update_turbo_state();
1619 policy->cpuinfo.max_freq = limits->turbo_disabled ?
1620 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1621 policy->cpuinfo.max_freq *= cpu->pstate.scaling;
1622
1623 intel_pstate_init_acpi_perf_limits(policy);
1624 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
1625 cpumask_set_cpu(policy->cpu, policy->cpus);
1626
1627 return 0;
1628}
1629
1630static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
1631{
1632 intel_pstate_exit_perf_limits(policy);
1633
1634 return 0;
1635}
1636
1637static struct cpufreq_driver intel_pstate_driver = {
1638 .flags = CPUFREQ_CONST_LOOPS,
1639 .verify = intel_pstate_verify_policy,
1640 .setpolicy = intel_pstate_set_policy,
1641 .resume = intel_pstate_hwp_set_policy,
1642 .get = intel_pstate_get,
1643 .init = intel_pstate_cpu_init,
1644 .exit = intel_pstate_cpu_exit,
1645 .stop_cpu = intel_pstate_stop_cpu,
1646 .name = "intel_pstate",
1647};
1648
1649static int no_load __initdata;
1650static int no_hwp __initdata;
1651static int hwp_only __initdata;
1652static unsigned int force_load __initdata;
1653
1654static int __init intel_pstate_msrs_not_valid(void)
1655{
1656 if (!pstate_funcs.get_max() ||
1657 !pstate_funcs.get_min() ||
1658 !pstate_funcs.get_turbo())
1659 return -ENODEV;
1660
1661 return 0;
1662}
1663
1664static void __init copy_pid_params(struct pstate_adjust_policy *policy)
1665{
1666 pid_params.sample_rate_ms = policy->sample_rate_ms;
1667 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
1668 pid_params.p_gain_pct = policy->p_gain_pct;
1669 pid_params.i_gain_pct = policy->i_gain_pct;
1670 pid_params.d_gain_pct = policy->d_gain_pct;
1671 pid_params.deadband = policy->deadband;
1672 pid_params.setpoint = policy->setpoint;
1673}
1674
1675static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
1676{
1677 pstate_funcs.get_max = funcs->get_max;
1678 pstate_funcs.get_max_physical = funcs->get_max_physical;
1679 pstate_funcs.get_min = funcs->get_min;
1680 pstate_funcs.get_turbo = funcs->get_turbo;
1681 pstate_funcs.get_scaling = funcs->get_scaling;
1682 pstate_funcs.get_val = funcs->get_val;
1683 pstate_funcs.get_vid = funcs->get_vid;
1684 pstate_funcs.get_target_pstate = funcs->get_target_pstate;
1685
1686}
1687
1688#ifdef CONFIG_ACPI
1689
1690static bool __init intel_pstate_no_acpi_pss(void)
1691{
1692 int i;
1693
1694 for_each_possible_cpu(i) {
1695 acpi_status status;
1696 union acpi_object *pss;
1697 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
1698 struct acpi_processor *pr = per_cpu(processors, i);
1699
1700 if (!pr)
1701 continue;
1702
1703 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
1704 if (ACPI_FAILURE(status))
1705 continue;
1706
1707 pss = buffer.pointer;
1708 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
1709 kfree(pss);
1710 return false;
1711 }
1712
1713 kfree(pss);
1714 }
1715
1716 return true;
1717}
1718
1719static bool __init intel_pstate_has_acpi_ppc(void)
1720{
1721 int i;
1722
1723 for_each_possible_cpu(i) {
1724 struct acpi_processor *pr = per_cpu(processors, i);
1725
1726 if (!pr)
1727 continue;
1728 if (acpi_has_method(pr->handle, "_PPC"))
1729 return true;
1730 }
1731 return false;
1732}
1733
1734enum {
1735 PSS,
1736 PPC,
1737};
1738
1739struct hw_vendor_info {
1740 u16 valid;
1741 char oem_id[ACPI_OEM_ID_SIZE];
1742 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
1743 int oem_pwr_table;
1744};
1745
1746
1747static struct hw_vendor_info vendor_info[] __initdata = {
1748 {1, "HP ", "ProLiant", PSS},
1749 {1, "ORACLE", "X4-2 ", PPC},
1750 {1, "ORACLE", "X4-2L ", PPC},
1751 {1, "ORACLE", "X4-2B ", PPC},
1752 {1, "ORACLE", "X3-2 ", PPC},
1753 {1, "ORACLE", "X3-2L ", PPC},
1754 {1, "ORACLE", "X3-2B ", PPC},
1755 {1, "ORACLE", "X4470M2 ", PPC},
1756 {1, "ORACLE", "X4270M3 ", PPC},
1757 {1, "ORACLE", "X4270M2 ", PPC},
1758 {1, "ORACLE", "X4170M2 ", PPC},
1759 {1, "ORACLE", "X4170 M3", PPC},
1760 {1, "ORACLE", "X4275 M3", PPC},
1761 {1, "ORACLE", "X6-2 ", PPC},
1762 {1, "ORACLE", "Sudbury ", PPC},
1763 {0, "", ""},
1764};
1765
1766static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
1767{
1768 struct acpi_table_header hdr;
1769 struct hw_vendor_info *v_info;
1770 const struct x86_cpu_id *id;
1771 u64 misc_pwr;
1772
1773 id = x86_match_cpu(intel_pstate_cpu_oob_ids);
1774 if (id) {
1775 rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
1776 if ( misc_pwr & (1 << 8))
1777 return true;
1778 }
1779
1780 if (acpi_disabled ||
1781 ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
1782 return false;
1783
1784 for (v_info = vendor_info; v_info->valid; v_info++) {
1785 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
1786 !strncmp(hdr.oem_table_id, v_info->oem_table_id,
1787 ACPI_OEM_TABLE_ID_SIZE))
1788 switch (v_info->oem_pwr_table) {
1789 case PSS:
1790 return intel_pstate_no_acpi_pss();
1791 case PPC:
1792 return intel_pstate_has_acpi_ppc() &&
1793 (!force_load);
1794 }
1795 }
1796
1797 return false;
1798}
1799#else
1800static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
1801static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
1802#endif
1803
1804static const struct x86_cpu_id hwp_support_ids[] __initconst = {
1805 { X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_HWP },
1806 {}
1807};
1808
1809static int __init intel_pstate_init(void)
1810{
1811 int cpu, rc = 0;
1812 const struct x86_cpu_id *id;
1813 struct cpu_defaults *cpu_def;
1814
1815 if (no_load)
1816 return -ENODEV;
1817
1818 if (x86_match_cpu(hwp_support_ids) && !no_hwp) {
1819 copy_cpu_funcs(&core_params.funcs);
1820 hwp_active++;
1821 goto hwp_cpu_matched;
1822 }
1823
1824 id = x86_match_cpu(intel_pstate_cpu_ids);
1825 if (!id)
1826 return -ENODEV;
1827
1828 cpu_def = (struct cpu_defaults *)id->driver_data;
1829
1830 copy_pid_params(&cpu_def->pid_policy);
1831 copy_cpu_funcs(&cpu_def->funcs);
1832
1833 if (intel_pstate_msrs_not_valid())
1834 return -ENODEV;
1835
1836hwp_cpu_matched:
1837
1838
1839
1840
1841 if (intel_pstate_platform_pwr_mgmt_exists())
1842 return -ENODEV;
1843
1844 pr_info("Intel P-state driver initializing\n");
1845
1846 all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
1847 if (!all_cpu_data)
1848 return -ENOMEM;
1849
1850 if (!hwp_active && hwp_only)
1851 goto out;
1852
1853 rc = cpufreq_register_driver(&intel_pstate_driver);
1854 if (rc)
1855 goto out;
1856
1857 intel_pstate_debug_expose_params();
1858 intel_pstate_sysfs_expose_params();
1859
1860 if (hwp_active)
1861 pr_info("HWP enabled\n");
1862
1863 return rc;
1864out:
1865 get_online_cpus();
1866 for_each_online_cpu(cpu) {
1867 if (all_cpu_data[cpu]) {
1868 intel_pstate_clear_update_util_hook(cpu);
1869 kfree(all_cpu_data[cpu]);
1870 }
1871 }
1872
1873 put_online_cpus();
1874 vfree(all_cpu_data);
1875 return -ENODEV;
1876}
1877device_initcall(intel_pstate_init);
1878
1879static int __init intel_pstate_setup(char *str)
1880{
1881 if (!str)
1882 return -EINVAL;
1883
1884 if (!strcmp(str, "disable"))
1885 no_load = 1;
1886 if (!strcmp(str, "no_hwp")) {
1887 pr_info("HWP disabled\n");
1888 no_hwp = 1;
1889 }
1890 if (!strcmp(str, "force"))
1891 force_load = 1;
1892 if (!strcmp(str, "hwp_only"))
1893 hwp_only = 1;
1894
1895#ifdef CONFIG_ACPI
1896 if (!strcmp(str, "support_acpi_ppc"))
1897 acpi_ppc = true;
1898#endif
1899
1900 return 0;
1901}
1902early_param("intel_pstate", intel_pstate_setup);
1903
1904MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
1905MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
1906MODULE_LICENSE("GPL");
1907