1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/module.h>
26#include <linux/thermal.h>
27#include <linux/cpufreq.h>
28#include <linux/err.h>
29#include <linux/idr.h>
30#include <linux/pm_opp.h>
31#include <linux/pm_qos.h>
32#include <linux/slab.h>
33#include <linux/cpu.h>
34#include <linux/cpu_cooling.h>
35
36#include <trace/events/thermal.h>
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60struct freq_table {
61 u32 frequency;
62 u32 power;
63};
64
65
66
67
68
69
70struct time_in_idle {
71 u64 time;
72 u64 timestamp;
73};
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94struct cpufreq_cooling_device {
95 int id;
96 u32 last_load;
97 unsigned int cpufreq_state;
98 unsigned int max_level;
99 struct freq_table *freq_table;
100 struct thermal_cooling_device *cdev;
101 struct cpufreq_policy *policy;
102 struct list_head node;
103 struct time_in_idle *idle_time;
104 struct freq_qos_request qos_req;
105};
106
107static DEFINE_IDA(cpufreq_ida);
108static DEFINE_MUTEX(cooling_list_lock);
109static LIST_HEAD(cpufreq_cdev_list);
110
111
112
113
114
115
116
117
118
119
120static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev,
121 unsigned int freq)
122{
123 struct freq_table *freq_table = cpufreq_cdev->freq_table;
124 unsigned long level;
125
126 for (level = 1; level <= cpufreq_cdev->max_level; level++)
127 if (freq > freq_table[level].frequency)
128 break;
129
130 return level - 1;
131}
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146static int update_freq_table(struct cpufreq_cooling_device *cpufreq_cdev,
147 u32 capacitance)
148{
149 struct freq_table *freq_table = cpufreq_cdev->freq_table;
150 struct dev_pm_opp *opp;
151 struct device *dev = NULL;
152 int num_opps = 0, cpu = cpufreq_cdev->policy->cpu, i;
153
154 dev = get_cpu_device(cpu);
155 if (unlikely(!dev)) {
156 dev_warn(&cpufreq_cdev->cdev->device,
157 "No cpu device for cpu %d\n", cpu);
158 return -ENODEV;
159 }
160
161 num_opps = dev_pm_opp_get_opp_count(dev);
162 if (num_opps < 0)
163 return num_opps;
164
165
166
167
168
169 if (num_opps != cpufreq_cdev->max_level + 1) {
170 dev_warn(dev, "Number of OPPs not matching with max_levels\n");
171 return -EINVAL;
172 }
173
174 for (i = 0; i <= cpufreq_cdev->max_level; i++) {
175 unsigned long freq = freq_table[i].frequency * 1000;
176 u32 freq_mhz = freq_table[i].frequency / 1000;
177 u64 power;
178 u32 voltage_mv;
179
180
181
182
183
184 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
185 if (IS_ERR(opp)) {
186 dev_err(dev, "failed to get opp for %lu frequency\n",
187 freq);
188 return -EINVAL;
189 }
190
191 voltage_mv = dev_pm_opp_get_voltage(opp) / 1000;
192 dev_pm_opp_put(opp);
193
194
195
196
197
198 power = (u64)capacitance * freq_mhz * voltage_mv * voltage_mv;
199 do_div(power, 1000000000);
200
201
202 freq_table[i].power = power;
203 }
204
205 return 0;
206}
207
208static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev,
209 u32 freq)
210{
211 int i;
212 struct freq_table *freq_table = cpufreq_cdev->freq_table;
213
214 for (i = 1; i <= cpufreq_cdev->max_level; i++)
215 if (freq > freq_table[i].frequency)
216 break;
217
218 return freq_table[i - 1].power;
219}
220
221static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev,
222 u32 power)
223{
224 int i;
225 struct freq_table *freq_table = cpufreq_cdev->freq_table;
226
227 for (i = 1; i <= cpufreq_cdev->max_level; i++)
228 if (power > freq_table[i].power)
229 break;
230
231 return freq_table[i - 1].frequency;
232}
233
234
235
236
237
238
239
240
241
242
243static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu,
244 int cpu_idx)
245{
246 u32 load;
247 u64 now, now_idle, delta_time, delta_idle;
248 struct time_in_idle *idle_time = &cpufreq_cdev->idle_time[cpu_idx];
249
250 now_idle = get_cpu_idle_time(cpu, &now, 0);
251 delta_idle = now_idle - idle_time->time;
252 delta_time = now - idle_time->timestamp;
253
254 if (delta_time <= delta_idle)
255 load = 0;
256 else
257 load = div64_u64(100 * (delta_time - delta_idle), delta_time);
258
259 idle_time->time = now_idle;
260 idle_time->timestamp = now;
261
262 return load;
263}
264
265
266
267
268
269
270
271
272
273static u32 get_dynamic_power(struct cpufreq_cooling_device *cpufreq_cdev,
274 unsigned long freq)
275{
276 u32 raw_cpu_power;
277
278 raw_cpu_power = cpu_freq_to_power(cpufreq_cdev, freq);
279 return (raw_cpu_power * cpufreq_cdev->last_load) / 100;
280}
281
282
283
284
285
286
287
288
289
290
291
292
293
294static int cpufreq_get_max_state(struct thermal_cooling_device *cdev,
295 unsigned long *state)
296{
297 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
298
299 *state = cpufreq_cdev->max_level;
300 return 0;
301}
302
303
304
305
306
307
308
309
310
311
312
313static int cpufreq_get_cur_state(struct thermal_cooling_device *cdev,
314 unsigned long *state)
315{
316 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
317
318 *state = cpufreq_cdev->cpufreq_state;
319
320 return 0;
321}
322
323
324
325
326
327
328
329
330
331
332
333static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev,
334 unsigned long state)
335{
336 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
337 int ret;
338
339
340 if (WARN_ON(state > cpufreq_cdev->max_level))
341 return -EINVAL;
342
343
344 if (cpufreq_cdev->cpufreq_state == state)
345 return 0;
346
347 cpufreq_cdev->cpufreq_state = state;
348
349 ret = freq_qos_update_request(&cpufreq_cdev->qos_req,
350 cpufreq_cdev->freq_table[state].frequency);
351 return ret < 0 ? ret : 0;
352}
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev,
378 struct thermal_zone_device *tz,
379 u32 *power)
380{
381 unsigned long freq;
382 int i = 0, cpu;
383 u32 total_load = 0;
384 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
385 struct cpufreq_policy *policy = cpufreq_cdev->policy;
386 u32 *load_cpu = NULL;
387
388 freq = cpufreq_quick_get(policy->cpu);
389
390 if (trace_thermal_power_cpu_get_power_enabled()) {
391 u32 ncpus = cpumask_weight(policy->related_cpus);
392
393 load_cpu = kcalloc(ncpus, sizeof(*load_cpu), GFP_KERNEL);
394 }
395
396 for_each_cpu(cpu, policy->related_cpus) {
397 u32 load;
398
399 if (cpu_online(cpu))
400 load = get_load(cpufreq_cdev, cpu, i);
401 else
402 load = 0;
403
404 total_load += load;
405 if (trace_thermal_power_cpu_limit_enabled() && load_cpu)
406 load_cpu[i] = load;
407
408 i++;
409 }
410
411 cpufreq_cdev->last_load = total_load;
412
413 *power = get_dynamic_power(cpufreq_cdev, freq);
414
415 if (load_cpu) {
416 trace_thermal_power_cpu_get_power(policy->related_cpus, freq,
417 load_cpu, i, *power);
418
419 kfree(load_cpu);
420 }
421
422 return 0;
423}
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440static int cpufreq_state2power(struct thermal_cooling_device *cdev,
441 struct thermal_zone_device *tz,
442 unsigned long state, u32 *power)
443{
444 unsigned int freq, num_cpus;
445 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
446
447
448 if (WARN_ON(state > cpufreq_cdev->max_level))
449 return -EINVAL;
450
451 num_cpus = cpumask_weight(cpufreq_cdev->policy->cpus);
452
453 freq = cpufreq_cdev->freq_table[state].frequency;
454 *power = cpu_freq_to_power(cpufreq_cdev, freq) * num_cpus;
455
456 return 0;
457}
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479static int cpufreq_power2state(struct thermal_cooling_device *cdev,
480 struct thermal_zone_device *tz, u32 power,
481 unsigned long *state)
482{
483 unsigned int cur_freq, target_freq;
484 u32 last_load, normalised_power;
485 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
486 struct cpufreq_policy *policy = cpufreq_cdev->policy;
487
488 cur_freq = cpufreq_quick_get(policy->cpu);
489 power = power > 0 ? power : 0;
490 last_load = cpufreq_cdev->last_load ?: 1;
491 normalised_power = (power * 100) / last_load;
492 target_freq = cpu_power_to_freq(cpufreq_cdev, normalised_power);
493
494 *state = get_level(cpufreq_cdev, target_freq);
495 trace_thermal_power_cpu_limit(policy->related_cpus, target_freq, *state,
496 power);
497 return 0;
498}
499
500
501
502static struct thermal_cooling_device_ops cpufreq_cooling_ops = {
503 .get_max_state = cpufreq_get_max_state,
504 .get_cur_state = cpufreq_get_cur_state,
505 .set_cur_state = cpufreq_set_cur_state,
506};
507
508static struct thermal_cooling_device_ops cpufreq_power_cooling_ops = {
509 .get_max_state = cpufreq_get_max_state,
510 .get_cur_state = cpufreq_get_cur_state,
511 .set_cur_state = cpufreq_set_cur_state,
512 .get_requested_power = cpufreq_get_requested_power,
513 .state2power = cpufreq_state2power,
514 .power2state = cpufreq_power2state,
515};
516
517static unsigned int find_next_max(struct cpufreq_frequency_table *table,
518 unsigned int prev_max)
519{
520 struct cpufreq_frequency_table *pos;
521 unsigned int max = 0;
522
523 cpufreq_for_each_valid_entry(pos, table) {
524 if (pos->frequency > max && pos->frequency < prev_max)
525 max = pos->frequency;
526 }
527
528 return max;
529}
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546static struct thermal_cooling_device *
547__cpufreq_cooling_register(struct device_node *np,
548 struct cpufreq_policy *policy, u32 capacitance)
549{
550 struct thermal_cooling_device *cdev;
551 struct cpufreq_cooling_device *cpufreq_cdev;
552 char dev_name[THERMAL_NAME_LENGTH];
553 unsigned int freq, i, num_cpus;
554 struct device *dev;
555 int ret;
556 struct thermal_cooling_device_ops *cooling_ops;
557
558 dev = get_cpu_device(policy->cpu);
559 if (unlikely(!dev)) {
560 pr_warn("No cpu device for cpu %d\n", policy->cpu);
561 return ERR_PTR(-ENODEV);
562 }
563
564
565 if (IS_ERR_OR_NULL(policy)) {
566 pr_err("%s: cpufreq policy isn't valid: %p\n", __func__, policy);
567 return ERR_PTR(-EINVAL);
568 }
569
570 i = cpufreq_table_count_valid_entries(policy);
571 if (!i) {
572 pr_debug("%s: CPUFreq table not found or has no valid entries\n",
573 __func__);
574 return ERR_PTR(-ENODEV);
575 }
576
577 cpufreq_cdev = kzalloc(sizeof(*cpufreq_cdev), GFP_KERNEL);
578 if (!cpufreq_cdev)
579 return ERR_PTR(-ENOMEM);
580
581 cpufreq_cdev->policy = policy;
582 num_cpus = cpumask_weight(policy->related_cpus);
583 cpufreq_cdev->idle_time = kcalloc(num_cpus,
584 sizeof(*cpufreq_cdev->idle_time),
585 GFP_KERNEL);
586 if (!cpufreq_cdev->idle_time) {
587 cdev = ERR_PTR(-ENOMEM);
588 goto free_cdev;
589 }
590
591
592 cpufreq_cdev->max_level = i - 1;
593
594 cpufreq_cdev->freq_table = kmalloc_array(i,
595 sizeof(*cpufreq_cdev->freq_table),
596 GFP_KERNEL);
597 if (!cpufreq_cdev->freq_table) {
598 cdev = ERR_PTR(-ENOMEM);
599 goto free_idle_time;
600 }
601
602 ret = ida_simple_get(&cpufreq_ida, 0, 0, GFP_KERNEL);
603 if (ret < 0) {
604 cdev = ERR_PTR(ret);
605 goto free_table;
606 }
607 cpufreq_cdev->id = ret;
608
609 snprintf(dev_name, sizeof(dev_name), "thermal-cpufreq-%d",
610 cpufreq_cdev->id);
611
612
613 for (i = 0, freq = -1; i <= cpufreq_cdev->max_level; i++) {
614 freq = find_next_max(policy->freq_table, freq);
615 cpufreq_cdev->freq_table[i].frequency = freq;
616
617
618 if (!freq)
619 pr_warn("%s: table has duplicate entries\n", __func__);
620 else
621 pr_debug("%s: freq:%u KHz\n", __func__, freq);
622 }
623
624 if (capacitance) {
625 ret = update_freq_table(cpufreq_cdev, capacitance);
626 if (ret) {
627 cdev = ERR_PTR(ret);
628 goto remove_ida;
629 }
630
631 cooling_ops = &cpufreq_power_cooling_ops;
632 } else {
633 cooling_ops = &cpufreq_cooling_ops;
634 }
635
636 ret = freq_qos_add_request(&policy->constraints,
637 &cpufreq_cdev->qos_req, FREQ_QOS_MAX,
638 cpufreq_cdev->freq_table[0].frequency);
639 if (ret < 0) {
640 pr_err("%s: Failed to add freq constraint (%d)\n", __func__,
641 ret);
642 cdev = ERR_PTR(ret);
643 goto remove_ida;
644 }
645
646 cdev = thermal_of_cooling_device_register(np, dev_name, cpufreq_cdev,
647 cooling_ops);
648 if (IS_ERR(cdev))
649 goto remove_qos_req;
650
651 cpufreq_cdev->cdev = cdev;
652
653 mutex_lock(&cooling_list_lock);
654 list_add(&cpufreq_cdev->node, &cpufreq_cdev_list);
655 mutex_unlock(&cooling_list_lock);
656
657 return cdev;
658
659remove_qos_req:
660 freq_qos_remove_request(&cpufreq_cdev->qos_req);
661remove_ida:
662 ida_simple_remove(&cpufreq_ida, cpufreq_cdev->id);
663free_table:
664 kfree(cpufreq_cdev->freq_table);
665free_idle_time:
666 kfree(cpufreq_cdev->idle_time);
667free_cdev:
668 kfree(cpufreq_cdev);
669 return cdev;
670}
671
672
673
674
675
676
677
678
679
680
681
682
683struct thermal_cooling_device *
684cpufreq_cooling_register(struct cpufreq_policy *policy)
685{
686 return __cpufreq_cooling_register(NULL, policy, 0);
687}
688EXPORT_SYMBOL_GPL(cpufreq_cooling_register);
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709struct thermal_cooling_device *
710of_cpufreq_cooling_register(struct cpufreq_policy *policy)
711{
712 struct device_node *np = of_get_cpu_node(policy->cpu, NULL);
713 struct thermal_cooling_device *cdev = NULL;
714 u32 capacitance = 0;
715
716 if (!np) {
717 pr_err("cpu_cooling: OF node not available for cpu%d\n",
718 policy->cpu);
719 return NULL;
720 }
721
722 if (of_find_property(np, "#cooling-cells", NULL)) {
723 of_property_read_u32(np, "dynamic-power-coefficient",
724 &capacitance);
725
726 cdev = __cpufreq_cooling_register(np, policy, capacitance);
727 if (IS_ERR(cdev)) {
728 pr_err("cpu_cooling: cpu%d is not running as cooling device: %ld\n",
729 policy->cpu, PTR_ERR(cdev));
730 cdev = NULL;
731 }
732 }
733
734 of_node_put(np);
735 return cdev;
736}
737EXPORT_SYMBOL_GPL(of_cpufreq_cooling_register);
738
739
740
741
742
743
744
745void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev)
746{
747 struct cpufreq_cooling_device *cpufreq_cdev;
748
749 if (!cdev)
750 return;
751
752 cpufreq_cdev = cdev->devdata;
753
754 mutex_lock(&cooling_list_lock);
755 list_del(&cpufreq_cdev->node);
756 mutex_unlock(&cooling_list_lock);
757
758 thermal_cooling_device_unregister(cpufreq_cdev->cdev);
759 freq_qos_remove_request(&cpufreq_cdev->qos_req);
760 ida_simple_remove(&cpufreq_ida, cpufreq_cdev->id);
761 kfree(cpufreq_cdev->idle_time);
762 kfree(cpufreq_cdev->freq_table);
763 kfree(cpufreq_cdev);
764}
765EXPORT_SYMBOL_GPL(cpufreq_cooling_unregister);
766