1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/module.h>
14#include <linux/thermal.h>
15#include <linux/cpufreq.h>
16#include <linux/err.h>
17#include <linux/idr.h>
18#include <linux/pm_opp.h>
19#include <linux/pm_qos.h>
20#include <linux/slab.h>
21#include <linux/cpu.h>
22#include <linux/cpu_cooling.h>
23
24#include <trace/events/thermal.h>
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48struct freq_table {
49 u32 frequency;
50 u32 power;
51};
52
53
54
55
56
57
58struct time_in_idle {
59 u64 time;
60 u64 timestamp;
61};
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82struct cpufreq_cooling_device {
83 int id;
84 u32 last_load;
85 unsigned int cpufreq_state;
86 unsigned int max_level;
87 struct freq_table *freq_table;
88 struct cpufreq_policy *policy;
89 struct list_head node;
90 struct time_in_idle *idle_time;
91 struct freq_qos_request qos_req;
92};
93
94static DEFINE_IDA(cpufreq_ida);
95static DEFINE_MUTEX(cooling_list_lock);
96static LIST_HEAD(cpufreq_cdev_list);
97
98
99
100
101
102
103
104
105
106
107static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev,
108 unsigned int freq)
109{
110 struct freq_table *freq_table = cpufreq_cdev->freq_table;
111 unsigned long level;
112
113 for (level = 1; level <= cpufreq_cdev->max_level; level++)
114 if (freq > freq_table[level].frequency)
115 break;
116
117 return level - 1;
118}
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133static int update_freq_table(struct cpufreq_cooling_device *cpufreq_cdev,
134 u32 capacitance)
135{
136 struct freq_table *freq_table = cpufreq_cdev->freq_table;
137 struct dev_pm_opp *opp;
138 struct device *dev = NULL;
139 int num_opps = 0, cpu = cpufreq_cdev->policy->cpu, i;
140
141 dev = get_cpu_device(cpu);
142 if (unlikely(!dev)) {
143 pr_warn("No cpu device for cpu %d\n", cpu);
144 return -ENODEV;
145 }
146
147 num_opps = dev_pm_opp_get_opp_count(dev);
148 if (num_opps < 0)
149 return num_opps;
150
151
152
153
154
155 if (num_opps != cpufreq_cdev->max_level + 1) {
156 dev_warn(dev, "Number of OPPs not matching with max_levels\n");
157 return -EINVAL;
158 }
159
160 for (i = 0; i <= cpufreq_cdev->max_level; i++) {
161 unsigned long freq = freq_table[i].frequency * 1000;
162 u32 freq_mhz = freq_table[i].frequency / 1000;
163 u64 power;
164 u32 voltage_mv;
165
166
167
168
169
170 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
171 if (IS_ERR(opp)) {
172 dev_err(dev, "failed to get opp for %lu frequency\n",
173 freq);
174 return -EINVAL;
175 }
176
177 voltage_mv = dev_pm_opp_get_voltage(opp) / 1000;
178 dev_pm_opp_put(opp);
179
180
181
182
183
184 power = (u64)capacitance * freq_mhz * voltage_mv * voltage_mv;
185 do_div(power, 1000000000);
186
187
188 freq_table[i].power = power;
189 }
190
191 return 0;
192}
193
194static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev,
195 u32 freq)
196{
197 int i;
198 struct freq_table *freq_table = cpufreq_cdev->freq_table;
199
200 for (i = 1; i <= cpufreq_cdev->max_level; i++)
201 if (freq > freq_table[i].frequency)
202 break;
203
204 return freq_table[i - 1].power;
205}
206
207static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev,
208 u32 power)
209{
210 int i;
211 struct freq_table *freq_table = cpufreq_cdev->freq_table;
212
213 for (i = 1; i <= cpufreq_cdev->max_level; i++)
214 if (power > freq_table[i].power)
215 break;
216
217 return freq_table[i - 1].frequency;
218}
219
220
221
222
223
224
225
226
227
228
229static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu,
230 int cpu_idx)
231{
232 u32 load;
233 u64 now, now_idle, delta_time, delta_idle;
234 struct time_in_idle *idle_time = &cpufreq_cdev->idle_time[cpu_idx];
235
236 now_idle = get_cpu_idle_time(cpu, &now, 0);
237 delta_idle = now_idle - idle_time->time;
238 delta_time = now - idle_time->timestamp;
239
240 if (delta_time <= delta_idle)
241 load = 0;
242 else
243 load = div64_u64(100 * (delta_time - delta_idle), delta_time);
244
245 idle_time->time = now_idle;
246 idle_time->timestamp = now;
247
248 return load;
249}
250
251
252
253
254
255
256
257
258
259static u32 get_dynamic_power(struct cpufreq_cooling_device *cpufreq_cdev,
260 unsigned long freq)
261{
262 u32 raw_cpu_power;
263
264 raw_cpu_power = cpu_freq_to_power(cpufreq_cdev, freq);
265 return (raw_cpu_power * cpufreq_cdev->last_load) / 100;
266}
267
268
269
270
271
272
273
274
275
276
277
278
279
280static int cpufreq_get_max_state(struct thermal_cooling_device *cdev,
281 unsigned long *state)
282{
283 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
284
285 *state = cpufreq_cdev->max_level;
286 return 0;
287}
288
289
290
291
292
293
294
295
296
297
298
299static int cpufreq_get_cur_state(struct thermal_cooling_device *cdev,
300 unsigned long *state)
301{
302 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
303
304 *state = cpufreq_cdev->cpufreq_state;
305
306 return 0;
307}
308
309
310
311
312
313
314
315
316
317
318
319static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev,
320 unsigned long state)
321{
322 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
323
324
325 if (WARN_ON(state > cpufreq_cdev->max_level))
326 return -EINVAL;
327
328
329 if (cpufreq_cdev->cpufreq_state == state)
330 return 0;
331
332 cpufreq_cdev->cpufreq_state = state;
333
334 return freq_qos_update_request(&cpufreq_cdev->qos_req,
335 cpufreq_cdev->freq_table[state].frequency);
336}
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev,
362 struct thermal_zone_device *tz,
363 u32 *power)
364{
365 unsigned long freq;
366 int i = 0, cpu;
367 u32 total_load = 0;
368 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
369 struct cpufreq_policy *policy = cpufreq_cdev->policy;
370 u32 *load_cpu = NULL;
371
372 freq = cpufreq_quick_get(policy->cpu);
373
374 if (trace_thermal_power_cpu_get_power_enabled()) {
375 u32 ncpus = cpumask_weight(policy->related_cpus);
376
377 load_cpu = kcalloc(ncpus, sizeof(*load_cpu), GFP_KERNEL);
378 }
379
380 for_each_cpu(cpu, policy->related_cpus) {
381 u32 load;
382
383 if (cpu_online(cpu))
384 load = get_load(cpufreq_cdev, cpu, i);
385 else
386 load = 0;
387
388 total_load += load;
389 if (load_cpu)
390 load_cpu[i] = load;
391
392 i++;
393 }
394
395 cpufreq_cdev->last_load = total_load;
396
397 *power = get_dynamic_power(cpufreq_cdev, freq);
398
399 if (load_cpu) {
400 trace_thermal_power_cpu_get_power(policy->related_cpus, freq,
401 load_cpu, i, *power);
402
403 kfree(load_cpu);
404 }
405
406 return 0;
407}
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424static int cpufreq_state2power(struct thermal_cooling_device *cdev,
425 struct thermal_zone_device *tz,
426 unsigned long state, u32 *power)
427{
428 unsigned int freq, num_cpus;
429 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
430
431
432 if (WARN_ON(state > cpufreq_cdev->max_level))
433 return -EINVAL;
434
435 num_cpus = cpumask_weight(cpufreq_cdev->policy->cpus);
436
437 freq = cpufreq_cdev->freq_table[state].frequency;
438 *power = cpu_freq_to_power(cpufreq_cdev, freq) * num_cpus;
439
440 return 0;
441}
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463static int cpufreq_power2state(struct thermal_cooling_device *cdev,
464 struct thermal_zone_device *tz, u32 power,
465 unsigned long *state)
466{
467 unsigned int target_freq;
468 u32 last_load, normalised_power;
469 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata;
470 struct cpufreq_policy *policy = cpufreq_cdev->policy;
471
472 last_load = cpufreq_cdev->last_load ?: 1;
473 normalised_power = (power * 100) / last_load;
474 target_freq = cpu_power_to_freq(cpufreq_cdev, normalised_power);
475
476 *state = get_level(cpufreq_cdev, target_freq);
477 trace_thermal_power_cpu_limit(policy->related_cpus, target_freq, *state,
478 power);
479 return 0;
480}
481
482
483
484static struct thermal_cooling_device_ops cpufreq_cooling_ops = {
485 .get_max_state = cpufreq_get_max_state,
486 .get_cur_state = cpufreq_get_cur_state,
487 .set_cur_state = cpufreq_set_cur_state,
488};
489
490static struct thermal_cooling_device_ops cpufreq_power_cooling_ops = {
491 .get_max_state = cpufreq_get_max_state,
492 .get_cur_state = cpufreq_get_cur_state,
493 .set_cur_state = cpufreq_set_cur_state,
494 .get_requested_power = cpufreq_get_requested_power,
495 .state2power = cpufreq_state2power,
496 .power2state = cpufreq_power2state,
497};
498
499static unsigned int find_next_max(struct cpufreq_frequency_table *table,
500 unsigned int prev_max)
501{
502 struct cpufreq_frequency_table *pos;
503 unsigned int max = 0;
504
505 cpufreq_for_each_valid_entry(pos, table) {
506 if (pos->frequency > max && pos->frequency < prev_max)
507 max = pos->frequency;
508 }
509
510 return max;
511}
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528static struct thermal_cooling_device *
529__cpufreq_cooling_register(struct device_node *np,
530 struct cpufreq_policy *policy, u32 capacitance)
531{
532 struct thermal_cooling_device *cdev;
533 struct cpufreq_cooling_device *cpufreq_cdev;
534 char dev_name[THERMAL_NAME_LENGTH];
535 unsigned int freq, i, num_cpus;
536 struct device *dev;
537 int ret;
538 struct thermal_cooling_device_ops *cooling_ops;
539
540 dev = get_cpu_device(policy->cpu);
541 if (unlikely(!dev)) {
542 pr_warn("No cpu device for cpu %d\n", policy->cpu);
543 return ERR_PTR(-ENODEV);
544 }
545
546
547 if (IS_ERR_OR_NULL(policy)) {
548 pr_err("%s: cpufreq policy isn't valid: %p\n", __func__, policy);
549 return ERR_PTR(-EINVAL);
550 }
551
552 i = cpufreq_table_count_valid_entries(policy);
553 if (!i) {
554 pr_debug("%s: CPUFreq table not found or has no valid entries\n",
555 __func__);
556 return ERR_PTR(-ENODEV);
557 }
558
559 cpufreq_cdev = kzalloc(sizeof(*cpufreq_cdev), GFP_KERNEL);
560 if (!cpufreq_cdev)
561 return ERR_PTR(-ENOMEM);
562
563 cpufreq_cdev->policy = policy;
564 num_cpus = cpumask_weight(policy->related_cpus);
565 cpufreq_cdev->idle_time = kcalloc(num_cpus,
566 sizeof(*cpufreq_cdev->idle_time),
567 GFP_KERNEL);
568 if (!cpufreq_cdev->idle_time) {
569 cdev = ERR_PTR(-ENOMEM);
570 goto free_cdev;
571 }
572
573
574 cpufreq_cdev->max_level = i - 1;
575
576 cpufreq_cdev->freq_table = kmalloc_array(i,
577 sizeof(*cpufreq_cdev->freq_table),
578 GFP_KERNEL);
579 if (!cpufreq_cdev->freq_table) {
580 cdev = ERR_PTR(-ENOMEM);
581 goto free_idle_time;
582 }
583
584 ret = ida_simple_get(&cpufreq_ida, 0, 0, GFP_KERNEL);
585 if (ret < 0) {
586 cdev = ERR_PTR(ret);
587 goto free_table;
588 }
589 cpufreq_cdev->id = ret;
590
591 snprintf(dev_name, sizeof(dev_name), "thermal-cpufreq-%d",
592 cpufreq_cdev->id);
593
594
595 for (i = 0, freq = -1; i <= cpufreq_cdev->max_level; i++) {
596 freq = find_next_max(policy->freq_table, freq);
597 cpufreq_cdev->freq_table[i].frequency = freq;
598
599
600 if (!freq)
601 pr_warn("%s: table has duplicate entries\n", __func__);
602 else
603 pr_debug("%s: freq:%u KHz\n", __func__, freq);
604 }
605
606 if (capacitance) {
607 ret = update_freq_table(cpufreq_cdev, capacitance);
608 if (ret) {
609 cdev = ERR_PTR(ret);
610 goto remove_ida;
611 }
612
613 cooling_ops = &cpufreq_power_cooling_ops;
614 } else {
615 cooling_ops = &cpufreq_cooling_ops;
616 }
617
618 ret = freq_qos_add_request(&policy->constraints,
619 &cpufreq_cdev->qos_req, FREQ_QOS_MAX,
620 cpufreq_cdev->freq_table[0].frequency);
621 if (ret < 0) {
622 pr_err("%s: Failed to add freq constraint (%d)\n", __func__,
623 ret);
624 cdev = ERR_PTR(ret);
625 goto remove_ida;
626 }
627
628 cdev = thermal_of_cooling_device_register(np, dev_name, cpufreq_cdev,
629 cooling_ops);
630 if (IS_ERR(cdev))
631 goto remove_qos_req;
632
633 mutex_lock(&cooling_list_lock);
634 list_add(&cpufreq_cdev->node, &cpufreq_cdev_list);
635 mutex_unlock(&cooling_list_lock);
636
637 return cdev;
638
639remove_qos_req:
640 freq_qos_remove_request(&cpufreq_cdev->qos_req);
641remove_ida:
642 ida_simple_remove(&cpufreq_ida, cpufreq_cdev->id);
643free_table:
644 kfree(cpufreq_cdev->freq_table);
645free_idle_time:
646 kfree(cpufreq_cdev->idle_time);
647free_cdev:
648 kfree(cpufreq_cdev);
649 return cdev;
650}
651
652
653
654
655
656
657
658
659
660
661
662
663struct thermal_cooling_device *
664cpufreq_cooling_register(struct cpufreq_policy *policy)
665{
666 return __cpufreq_cooling_register(NULL, policy, 0);
667}
668EXPORT_SYMBOL_GPL(cpufreq_cooling_register);
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689struct thermal_cooling_device *
690of_cpufreq_cooling_register(struct cpufreq_policy *policy)
691{
692 struct device_node *np = of_get_cpu_node(policy->cpu, NULL);
693 struct thermal_cooling_device *cdev = NULL;
694 u32 capacitance = 0;
695
696 if (!np) {
697 pr_err("cpu_cooling: OF node not available for cpu%d\n",
698 policy->cpu);
699 return NULL;
700 }
701
702 if (of_find_property(np, "#cooling-cells", NULL)) {
703 of_property_read_u32(np, "dynamic-power-coefficient",
704 &capacitance);
705
706 cdev = __cpufreq_cooling_register(np, policy, capacitance);
707 if (IS_ERR(cdev)) {
708 pr_err("cpu_cooling: cpu%d failed to register as cooling device: %ld\n",
709 policy->cpu, PTR_ERR(cdev));
710 cdev = NULL;
711 }
712 }
713
714 of_node_put(np);
715 return cdev;
716}
717EXPORT_SYMBOL_GPL(of_cpufreq_cooling_register);
718
719
720
721
722
723
724
725void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev)
726{
727 struct cpufreq_cooling_device *cpufreq_cdev;
728
729 if (!cdev)
730 return;
731
732 cpufreq_cdev = cdev->devdata;
733
734 mutex_lock(&cooling_list_lock);
735 list_del(&cpufreq_cdev->node);
736 mutex_unlock(&cooling_list_lock);
737
738 thermal_cooling_device_unregister(cdev);
739 freq_qos_remove_request(&cpufreq_cdev->qos_req);
740 ida_simple_remove(&cpufreq_ida, cpufreq_cdev->id);
741 kfree(cpufreq_cdev->idle_time);
742 kfree(cpufreq_cdev->freq_table);
743 kfree(cpufreq_cdev);
744}
745EXPORT_SYMBOL_GPL(cpufreq_cooling_unregister);
746