1
2
3
4
5
6
7#include <linux/clk.h>
8#include <linux/devfreq.h>
9#include <linux/devfreq-event.h>
10#include <linux/device.h>
11#include <linux/interrupt.h>
12#include <linux/io.h>
13#include <linux/mfd/syscon.h>
14#include <linux/module.h>
15#include <linux/moduleparam.h>
16#include <linux/of_device.h>
17#include <linux/pm_opp.h>
18#include <linux/platform_device.h>
19#include <linux/regmap.h>
20#include <linux/regulator/consumer.h>
21#include <linux/slab.h>
22#include "../jedec_ddr.h"
23#include "../of_memory.h"
24
25static int irqmode;
26module_param(irqmode, int, 0644);
27MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)");
28
29#define EXYNOS5_DREXI_TIMINGAREF (0x0030)
30#define EXYNOS5_DREXI_TIMINGROW0 (0x0034)
31#define EXYNOS5_DREXI_TIMINGDATA0 (0x0038)
32#define EXYNOS5_DREXI_TIMINGPOWER0 (0x003C)
33#define EXYNOS5_DREXI_TIMINGROW1 (0x00E4)
34#define EXYNOS5_DREXI_TIMINGDATA1 (0x00E8)
35#define EXYNOS5_DREXI_TIMINGPOWER1 (0x00EC)
36#define CDREX_PAUSE (0x2091c)
37#define CDREX_LPDDR3PHY_CON3 (0x20a20)
38#define CDREX_LPDDR3PHY_CLKM_SRC (0x20700)
39#define EXYNOS5_TIMING_SET_SWI BIT(28)
40#define USE_MX_MSPLL_TIMINGS (1)
41#define USE_BPLL_TIMINGS (0)
42#define EXYNOS5_AREF_NORMAL (0x2e)
43
44#define DREX_PPCCLKCON (0x0130)
45#define DREX_PEREV2CONFIG (0x013c)
46#define DREX_PMNC_PPC (0xE000)
47#define DREX_CNTENS_PPC (0xE010)
48#define DREX_CNTENC_PPC (0xE020)
49#define DREX_INTENS_PPC (0xE030)
50#define DREX_INTENC_PPC (0xE040)
51#define DREX_FLAG_PPC (0xE050)
52#define DREX_PMCNT2_PPC (0xE130)
53
54
55
56
57
58
59#define CC_RESET BIT(2)
60
61
62
63
64
65#define PPC_COUNTER_RESET BIT(1)
66
67
68
69
70
71#define PPC_ENABLE BIT(0)
72
73
74
75
76
77#define PEREV_CLK_EN BIT(0)
78
79
80
81
82
83#define PERF_CNT2 BIT(2)
84#define PERF_CCNT BIT(31)
85
86
87
88
89
90
91
92
93#define READ_TRANSFER_CH0 (0x6d)
94#define READ_TRANSFER_CH1 (0x6f)
95
96#define PERF_COUNTER_START_VALUE 0xff000000
97#define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
98
99
100
101
102
103
104
105
106struct dmc_opp_table {
107 u32 freq_hz;
108 u32 volt_uv;
109};
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152struct exynos5_dmc {
153 struct device *dev;
154 struct devfreq *df;
155 struct devfreq_simple_ondemand_data gov_data;
156 void __iomem *base_drexi0;
157 void __iomem *base_drexi1;
158 struct regmap *clk_regmap;
159
160 struct mutex lock;
161 unsigned long curr_rate;
162 unsigned long curr_volt;
163 struct dmc_opp_table *opp;
164 int opp_count;
165 u32 timings_arr_size;
166 u32 *timing_row;
167 u32 *timing_data;
168 u32 *timing_power;
169 const struct lpddr3_timings *timings;
170 const struct lpddr3_min_tck *min_tck;
171 u32 bypass_timing_row;
172 u32 bypass_timing_data;
173 u32 bypass_timing_power;
174 struct regulator *vdd_mif;
175 struct clk *fout_spll;
176 struct clk *fout_bpll;
177 struct clk *mout_spll;
178 struct clk *mout_bpll;
179 struct clk *mout_mclk_cdrex;
180 struct clk *mout_mx_mspll_ccore;
181 struct devfreq_event_dev **counter;
182 int num_counters;
183 u64 last_overflow_ts[2];
184 unsigned long load;
185 unsigned long total;
186 bool in_irq_mode;
187};
188
189#define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
190 { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
191
192#define TIMING_VAL2REG(timing, t_val) \
193({ \
194 u32 __val; \
195 __val = (t_val) << (timing)->bit_beg; \
196 __val; \
197})
198
199struct timing_reg {
200 char *name;
201 int bit_beg;
202 int bit_end;
203 unsigned int val;
204};
205
206static const struct timing_reg timing_row_reg_fields[] = {
207 TIMING_FIELD("tRFC", 24, 31),
208 TIMING_FIELD("tRRD", 20, 23),
209 TIMING_FIELD("tRP", 16, 19),
210 TIMING_FIELD("tRCD", 12, 15),
211 TIMING_FIELD("tRC", 6, 11),
212 TIMING_FIELD("tRAS", 0, 5),
213};
214
215static const struct timing_reg timing_data_reg_fields[] = {
216 TIMING_FIELD("tWTR", 28, 31),
217 TIMING_FIELD("tWR", 24, 27),
218 TIMING_FIELD("tRTP", 20, 23),
219 TIMING_FIELD("tW2W-C2C", 14, 14),
220 TIMING_FIELD("tR2R-C2C", 12, 12),
221 TIMING_FIELD("WL", 8, 11),
222 TIMING_FIELD("tDQSCK", 4, 7),
223 TIMING_FIELD("RL", 0, 3),
224};
225
226static const struct timing_reg timing_power_reg_fields[] = {
227 TIMING_FIELD("tFAW", 26, 31),
228 TIMING_FIELD("tXSR", 16, 25),
229 TIMING_FIELD("tXP", 8, 15),
230 TIMING_FIELD("tCKE", 4, 7),
231 TIMING_FIELD("tMRD", 0, 3),
232};
233
234#define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \
235 ARRAY_SIZE(timing_data_reg_fields) + \
236 ARRAY_SIZE(timing_power_reg_fields))
237
238static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
239{
240 int i, ret;
241
242 for (i = 0; i < dmc->num_counters; i++) {
243 if (!dmc->counter[i])
244 continue;
245 ret = devfreq_event_set_event(dmc->counter[i]);
246 if (ret < 0)
247 return ret;
248 }
249 return 0;
250}
251
252static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
253{
254 int i, ret;
255
256 for (i = 0; i < dmc->num_counters; i++) {
257 if (!dmc->counter[i])
258 continue;
259 ret = devfreq_event_enable_edev(dmc->counter[i]);
260 if (ret < 0)
261 return ret;
262 }
263 return 0;
264}
265
266static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
267{
268 int i, ret;
269
270 for (i = 0; i < dmc->num_counters; i++) {
271 if (!dmc->counter[i])
272 continue;
273 ret = devfreq_event_disable_edev(dmc->counter[i]);
274 if (ret < 0)
275 return ret;
276 }
277 return 0;
278}
279
280
281
282
283
284
285
286
287
288static int find_target_freq_idx(struct exynos5_dmc *dmc,
289 unsigned long target_rate)
290{
291 int i;
292
293 for (i = dmc->opp_count - 1; i >= 0; i--)
294 if (dmc->opp[i].freq_hz <= target_rate)
295 return i;
296
297 return -EINVAL;
298}
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
314{
315 unsigned int reg;
316 int ret;
317
318 ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, ®);
319 if (ret)
320 return ret;
321
322 if (set)
323 reg |= EXYNOS5_TIMING_SET_SWI;
324 else
325 reg &= ~EXYNOS5_TIMING_SET_SWI;
326
327 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);
328
329 return 0;
330}
331
332
333
334
335
336
337
338
339static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
340 struct devfreq_dev_profile *profile)
341{
342 int i, ret;
343 int idx;
344 unsigned long freq;
345
346 ret = dev_pm_opp_of_add_table(dmc->dev);
347 if (ret < 0) {
348 dev_err(dmc->dev, "Failed to get OPP table\n");
349 return ret;
350 }
351
352 dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);
353
354 dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,
355 sizeof(struct dmc_opp_table), GFP_KERNEL);
356 if (!dmc->opp)
357 goto err_opp;
358
359 idx = dmc->opp_count - 1;
360 for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
361 struct dev_pm_opp *opp;
362
363 opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);
364 if (IS_ERR(opp))
365 goto err_opp;
366
367 dmc->opp[idx - i].freq_hz = freq;
368 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
369
370 dev_pm_opp_put(opp);
371 }
372
373 return 0;
374
375err_opp:
376 dev_pm_opp_of_remove_table(dmc->dev);
377
378 return -EINVAL;
379}
380
381
382
383
384
385
386
387
388
389static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
390{
391 writel(EXYNOS5_AREF_NORMAL,
392 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
393
394 writel(dmc->bypass_timing_row,
395 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
396 writel(dmc->bypass_timing_row,
397 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
398 writel(dmc->bypass_timing_data,
399 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
400 writel(dmc->bypass_timing_data,
401 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
402 writel(dmc->bypass_timing_power,
403 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
404 writel(dmc->bypass_timing_power,
405 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
406}
407
408
409
410
411
412
413
414
415
416
417
418
419static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
420 unsigned long target_rate)
421{
422 int idx;
423
424 for (idx = dmc->opp_count - 1; idx >= 0; idx--)
425 if (dmc->opp[idx].freq_hz <= target_rate)
426 break;
427
428 if (idx < 0)
429 return -EINVAL;
430
431 writel(EXYNOS5_AREF_NORMAL,
432 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
433
434 writel(dmc->timing_row[idx],
435 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
436 writel(dmc->timing_row[idx],
437 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
438 writel(dmc->timing_data[idx],
439 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
440 writel(dmc->timing_data[idx],
441 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
442 writel(dmc->timing_power[idx],
443 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
444 writel(dmc->timing_power[idx],
445 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
446
447 return 0;
448}
449
450
451
452
453
454
455
456
457
458
459
460static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
461 unsigned long target_volt)
462{
463 int ret = 0;
464
465 if (dmc->curr_volt <= target_volt)
466 return 0;
467
468 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
469 target_volt);
470 if (!ret)
471 dmc->curr_volt = target_volt;
472
473 return ret;
474}
475
476
477
478
479
480
481
482
483
484
485
486static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
487 unsigned long target_volt)
488{
489 int ret = 0;
490
491 if (dmc->curr_volt >= target_volt)
492 return 0;
493
494 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
495 target_volt);
496 if (!ret)
497 dmc->curr_volt = target_volt;
498
499 return ret;
500}
501
502
503
504
505
506
507
508
509static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
510 unsigned long target_rate)
511{
512 int idx = find_target_freq_idx(dmc, target_rate);
513
514 if (idx < 0)
515 return -EINVAL;
516
517 exynos5_set_bypass_dram_timings(dmc);
518
519 return 0;
520}
521
522
523
524
525
526
527
528
529
530
531
532
533static int
534exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
535 unsigned long target_rate,
536 unsigned long target_volt)
537{
538 int ret;
539
540
541
542
543
544
545 ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
546 if (ret)
547 return ret;
548
549
550
551
552 ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
553 if (ret)
554 return ret;
555
556
557
558
559 ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
560
561 return ret;
562}
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587static int
588exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
589 unsigned long target_rate,
590 unsigned long target_volt)
591{
592 int ret;
593
594 ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
595 target_volt);
596 if (ret)
597 return ret;
598
599
600
601
602
603 clk_prepare_enable(dmc->fout_spll);
604 clk_prepare_enable(dmc->mout_spll);
605 clk_prepare_enable(dmc->mout_mx_mspll_ccore);
606
607 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);
608 if (ret)
609 goto disable_clocks;
610
611
612
613
614
615
616 exynos5_dram_change_timings(dmc, target_rate);
617
618 clk_set_rate(dmc->fout_bpll, target_rate);
619
620 ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
621 if (ret)
622 goto disable_clocks;
623
624 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);
625 if (ret)
626 goto disable_clocks;
627
628
629
630
631
632 ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
633
634disable_clocks:
635 clk_disable_unprepare(dmc->mout_mx_mspll_ccore);
636 clk_disable_unprepare(dmc->mout_spll);
637 clk_disable_unprepare(dmc->fout_spll);
638
639 return ret;
640}
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
658 unsigned long *freq,
659 unsigned long *target_rate,
660 unsigned long *target_volt, u32 flags)
661{
662 struct dev_pm_opp *opp;
663
664 opp = devfreq_recommended_opp(dmc->dev, freq, flags);
665 if (IS_ERR(opp))
666 return PTR_ERR(opp);
667
668 *target_rate = dev_pm_opp_get_freq(opp);
669 *target_volt = dev_pm_opp_get_voltage(opp);
670 dev_pm_opp_put(opp);
671
672 return 0;
673}
674
675
676
677
678
679
680
681
682
683
684
685
686
687static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
688 u32 flags)
689{
690 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
691 unsigned long target_rate = 0;
692 unsigned long target_volt = 0;
693 int ret;
694
695 ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,
696 flags);
697
698 if (ret)
699 return ret;
700
701 if (target_rate == dmc->curr_rate)
702 return 0;
703
704 mutex_lock(&dmc->lock);
705
706 ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
707
708 if (ret) {
709 mutex_unlock(&dmc->lock);
710 return ret;
711 }
712
713 dmc->curr_rate = target_rate;
714
715 mutex_unlock(&dmc->lock);
716 return 0;
717}
718
719
720
721
722
723
724
725
726
727
728
729static int exynos5_counters_get(struct exynos5_dmc *dmc,
730 unsigned long *load_count,
731 unsigned long *total_count)
732{
733 unsigned long total = 0;
734 struct devfreq_event_data event;
735 int ret, i;
736
737 *load_count = 0;
738
739
740 for (i = 0; i < dmc->num_counters; i++) {
741 if (!dmc->counter[i])
742 continue;
743
744 ret = devfreq_event_get_event(dmc->counter[i], &event);
745 if (ret < 0)
746 return ret;
747
748 *load_count += event.load_count;
749
750 if (total < event.total_count)
751 total = event.total_count;
752 }
753
754 *total_count = total;
755
756 return 0;
757}
758
759
760
761
762
763
764
765
766
767static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
768 u32 beg_value)
769{
770
771 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);
772 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);
773
774
775 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);
776 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);
777
778
779 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
780 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
781
782
783 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);
784 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);
785
786
787
788
789
790 writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);
791 writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);
792
793
794 writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);
795 writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);
796}
797
798
799
800
801
802
803
804
805
806
807static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
808{
809
810
811
812
813
814
815
816
817
818
819 if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
820
821
822
823
824 dmc->load = 70;
825 dmc->total = 100;
826 } else {
827
828
829
830
831 dmc->load = 35;
832 dmc->total = 100;
833 }
834
835 dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
836}
837
838
839
840
841
842
843
844
845static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
846{
847 u32 val;
848 u64 diff_ts, ts;
849
850 ts = ktime_get_ns();
851
852
853 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
854 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
855
856
857 val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);
858 if (val) {
859 diff_ts = ts - dmc->last_overflow_ts[0];
860 dmc->last_overflow_ts[0] = ts;
861 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n", val);
862 } else {
863 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);
864 diff_ts = ts - dmc->last_overflow_ts[1];
865 dmc->last_overflow_ts[1] = ts;
866 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n", val);
867 }
868
869 exynos5_dmc_perf_events_calc(dmc, diff_ts);
870
871 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
872}
873
874
875
876
877
878
879
880static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
881{
882 u64 ts;
883
884
885 writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);
886 writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);
887
888
889 writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);
890 writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);
891
892 ts = ktime_get_ns();
893 dmc->last_overflow_ts[0] = ts;
894 dmc->last_overflow_ts[1] = ts;
895
896
897 dmc->load = 99;
898 dmc->total = 100;
899}
900
901
902
903
904
905
906
907static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
908{
909
910 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
911 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
912
913
914 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);
915 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);
916
917
918 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);
919 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);
920
921
922 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
923 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
924}
925
926
927
928
929
930
931
932
933
934
935static int exynos5_dmc_get_status(struct device *dev,
936 struct devfreq_dev_status *stat)
937{
938 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
939 unsigned long load, total;
940 int ret;
941
942 if (dmc->in_irq_mode) {
943 mutex_lock(&dmc->lock);
944 stat->current_frequency = dmc->curr_rate;
945 mutex_unlock(&dmc->lock);
946
947 stat->busy_time = dmc->load;
948 stat->total_time = dmc->total;
949 } else {
950 ret = exynos5_counters_get(dmc, &load, &total);
951 if (ret < 0)
952 return -EINVAL;
953
954
955 stat->busy_time = load >> 10;
956 stat->total_time = total >> 10;
957
958 ret = exynos5_counters_set_event(dmc);
959 if (ret < 0) {
960 dev_err(dev, "could not set event counter\n");
961 return ret;
962 }
963 }
964
965 return 0;
966}
967
968
969
970
971
972
973
974
975
976
977static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
978{
979 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
980
981 mutex_lock(&dmc->lock);
982 *freq = dmc->curr_rate;
983 mutex_unlock(&dmc->lock);
984
985 return 0;
986}
987
988
989
990
991
992
993static struct devfreq_dev_profile exynos5_dmc_df_profile = {
994 .timer = DEVFREQ_TIMER_DELAYED,
995 .target = exynos5_dmc_target,
996 .get_dev_status = exynos5_dmc_get_status,
997 .get_cur_freq = exynos5_dmc_get_cur_freq,
998};
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012static unsigned long
1013exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
1014 unsigned long bootloader_init_freq)
1015{
1016 unsigned long aligned_freq;
1017 int idx;
1018
1019 idx = find_target_freq_idx(dmc, bootloader_init_freq);
1020 if (idx >= 0)
1021 aligned_freq = dmc->opp[idx].freq_hz;
1022 else
1023 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
1024
1025 return aligned_freq;
1026}
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
1042 u32 *reg_timing_data, u32 *reg_timing_power,
1043 u32 clk_period_ps)
1044{
1045 u32 val;
1046 const struct timing_reg *reg;
1047
1048 if (clk_period_ps == 0)
1049 return -EINVAL;
1050
1051 *reg_timing_row = 0;
1052 *reg_timing_data = 0;
1053 *reg_timing_power = 0;
1054
1055 val = dmc->timings->tRFC / clk_period_ps;
1056 val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1057 val = max(val, dmc->min_tck->tRFC);
1058 reg = &timing_row_reg_fields[0];
1059 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1060
1061 val = dmc->timings->tRRD / clk_period_ps;
1062 val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1063 val = max(val, dmc->min_tck->tRRD);
1064 reg = &timing_row_reg_fields[1];
1065 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1066
1067 val = dmc->timings->tRPab / clk_period_ps;
1068 val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1069 val = max(val, dmc->min_tck->tRPab);
1070 reg = &timing_row_reg_fields[2];
1071 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1072
1073 val = dmc->timings->tRCD / clk_period_ps;
1074 val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1075 val = max(val, dmc->min_tck->tRCD);
1076 reg = &timing_row_reg_fields[3];
1077 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1078
1079 val = dmc->timings->tRC / clk_period_ps;
1080 val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1081 val = max(val, dmc->min_tck->tRC);
1082 reg = &timing_row_reg_fields[4];
1083 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1084
1085 val = dmc->timings->tRAS / clk_period_ps;
1086 val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1087 val = max(val, dmc->min_tck->tRAS);
1088 reg = &timing_row_reg_fields[5];
1089 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1090
1091
1092 val = dmc->timings->tWTR / clk_period_ps;
1093 val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1094 val = max(val, dmc->min_tck->tWTR);
1095 reg = &timing_data_reg_fields[0];
1096 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1097
1098 val = dmc->timings->tWR / clk_period_ps;
1099 val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1100 val = max(val, dmc->min_tck->tWR);
1101 reg = &timing_data_reg_fields[1];
1102 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1103
1104 val = dmc->timings->tRTP / clk_period_ps;
1105 val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1106 val = max(val, dmc->min_tck->tRTP);
1107 reg = &timing_data_reg_fields[2];
1108 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1109
1110 val = dmc->timings->tW2W_C2C / clk_period_ps;
1111 val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1112 val = max(val, dmc->min_tck->tW2W_C2C);
1113 reg = &timing_data_reg_fields[3];
1114 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1115
1116 val = dmc->timings->tR2R_C2C / clk_period_ps;
1117 val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1118 val = max(val, dmc->min_tck->tR2R_C2C);
1119 reg = &timing_data_reg_fields[4];
1120 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1121
1122 val = dmc->timings->tWL / clk_period_ps;
1123 val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1124 val = max(val, dmc->min_tck->tWL);
1125 reg = &timing_data_reg_fields[5];
1126 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1127
1128 val = dmc->timings->tDQSCK / clk_period_ps;
1129 val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1130 val = max(val, dmc->min_tck->tDQSCK);
1131 reg = &timing_data_reg_fields[6];
1132 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1133
1134 val = dmc->timings->tRL / clk_period_ps;
1135 val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1136 val = max(val, dmc->min_tck->tRL);
1137 reg = &timing_data_reg_fields[7];
1138 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1139
1140
1141 val = dmc->timings->tFAW / clk_period_ps;
1142 val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1143 val = max(val, dmc->min_tck->tFAW);
1144 reg = &timing_power_reg_fields[0];
1145 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1146
1147 val = dmc->timings->tXSR / clk_period_ps;
1148 val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1149 val = max(val, dmc->min_tck->tXSR);
1150 reg = &timing_power_reg_fields[1];
1151 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1152
1153 val = dmc->timings->tXP / clk_period_ps;
1154 val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1155 val = max(val, dmc->min_tck->tXP);
1156 reg = &timing_power_reg_fields[2];
1157 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1158
1159 val = dmc->timings->tCKE / clk_period_ps;
1160 val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1161 val = max(val, dmc->min_tck->tCKE);
1162 reg = &timing_power_reg_fields[3];
1163 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1164
1165 val = dmc->timings->tMRD / clk_period_ps;
1166 val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1167 val = max(val, dmc->min_tck->tMRD);
1168 reg = &timing_power_reg_fields[4];
1169 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1170
1171 return 0;
1172}
1173
1174
1175
1176
1177
1178
1179
1180static int of_get_dram_timings(struct exynos5_dmc *dmc)
1181{
1182 int ret = 0;
1183 int idx;
1184 struct device_node *np_ddr;
1185 u32 freq_mhz, clk_period_ps;
1186
1187 np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);
1188 if (!np_ddr) {
1189 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1190 return -EINVAL;
1191 }
1192
1193 dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1194 sizeof(u32), GFP_KERNEL);
1195 if (!dmc->timing_row)
1196 return -ENOMEM;
1197
1198 dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1199 sizeof(u32), GFP_KERNEL);
1200 if (!dmc->timing_data)
1201 return -ENOMEM;
1202
1203 dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1204 sizeof(u32), GFP_KERNEL);
1205 if (!dmc->timing_power)
1206 return -ENOMEM;
1207
1208 dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,
1209 DDR_TYPE_LPDDR3,
1210 &dmc->timings_arr_size);
1211 if (!dmc->timings) {
1212 of_node_put(np_ddr);
1213 dev_warn(dmc->dev, "could not get timings from DT\n");
1214 return -EINVAL;
1215 }
1216
1217 dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);
1218 if (!dmc->min_tck) {
1219 of_node_put(np_ddr);
1220 dev_warn(dmc->dev, "could not get tck from DT\n");
1221 return -EINVAL;
1222 }
1223
1224
1225 for (idx = 0; idx < dmc->opp_count; idx++) {
1226 freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1227 clk_period_ps = 1000000 / freq_mhz;
1228
1229 ret = create_timings_aligned(dmc, &dmc->timing_row[idx],
1230 &dmc->timing_data[idx],
1231 &dmc->timing_power[idx],
1232 clk_period_ps);
1233 }
1234
1235 of_node_put(np_ddr);
1236
1237
1238 dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1239 dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1240 dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1241
1242 return ret;
1243}
1244
1245
1246
1247
1248
1249
1250
1251
1252static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1253{
1254 int ret;
1255 unsigned long target_volt = 0;
1256 unsigned long target_rate = 0;
1257 unsigned int tmp;
1258
1259 dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");
1260 if (IS_ERR(dmc->fout_spll))
1261 return PTR_ERR(dmc->fout_spll);
1262
1263 dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");
1264 if (IS_ERR(dmc->fout_bpll))
1265 return PTR_ERR(dmc->fout_bpll);
1266
1267 dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");
1268 if (IS_ERR(dmc->mout_mclk_cdrex))
1269 return PTR_ERR(dmc->mout_mclk_cdrex);
1270
1271 dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");
1272 if (IS_ERR(dmc->mout_bpll))
1273 return PTR_ERR(dmc->mout_bpll);
1274
1275 dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,
1276 "mout_mx_mspll_ccore");
1277 if (IS_ERR(dmc->mout_mx_mspll_ccore))
1278 return PTR_ERR(dmc->mout_mx_mspll_ccore);
1279
1280 dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");
1281 if (IS_ERR(dmc->mout_spll)) {
1282 dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");
1283 if (IS_ERR(dmc->mout_spll))
1284 return PTR_ERR(dmc->mout_spll);
1285 }
1286
1287
1288
1289
1290 dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);
1291 dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);
1292 exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1293
1294 ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,
1295 &target_volt, 0);
1296 if (ret)
1297 return ret;
1298
1299 dmc->curr_volt = target_volt;
1300
1301 clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);
1302
1303 clk_prepare_enable(dmc->fout_bpll);
1304 clk_prepare_enable(dmc->mout_bpll);
1305
1306
1307
1308
1309
1310 regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);
1311 tmp &= ~(BIT(1) | BIT(0));
1312 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);
1313
1314 return 0;
1315}
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1327{
1328 int counters_size;
1329 int ret, i;
1330
1331 dmc->num_counters = devfreq_event_get_edev_count(dmc->dev,
1332 "devfreq-events");
1333 if (dmc->num_counters < 0) {
1334 dev_err(dmc->dev, "could not get devfreq-event counters\n");
1335 return dmc->num_counters;
1336 }
1337
1338 counters_size = sizeof(struct devfreq_event_dev) * dmc->num_counters;
1339 dmc->counter = devm_kzalloc(dmc->dev, counters_size, GFP_KERNEL);
1340 if (!dmc->counter)
1341 return -ENOMEM;
1342
1343 for (i = 0; i < dmc->num_counters; i++) {
1344 dmc->counter[i] =
1345 devfreq_event_get_edev_by_phandle(dmc->dev,
1346 "devfreq-events", i);
1347 if (IS_ERR_OR_NULL(dmc->counter[i]))
1348 return -EPROBE_DEFER;
1349 }
1350
1351 ret = exynos5_counters_enable_edev(dmc);
1352 if (ret < 0) {
1353 dev_err(dmc->dev, "could not enable event counter\n");
1354 return ret;
1355 }
1356
1357 ret = exynos5_counters_set_event(dmc);
1358 if (ret < 0) {
1359 exynos5_counters_disable_edev(dmc);
1360 dev_err(dmc->dev, "could not set event counter\n");
1361 return ret;
1362 }
1363
1364 return 0;
1365}
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1377{
1378 unsigned int val;
1379 int ret;
1380
1381 ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);
1382 if (ret)
1383 return ret;
1384
1385 val |= 1UL;
1386 regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);
1387
1388 return 0;
1389}
1390
1391static irqreturn_t dmc_irq_thread(int irq, void *priv)
1392{
1393 int res;
1394 struct exynos5_dmc *dmc = priv;
1395
1396 mutex_lock(&dmc->df->lock);
1397 exynos5_dmc_perf_events_check(dmc);
1398 res = update_devfreq(dmc->df);
1399 mutex_unlock(&dmc->df->lock);
1400
1401 if (res)
1402 dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1403
1404 return IRQ_HANDLED;
1405}
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417static int exynos5_dmc_probe(struct platform_device *pdev)
1418{
1419 int ret = 0;
1420 struct device *dev = &pdev->dev;
1421 struct device_node *np = dev->of_node;
1422 struct exynos5_dmc *dmc;
1423 int irq[2];
1424
1425 dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);
1426 if (!dmc)
1427 return -ENOMEM;
1428
1429 mutex_init(&dmc->lock);
1430
1431 dmc->dev = dev;
1432 platform_set_drvdata(pdev, dmc);
1433
1434 dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);
1435 if (IS_ERR(dmc->base_drexi0))
1436 return PTR_ERR(dmc->base_drexi0);
1437
1438 dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);
1439 if (IS_ERR(dmc->base_drexi1))
1440 return PTR_ERR(dmc->base_drexi1);
1441
1442 dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1443 "samsung,syscon-clk");
1444 if (IS_ERR(dmc->clk_regmap))
1445 return PTR_ERR(dmc->clk_regmap);
1446
1447 ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);
1448 if (ret) {
1449 dev_warn(dev, "couldn't initialize frequency settings\n");
1450 return ret;
1451 }
1452
1453 dmc->vdd_mif = devm_regulator_get(dev, "vdd");
1454 if (IS_ERR(dmc->vdd_mif)) {
1455 ret = PTR_ERR(dmc->vdd_mif);
1456 return ret;
1457 }
1458
1459 ret = exynos5_dmc_init_clks(dmc);
1460 if (ret)
1461 return ret;
1462
1463 ret = of_get_dram_timings(dmc);
1464 if (ret) {
1465 dev_warn(dev, "couldn't initialize timings settings\n");
1466 goto remove_clocks;
1467 }
1468
1469 ret = exynos5_dmc_set_pause_on_switching(dmc);
1470 if (ret) {
1471 dev_warn(dev, "couldn't get access to PAUSE register\n");
1472 goto remove_clocks;
1473 }
1474
1475
1476 irq[0] = platform_get_irq_byname(pdev, "drex_0");
1477 irq[1] = platform_get_irq_byname(pdev, "drex_1");
1478 if (irq[0] > 0 && irq[1] > 0 && irqmode) {
1479 ret = devm_request_threaded_irq(dev, irq[0], NULL,
1480 dmc_irq_thread, IRQF_ONESHOT,
1481 dev_name(dev), dmc);
1482 if (ret) {
1483 dev_err(dev, "couldn't grab IRQ\n");
1484 goto remove_clocks;
1485 }
1486
1487 ret = devm_request_threaded_irq(dev, irq[1], NULL,
1488 dmc_irq_thread, IRQF_ONESHOT,
1489 dev_name(dev), dmc);
1490 if (ret) {
1491 dev_err(dev, "couldn't grab IRQ\n");
1492 goto remove_clocks;
1493 }
1494
1495
1496
1497
1498
1499 dmc->gov_data.upthreshold = 55;
1500 dmc->gov_data.downdifferential = 5;
1501
1502 exynos5_dmc_enable_perf_events(dmc);
1503
1504 dmc->in_irq_mode = 1;
1505 } else {
1506 ret = exynos5_performance_counters_init(dmc);
1507 if (ret) {
1508 dev_warn(dev, "couldn't probe performance counters\n");
1509 goto remove_clocks;
1510 }
1511
1512
1513
1514
1515
1516 dmc->gov_data.upthreshold = 10;
1517 dmc->gov_data.downdifferential = 5;
1518
1519 exynos5_dmc_df_profile.polling_ms = 100;
1520 }
1521
1522 dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,
1523 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1524 &dmc->gov_data);
1525
1526 if (IS_ERR(dmc->df)) {
1527 ret = PTR_ERR(dmc->df);
1528 goto err_devfreq_add;
1529 }
1530
1531 if (dmc->in_irq_mode)
1532 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1533
1534 dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);
1535
1536 return 0;
1537
1538err_devfreq_add:
1539 if (dmc->in_irq_mode)
1540 exynos5_dmc_disable_perf_events(dmc);
1541 else
1542 exynos5_counters_disable_edev(dmc);
1543remove_clocks:
1544 clk_disable_unprepare(dmc->mout_bpll);
1545 clk_disable_unprepare(dmc->fout_bpll);
1546
1547 return ret;
1548}
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558static int exynos5_dmc_remove(struct platform_device *pdev)
1559{
1560 struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);
1561
1562 if (dmc->in_irq_mode)
1563 exynos5_dmc_disable_perf_events(dmc);
1564 else
1565 exynos5_counters_disable_edev(dmc);
1566
1567 clk_disable_unprepare(dmc->mout_bpll);
1568 clk_disable_unprepare(dmc->fout_bpll);
1569
1570 dev_pm_opp_remove_table(dmc->dev);
1571
1572 return 0;
1573}
1574
1575static const struct of_device_id exynos5_dmc_of_match[] = {
1576 { .compatible = "samsung,exynos5422-dmc", },
1577 { },
1578};
1579MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1580
1581static struct platform_driver exynos5_dmc_platdrv = {
1582 .probe = exynos5_dmc_probe,
1583 .remove = exynos5_dmc_remove,
1584 .driver = {
1585 .name = "exynos5-dmc",
1586 .of_match_table = exynos5_dmc_of_match,
1587 },
1588};
1589module_platform_driver(exynos5_dmc_platdrv);
1590MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1591MODULE_LICENSE("GPL v2");
1592MODULE_AUTHOR("Lukasz Luba");
1593