1
2
3
4
5
6
7#include <linux/clk.h>
8#include <linux/devfreq.h>
9#include <linux/devfreq-event.h>
10#include <linux/device.h>
11#include <linux/interrupt.h>
12#include <linux/io.h>
13#include <linux/mfd/syscon.h>
14#include <linux/module.h>
15#include <linux/of_device.h>
16#include <linux/pm_opp.h>
17#include <linux/platform_device.h>
18#include <linux/regmap.h>
19#include <linux/regulator/consumer.h>
20#include <linux/slab.h>
21#include "../jedec_ddr.h"
22#include "../of_memory.h"
23
24#define EXYNOS5_DREXI_TIMINGAREF (0x0030)
25#define EXYNOS5_DREXI_TIMINGROW0 (0x0034)
26#define EXYNOS5_DREXI_TIMINGDATA0 (0x0038)
27#define EXYNOS5_DREXI_TIMINGPOWER0 (0x003C)
28#define EXYNOS5_DREXI_TIMINGROW1 (0x00E4)
29#define EXYNOS5_DREXI_TIMINGDATA1 (0x00E8)
30#define EXYNOS5_DREXI_TIMINGPOWER1 (0x00EC)
31#define CDREX_PAUSE (0x2091c)
32#define CDREX_LPDDR3PHY_CON3 (0x20a20)
33#define CDREX_LPDDR3PHY_CLKM_SRC (0x20700)
34#define EXYNOS5_TIMING_SET_SWI BIT(28)
35#define USE_MX_MSPLL_TIMINGS (1)
36#define USE_BPLL_TIMINGS (0)
37#define EXYNOS5_AREF_NORMAL (0x2e)
38
39#define DREX_PPCCLKCON (0x0130)
40#define DREX_PEREV2CONFIG (0x013c)
41#define DREX_PMNC_PPC (0xE000)
42#define DREX_CNTENS_PPC (0xE010)
43#define DREX_CNTENC_PPC (0xE020)
44#define DREX_INTENS_PPC (0xE030)
45#define DREX_INTENC_PPC (0xE040)
46#define DREX_FLAG_PPC (0xE050)
47#define DREX_PMCNT2_PPC (0xE130)
48
49
50
51
52
53
54#define CC_RESET BIT(2)
55
56
57
58
59
60#define PPC_COUNTER_RESET BIT(1)
61
62
63
64
65
66#define PPC_ENABLE BIT(0)
67
68
69
70
71
72#define PEREV_CLK_EN BIT(0)
73
74
75
76
77
78#define PERF_CNT2 BIT(2)
79#define PERF_CCNT BIT(31)
80
81
82
83
84
85
86
87
88#define READ_TRANSFER_CH0 (0x6d)
89#define READ_TRANSFER_CH1 (0x6f)
90
91#define PERF_COUNTER_START_VALUE 0xff000000
92#define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
93
94
95
96
97
98
99struct dmc_opp_table {
100 u32 freq_hz;
101 u32 volt_uv;
102};
103
104
105
106
107
108
109
110struct exynos5_dmc {
111 struct device *dev;
112 struct devfreq *df;
113 struct devfreq_simple_ondemand_data gov_data;
114 void __iomem *base_drexi0;
115 void __iomem *base_drexi1;
116 struct regmap *clk_regmap;
117 struct mutex lock;
118 unsigned long curr_rate;
119 unsigned long curr_volt;
120 unsigned long bypass_rate;
121 struct dmc_opp_table *opp;
122 struct dmc_opp_table opp_bypass;
123 int opp_count;
124 u32 timings_arr_size;
125 u32 *timing_row;
126 u32 *timing_data;
127 u32 *timing_power;
128 const struct lpddr3_timings *timings;
129 const struct lpddr3_min_tck *min_tck;
130 u32 bypass_timing_row;
131 u32 bypass_timing_data;
132 u32 bypass_timing_power;
133 struct regulator *vdd_mif;
134 struct clk *fout_spll;
135 struct clk *fout_bpll;
136 struct clk *mout_spll;
137 struct clk *mout_bpll;
138 struct clk *mout_mclk_cdrex;
139 struct clk *mout_mx_mspll_ccore;
140 struct clk *mx_mspll_ccore_phy;
141 struct clk *mout_mx_mspll_ccore_phy;
142 struct devfreq_event_dev **counter;
143 int num_counters;
144 u64 last_overflow_ts[2];
145 unsigned long load;
146 unsigned long total;
147 bool in_irq_mode;
148};
149
150#define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
151 { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
152
153#define TIMING_VAL2REG(timing, t_val) \
154({ \
155 u32 __val; \
156 __val = (t_val) << (timing)->bit_beg; \
157 __val; \
158})
159
160struct timing_reg {
161 char *name;
162 int bit_beg;
163 int bit_end;
164 unsigned int val;
165};
166
167static const struct timing_reg timing_row[] = {
168 TIMING_FIELD("tRFC", 24, 31),
169 TIMING_FIELD("tRRD", 20, 23),
170 TIMING_FIELD("tRP", 16, 19),
171 TIMING_FIELD("tRCD", 12, 15),
172 TIMING_FIELD("tRC", 6, 11),
173 TIMING_FIELD("tRAS", 0, 5),
174};
175
176static const struct timing_reg timing_data[] = {
177 TIMING_FIELD("tWTR", 28, 31),
178 TIMING_FIELD("tWR", 24, 27),
179 TIMING_FIELD("tRTP", 20, 23),
180 TIMING_FIELD("tW2W-C2C", 14, 14),
181 TIMING_FIELD("tR2R-C2C", 12, 12),
182 TIMING_FIELD("WL", 8, 11),
183 TIMING_FIELD("tDQSCK", 4, 7),
184 TIMING_FIELD("RL", 0, 3),
185};
186
187static const struct timing_reg timing_power[] = {
188 TIMING_FIELD("tFAW", 26, 31),
189 TIMING_FIELD("tXSR", 16, 25),
190 TIMING_FIELD("tXP", 8, 15),
191 TIMING_FIELD("tCKE", 4, 7),
192 TIMING_FIELD("tMRD", 0, 3),
193};
194
195#define TIMING_COUNT (ARRAY_SIZE(timing_row) + ARRAY_SIZE(timing_data) + \
196 ARRAY_SIZE(timing_power))
197
198static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
199{
200 int i, ret;
201
202 for (i = 0; i < dmc->num_counters; i++) {
203 if (!dmc->counter[i])
204 continue;
205 ret = devfreq_event_set_event(dmc->counter[i]);
206 if (ret < 0)
207 return ret;
208 }
209 return 0;
210}
211
212static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
213{
214 int i, ret;
215
216 for (i = 0; i < dmc->num_counters; i++) {
217 if (!dmc->counter[i])
218 continue;
219 ret = devfreq_event_enable_edev(dmc->counter[i]);
220 if (ret < 0)
221 return ret;
222 }
223 return 0;
224}
225
226static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
227{
228 int i, ret;
229
230 for (i = 0; i < dmc->num_counters; i++) {
231 if (!dmc->counter[i])
232 continue;
233 ret = devfreq_event_disable_edev(dmc->counter[i]);
234 if (ret < 0)
235 return ret;
236 }
237 return 0;
238}
239
240
241
242
243
244
245
246
247
248static int find_target_freq_idx(struct exynos5_dmc *dmc,
249 unsigned long target_rate)
250{
251 int i;
252
253 for (i = dmc->opp_count - 1; i >= 0; i--)
254 if (dmc->opp[i].freq_hz <= target_rate)
255 return i;
256
257 return -EINVAL;
258}
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273static void exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
274{
275 unsigned int reg;
276 int ret;
277
278 ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, ®);
279
280 if (set)
281 reg |= EXYNOS5_TIMING_SET_SWI;
282 else
283 reg &= ~EXYNOS5_TIMING_SET_SWI;
284
285 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);
286}
287
288
289
290
291
292
293
294
295static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
296 struct devfreq_dev_profile *profile)
297{
298 int i, ret;
299 int idx;
300 unsigned long freq;
301
302 ret = dev_pm_opp_of_add_table(dmc->dev);
303 if (ret < 0) {
304 dev_err(dmc->dev, "Failed to get OPP table\n");
305 return ret;
306 }
307
308 dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);
309
310 dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,
311 sizeof(struct dmc_opp_table), GFP_KERNEL);
312 if (!dmc->opp)
313 goto err_opp;
314
315 idx = dmc->opp_count - 1;
316 for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
317 struct dev_pm_opp *opp;
318
319 opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);
320 if (IS_ERR(opp))
321 goto err_opp;
322
323 dmc->opp[idx - i].freq_hz = freq;
324 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
325
326 dev_pm_opp_put(opp);
327 }
328
329 return 0;
330
331err_opp:
332 dev_pm_opp_of_remove_table(dmc->dev);
333
334 return -EINVAL;
335}
336
337
338
339
340
341
342
343
344
345
346static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
347{
348 writel(EXYNOS5_AREF_NORMAL,
349 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
350
351 writel(dmc->bypass_timing_row,
352 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
353 writel(dmc->bypass_timing_row,
354 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
355 writel(dmc->bypass_timing_data,
356 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
357 writel(dmc->bypass_timing_data,
358 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
359 writel(dmc->bypass_timing_power,
360 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
361 writel(dmc->bypass_timing_power,
362 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
363}
364
365
366
367
368
369
370
371
372
373
374
375
376static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
377 unsigned long target_rate)
378{
379 int idx;
380
381 for (idx = dmc->opp_count - 1; idx >= 0; idx--)
382 if (dmc->opp[idx].freq_hz <= target_rate)
383 break;
384
385 if (idx < 0)
386 return -EINVAL;
387
388 writel(EXYNOS5_AREF_NORMAL,
389 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
390
391 writel(dmc->timing_row[idx],
392 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
393 writel(dmc->timing_row[idx],
394 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
395 writel(dmc->timing_data[idx],
396 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
397 writel(dmc->timing_data[idx],
398 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
399 writel(dmc->timing_power[idx],
400 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
401 writel(dmc->timing_power[idx],
402 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
403
404 return 0;
405}
406
407
408
409
410
411
412
413
414
415
416
417static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
418 unsigned long target_volt)
419{
420 int ret = 0;
421
422 if (dmc->curr_volt <= target_volt)
423 return 0;
424
425 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
426 target_volt);
427 if (!ret)
428 dmc->curr_volt = target_volt;
429
430 return ret;
431}
432
433
434
435
436
437
438
439
440
441
442
443static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
444 unsigned long target_volt)
445{
446 int ret = 0;
447 unsigned long bypass_volt = dmc->opp_bypass.volt_uv;
448
449 target_volt = max(bypass_volt, target_volt);
450
451 if (dmc->curr_volt >= target_volt)
452 return 0;
453
454 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
455 target_volt);
456 if (!ret)
457 dmc->curr_volt = target_volt;
458
459 return ret;
460}
461
462
463
464
465
466
467
468
469static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
470 unsigned long target_rate)
471{
472 int idx = find_target_freq_idx(dmc, target_rate);
473
474 if (idx < 0)
475 return -EINVAL;
476
477 exynos5_set_bypass_dram_timings(dmc);
478
479 return 0;
480}
481
482
483
484
485
486
487
488
489
490
491
492
493static int
494exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
495 unsigned long target_rate,
496 unsigned long target_volt)
497{
498 int ret;
499
500
501
502
503
504
505 ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
506 if (ret)
507 return ret;
508
509
510
511
512 ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
513 if (ret)
514 return ret;
515
516
517
518
519 exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
520
521 return ret;
522}
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547static int
548exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
549 unsigned long target_rate,
550 unsigned long target_volt)
551{
552 int ret;
553
554 ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
555 target_volt);
556 if (ret)
557 return ret;
558
559
560
561
562
563 clk_prepare_enable(dmc->fout_spll);
564 clk_prepare_enable(dmc->mout_spll);
565 clk_prepare_enable(dmc->mout_mx_mspll_ccore);
566
567 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);
568 if (ret)
569 goto disable_clocks;
570
571
572
573
574
575
576 exynos5_dram_change_timings(dmc, target_rate);
577
578 clk_set_rate(dmc->fout_bpll, target_rate);
579
580 exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
581
582 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);
583 if (ret)
584 goto disable_clocks;
585
586
587
588
589
590 ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
591
592disable_clocks:
593 clk_disable_unprepare(dmc->mout_mx_mspll_ccore);
594 clk_disable_unprepare(dmc->mout_spll);
595 clk_disable_unprepare(dmc->fout_spll);
596
597 return ret;
598}
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
615 unsigned long *freq,
616 unsigned long *target_rate,
617 unsigned long *target_volt, u32 flags)
618{
619 struct dev_pm_opp *opp;
620
621 opp = devfreq_recommended_opp(dmc->dev, freq, flags);
622 if (IS_ERR(opp))
623 return PTR_ERR(opp);
624
625 *target_rate = dev_pm_opp_get_freq(opp);
626 *target_volt = dev_pm_opp_get_voltage(opp);
627 dev_pm_opp_put(opp);
628
629 return 0;
630}
631
632
633
634
635
636
637
638
639
640
641
642
643
644static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
645 u32 flags)
646{
647 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
648 unsigned long target_rate = 0;
649 unsigned long target_volt = 0;
650 int ret;
651
652 ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,
653 flags);
654
655 if (ret)
656 return ret;
657
658 if (target_rate == dmc->curr_rate)
659 return 0;
660
661 mutex_lock(&dmc->lock);
662
663 ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
664
665 if (ret) {
666 mutex_unlock(&dmc->lock);
667 return ret;
668 }
669
670 dmc->curr_rate = target_rate;
671
672 mutex_unlock(&dmc->lock);
673 return 0;
674}
675
676
677
678
679
680
681
682
683
684
685
686static int exynos5_counters_get(struct exynos5_dmc *dmc,
687 unsigned long *load_count,
688 unsigned long *total_count)
689{
690 unsigned long total = 0;
691 struct devfreq_event_data event;
692 int ret, i;
693
694 *load_count = 0;
695
696
697 for (i = 0; i < dmc->num_counters; i++) {
698 if (!dmc->counter[i])
699 continue;
700
701 ret = devfreq_event_get_event(dmc->counter[i], &event);
702 if (ret < 0)
703 return ret;
704
705 *load_count += event.load_count;
706
707 if (total < event.total_count)
708 total = event.total_count;
709 }
710
711 *total_count = total;
712
713 return 0;
714}
715
716
717
718
719
720
721
722
723
724static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
725 u32 beg_value)
726{
727
728 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);
729 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);
730
731
732 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);
733 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);
734
735
736 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
737 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
738
739
740 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);
741 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);
742
743
744
745
746
747 writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);
748 writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);
749
750
751 writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);
752 writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);
753}
754
755
756
757
758
759
760
761
762
763
764static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
765{
766
767
768
769
770
771
772
773
774
775
776 if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
777
778
779
780
781 dmc->load = 70;
782 dmc->total = 100;
783 } else {
784
785
786
787
788 dmc->load = 35;
789 dmc->total = 100;
790 }
791
792 dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
793}
794
795
796
797
798
799
800
801
802static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
803{
804 u32 val;
805 u64 diff_ts, ts;
806
807 ts = ktime_get_ns();
808
809
810 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
811 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
812
813
814 val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);
815 if (val) {
816 diff_ts = ts - dmc->last_overflow_ts[0];
817 dmc->last_overflow_ts[0] = ts;
818 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n", val);
819 } else {
820 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);
821 diff_ts = ts - dmc->last_overflow_ts[1];
822 dmc->last_overflow_ts[1] = ts;
823 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n", val);
824 }
825
826 exynos5_dmc_perf_events_calc(dmc, diff_ts);
827
828 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
829}
830
831
832
833
834
835
836
837static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
838{
839 u64 ts;
840
841
842 writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);
843 writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);
844
845
846 writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);
847 writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);
848
849 ts = ktime_get_ns();
850 dmc->last_overflow_ts[0] = ts;
851 dmc->last_overflow_ts[1] = ts;
852
853
854 dmc->load = 99;
855 dmc->total = 100;
856}
857
858
859
860
861
862
863
864static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
865{
866
867 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
868 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
869
870
871 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);
872 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);
873
874
875 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);
876 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);
877
878
879 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
880 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
881}
882
883
884
885
886
887
888
889
890
891
892static int exynos5_dmc_get_status(struct device *dev,
893 struct devfreq_dev_status *stat)
894{
895 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
896 unsigned long load, total;
897 int ret;
898
899 if (dmc->in_irq_mode) {
900 stat->current_frequency = dmc->curr_rate;
901 stat->busy_time = dmc->load;
902 stat->total_time = dmc->total;
903 } else {
904 ret = exynos5_counters_get(dmc, &load, &total);
905 if (ret < 0)
906 return -EINVAL;
907
908
909 stat->busy_time = load >> 10;
910 stat->total_time = total >> 10;
911
912 ret = exynos5_counters_set_event(dmc);
913 if (ret < 0) {
914 dev_err(dev, "could not set event counter\n");
915 return ret;
916 }
917 }
918
919 return 0;
920}
921
922
923
924
925
926
927
928
929
930
931static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
932{
933 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
934
935 mutex_lock(&dmc->lock);
936 *freq = dmc->curr_rate;
937 mutex_unlock(&dmc->lock);
938
939 return 0;
940}
941
942
943
944
945
946
947static struct devfreq_dev_profile exynos5_dmc_df_profile = {
948 .target = exynos5_dmc_target,
949 .get_dev_status = exynos5_dmc_get_status,
950 .get_cur_freq = exynos5_dmc_get_cur_freq,
951};
952
953
954
955
956
957
958
959
960
961
962
963
964
965static unsigned long
966exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
967 unsigned long bootloader_init_freq)
968{
969 unsigned long aligned_freq;
970 int idx;
971
972 idx = find_target_freq_idx(dmc, bootloader_init_freq);
973 if (idx >= 0)
974 aligned_freq = dmc->opp[idx].freq_hz;
975 else
976 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
977
978 return aligned_freq;
979}
980
981
982
983
984
985
986
987
988
989
990
991
992static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
993 u32 *reg_timing_data, u32 *reg_timing_power,
994 u32 clk_period_ps)
995{
996 u32 val;
997 const struct timing_reg *reg;
998
999 if (clk_period_ps == 0)
1000 return -EINVAL;
1001
1002 *reg_timing_row = 0;
1003 *reg_timing_data = 0;
1004 *reg_timing_power = 0;
1005
1006 val = dmc->timings->tRFC / clk_period_ps;
1007 val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1008 val = max(val, dmc->min_tck->tRFC);
1009 reg = &timing_row[0];
1010 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1011
1012 val = dmc->timings->tRRD / clk_period_ps;
1013 val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1014 val = max(val, dmc->min_tck->tRRD);
1015 reg = &timing_row[1];
1016 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1017
1018 val = dmc->timings->tRPab / clk_period_ps;
1019 val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1020 val = max(val, dmc->min_tck->tRPab);
1021 reg = &timing_row[2];
1022 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1023
1024 val = dmc->timings->tRCD / clk_period_ps;
1025 val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1026 val = max(val, dmc->min_tck->tRCD);
1027 reg = &timing_row[3];
1028 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1029
1030 val = dmc->timings->tRC / clk_period_ps;
1031 val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1032 val = max(val, dmc->min_tck->tRC);
1033 reg = &timing_row[4];
1034 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1035
1036 val = dmc->timings->tRAS / clk_period_ps;
1037 val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1038 val = max(val, dmc->min_tck->tRAS);
1039 reg = &timing_row[5];
1040 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1041
1042
1043 val = dmc->timings->tWTR / clk_period_ps;
1044 val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1045 val = max(val, dmc->min_tck->tWTR);
1046 reg = &timing_data[0];
1047 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1048
1049 val = dmc->timings->tWR / clk_period_ps;
1050 val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1051 val = max(val, dmc->min_tck->tWR);
1052 reg = &timing_data[1];
1053 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1054
1055 val = dmc->timings->tRTP / clk_period_ps;
1056 val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1057 val = max(val, dmc->min_tck->tRTP);
1058 reg = &timing_data[2];
1059 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1060
1061 val = dmc->timings->tW2W_C2C / clk_period_ps;
1062 val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1063 val = max(val, dmc->min_tck->tW2W_C2C);
1064 reg = &timing_data[3];
1065 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1066
1067 val = dmc->timings->tR2R_C2C / clk_period_ps;
1068 val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1069 val = max(val, dmc->min_tck->tR2R_C2C);
1070 reg = &timing_data[4];
1071 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1072
1073 val = dmc->timings->tWL / clk_period_ps;
1074 val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1075 val = max(val, dmc->min_tck->tWL);
1076 reg = &timing_data[5];
1077 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1078
1079 val = dmc->timings->tDQSCK / clk_period_ps;
1080 val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1081 val = max(val, dmc->min_tck->tDQSCK);
1082 reg = &timing_data[6];
1083 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1084
1085 val = dmc->timings->tRL / clk_period_ps;
1086 val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1087 val = max(val, dmc->min_tck->tRL);
1088 reg = &timing_data[7];
1089 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1090
1091
1092 val = dmc->timings->tFAW / clk_period_ps;
1093 val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1094 val = max(val, dmc->min_tck->tFAW);
1095 reg = &timing_power[0];
1096 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1097
1098 val = dmc->timings->tXSR / clk_period_ps;
1099 val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1100 val = max(val, dmc->min_tck->tXSR);
1101 reg = &timing_power[1];
1102 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1103
1104 val = dmc->timings->tXP / clk_period_ps;
1105 val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1106 val = max(val, dmc->min_tck->tXP);
1107 reg = &timing_power[2];
1108 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1109
1110 val = dmc->timings->tCKE / clk_period_ps;
1111 val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1112 val = max(val, dmc->min_tck->tCKE);
1113 reg = &timing_power[3];
1114 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1115
1116 val = dmc->timings->tMRD / clk_period_ps;
1117 val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1118 val = max(val, dmc->min_tck->tMRD);
1119 reg = &timing_power[4];
1120 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1121
1122 return 0;
1123}
1124
1125
1126
1127
1128
1129
1130
1131static int of_get_dram_timings(struct exynos5_dmc *dmc)
1132{
1133 int ret = 0;
1134 int idx;
1135 struct device_node *np_ddr;
1136 u32 freq_mhz, clk_period_ps;
1137
1138 np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);
1139 if (!np_ddr) {
1140 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1141 return -EINVAL;
1142 }
1143
1144 dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1145 sizeof(u32), GFP_KERNEL);
1146 if (!dmc->timing_row)
1147 return -ENOMEM;
1148
1149 dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1150 sizeof(u32), GFP_KERNEL);
1151 if (!dmc->timing_data)
1152 return -ENOMEM;
1153
1154 dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1155 sizeof(u32), GFP_KERNEL);
1156 if (!dmc->timing_power)
1157 return -ENOMEM;
1158
1159 dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,
1160 DDR_TYPE_LPDDR3,
1161 &dmc->timings_arr_size);
1162 if (!dmc->timings) {
1163 of_node_put(np_ddr);
1164 dev_warn(dmc->dev, "could not get timings from DT\n");
1165 return -EINVAL;
1166 }
1167
1168 dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);
1169 if (!dmc->min_tck) {
1170 of_node_put(np_ddr);
1171 dev_warn(dmc->dev, "could not get tck from DT\n");
1172 return -EINVAL;
1173 }
1174
1175
1176 for (idx = 0; idx < dmc->opp_count; idx++) {
1177 freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1178 clk_period_ps = 1000000 / freq_mhz;
1179
1180 ret = create_timings_aligned(dmc, &dmc->timing_row[idx],
1181 &dmc->timing_data[idx],
1182 &dmc->timing_power[idx],
1183 clk_period_ps);
1184 }
1185
1186 of_node_put(np_ddr);
1187
1188
1189 dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1190 dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1191 dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1192
1193 return ret;
1194}
1195
1196
1197
1198
1199
1200
1201
1202
1203static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1204{
1205 int ret;
1206 unsigned long target_volt = 0;
1207 unsigned long target_rate = 0;
1208 unsigned int tmp;
1209
1210 dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");
1211 if (IS_ERR(dmc->fout_spll))
1212 return PTR_ERR(dmc->fout_spll);
1213
1214 dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");
1215 if (IS_ERR(dmc->fout_bpll))
1216 return PTR_ERR(dmc->fout_bpll);
1217
1218 dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");
1219 if (IS_ERR(dmc->mout_mclk_cdrex))
1220 return PTR_ERR(dmc->mout_mclk_cdrex);
1221
1222 dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");
1223 if (IS_ERR(dmc->mout_bpll))
1224 return PTR_ERR(dmc->mout_bpll);
1225
1226 dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,
1227 "mout_mx_mspll_ccore");
1228 if (IS_ERR(dmc->mout_mx_mspll_ccore))
1229 return PTR_ERR(dmc->mout_mx_mspll_ccore);
1230
1231 dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");
1232 if (IS_ERR(dmc->mout_spll)) {
1233 dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");
1234 if (IS_ERR(dmc->mout_spll))
1235 return PTR_ERR(dmc->mout_spll);
1236 }
1237
1238
1239
1240
1241 dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);
1242 dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);
1243 exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1244
1245 ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,
1246 &target_volt, 0);
1247 if (ret)
1248 return ret;
1249
1250 dmc->curr_volt = target_volt;
1251
1252 clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);
1253
1254 dmc->bypass_rate = clk_get_rate(dmc->mout_mx_mspll_ccore);
1255
1256 clk_prepare_enable(dmc->fout_bpll);
1257 clk_prepare_enable(dmc->mout_bpll);
1258
1259
1260
1261
1262
1263 regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);
1264 tmp &= ~(BIT(1) | BIT(0));
1265 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);
1266
1267 return 0;
1268}
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1280{
1281 int counters_size;
1282 int ret, i;
1283
1284 dmc->num_counters = devfreq_event_get_edev_count(dmc->dev);
1285 if (dmc->num_counters < 0) {
1286 dev_err(dmc->dev, "could not get devfreq-event counters\n");
1287 return dmc->num_counters;
1288 }
1289
1290 counters_size = sizeof(struct devfreq_event_dev) * dmc->num_counters;
1291 dmc->counter = devm_kzalloc(dmc->dev, counters_size, GFP_KERNEL);
1292 if (!dmc->counter)
1293 return -ENOMEM;
1294
1295 for (i = 0; i < dmc->num_counters; i++) {
1296 dmc->counter[i] =
1297 devfreq_event_get_edev_by_phandle(dmc->dev, i);
1298 if (IS_ERR_OR_NULL(dmc->counter[i]))
1299 return -EPROBE_DEFER;
1300 }
1301
1302 ret = exynos5_counters_enable_edev(dmc);
1303 if (ret < 0) {
1304 dev_err(dmc->dev, "could not enable event counter\n");
1305 return ret;
1306 }
1307
1308 ret = exynos5_counters_set_event(dmc);
1309 if (ret < 0) {
1310 exynos5_counters_disable_edev(dmc);
1311 dev_err(dmc->dev, "could not set event counter\n");
1312 return ret;
1313 }
1314
1315 return 0;
1316}
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1329{
1330 unsigned int val;
1331 int ret;
1332
1333 ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);
1334 if (ret)
1335 return ret;
1336
1337 val |= 1UL;
1338 regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);
1339
1340 return 0;
1341}
1342
1343static irqreturn_t dmc_irq_thread(int irq, void *priv)
1344{
1345 int res;
1346 struct exynos5_dmc *dmc = priv;
1347
1348 mutex_lock(&dmc->df->lock);
1349 exynos5_dmc_perf_events_check(dmc);
1350 res = update_devfreq(dmc->df);
1351 mutex_unlock(&dmc->df->lock);
1352
1353 if (res)
1354 dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1355
1356 return IRQ_HANDLED;
1357}
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369static int exynos5_dmc_probe(struct platform_device *pdev)
1370{
1371 int ret = 0;
1372 struct device *dev = &pdev->dev;
1373 struct device_node *np = dev->of_node;
1374 struct exynos5_dmc *dmc;
1375 int irq[2];
1376
1377 dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);
1378 if (!dmc)
1379 return -ENOMEM;
1380
1381 mutex_init(&dmc->lock);
1382
1383 dmc->dev = dev;
1384 platform_set_drvdata(pdev, dmc);
1385
1386 dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);
1387 if (IS_ERR(dmc->base_drexi0))
1388 return PTR_ERR(dmc->base_drexi0);
1389
1390 dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);
1391 if (IS_ERR(dmc->base_drexi1))
1392 return PTR_ERR(dmc->base_drexi1);
1393
1394 dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1395 "samsung,syscon-clk");
1396 if (IS_ERR(dmc->clk_regmap))
1397 return PTR_ERR(dmc->clk_regmap);
1398
1399 ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);
1400 if (ret) {
1401 dev_warn(dev, "couldn't initialize frequency settings\n");
1402 return ret;
1403 }
1404
1405 dmc->vdd_mif = devm_regulator_get(dev, "vdd");
1406 if (IS_ERR(dmc->vdd_mif)) {
1407 ret = PTR_ERR(dmc->vdd_mif);
1408 return ret;
1409 }
1410
1411 ret = exynos5_dmc_init_clks(dmc);
1412 if (ret)
1413 return ret;
1414
1415 ret = of_get_dram_timings(dmc);
1416 if (ret) {
1417 dev_warn(dev, "couldn't initialize timings settings\n");
1418 goto remove_clocks;
1419 }
1420
1421 ret = exynos5_dmc_set_pause_on_switching(dmc);
1422 if (ret) {
1423 dev_warn(dev, "couldn't get access to PAUSE register\n");
1424 goto remove_clocks;
1425 }
1426
1427
1428 irq[0] = platform_get_irq_byname(pdev, "drex_0");
1429 irq[1] = platform_get_irq_byname(pdev, "drex_1");
1430 if (irq[0] > 0 && irq[1] > 0) {
1431 ret = devm_request_threaded_irq(dev, irq[0], NULL,
1432 dmc_irq_thread, IRQF_ONESHOT,
1433 dev_name(dev), dmc);
1434 if (ret) {
1435 dev_err(dev, "couldn't grab IRQ\n");
1436 goto remove_clocks;
1437 }
1438
1439 ret = devm_request_threaded_irq(dev, irq[1], NULL,
1440 dmc_irq_thread, IRQF_ONESHOT,
1441 dev_name(dev), dmc);
1442 if (ret) {
1443 dev_err(dev, "couldn't grab IRQ\n");
1444 goto remove_clocks;
1445 }
1446
1447
1448
1449
1450
1451 dmc->gov_data.upthreshold = 55;
1452 dmc->gov_data.downdifferential = 5;
1453
1454 exynos5_dmc_enable_perf_events(dmc);
1455
1456 dmc->in_irq_mode = 1;
1457 } else {
1458 ret = exynos5_performance_counters_init(dmc);
1459 if (ret) {
1460 dev_warn(dev, "couldn't probe performance counters\n");
1461 goto remove_clocks;
1462 }
1463
1464
1465
1466
1467
1468 dmc->gov_data.upthreshold = 30;
1469 dmc->gov_data.downdifferential = 5;
1470
1471 exynos5_dmc_df_profile.polling_ms = 500;
1472 }
1473
1474
1475 dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,
1476 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1477 &dmc->gov_data);
1478
1479 if (IS_ERR(dmc->df)) {
1480 ret = PTR_ERR(dmc->df);
1481 goto err_devfreq_add;
1482 }
1483
1484 if (dmc->in_irq_mode)
1485 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1486
1487 dev_info(dev, "DMC initialized\n");
1488
1489 return 0;
1490
1491err_devfreq_add:
1492 if (dmc->in_irq_mode)
1493 exynos5_dmc_disable_perf_events(dmc);
1494 else
1495 exynos5_counters_disable_edev(dmc);
1496remove_clocks:
1497 clk_disable_unprepare(dmc->mout_bpll);
1498 clk_disable_unprepare(dmc->fout_bpll);
1499
1500 return ret;
1501}
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511static int exynos5_dmc_remove(struct platform_device *pdev)
1512{
1513 struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);
1514
1515 if (dmc->in_irq_mode)
1516 exynos5_dmc_disable_perf_events(dmc);
1517 else
1518 exynos5_counters_disable_edev(dmc);
1519
1520 clk_disable_unprepare(dmc->mout_bpll);
1521 clk_disable_unprepare(dmc->fout_bpll);
1522
1523 dev_pm_opp_remove_table(dmc->dev);
1524
1525 return 0;
1526}
1527
1528static const struct of_device_id exynos5_dmc_of_match[] = {
1529 { .compatible = "samsung,exynos5422-dmc", },
1530 { },
1531};
1532MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1533
1534static struct platform_driver exynos5_dmc_platdrv = {
1535 .probe = exynos5_dmc_probe,
1536 .remove = exynos5_dmc_remove,
1537 .driver = {
1538 .name = "exynos5-dmc",
1539 .of_match_table = exynos5_dmc_of_match,
1540 },
1541};
1542module_platform_driver(exynos5_dmc_platdrv);
1543MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1544MODULE_LICENSE("GPL v2");
1545MODULE_AUTHOR("Lukasz Luba");
1546