1
2
3
4
5
6
7
8
9#include <linux/clk.h>
10#include <linux/delay.h>
11#include <linux/interrupt.h>
12#include <linux/kernel.h>
13#include <linux/module.h>
14#include <linux/of.h>
15#include <linux/platform_device.h>
16#include <linux/slab.h>
17#include <linux/sort.h>
18
19#include <soc/tegra/fuse.h>
20
21#include "mc.h"
22
23#define MC_INTSTATUS 0x000
24
25#define MC_INTMASK 0x004
26
27#define MC_ERR_STATUS 0x08
28#define MC_ERR_STATUS_TYPE_SHIFT 28
29#define MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE (6 << MC_ERR_STATUS_TYPE_SHIFT)
30#define MC_ERR_STATUS_TYPE_MASK (0x7 << MC_ERR_STATUS_TYPE_SHIFT)
31#define MC_ERR_STATUS_READABLE (1 << 27)
32#define MC_ERR_STATUS_WRITABLE (1 << 26)
33#define MC_ERR_STATUS_NONSECURE (1 << 25)
34#define MC_ERR_STATUS_ADR_HI_SHIFT 20
35#define MC_ERR_STATUS_ADR_HI_MASK 0x3
36#define MC_ERR_STATUS_SECURITY (1 << 17)
37#define MC_ERR_STATUS_RW (1 << 16)
38
39#define MC_ERR_ADR 0x0c
40
41#define MC_DECERR_EMEM_OTHERS_STATUS 0x58
42#define MC_SECURITY_VIOLATION_STATUS 0x74
43
44#define MC_EMEM_ARB_CFG 0x90
45#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x) (((x) & 0x1ff) << 0)
46#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK 0x1ff
47#define MC_EMEM_ARB_MISC0 0xd8
48
49#define MC_EMEM_ADR_CFG 0x54
50#define MC_EMEM_ADR_CFG_EMEM_NUMDEV BIT(0)
51
52static const struct of_device_id tegra_mc_of_match[] = {
53#ifdef CONFIG_ARCH_TEGRA_2x_SOC
54 { .compatible = "nvidia,tegra20-mc", .data = &tegra20_mc_soc },
55#endif
56#ifdef CONFIG_ARCH_TEGRA_3x_SOC
57 { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
58#endif
59#ifdef CONFIG_ARCH_TEGRA_114_SOC
60 { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
61#endif
62#ifdef CONFIG_ARCH_TEGRA_124_SOC
63 { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
64#endif
65#ifdef CONFIG_ARCH_TEGRA_132_SOC
66 { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
67#endif
68#ifdef CONFIG_ARCH_TEGRA_210_SOC
69 { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
70#endif
71 { }
72};
73MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
74
75static int terga_mc_block_dma_common(struct tegra_mc *mc,
76 const struct tegra_mc_reset *rst)
77{
78 unsigned long flags;
79 u32 value;
80
81 spin_lock_irqsave(&mc->lock, flags);
82
83 value = mc_readl(mc, rst->control) | BIT(rst->bit);
84 mc_writel(mc, value, rst->control);
85
86 spin_unlock_irqrestore(&mc->lock, flags);
87
88 return 0;
89}
90
91static bool terga_mc_dma_idling_common(struct tegra_mc *mc,
92 const struct tegra_mc_reset *rst)
93{
94 return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
95}
96
97static int terga_mc_unblock_dma_common(struct tegra_mc *mc,
98 const struct tegra_mc_reset *rst)
99{
100 unsigned long flags;
101 u32 value;
102
103 spin_lock_irqsave(&mc->lock, flags);
104
105 value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
106 mc_writel(mc, value, rst->control);
107
108 spin_unlock_irqrestore(&mc->lock, flags);
109
110 return 0;
111}
112
113static int terga_mc_reset_status_common(struct tegra_mc *mc,
114 const struct tegra_mc_reset *rst)
115{
116 return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
117}
118
119const struct tegra_mc_reset_ops terga_mc_reset_ops_common = {
120 .block_dma = terga_mc_block_dma_common,
121 .dma_idling = terga_mc_dma_idling_common,
122 .unblock_dma = terga_mc_unblock_dma_common,
123 .reset_status = terga_mc_reset_status_common,
124};
125
126static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
127{
128 return container_of(rcdev, struct tegra_mc, reset);
129}
130
131static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
132 unsigned long id)
133{
134 unsigned int i;
135
136 for (i = 0; i < mc->soc->num_resets; i++)
137 if (mc->soc->resets[i].id == id)
138 return &mc->soc->resets[i];
139
140 return NULL;
141}
142
143static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
144 unsigned long id)
145{
146 struct tegra_mc *mc = reset_to_mc(rcdev);
147 const struct tegra_mc_reset_ops *rst_ops;
148 const struct tegra_mc_reset *rst;
149 int retries = 500;
150 int err;
151
152 rst = tegra_mc_reset_find(mc, id);
153 if (!rst)
154 return -ENODEV;
155
156 rst_ops = mc->soc->reset_ops;
157 if (!rst_ops)
158 return -ENODEV;
159
160 if (rst_ops->block_dma) {
161
162 err = rst_ops->block_dma(mc, rst);
163 if (err) {
164 dev_err(mc->dev, "Failed to block %s DMA: %d\n",
165 rst->name, err);
166 return err;
167 }
168 }
169
170 if (rst_ops->dma_idling) {
171
172 while (!rst_ops->dma_idling(mc, rst)) {
173 if (!retries--) {
174 dev_err(mc->dev, "Failed to flush %s DMA\n",
175 rst->name);
176 return -EBUSY;
177 }
178
179 usleep_range(10, 100);
180 }
181 }
182
183 if (rst_ops->hotreset_assert) {
184
185 err = rst_ops->hotreset_assert(mc, rst);
186 if (err) {
187 dev_err(mc->dev, "Failed to hot reset %s: %d\n",
188 rst->name, err);
189 return err;
190 }
191 }
192
193 return 0;
194}
195
196static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
197 unsigned long id)
198{
199 struct tegra_mc *mc = reset_to_mc(rcdev);
200 const struct tegra_mc_reset_ops *rst_ops;
201 const struct tegra_mc_reset *rst;
202 int err;
203
204 rst = tegra_mc_reset_find(mc, id);
205 if (!rst)
206 return -ENODEV;
207
208 rst_ops = mc->soc->reset_ops;
209 if (!rst_ops)
210 return -ENODEV;
211
212 if (rst_ops->hotreset_deassert) {
213
214 err = rst_ops->hotreset_deassert(mc, rst);
215 if (err) {
216 dev_err(mc->dev, "Failed to deassert hot reset %s: %d\n",
217 rst->name, err);
218 return err;
219 }
220 }
221
222 if (rst_ops->unblock_dma) {
223
224 err = rst_ops->unblock_dma(mc, rst);
225 if (err) {
226 dev_err(mc->dev, "Failed to unblock %s DMA : %d\n",
227 rst->name, err);
228 return err;
229 }
230 }
231
232 return 0;
233}
234
235static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
236 unsigned long id)
237{
238 struct tegra_mc *mc = reset_to_mc(rcdev);
239 const struct tegra_mc_reset_ops *rst_ops;
240 const struct tegra_mc_reset *rst;
241
242 rst = tegra_mc_reset_find(mc, id);
243 if (!rst)
244 return -ENODEV;
245
246 rst_ops = mc->soc->reset_ops;
247 if (!rst_ops)
248 return -ENODEV;
249
250 return rst_ops->reset_status(mc, rst);
251}
252
253static const struct reset_control_ops tegra_mc_reset_ops = {
254 .assert = tegra_mc_hotreset_assert,
255 .deassert = tegra_mc_hotreset_deassert,
256 .status = tegra_mc_hotreset_status,
257};
258
259static int tegra_mc_reset_setup(struct tegra_mc *mc)
260{
261 int err;
262
263 mc->reset.ops = &tegra_mc_reset_ops;
264 mc->reset.owner = THIS_MODULE;
265 mc->reset.of_node = mc->dev->of_node;
266 mc->reset.of_reset_n_cells = 1;
267 mc->reset.nr_resets = mc->soc->num_resets;
268
269 err = reset_controller_register(&mc->reset);
270 if (err < 0)
271 return err;
272
273 return 0;
274}
275
276static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
277{
278 unsigned long long tick;
279 unsigned int i;
280 u32 value;
281
282
283 tick = mc->tick * clk_get_rate(mc->clk);
284 do_div(tick, NSEC_PER_SEC);
285
286 value = readl(mc->regs + MC_EMEM_ARB_CFG);
287 value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
288 value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
289 writel(value, mc->regs + MC_EMEM_ARB_CFG);
290
291
292 for (i = 0; i < mc->soc->num_clients; i++) {
293 const struct tegra_mc_la *la = &mc->soc->clients[i].la;
294 u32 value;
295
296 value = readl(mc->regs + la->reg);
297 value &= ~(la->mask << la->shift);
298 value |= (la->def & la->mask) << la->shift;
299 writel(value, mc->regs + la->reg);
300 }
301
302 return 0;
303}
304
305void tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
306{
307 unsigned int i;
308 struct tegra_mc_timing *timing = NULL;
309
310 for (i = 0; i < mc->num_timings; i++) {
311 if (mc->timings[i].rate == rate) {
312 timing = &mc->timings[i];
313 break;
314 }
315 }
316
317 if (!timing) {
318 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
319 rate);
320 return;
321 }
322
323 for (i = 0; i < mc->soc->num_emem_regs; ++i)
324 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
325}
326
327unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
328{
329 u8 dram_count;
330
331 dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
332 dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
333 dram_count++;
334
335 return dram_count;
336}
337
338static int load_one_timing(struct tegra_mc *mc,
339 struct tegra_mc_timing *timing,
340 struct device_node *node)
341{
342 int err;
343 u32 tmp;
344
345 err = of_property_read_u32(node, "clock-frequency", &tmp);
346 if (err) {
347 dev_err(mc->dev,
348 "timing %s: failed to read rate\n", node->name);
349 return err;
350 }
351
352 timing->rate = tmp;
353 timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
354 sizeof(u32), GFP_KERNEL);
355 if (!timing->emem_data)
356 return -ENOMEM;
357
358 err = of_property_read_u32_array(node, "nvidia,emem-configuration",
359 timing->emem_data,
360 mc->soc->num_emem_regs);
361 if (err) {
362 dev_err(mc->dev,
363 "timing %s: failed to read EMEM configuration\n",
364 node->name);
365 return err;
366 }
367
368 return 0;
369}
370
371static int load_timings(struct tegra_mc *mc, struct device_node *node)
372{
373 struct device_node *child;
374 struct tegra_mc_timing *timing;
375 int child_count = of_get_child_count(node);
376 int i = 0, err;
377
378 mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
379 GFP_KERNEL);
380 if (!mc->timings)
381 return -ENOMEM;
382
383 mc->num_timings = child_count;
384
385 for_each_child_of_node(node, child) {
386 timing = &mc->timings[i++];
387
388 err = load_one_timing(mc, timing, child);
389 if (err) {
390 of_node_put(child);
391 return err;
392 }
393 }
394
395 return 0;
396}
397
398static int tegra_mc_setup_timings(struct tegra_mc *mc)
399{
400 struct device_node *node;
401 u32 ram_code, node_ram_code;
402 int err;
403
404 ram_code = tegra_read_ram_code();
405
406 mc->num_timings = 0;
407
408 for_each_child_of_node(mc->dev->of_node, node) {
409 err = of_property_read_u32(node, "nvidia,ram-code",
410 &node_ram_code);
411 if (err || (node_ram_code != ram_code))
412 continue;
413
414 err = load_timings(mc, node);
415 of_node_put(node);
416 if (err)
417 return err;
418 break;
419 }
420
421 if (mc->num_timings == 0)
422 dev_warn(mc->dev,
423 "no memory timings for RAM code %u registered\n",
424 ram_code);
425
426 return 0;
427}
428
429static const char *const status_names[32] = {
430 [ 1] = "External interrupt",
431 [ 6] = "EMEM address decode error",
432 [ 7] = "GART page fault",
433 [ 8] = "Security violation",
434 [ 9] = "EMEM arbitration error",
435 [10] = "Page fault",
436 [11] = "Invalid APB ASID update",
437 [12] = "VPR violation",
438 [13] = "Secure carveout violation",
439 [16] = "MTS carveout violation",
440};
441
442static const char *const error_names[8] = {
443 [2] = "EMEM decode error",
444 [3] = "TrustZone violation",
445 [4] = "Carveout violation",
446 [6] = "SMMU translation error",
447};
448
449static irqreturn_t tegra_mc_irq(int irq, void *data)
450{
451 struct tegra_mc *mc = data;
452 unsigned long status;
453 unsigned int bit;
454
455
456 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
457 if (!status)
458 return IRQ_NONE;
459
460 for_each_set_bit(bit, &status, 32) {
461 const char *error = status_names[bit] ?: "unknown";
462 const char *client = "unknown", *desc;
463 const char *direction, *secure;
464 phys_addr_t addr = 0;
465 unsigned int i;
466 char perm[7];
467 u8 id, type;
468 u32 value;
469
470 value = mc_readl(mc, MC_ERR_STATUS);
471
472#ifdef CONFIG_PHYS_ADDR_T_64BIT
473 if (mc->soc->num_address_bits > 32) {
474 addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
475 MC_ERR_STATUS_ADR_HI_MASK);
476 addr <<= 32;
477 }
478#endif
479
480 if (value & MC_ERR_STATUS_RW)
481 direction = "write";
482 else
483 direction = "read";
484
485 if (value & MC_ERR_STATUS_SECURITY)
486 secure = "secure ";
487 else
488 secure = "";
489
490 id = value & mc->soc->client_id_mask;
491
492 for (i = 0; i < mc->soc->num_clients; i++) {
493 if (mc->soc->clients[i].id == id) {
494 client = mc->soc->clients[i].name;
495 break;
496 }
497 }
498
499 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
500 MC_ERR_STATUS_TYPE_SHIFT;
501 desc = error_names[type];
502
503 switch (value & MC_ERR_STATUS_TYPE_MASK) {
504 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
505 perm[0] = ' ';
506 perm[1] = '[';
507
508 if (value & MC_ERR_STATUS_READABLE)
509 perm[2] = 'R';
510 else
511 perm[2] = '-';
512
513 if (value & MC_ERR_STATUS_WRITABLE)
514 perm[3] = 'W';
515 else
516 perm[3] = '-';
517
518 if (value & MC_ERR_STATUS_NONSECURE)
519 perm[4] = '-';
520 else
521 perm[4] = 'S';
522
523 perm[5] = ']';
524 perm[6] = '\0';
525 break;
526
527 default:
528 perm[0] = '\0';
529 break;
530 }
531
532 value = mc_readl(mc, MC_ERR_ADR);
533 addr |= value;
534
535 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
536 client, secure, direction, &addr, error,
537 desc, perm);
538 }
539
540
541 mc_writel(mc, status, MC_INTSTATUS);
542
543 return IRQ_HANDLED;
544}
545
546static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
547{
548 struct tegra_mc *mc = data;
549 unsigned long status;
550 unsigned int bit;
551
552
553 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
554 if (!status)
555 return IRQ_NONE;
556
557 for_each_set_bit(bit, &status, 32) {
558 const char *direction = "read", *secure = "";
559 const char *error = status_names[bit];
560 const char *client, *desc;
561 phys_addr_t addr;
562 u32 value, reg;
563 u8 id, type;
564
565 switch (BIT(bit)) {
566 case MC_INT_DECERR_EMEM:
567 reg = MC_DECERR_EMEM_OTHERS_STATUS;
568 value = mc_readl(mc, reg);
569
570 id = value & mc->soc->client_id_mask;
571 desc = error_names[2];
572
573 if (value & BIT(31))
574 direction = "write";
575 break;
576
577 case MC_INT_INVALID_GART_PAGE:
578 dev_err_ratelimited(mc->dev, "%s\n", error);
579 continue;
580
581 case MC_INT_SECURITY_VIOLATION:
582 reg = MC_SECURITY_VIOLATION_STATUS;
583 value = mc_readl(mc, reg);
584
585 id = value & mc->soc->client_id_mask;
586 type = (value & BIT(30)) ? 4 : 3;
587 desc = error_names[type];
588 secure = "secure ";
589
590 if (value & BIT(31))
591 direction = "write";
592 break;
593
594 default:
595 continue;
596 }
597
598 client = mc->soc->clients[id].name;
599 addr = mc_readl(mc, reg + sizeof(u32));
600
601 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
602 client, secure, direction, &addr, error,
603 desc);
604 }
605
606
607 mc_writel(mc, status, MC_INTSTATUS);
608
609 return IRQ_HANDLED;
610}
611
612static int tegra_mc_probe(struct platform_device *pdev)
613{
614 const struct of_device_id *match;
615 struct resource *res;
616 struct tegra_mc *mc;
617 void *isr;
618 int err;
619
620 match = of_match_node(tegra_mc_of_match, pdev->dev.of_node);
621 if (!match)
622 return -ENODEV;
623
624 mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
625 if (!mc)
626 return -ENOMEM;
627
628 platform_set_drvdata(pdev, mc);
629 spin_lock_init(&mc->lock);
630 mc->soc = match->data;
631 mc->dev = &pdev->dev;
632
633
634 mc->tick = 30;
635
636 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
637 mc->regs = devm_ioremap_resource(&pdev->dev, res);
638 if (IS_ERR(mc->regs))
639 return PTR_ERR(mc->regs);
640
641#ifdef CONFIG_ARCH_TEGRA_2x_SOC
642 if (mc->soc == &tegra20_mc_soc) {
643 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
644 mc->regs2 = devm_ioremap_resource(&pdev->dev, res);
645 if (IS_ERR(mc->regs2))
646 return PTR_ERR(mc->regs2);
647
648 isr = tegra20_mc_irq;
649 } else
650#endif
651 {
652 mc->clk = devm_clk_get(&pdev->dev, "mc");
653 if (IS_ERR(mc->clk)) {
654 dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
655 PTR_ERR(mc->clk));
656 return PTR_ERR(mc->clk);
657 }
658
659 err = tegra_mc_setup_latency_allowance(mc);
660 if (err < 0) {
661 dev_err(&pdev->dev, "failed to setup latency allowance: %d\n",
662 err);
663 return err;
664 }
665
666 isr = tegra_mc_irq;
667 }
668
669 err = tegra_mc_setup_timings(mc);
670 if (err < 0) {
671 dev_err(&pdev->dev, "failed to setup timings: %d\n", err);
672 return err;
673 }
674
675 err = tegra_mc_reset_setup(mc);
676 if (err < 0) {
677 dev_err(&pdev->dev, "failed to register reset controller: %d\n",
678 err);
679 return err;
680 }
681
682 mc->irq = platform_get_irq(pdev, 0);
683 if (mc->irq < 0) {
684 dev_err(&pdev->dev, "interrupt not specified\n");
685 return mc->irq;
686 }
687
688 WARN(!mc->soc->client_id_mask, "Missing client ID mask for this SoC\n");
689
690 mc_writel(mc, mc->soc->intmask, MC_INTMASK);
691
692 err = devm_request_irq(&pdev->dev, mc->irq, isr, IRQF_SHARED,
693 dev_name(&pdev->dev), mc);
694 if (err < 0) {
695 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
696 err);
697 return err;
698 }
699
700 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU)) {
701 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
702 if (IS_ERR(mc->smmu)) {
703 dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
704 PTR_ERR(mc->smmu));
705 return PTR_ERR(mc->smmu);
706 }
707 }
708
709 return 0;
710}
711
712static struct platform_driver tegra_mc_driver = {
713 .driver = {
714 .name = "tegra-mc",
715 .of_match_table = tegra_mc_of_match,
716 .suppress_bind_attrs = true,
717 },
718 .prevent_deferred_probe = true,
719 .probe = tegra_mc_probe,
720};
721
722static int tegra_mc_init(void)
723{
724 return platform_driver_register(&tegra_mc_driver);
725}
726arch_initcall(tegra_mc_init);
727
728MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
729MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
730MODULE_LICENSE("GPL v2");
731
