1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/init.h>
19#include <linux/irq.h>
20#include <linux/interrupt.h>
21#include <linux/irqchip.h>
22#include <linux/irqchip/chained_irq.h>
23#include <linux/cpu.h>
24#include <linux/io.h>
25#include <linux/of_address.h>
26#include <linux/of_irq.h>
27#include <linux/of_pci.h>
28#include <linux/irqdomain.h>
29#include <linux/slab.h>
30#include <linux/syscore_ops.h>
31#include <linux/msi.h>
32#include <asm/mach/arch.h>
33#include <asm/exception.h>
34#include <asm/smp_plat.h>
35#include <asm/mach/irq.h>
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118#define ARMADA_370_XP_INT_CONTROL (0x00)
119#define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04)
120#define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30)
121#define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34)
122#define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4)
123#define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF
124#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid)
125
126
127#define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08)
128#define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c)
129#define ARMADA_375_PPI_CAUSE (0x10)
130#define ARMADA_370_XP_CPU_INTACK_OFFS (0x44)
131#define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48)
132#define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C)
133#define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54)
134#define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu)
135
136#define ARMADA_370_XP_MAX_PER_CPU_IRQS (28)
137
138#define IPI_DOORBELL_START (0)
139#define IPI_DOORBELL_END (8)
140#define IPI_DOORBELL_MASK 0xFF
141#define PCI_MSI_DOORBELL_START (16)
142#define PCI_MSI_DOORBELL_NR (16)
143#define PCI_MSI_DOORBELL_END (32)
144#define PCI_MSI_DOORBELL_MASK 0xFFFF0000
145
146static void __iomem *per_cpu_int_base;
147static void __iomem *main_int_base;
148static struct irq_domain *armada_370_xp_mpic_domain;
149static u32 doorbell_mask_reg;
150static int parent_irq;
151#ifdef CONFIG_PCI_MSI
152static struct irq_domain *armada_370_xp_msi_domain;
153static struct irq_domain *armada_370_xp_msi_inner_domain;
154static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR);
155static DEFINE_MUTEX(msi_used_lock);
156static phys_addr_t msi_doorbell_addr;
157#endif
158
159static inline bool is_percpu_irq(irq_hw_number_t irq)
160{
161 if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS)
162 return true;
163
164 return false;
165}
166
167
168
169
170
171
172static void armada_370_xp_irq_mask(struct irq_data *d)
173{
174 irq_hw_number_t hwirq = irqd_to_hwirq(d);
175
176 if (!is_percpu_irq(hwirq))
177 writel(hwirq, main_int_base +
178 ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS);
179 else
180 writel(hwirq, per_cpu_int_base +
181 ARMADA_370_XP_INT_SET_MASK_OFFS);
182}
183
184static void armada_370_xp_irq_unmask(struct irq_data *d)
185{
186 irq_hw_number_t hwirq = irqd_to_hwirq(d);
187
188 if (!is_percpu_irq(hwirq))
189 writel(hwirq, main_int_base +
190 ARMADA_370_XP_INT_SET_ENABLE_OFFS);
191 else
192 writel(hwirq, per_cpu_int_base +
193 ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
194}
195
196#ifdef CONFIG_PCI_MSI
197
198static struct irq_chip armada_370_xp_msi_irq_chip = {
199 .name = "MPIC MSI",
200 .irq_mask = pci_msi_mask_irq,
201 .irq_unmask = pci_msi_unmask_irq,
202};
203
204static struct msi_domain_info armada_370_xp_msi_domain_info = {
205 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
206 MSI_FLAG_MULTI_PCI_MSI),
207 .chip = &armada_370_xp_msi_irq_chip,
208};
209
210static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
211{
212 msg->address_lo = lower_32_bits(msi_doorbell_addr);
213 msg->address_hi = upper_32_bits(msi_doorbell_addr);
214 msg->data = 0xf00 | (data->hwirq + PCI_MSI_DOORBELL_START);
215}
216
217static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data,
218 const struct cpumask *mask, bool force)
219{
220 return -EINVAL;
221}
222
223static struct irq_chip armada_370_xp_msi_bottom_irq_chip = {
224 .name = "MPIC MSI",
225 .irq_compose_msi_msg = armada_370_xp_compose_msi_msg,
226 .irq_set_affinity = armada_370_xp_msi_set_affinity,
227};
228
229static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq,
230 unsigned int nr_irqs, void *args)
231{
232 int hwirq, i;
233
234 mutex_lock(&msi_used_lock);
235
236 hwirq = bitmap_find_next_zero_area(msi_used, PCI_MSI_DOORBELL_NR,
237 0, nr_irqs, 0);
238 if (hwirq >= PCI_MSI_DOORBELL_NR) {
239 mutex_unlock(&msi_used_lock);
240 return -ENOSPC;
241 }
242
243 bitmap_set(msi_used, hwirq, nr_irqs);
244 mutex_unlock(&msi_used_lock);
245
246 for (i = 0; i < nr_irqs; i++) {
247 irq_domain_set_info(domain, virq + i, hwirq + i,
248 &armada_370_xp_msi_bottom_irq_chip,
249 domain->host_data, handle_simple_irq,
250 NULL, NULL);
251 }
252
253 return hwirq;
254}
255
256static void armada_370_xp_msi_free(struct irq_domain *domain,
257 unsigned int virq, unsigned int nr_irqs)
258{
259 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
260
261 mutex_lock(&msi_used_lock);
262 bitmap_clear(msi_used, d->hwirq, nr_irqs);
263 mutex_unlock(&msi_used_lock);
264}
265
266static const struct irq_domain_ops armada_370_xp_msi_domain_ops = {
267 .alloc = armada_370_xp_msi_alloc,
268 .free = armada_370_xp_msi_free,
269};
270
271static int armada_370_xp_msi_init(struct device_node *node,
272 phys_addr_t main_int_phys_base)
273{
274 u32 reg;
275
276 msi_doorbell_addr = main_int_phys_base +
277 ARMADA_370_XP_SW_TRIG_INT_OFFS;
278
279 armada_370_xp_msi_inner_domain =
280 irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR,
281 &armada_370_xp_msi_domain_ops, NULL);
282 if (!armada_370_xp_msi_inner_domain)
283 return -ENOMEM;
284
285 armada_370_xp_msi_domain =
286 pci_msi_create_irq_domain(of_node_to_fwnode(node),
287 &armada_370_xp_msi_domain_info,
288 armada_370_xp_msi_inner_domain);
289 if (!armada_370_xp_msi_domain) {
290 irq_domain_remove(armada_370_xp_msi_inner_domain);
291 return -ENOMEM;
292 }
293
294 reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS)
295 | PCI_MSI_DOORBELL_MASK;
296
297 writel(reg, per_cpu_int_base +
298 ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
299
300
301 writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
302
303 return 0;
304}
305#else
306static inline int armada_370_xp_msi_init(struct device_node *node,
307 phys_addr_t main_int_phys_base)
308{
309 return 0;
310}
311#endif
312
313#ifdef CONFIG_SMP
314static DEFINE_RAW_SPINLOCK(irq_controller_lock);
315
316static int armada_xp_set_affinity(struct irq_data *d,
317 const struct cpumask *mask_val, bool force)
318{
319 irq_hw_number_t hwirq = irqd_to_hwirq(d);
320 unsigned long reg, mask;
321 int cpu;
322
323
324 cpu = cpumask_any_and(mask_val, cpu_online_mask);
325 mask = 1UL << cpu_logical_map(cpu);
326
327 raw_spin_lock(&irq_controller_lock);
328 reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq));
329 reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask;
330 writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq));
331 raw_spin_unlock(&irq_controller_lock);
332
333 return IRQ_SET_MASK_OK;
334}
335#endif
336
337static struct irq_chip armada_370_xp_irq_chip = {
338 .name = "MPIC",
339 .irq_mask = armada_370_xp_irq_mask,
340 .irq_mask_ack = armada_370_xp_irq_mask,
341 .irq_unmask = armada_370_xp_irq_unmask,
342#ifdef CONFIG_SMP
343 .irq_set_affinity = armada_xp_set_affinity,
344#endif
345 .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
346};
347
348static int armada_370_xp_mpic_irq_map(struct irq_domain *h,
349 unsigned int virq, irq_hw_number_t hw)
350{
351 armada_370_xp_irq_mask(irq_get_irq_data(virq));
352 if (!is_percpu_irq(hw))
353 writel(hw, per_cpu_int_base +
354 ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
355 else
356 writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS);
357 irq_set_status_flags(virq, IRQ_LEVEL);
358
359 if (is_percpu_irq(hw)) {
360 irq_set_percpu_devid(virq);
361 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
362 handle_percpu_devid_irq);
363 } else {
364 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
365 handle_level_irq);
366 }
367 irq_set_probe(virq);
368
369 return 0;
370}
371
372static void armada_xp_mpic_smp_cpu_init(void)
373{
374 u32 control;
375 int nr_irqs, i;
376
377 control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL);
378 nr_irqs = (control >> 2) & 0x3ff;
379
380 for (i = 0; i < nr_irqs; i++)
381 writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS);
382
383
384 writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
385
386
387 writel(IPI_DOORBELL_MASK, per_cpu_int_base +
388 ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
389
390
391 writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
392}
393
394static void armada_xp_mpic_perf_init(void)
395{
396 unsigned long cpuid = cpu_logical_map(smp_processor_id());
397
398
399 writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid),
400 per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS);
401}
402
403#ifdef CONFIG_SMP
404static void armada_mpic_send_doorbell(const struct cpumask *mask,
405 unsigned int irq)
406{
407 int cpu;
408 unsigned long map = 0;
409
410
411 for_each_cpu(cpu, mask)
412 map |= 1 << cpu_logical_map(cpu);
413
414
415
416
417
418 dsb();
419
420
421 writel((map << 8) | irq, main_int_base +
422 ARMADA_370_XP_SW_TRIG_INT_OFFS);
423}
424
425static void armada_xp_mpic_reenable_percpu(void)
426{
427 unsigned int irq;
428
429
430 for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) {
431 struct irq_data *data;
432 int virq;
433
434 virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq);
435 if (virq == 0)
436 continue;
437
438 data = irq_get_irq_data(virq);
439
440 if (!irq_percpu_is_enabled(virq))
441 continue;
442
443 armada_370_xp_irq_unmask(data);
444 }
445}
446
447static int armada_xp_mpic_starting_cpu(unsigned int cpu)
448{
449 armada_xp_mpic_perf_init();
450 armada_xp_mpic_smp_cpu_init();
451 armada_xp_mpic_reenable_percpu();
452 return 0;
453}
454
455static int mpic_cascaded_starting_cpu(unsigned int cpu)
456{
457 armada_xp_mpic_perf_init();
458 armada_xp_mpic_reenable_percpu();
459 enable_percpu_irq(parent_irq, IRQ_TYPE_NONE);
460 return 0;
461}
462#endif
463
464static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = {
465 .map = armada_370_xp_mpic_irq_map,
466 .xlate = irq_domain_xlate_onecell,
467};
468
469#ifdef CONFIG_PCI_MSI
470static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained)
471{
472 u32 msimask, msinr;
473
474 msimask = readl_relaxed(per_cpu_int_base +
475 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS)
476 & PCI_MSI_DOORBELL_MASK;
477
478 writel(~msimask, per_cpu_int_base +
479 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
480
481 for (msinr = PCI_MSI_DOORBELL_START;
482 msinr < PCI_MSI_DOORBELL_END; msinr++) {
483 int irq;
484
485 if (!(msimask & BIT(msinr)))
486 continue;
487
488 if (is_chained) {
489 irq = irq_find_mapping(armada_370_xp_msi_inner_domain,
490 msinr - PCI_MSI_DOORBELL_START);
491 generic_handle_irq(irq);
492 } else {
493 irq = msinr - PCI_MSI_DOORBELL_START;
494 handle_domain_irq(armada_370_xp_msi_inner_domain,
495 irq, regs);
496 }
497 }
498}
499#else
500static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {}
501#endif
502
503static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc)
504{
505 struct irq_chip *chip = irq_desc_get_chip(desc);
506 unsigned long irqmap, irqn, irqsrc, cpuid;
507 unsigned int cascade_irq;
508
509 chained_irq_enter(chip, desc);
510
511 irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE);
512 cpuid = cpu_logical_map(smp_processor_id());
513
514 for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) {
515 irqsrc = readl_relaxed(main_int_base +
516 ARMADA_370_XP_INT_SOURCE_CTL(irqn));
517
518
519
520
521 if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid)))
522 continue;
523
524 if (irqn == 1) {
525 armada_370_xp_handle_msi_irq(NULL, true);
526 continue;
527 }
528
529 cascade_irq = irq_find_mapping(armada_370_xp_mpic_domain, irqn);
530 generic_handle_irq(cascade_irq);
531 }
532
533 chained_irq_exit(chip, desc);
534}
535
536static void __exception_irq_entry
537armada_370_xp_handle_irq(struct pt_regs *regs)
538{
539 u32 irqstat, irqnr;
540
541 do {
542 irqstat = readl_relaxed(per_cpu_int_base +
543 ARMADA_370_XP_CPU_INTACK_OFFS);
544 irqnr = irqstat & 0x3FF;
545
546 if (irqnr > 1022)
547 break;
548
549 if (irqnr > 1) {
550 handle_domain_irq(armada_370_xp_mpic_domain,
551 irqnr, regs);
552 continue;
553 }
554
555
556 if (irqnr == 1)
557 armada_370_xp_handle_msi_irq(regs, false);
558
559#ifdef CONFIG_SMP
560
561 if (irqnr == 0) {
562 u32 ipimask, ipinr;
563
564 ipimask = readl_relaxed(per_cpu_int_base +
565 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS)
566 & IPI_DOORBELL_MASK;
567
568 writel(~ipimask, per_cpu_int_base +
569 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
570
571
572 for (ipinr = IPI_DOORBELL_START;
573 ipinr < IPI_DOORBELL_END; ipinr++) {
574 if (ipimask & (0x1 << ipinr))
575 handle_IPI(ipinr, regs);
576 }
577 continue;
578 }
579#endif
580
581 } while (1);
582}
583
584static int armada_370_xp_mpic_suspend(void)
585{
586 doorbell_mask_reg = readl(per_cpu_int_base +
587 ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
588 return 0;
589}
590
591static void armada_370_xp_mpic_resume(void)
592{
593 int nirqs;
594 irq_hw_number_t irq;
595
596
597 nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff;
598 for (irq = 0; irq < nirqs; irq++) {
599 struct irq_data *data;
600 int virq;
601
602 virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq);
603 if (virq == 0)
604 continue;
605
606 data = irq_get_irq_data(virq);
607
608 if (!is_percpu_irq(irq)) {
609
610 writel(irq, per_cpu_int_base +
611 ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
612 if (!irqd_irq_disabled(data))
613 armada_370_xp_irq_unmask(data);
614 } else {
615
616 writel(irq, main_int_base +
617 ARMADA_370_XP_INT_SET_ENABLE_OFFS);
618
619
620
621
622
623
624 if (irq_percpu_is_enabled(virq))
625 armada_370_xp_irq_unmask(data);
626 }
627 }
628
629
630 writel(doorbell_mask_reg,
631 per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
632 if (doorbell_mask_reg & IPI_DOORBELL_MASK)
633 writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
634 if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK)
635 writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
636}
637
638static struct syscore_ops armada_370_xp_mpic_syscore_ops = {
639 .suspend = armada_370_xp_mpic_suspend,
640 .resume = armada_370_xp_mpic_resume,
641};
642
643static int __init armada_370_xp_mpic_of_init(struct device_node *node,
644 struct device_node *parent)
645{
646 struct resource main_int_res, per_cpu_int_res;
647 int nr_irqs, i;
648 u32 control;
649
650 BUG_ON(of_address_to_resource(node, 0, &main_int_res));
651 BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res));
652
653 BUG_ON(!request_mem_region(main_int_res.start,
654 resource_size(&main_int_res),
655 node->full_name));
656 BUG_ON(!request_mem_region(per_cpu_int_res.start,
657 resource_size(&per_cpu_int_res),
658 node->full_name));
659
660 main_int_base = ioremap(main_int_res.start,
661 resource_size(&main_int_res));
662 BUG_ON(!main_int_base);
663
664 per_cpu_int_base = ioremap(per_cpu_int_res.start,
665 resource_size(&per_cpu_int_res));
666 BUG_ON(!per_cpu_int_base);
667
668 control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL);
669 nr_irqs = (control >> 2) & 0x3ff;
670
671 for (i = 0; i < nr_irqs; i++)
672 writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS);
673
674 armada_370_xp_mpic_domain =
675 irq_domain_add_linear(node, nr_irqs,
676 &armada_370_xp_mpic_irq_ops, NULL);
677 BUG_ON(!armada_370_xp_mpic_domain);
678 irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED);
679
680
681 armada_xp_mpic_perf_init();
682 armada_xp_mpic_smp_cpu_init();
683
684 armada_370_xp_msi_init(node, main_int_res.start);
685
686 parent_irq = irq_of_parse_and_map(node, 0);
687 if (parent_irq <= 0) {
688 irq_set_default_host(armada_370_xp_mpic_domain);
689 set_handle_irq(armada_370_xp_handle_irq);
690#ifdef CONFIG_SMP
691 set_smp_cross_call(armada_mpic_send_doorbell);
692 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING,
693 "irqchip/armada/ipi:starting",
694 armada_xp_mpic_starting_cpu, NULL);
695#endif
696 } else {
697#ifdef CONFIG_SMP
698 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING,
699 "irqchip/armada/cascade:starting",
700 mpic_cascaded_starting_cpu, NULL);
701#endif
702 irq_set_chained_handler(parent_irq,
703 armada_370_xp_mpic_handle_cascade_irq);
704 }
705
706 register_syscore_ops(&armada_370_xp_mpic_syscore_ops);
707
708 return 0;
709}
710
711IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init);
712