1
2
3
4
5
6
7
8
9
10
11#include <linux/irq.h>
12#include <linux/msi.h>
13#include <linux/module.h>
14#include <linux/interrupt.h>
15#include <linux/kernel_stat.h>
16#include <linux/irqdomain.h>
17
18#include <trace/events/irq.h>
19
20#include "internals.h"
21
22static irqreturn_t bad_chained_irq(int irq, void *dev_id)
23{
24 WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
25 return IRQ_NONE;
26}
27
28
29
30
31
32struct irqaction chained_action = {
33 .handler = bad_chained_irq,
34};
35
36
37
38
39
40
41int irq_set_chip(unsigned int irq, const struct irq_chip *chip)
42{
43 unsigned long flags;
44 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
45
46 if (!desc)
47 return -EINVAL;
48
49 desc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip);
50 irq_put_desc_unlock(desc, flags);
51
52
53
54
55 irq_mark_irq(irq);
56 return 0;
57}
58EXPORT_SYMBOL(irq_set_chip);
59
60
61
62
63
64
65int irq_set_irq_type(unsigned int irq, unsigned int type)
66{
67 unsigned long flags;
68 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
69 int ret = 0;
70
71 if (!desc)
72 return -EINVAL;
73
74 ret = __irq_set_trigger(desc, type);
75 irq_put_desc_busunlock(desc, flags);
76 return ret;
77}
78EXPORT_SYMBOL(irq_set_irq_type);
79
80
81
82
83
84
85
86
87int irq_set_handler_data(unsigned int irq, void *data)
88{
89 unsigned long flags;
90 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
91
92 if (!desc)
93 return -EINVAL;
94 desc->irq_common_data.handler_data = data;
95 irq_put_desc_unlock(desc, flags);
96 return 0;
97}
98EXPORT_SYMBOL(irq_set_handler_data);
99
100
101
102
103
104
105
106
107
108int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
109 struct msi_desc *entry)
110{
111 unsigned long flags;
112 struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
113
114 if (!desc)
115 return -EINVAL;
116 desc->irq_common_data.msi_desc = entry;
117 if (entry && !irq_offset)
118 entry->irq = irq_base;
119 irq_put_desc_unlock(desc, flags);
120 return 0;
121}
122
123
124
125
126
127
128
129
130int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
131{
132 return irq_set_msi_desc_off(irq, 0, entry);
133}
134
135
136
137
138
139
140
141
142int irq_set_chip_data(unsigned int irq, void *data)
143{
144 unsigned long flags;
145 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
146
147 if (!desc)
148 return -EINVAL;
149 desc->irq_data.chip_data = data;
150 irq_put_desc_unlock(desc, flags);
151 return 0;
152}
153EXPORT_SYMBOL(irq_set_chip_data);
154
155struct irq_data *irq_get_irq_data(unsigned int irq)
156{
157 struct irq_desc *desc = irq_to_desc(irq);
158
159 return desc ? &desc->irq_data : NULL;
160}
161EXPORT_SYMBOL_GPL(irq_get_irq_data);
162
163static void irq_state_clr_disabled(struct irq_desc *desc)
164{
165 irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
166}
167
168static void irq_state_clr_masked(struct irq_desc *desc)
169{
170 irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
171}
172
173static void irq_state_clr_started(struct irq_desc *desc)
174{
175 irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
176}
177
178static void irq_state_set_started(struct irq_desc *desc)
179{
180 irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
181}
182
183enum {
184 IRQ_STARTUP_NORMAL,
185 IRQ_STARTUP_MANAGED,
186 IRQ_STARTUP_ABORT,
187};
188
189#ifdef CONFIG_SMP
190static int
191__irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
192{
193 struct irq_data *d = irq_desc_get_irq_data(desc);
194
195 if (!irqd_affinity_is_managed(d))
196 return IRQ_STARTUP_NORMAL;
197
198 irqd_clr_managed_shutdown(d);
199
200 if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) {
201
202
203
204
205
206
207 if (WARN_ON_ONCE(force))
208 return IRQ_STARTUP_ABORT;
209
210
211
212
213
214
215 return IRQ_STARTUP_ABORT;
216 }
217
218
219
220
221 if (WARN_ON(irq_domain_activate_irq(d, false)))
222 return IRQ_STARTUP_ABORT;
223 return IRQ_STARTUP_MANAGED;
224}
225#else
226static __always_inline int
227__irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
228{
229 return IRQ_STARTUP_NORMAL;
230}
231#endif
232
233static int __irq_startup(struct irq_desc *desc)
234{
235 struct irq_data *d = irq_desc_get_irq_data(desc);
236 int ret = 0;
237
238
239 WARN_ON_ONCE(!irqd_is_activated(d));
240
241 if (d->chip->irq_startup) {
242 ret = d->chip->irq_startup(d);
243 irq_state_clr_disabled(desc);
244 irq_state_clr_masked(desc);
245 } else {
246 irq_enable(desc);
247 }
248 irq_state_set_started(desc);
249 return ret;
250}
251
252int irq_startup(struct irq_desc *desc, bool resend, bool force)
253{
254 struct irq_data *d = irq_desc_get_irq_data(desc);
255 struct cpumask *aff = irq_data_get_affinity_mask(d);
256 int ret = 0;
257
258 desc->depth = 0;
259
260 if (irqd_is_started(d)) {
261 irq_enable(desc);
262 } else {
263 switch (__irq_startup_managed(desc, aff, force)) {
264 case IRQ_STARTUP_NORMAL:
265 if (d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP)
266 irq_setup_affinity(desc);
267 ret = __irq_startup(desc);
268 if (!(d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP))
269 irq_setup_affinity(desc);
270 break;
271 case IRQ_STARTUP_MANAGED:
272 irq_do_set_affinity(d, aff, false);
273 ret = __irq_startup(desc);
274 break;
275 case IRQ_STARTUP_ABORT:
276 irqd_set_managed_shutdown(d);
277 return 0;
278 }
279 }
280 if (resend)
281 check_irq_resend(desc, false);
282
283 return ret;
284}
285
286int irq_activate(struct irq_desc *desc)
287{
288 struct irq_data *d = irq_desc_get_irq_data(desc);
289
290 if (!irqd_affinity_is_managed(d))
291 return irq_domain_activate_irq(d, false);
292 return 0;
293}
294
295int irq_activate_and_startup(struct irq_desc *desc, bool resend)
296{
297 if (WARN_ON(irq_activate(desc)))
298 return 0;
299 return irq_startup(desc, resend, IRQ_START_FORCE);
300}
301
302static void __irq_disable(struct irq_desc *desc, bool mask);
303
304void irq_shutdown(struct irq_desc *desc)
305{
306 if (irqd_is_started(&desc->irq_data)) {
307 desc->depth = 1;
308 if (desc->irq_data.chip->irq_shutdown) {
309 desc->irq_data.chip->irq_shutdown(&desc->irq_data);
310 irq_state_set_disabled(desc);
311 irq_state_set_masked(desc);
312 } else {
313 __irq_disable(desc, true);
314 }
315 irq_state_clr_started(desc);
316 }
317}
318
319
320void irq_shutdown_and_deactivate(struct irq_desc *desc)
321{
322 irq_shutdown(desc);
323
324
325
326
327
328
329 irq_domain_deactivate_irq(&desc->irq_data);
330}
331
332void irq_enable(struct irq_desc *desc)
333{
334 if (!irqd_irq_disabled(&desc->irq_data)) {
335 unmask_irq(desc);
336 } else {
337 irq_state_clr_disabled(desc);
338 if (desc->irq_data.chip->irq_enable) {
339 desc->irq_data.chip->irq_enable(&desc->irq_data);
340 irq_state_clr_masked(desc);
341 } else {
342 unmask_irq(desc);
343 }
344 }
345}
346
347static void __irq_disable(struct irq_desc *desc, bool mask)
348{
349 if (irqd_irq_disabled(&desc->irq_data)) {
350 if (mask)
351 mask_irq(desc);
352 } else {
353 irq_state_set_disabled(desc);
354 if (desc->irq_data.chip->irq_disable) {
355 desc->irq_data.chip->irq_disable(&desc->irq_data);
356 irq_state_set_masked(desc);
357 } else if (mask) {
358 mask_irq(desc);
359 }
360 }
361}
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383void irq_disable(struct irq_desc *desc)
384{
385 __irq_disable(desc, irq_settings_disable_unlazy(desc));
386}
387
388void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
389{
390 if (desc->irq_data.chip->irq_enable)
391 desc->irq_data.chip->irq_enable(&desc->irq_data);
392 else
393 desc->irq_data.chip->irq_unmask(&desc->irq_data);
394 cpumask_set_cpu(cpu, desc->percpu_enabled);
395}
396
397void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
398{
399 if (desc->irq_data.chip->irq_disable)
400 desc->irq_data.chip->irq_disable(&desc->irq_data);
401 else
402 desc->irq_data.chip->irq_mask(&desc->irq_data);
403 cpumask_clear_cpu(cpu, desc->percpu_enabled);
404}
405
406static inline void mask_ack_irq(struct irq_desc *desc)
407{
408 if (desc->irq_data.chip->irq_mask_ack) {
409 desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
410 irq_state_set_masked(desc);
411 } else {
412 mask_irq(desc);
413 if (desc->irq_data.chip->irq_ack)
414 desc->irq_data.chip->irq_ack(&desc->irq_data);
415 }
416}
417
418void mask_irq(struct irq_desc *desc)
419{
420 if (irqd_irq_masked(&desc->irq_data))
421 return;
422
423 if (desc->irq_data.chip->irq_mask) {
424 desc->irq_data.chip->irq_mask(&desc->irq_data);
425 irq_state_set_masked(desc);
426 }
427}
428
429void unmask_irq(struct irq_desc *desc)
430{
431 if (!irqd_irq_masked(&desc->irq_data))
432 return;
433
434 if (desc->irq_data.chip->irq_unmask) {
435 desc->irq_data.chip->irq_unmask(&desc->irq_data);
436 irq_state_clr_masked(desc);
437 }
438}
439
440void unmask_threaded_irq(struct irq_desc *desc)
441{
442 struct irq_chip *chip = desc->irq_data.chip;
443
444 if (chip->flags & IRQCHIP_EOI_THREADED)
445 chip->irq_eoi(&desc->irq_data);
446
447 unmask_irq(desc);
448}
449
450
451
452
453
454
455
456
457
458void handle_nested_irq(unsigned int irq)
459{
460 struct irq_desc *desc = irq_to_desc(irq);
461 struct irqaction *action;
462 irqreturn_t action_ret;
463
464 might_sleep();
465
466 raw_spin_lock_irq(&desc->lock);
467
468 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
469
470 action = desc->action;
471 if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {
472 desc->istate |= IRQS_PENDING;
473 goto out_unlock;
474 }
475
476 kstat_incr_irqs_this_cpu(desc);
477 irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
478 raw_spin_unlock_irq(&desc->lock);
479
480 action_ret = IRQ_NONE;
481 for_each_action_of_desc(desc, action)
482 action_ret |= action->thread_fn(action->irq, action->dev_id);
483
484 if (!irq_settings_no_debug(desc))
485 note_interrupt(desc, action_ret);
486
487 raw_spin_lock_irq(&desc->lock);
488 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
489
490out_unlock:
491 raw_spin_unlock_irq(&desc->lock);
492}
493EXPORT_SYMBOL_GPL(handle_nested_irq);
494
495static bool irq_check_poll(struct irq_desc *desc)
496{
497 if (!(desc->istate & IRQS_POLL_INPROGRESS))
498 return false;
499 return irq_wait_for_poll(desc);
500}
501
502static bool irq_may_run(struct irq_desc *desc)
503{
504 unsigned int mask = IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED;
505
506
507
508
509
510 if (!irqd_has_set(&desc->irq_data, mask))
511 return true;
512
513
514
515
516
517
518 if (irq_pm_check_wakeup(desc))
519 return false;
520
521
522
523
524 return irq_check_poll(desc);
525}
526
527
528
529
530
531
532
533
534
535
536
537
538void handle_simple_irq(struct irq_desc *desc)
539{
540 raw_spin_lock(&desc->lock);
541
542 if (!irq_may_run(desc))
543 goto out_unlock;
544
545 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
546
547 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
548 desc->istate |= IRQS_PENDING;
549 goto out_unlock;
550 }
551
552 kstat_incr_irqs_this_cpu(desc);
553 handle_irq_event(desc);
554
555out_unlock:
556 raw_spin_unlock(&desc->lock);
557}
558EXPORT_SYMBOL_GPL(handle_simple_irq);
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573void handle_untracked_irq(struct irq_desc *desc)
574{
575 raw_spin_lock(&desc->lock);
576
577 if (!irq_may_run(desc))
578 goto out_unlock;
579
580 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
581
582 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
583 desc->istate |= IRQS_PENDING;
584 goto out_unlock;
585 }
586
587 desc->istate &= ~IRQS_PENDING;
588 irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
589 raw_spin_unlock(&desc->lock);
590
591 __handle_irq_event_percpu(desc);
592
593 raw_spin_lock(&desc->lock);
594 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
595
596out_unlock:
597 raw_spin_unlock(&desc->lock);
598}
599EXPORT_SYMBOL_GPL(handle_untracked_irq);
600
601
602
603
604
605static void cond_unmask_irq(struct irq_desc *desc)
606{
607
608
609
610
611
612
613
614 if (!irqd_irq_disabled(&desc->irq_data) &&
615 irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
616 unmask_irq(desc);
617}
618
619
620
621
622
623
624
625
626
627
628void handle_level_irq(struct irq_desc *desc)
629{
630 raw_spin_lock(&desc->lock);
631 mask_ack_irq(desc);
632
633 if (!irq_may_run(desc))
634 goto out_unlock;
635
636 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
637
638
639
640
641
642 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
643 desc->istate |= IRQS_PENDING;
644 goto out_unlock;
645 }
646
647 kstat_incr_irqs_this_cpu(desc);
648 handle_irq_event(desc);
649
650 cond_unmask_irq(desc);
651
652out_unlock:
653 raw_spin_unlock(&desc->lock);
654}
655EXPORT_SYMBOL_GPL(handle_level_irq);
656
657static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
658{
659 if (!(desc->istate & IRQS_ONESHOT)) {
660 chip->irq_eoi(&desc->irq_data);
661 return;
662 }
663
664
665
666
667
668
669 if (!irqd_irq_disabled(&desc->irq_data) &&
670 irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
671 chip->irq_eoi(&desc->irq_data);
672 unmask_irq(desc);
673 } else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
674 chip->irq_eoi(&desc->irq_data);
675 }
676}
677
678
679
680
681
682
683
684
685
686
687void handle_fasteoi_irq(struct irq_desc *desc)
688{
689 struct irq_chip *chip = desc->irq_data.chip;
690
691 raw_spin_lock(&desc->lock);
692
693 if (!irq_may_run(desc))
694 goto out;
695
696 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
697
698
699
700
701
702 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
703 desc->istate |= IRQS_PENDING;
704 mask_irq(desc);
705 goto out;
706 }
707
708 kstat_incr_irqs_this_cpu(desc);
709 if (desc->istate & IRQS_ONESHOT)
710 mask_irq(desc);
711
712 handle_irq_event(desc);
713
714 cond_unmask_eoi_irq(desc, chip);
715
716 raw_spin_unlock(&desc->lock);
717 return;
718out:
719 if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
720 chip->irq_eoi(&desc->irq_data);
721 raw_spin_unlock(&desc->lock);
722}
723EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
724
725
726
727
728
729
730
731
732
733
734
735
736
737void handle_fasteoi_nmi(struct irq_desc *desc)
738{
739 struct irq_chip *chip = irq_desc_get_chip(desc);
740 struct irqaction *action = desc->action;
741 unsigned int irq = irq_desc_get_irq(desc);
742 irqreturn_t res;
743
744 __kstat_incr_irqs_this_cpu(desc);
745
746 trace_irq_handler_entry(irq, action);
747
748
749
750 res = action->handler(irq, action->dev_id);
751 trace_irq_handler_exit(irq, action, res);
752
753 if (chip->irq_eoi)
754 chip->irq_eoi(&desc->irq_data);
755}
756EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773void handle_edge_irq(struct irq_desc *desc)
774{
775 raw_spin_lock(&desc->lock);
776
777 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
778
779 if (!irq_may_run(desc)) {
780 desc->istate |= IRQS_PENDING;
781 mask_ack_irq(desc);
782 goto out_unlock;
783 }
784
785
786
787
788
789 if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
790 desc->istate |= IRQS_PENDING;
791 mask_ack_irq(desc);
792 goto out_unlock;
793 }
794
795 kstat_incr_irqs_this_cpu(desc);
796
797
798 desc->irq_data.chip->irq_ack(&desc->irq_data);
799
800 do {
801 if (unlikely(!desc->action)) {
802 mask_irq(desc);
803 goto out_unlock;
804 }
805
806
807
808
809
810
811 if (unlikely(desc->istate & IRQS_PENDING)) {
812 if (!irqd_irq_disabled(&desc->irq_data) &&
813 irqd_irq_masked(&desc->irq_data))
814 unmask_irq(desc);
815 }
816
817 handle_irq_event(desc);
818
819 } while ((desc->istate & IRQS_PENDING) &&
820 !irqd_irq_disabled(&desc->irq_data));
821
822out_unlock:
823 raw_spin_unlock(&desc->lock);
824}
825EXPORT_SYMBOL(handle_edge_irq);
826
827#ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
828
829
830
831
832
833
834
835void handle_edge_eoi_irq(struct irq_desc *desc)
836{
837 struct irq_chip *chip = irq_desc_get_chip(desc);
838
839 raw_spin_lock(&desc->lock);
840
841 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
842
843 if (!irq_may_run(desc)) {
844 desc->istate |= IRQS_PENDING;
845 goto out_eoi;
846 }
847
848
849
850
851
852 if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
853 desc->istate |= IRQS_PENDING;
854 goto out_eoi;
855 }
856
857 kstat_incr_irqs_this_cpu(desc);
858
859 do {
860 if (unlikely(!desc->action))
861 goto out_eoi;
862
863 handle_irq_event(desc);
864
865 } while ((desc->istate & IRQS_PENDING) &&
866 !irqd_irq_disabled(&desc->irq_data));
867
868out_eoi:
869 chip->irq_eoi(&desc->irq_data);
870 raw_spin_unlock(&desc->lock);
871}
872#endif
873
874
875
876
877
878
879
880void handle_percpu_irq(struct irq_desc *desc)
881{
882 struct irq_chip *chip = irq_desc_get_chip(desc);
883
884
885
886
887
888 __kstat_incr_irqs_this_cpu(desc);
889
890 if (chip->irq_ack)
891 chip->irq_ack(&desc->irq_data);
892
893 handle_irq_event_percpu(desc);
894
895 if (chip->irq_eoi)
896 chip->irq_eoi(&desc->irq_data);
897}
898
899
900
901
902
903
904
905
906
907
908
909
910void handle_percpu_devid_irq(struct irq_desc *desc)
911{
912 struct irq_chip *chip = irq_desc_get_chip(desc);
913 struct irqaction *action = desc->action;
914 unsigned int irq = irq_desc_get_irq(desc);
915 irqreturn_t res;
916
917
918
919
920
921 __kstat_incr_irqs_this_cpu(desc);
922
923 if (chip->irq_ack)
924 chip->irq_ack(&desc->irq_data);
925
926 if (likely(action)) {
927 trace_irq_handler_entry(irq, action);
928 res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
929 trace_irq_handler_exit(irq, action, res);
930 } else {
931 unsigned int cpu = smp_processor_id();
932 bool enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
933
934 if (enabled)
935 irq_percpu_disable(desc, cpu);
936
937 pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
938 enabled ? " and unmasked" : "", irq, cpu);
939 }
940
941 if (chip->irq_eoi)
942 chip->irq_eoi(&desc->irq_data);
943}
944
945
946
947
948
949
950
951
952
953void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc)
954{
955 struct irq_chip *chip = irq_desc_get_chip(desc);
956 struct irqaction *action = desc->action;
957 unsigned int irq = irq_desc_get_irq(desc);
958 irqreturn_t res;
959
960 __kstat_incr_irqs_this_cpu(desc);
961
962 trace_irq_handler_entry(irq, action);
963 res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
964 trace_irq_handler_exit(irq, action, res);
965
966 if (chip->irq_eoi)
967 chip->irq_eoi(&desc->irq_data);
968}
969
970static void
971__irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
972 int is_chained, const char *name)
973{
974 if (!handle) {
975 handle = handle_bad_irq;
976 } else {
977 struct irq_data *irq_data = &desc->irq_data;
978#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
979
980
981
982
983
984
985
986 while (irq_data) {
987 if (irq_data->chip != &no_irq_chip)
988 break;
989
990
991
992
993
994 if (WARN_ON(is_chained))
995 return;
996
997 irq_data = irq_data->parent_data;
998 }
999#endif
1000 if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
1001 return;
1002 }
1003
1004
1005 if (handle == handle_bad_irq) {
1006 if (desc->irq_data.chip != &no_irq_chip)
1007 mask_ack_irq(desc);
1008 irq_state_set_disabled(desc);
1009 if (is_chained) {
1010 desc->action = NULL;
1011 WARN_ON(irq_chip_pm_put(irq_desc_get_irq_data(desc)));
1012 }
1013 desc->depth = 1;
1014 }
1015 desc->handle_irq = handle;
1016 desc->name = name;
1017
1018 if (handle != handle_bad_irq && is_chained) {
1019 unsigned int type = irqd_get_trigger_type(&desc->irq_data);
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029 if (type != IRQ_TYPE_NONE) {
1030 __irq_set_trigger(desc, type);
1031 desc->handle_irq = handle;
1032 }
1033
1034 irq_settings_set_noprobe(desc);
1035 irq_settings_set_norequest(desc);
1036 irq_settings_set_nothread(desc);
1037 desc->action = &chained_action;
1038 WARN_ON(irq_chip_pm_get(irq_desc_get_irq_data(desc)));
1039 irq_activate_and_startup(desc, IRQ_RESEND);
1040 }
1041}
1042
1043void
1044__irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
1045 const char *name)
1046{
1047 unsigned long flags;
1048 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1049
1050 if (!desc)
1051 return;
1052
1053 __irq_do_set_handler(desc, handle, is_chained, name);
1054 irq_put_desc_busunlock(desc, flags);
1055}
1056EXPORT_SYMBOL_GPL(__irq_set_handler);
1057
1058void
1059irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
1060 void *data)
1061{
1062 unsigned long flags;
1063 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1064
1065 if (!desc)
1066 return;
1067
1068 desc->irq_common_data.handler_data = data;
1069 __irq_do_set_handler(desc, handle, 1, NULL);
1070
1071 irq_put_desc_busunlock(desc, flags);
1072}
1073EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
1074
1075void
1076irq_set_chip_and_handler_name(unsigned int irq, const struct irq_chip *chip,
1077 irq_flow_handler_t handle, const char *name)
1078{
1079 irq_set_chip(irq, chip);
1080 __irq_set_handler(irq, handle, 0, name);
1081}
1082EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
1083
1084void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
1085{
1086 unsigned long flags, trigger, tmp;
1087 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
1088
1089 if (!desc)
1090 return;
1091
1092
1093
1094
1095
1096 WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
1097
1098 irq_settings_clr_and_set(desc, clr, set);
1099
1100 trigger = irqd_get_trigger_type(&desc->irq_data);
1101
1102 irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
1103 IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
1104 if (irq_settings_has_no_balance_set(desc))
1105 irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1106 if (irq_settings_is_per_cpu(desc))
1107 irqd_set(&desc->irq_data, IRQD_PER_CPU);
1108 if (irq_settings_can_move_pcntxt(desc))
1109 irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
1110 if (irq_settings_is_level(desc))
1111 irqd_set(&desc->irq_data, IRQD_LEVEL);
1112
1113 tmp = irq_settings_get_trigger_mask(desc);
1114 if (tmp != IRQ_TYPE_NONE)
1115 trigger = tmp;
1116
1117 irqd_set(&desc->irq_data, trigger);
1118
1119 irq_put_desc_unlock(desc, flags);
1120}
1121EXPORT_SYMBOL_GPL(irq_modify_status);
1122
1123#ifdef CONFIG_DEPRECATED_IRQ_CPU_ONOFFLINE
1124
1125
1126
1127
1128
1129
1130void irq_cpu_online(void)
1131{
1132 struct irq_desc *desc;
1133 struct irq_chip *chip;
1134 unsigned long flags;
1135 unsigned int irq;
1136
1137 for_each_active_irq(irq) {
1138 desc = irq_to_desc(irq);
1139 if (!desc)
1140 continue;
1141
1142 raw_spin_lock_irqsave(&desc->lock, flags);
1143
1144 chip = irq_data_get_irq_chip(&desc->irq_data);
1145 if (chip && chip->irq_cpu_online &&
1146 (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1147 !irqd_irq_disabled(&desc->irq_data)))
1148 chip->irq_cpu_online(&desc->irq_data);
1149
1150 raw_spin_unlock_irqrestore(&desc->lock, flags);
1151 }
1152}
1153
1154
1155
1156
1157
1158
1159
1160void irq_cpu_offline(void)
1161{
1162 struct irq_desc *desc;
1163 struct irq_chip *chip;
1164 unsigned long flags;
1165 unsigned int irq;
1166
1167 for_each_active_irq(irq) {
1168 desc = irq_to_desc(irq);
1169 if (!desc)
1170 continue;
1171
1172 raw_spin_lock_irqsave(&desc->lock, flags);
1173
1174 chip = irq_data_get_irq_chip(&desc->irq_data);
1175 if (chip && chip->irq_cpu_offline &&
1176 (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1177 !irqd_irq_disabled(&desc->irq_data)))
1178 chip->irq_cpu_offline(&desc->irq_data);
1179
1180 raw_spin_unlock_irqrestore(&desc->lock, flags);
1181 }
1182}
1183#endif
1184
1185#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1186
1187#ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198void handle_fasteoi_ack_irq(struct irq_desc *desc)
1199{
1200 struct irq_chip *chip = desc->irq_data.chip;
1201
1202 raw_spin_lock(&desc->lock);
1203
1204 if (!irq_may_run(desc))
1205 goto out;
1206
1207 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1208
1209
1210
1211
1212
1213 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1214 desc->istate |= IRQS_PENDING;
1215 mask_irq(desc);
1216 goto out;
1217 }
1218
1219 kstat_incr_irqs_this_cpu(desc);
1220 if (desc->istate & IRQS_ONESHOT)
1221 mask_irq(desc);
1222
1223
1224 desc->irq_data.chip->irq_ack(&desc->irq_data);
1225
1226 handle_irq_event(desc);
1227
1228 cond_unmask_eoi_irq(desc, chip);
1229
1230 raw_spin_unlock(&desc->lock);
1231 return;
1232out:
1233 if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1234 chip->irq_eoi(&desc->irq_data);
1235 raw_spin_unlock(&desc->lock);
1236}
1237EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249void handle_fasteoi_mask_irq(struct irq_desc *desc)
1250{
1251 struct irq_chip *chip = desc->irq_data.chip;
1252
1253 raw_spin_lock(&desc->lock);
1254 mask_ack_irq(desc);
1255
1256 if (!irq_may_run(desc))
1257 goto out;
1258
1259 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1260
1261
1262
1263
1264
1265 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1266 desc->istate |= IRQS_PENDING;
1267 mask_irq(desc);
1268 goto out;
1269 }
1270
1271 kstat_incr_irqs_this_cpu(desc);
1272 if (desc->istate & IRQS_ONESHOT)
1273 mask_irq(desc);
1274
1275 handle_irq_event(desc);
1276
1277 cond_unmask_eoi_irq(desc, chip);
1278
1279 raw_spin_unlock(&desc->lock);
1280 return;
1281out:
1282 if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1283 chip->irq_eoi(&desc->irq_data);
1284 raw_spin_unlock(&desc->lock);
1285}
1286EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
1287
1288#endif
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299int irq_chip_set_parent_state(struct irq_data *data,
1300 enum irqchip_irq_state which,
1301 bool val)
1302{
1303 data = data->parent_data;
1304
1305 if (!data || !data->chip->irq_set_irqchip_state)
1306 return 0;
1307
1308 return data->chip->irq_set_irqchip_state(data, which, val);
1309}
1310EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321int irq_chip_get_parent_state(struct irq_data *data,
1322 enum irqchip_irq_state which,
1323 bool *state)
1324{
1325 data = data->parent_data;
1326
1327 if (!data || !data->chip->irq_get_irqchip_state)
1328 return 0;
1329
1330 return data->chip->irq_get_irqchip_state(data, which, state);
1331}
1332EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
1333
1334
1335
1336
1337
1338
1339void irq_chip_enable_parent(struct irq_data *data)
1340{
1341 data = data->parent_data;
1342 if (data->chip->irq_enable)
1343 data->chip->irq_enable(data);
1344 else
1345 data->chip->irq_unmask(data);
1346}
1347EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
1348
1349
1350
1351
1352
1353
1354void irq_chip_disable_parent(struct irq_data *data)
1355{
1356 data = data->parent_data;
1357 if (data->chip->irq_disable)
1358 data->chip->irq_disable(data);
1359 else
1360 data->chip->irq_mask(data);
1361}
1362EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
1363
1364
1365
1366
1367
1368void irq_chip_ack_parent(struct irq_data *data)
1369{
1370 data = data->parent_data;
1371 data->chip->irq_ack(data);
1372}
1373EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
1374
1375
1376
1377
1378
1379void irq_chip_mask_parent(struct irq_data *data)
1380{
1381 data = data->parent_data;
1382 data->chip->irq_mask(data);
1383}
1384EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
1385
1386
1387
1388
1389
1390void irq_chip_mask_ack_parent(struct irq_data *data)
1391{
1392 data = data->parent_data;
1393 data->chip->irq_mask_ack(data);
1394}
1395EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
1396
1397
1398
1399
1400
1401void irq_chip_unmask_parent(struct irq_data *data)
1402{
1403 data = data->parent_data;
1404 data->chip->irq_unmask(data);
1405}
1406EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
1407
1408
1409
1410
1411
1412void irq_chip_eoi_parent(struct irq_data *data)
1413{
1414 data = data->parent_data;
1415 data->chip->irq_eoi(data);
1416}
1417EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427int irq_chip_set_affinity_parent(struct irq_data *data,
1428 const struct cpumask *dest, bool force)
1429{
1430 data = data->parent_data;
1431 if (data->chip->irq_set_affinity)
1432 return data->chip->irq_set_affinity(data, dest, force);
1433
1434 return -ENOSYS;
1435}
1436EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
1437
1438
1439
1440
1441
1442
1443
1444
1445int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
1446{
1447 data = data->parent_data;
1448
1449 if (data->chip->irq_set_type)
1450 return data->chip->irq_set_type(data, type);
1451
1452 return -ENOSYS;
1453}
1454EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
1455
1456
1457
1458
1459
1460
1461
1462
1463int irq_chip_retrigger_hierarchy(struct irq_data *data)
1464{
1465 for (data = data->parent_data; data; data = data->parent_data)
1466 if (data->chip && data->chip->irq_retrigger)
1467 return data->chip->irq_retrigger(data);
1468
1469 return 0;
1470}
1471EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
1472
1473
1474
1475
1476
1477
1478int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
1479{
1480 data = data->parent_data;
1481 if (data->chip->irq_set_vcpu_affinity)
1482 return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
1483
1484 return -ENOSYS;
1485}
1486EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
1487
1488
1489
1490
1491
1492
1493
1494int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
1495{
1496 data = data->parent_data;
1497
1498 if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
1499 return 0;
1500
1501 if (data->chip->irq_set_wake)
1502 return data->chip->irq_set_wake(data, on);
1503
1504 return -ENOSYS;
1505}
1506EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
1507
1508
1509
1510
1511
1512int irq_chip_request_resources_parent(struct irq_data *data)
1513{
1514 data = data->parent_data;
1515
1516 if (data->chip->irq_request_resources)
1517 return data->chip->irq_request_resources(data);
1518
1519 return -ENOSYS;
1520}
1521EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
1522
1523
1524
1525
1526
1527void irq_chip_release_resources_parent(struct irq_data *data)
1528{
1529 data = data->parent_data;
1530 if (data->chip->irq_release_resources)
1531 data->chip->irq_release_resources(data);
1532}
1533EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
1534#endif
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1546{
1547 struct irq_data *pos;
1548
1549 for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
1550 if (data->chip && data->chip->irq_compose_msi_msg)
1551 pos = data;
1552 }
1553
1554 if (!pos)
1555 return -ENOSYS;
1556
1557 pos->chip->irq_compose_msi_msg(pos, msg);
1558 return 0;
1559}
1560
1561static struct device *irq_get_parent_device(struct irq_data *data)
1562{
1563 if (data->domain)
1564 return data->domain->dev;
1565
1566 return NULL;
1567}
1568
1569
1570
1571
1572
1573
1574
1575
1576int irq_chip_pm_get(struct irq_data *data)
1577{
1578 struct device *dev = irq_get_parent_device(data);
1579 int retval = 0;
1580
1581 if (IS_ENABLED(CONFIG_PM) && dev)
1582 retval = pm_runtime_resume_and_get(dev);
1583
1584 return retval;
1585}
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595int irq_chip_pm_put(struct irq_data *data)
1596{
1597 struct device *dev = irq_get_parent_device(data);
1598 int retval = 0;
1599
1600 if (IS_ENABLED(CONFIG_PM) && dev)
1601 retval = pm_runtime_put(dev);
1602
1603 return (retval < 0) ? retval : 0;
1604}
1605