1
2
3
4
5
6
7
8
9#define pr_fmt(fmt) "genirq: " fmt
10
11#include <linux/irq.h>
12#include <linux/kthread.h>
13#include <linux/module.h>
14#include <linux/random.h>
15#include <linux/interrupt.h>
16#include <linux/irqdomain.h>
17#include <linux/slab.h>
18#include <linux/sched.h>
19#include <linux/sched/rt.h>
20#include <linux/sched/task.h>
21#include <linux/sched/isolation.h>
22#include <uapi/linux/sched/types.h>
23#include <linux/task_work.h>
24
25#include "internals.h"
26
27#if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
28__read_mostly bool force_irqthreads;
29EXPORT_SYMBOL_GPL(force_irqthreads);
30
31static int __init setup_forced_irqthreads(char *arg)
32{
33 force_irqthreads = true;
34 return 0;
35}
36early_param("threadirqs", setup_forced_irqthreads);
37#endif
38
39static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)
40{
41 struct irq_data *irqd = irq_desc_get_irq_data(desc);
42 bool inprogress;
43
44 do {
45 unsigned long flags;
46
47
48
49
50
51 while (irqd_irq_inprogress(&desc->irq_data))
52 cpu_relax();
53
54
55 raw_spin_lock_irqsave(&desc->lock, flags);
56 inprogress = irqd_irq_inprogress(&desc->irq_data);
57
58
59
60
61
62
63 if (!inprogress && sync_chip) {
64
65
66
67
68 __irq_get_irqchip_state(irqd, IRQCHIP_STATE_ACTIVE,
69 &inprogress);
70 }
71 raw_spin_unlock_irqrestore(&desc->lock, flags);
72
73
74 } while (inprogress);
75}
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99bool synchronize_hardirq(unsigned int irq)
100{
101 struct irq_desc *desc = irq_to_desc(irq);
102
103 if (desc) {
104 __synchronize_hardirq(desc, false);
105 return !atomic_read(&desc->threads_active);
106 }
107
108 return true;
109}
110EXPORT_SYMBOL(synchronize_hardirq);
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127void synchronize_irq(unsigned int irq)
128{
129 struct irq_desc *desc = irq_to_desc(irq);
130
131 if (desc) {
132 __synchronize_hardirq(desc, true);
133
134
135
136
137
138 wait_event(desc->wait_for_threads,
139 !atomic_read(&desc->threads_active));
140 }
141}
142EXPORT_SYMBOL(synchronize_irq);
143
144#ifdef CONFIG_SMP
145cpumask_var_t irq_default_affinity;
146
147static bool __irq_can_set_affinity(struct irq_desc *desc)
148{
149 if (!desc || !irqd_can_balance(&desc->irq_data) ||
150 !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
151 return false;
152 return true;
153}
154
155
156
157
158
159
160int irq_can_set_affinity(unsigned int irq)
161{
162 return __irq_can_set_affinity(irq_to_desc(irq));
163}
164
165
166
167
168
169
170
171
172bool irq_can_set_affinity_usr(unsigned int irq)
173{
174 struct irq_desc *desc = irq_to_desc(irq);
175
176 return __irq_can_set_affinity(desc) &&
177 !irqd_affinity_is_managed(&desc->irq_data);
178}
179
180
181
182
183
184
185
186
187
188
189void irq_set_thread_affinity(struct irq_desc *desc)
190{
191 struct irqaction *action;
192
193 for_each_action_of_desc(desc, action)
194 if (action->thread)
195 set_bit(IRQTF_AFFINITY, &action->thread_flags);
196}
197
198#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
199static void irq_validate_effective_affinity(struct irq_data *data)
200{
201 const struct cpumask *m = irq_data_get_effective_affinity_mask(data);
202 struct irq_chip *chip = irq_data_get_irq_chip(data);
203
204 if (!cpumask_empty(m))
205 return;
206 pr_warn_once("irq_chip %s did not update eff. affinity mask of irq %u\n",
207 chip->name, data->irq);
208}
209
210static inline void irq_init_effective_affinity(struct irq_data *data,
211 const struct cpumask *mask)
212{
213 cpumask_copy(irq_data_get_effective_affinity_mask(data), mask);
214}
215#else
216static inline void irq_validate_effective_affinity(struct irq_data *data) { }
217static inline void irq_init_effective_affinity(struct irq_data *data,
218 const struct cpumask *mask) { }
219#endif
220
221int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
222 bool force)
223{
224 struct irq_desc *desc = irq_data_to_desc(data);
225 struct irq_chip *chip = irq_data_get_irq_chip(data);
226 int ret;
227
228 if (!chip || !chip->irq_set_affinity)
229 return -EINVAL;
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250 if (irqd_affinity_is_managed(data) &&
251 housekeeping_enabled(HK_FLAG_MANAGED_IRQ)) {
252 const struct cpumask *hk_mask, *prog_mask;
253
254 static DEFINE_RAW_SPINLOCK(tmp_mask_lock);
255 static struct cpumask tmp_mask;
256
257 hk_mask = housekeeping_cpumask(HK_FLAG_MANAGED_IRQ);
258
259 raw_spin_lock(&tmp_mask_lock);
260 cpumask_and(&tmp_mask, mask, hk_mask);
261 if (!cpumask_intersects(&tmp_mask, cpu_online_mask))
262 prog_mask = mask;
263 else
264 prog_mask = &tmp_mask;
265 ret = chip->irq_set_affinity(data, prog_mask, force);
266 raw_spin_unlock(&tmp_mask_lock);
267 } else {
268 ret = chip->irq_set_affinity(data, mask, force);
269 }
270 switch (ret) {
271 case IRQ_SET_MASK_OK:
272 case IRQ_SET_MASK_OK_DONE:
273 cpumask_copy(desc->irq_common_data.affinity, mask);
274 fallthrough;
275 case IRQ_SET_MASK_OK_NOCOPY:
276 irq_validate_effective_affinity(data);
277 irq_set_thread_affinity(desc);
278 ret = 0;
279 }
280
281 return ret;
282}
283
284#ifdef CONFIG_GENERIC_PENDING_IRQ
285static inline int irq_set_affinity_pending(struct irq_data *data,
286 const struct cpumask *dest)
287{
288 struct irq_desc *desc = irq_data_to_desc(data);
289
290 irqd_set_move_pending(data);
291 irq_copy_pending(desc, dest);
292 return 0;
293}
294#else
295static inline int irq_set_affinity_pending(struct irq_data *data,
296 const struct cpumask *dest)
297{
298 return -EBUSY;
299}
300#endif
301
302static int irq_try_set_affinity(struct irq_data *data,
303 const struct cpumask *dest, bool force)
304{
305 int ret = irq_do_set_affinity(data, dest, force);
306
307
308
309
310
311
312 if (ret == -EBUSY && !force)
313 ret = irq_set_affinity_pending(data, dest);
314 return ret;
315}
316
317static bool irq_set_affinity_deactivated(struct irq_data *data,
318 const struct cpumask *mask, bool force)
319{
320 struct irq_desc *desc = irq_data_to_desc(data);
321
322
323
324
325
326
327
328
329
330
331 if (!IS_ENABLED(CONFIG_IRQ_DOMAIN_HIERARCHY) ||
332 irqd_is_activated(data) || !irqd_affinity_on_activate(data))
333 return false;
334
335 cpumask_copy(desc->irq_common_data.affinity, mask);
336 irq_init_effective_affinity(data, mask);
337 irqd_set(data, IRQD_AFFINITY_SET);
338 return true;
339}
340
341int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
342 bool force)
343{
344 struct irq_chip *chip = irq_data_get_irq_chip(data);
345 struct irq_desc *desc = irq_data_to_desc(data);
346 int ret = 0;
347
348 if (!chip || !chip->irq_set_affinity)
349 return -EINVAL;
350
351 if (irq_set_affinity_deactivated(data, mask, force))
352 return 0;
353
354 if (irq_can_move_pcntxt(data) && !irqd_is_setaffinity_pending(data)) {
355 ret = irq_try_set_affinity(data, mask, force);
356 } else {
357 irqd_set_move_pending(data);
358 irq_copy_pending(desc, mask);
359 }
360
361 if (desc->affinity_notify) {
362 kref_get(&desc->affinity_notify->kref);
363 if (!schedule_work(&desc->affinity_notify->work)) {
364
365 kref_put(&desc->affinity_notify->kref,
366 desc->affinity_notify->release);
367 }
368 }
369 irqd_set(data, IRQD_AFFINITY_SET);
370
371 return ret;
372}
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389int irq_update_affinity_desc(unsigned int irq,
390 struct irq_affinity_desc *affinity)
391{
392 struct irq_desc *desc;
393 unsigned long flags;
394 bool activated;
395 int ret = 0;
396
397
398
399
400
401 if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
402 return -EOPNOTSUPP;
403
404 desc = irq_get_desc_buslock(irq, &flags, 0);
405 if (!desc)
406 return -EINVAL;
407
408
409 if (irqd_is_started(&desc->irq_data)) {
410 ret = -EBUSY;
411 goto out_unlock;
412 }
413
414
415 if (irqd_affinity_is_managed(&desc->irq_data)) {
416 ret = -EBUSY;
417 goto out_unlock;
418 }
419
420
421
422
423
424 activated = irqd_is_activated(&desc->irq_data);
425 if (activated)
426 irq_domain_deactivate_irq(&desc->irq_data);
427
428 if (affinity->is_managed) {
429 irqd_set(&desc->irq_data, IRQD_AFFINITY_MANAGED);
430 irqd_set(&desc->irq_data, IRQD_MANAGED_SHUTDOWN);
431 }
432
433 cpumask_copy(desc->irq_common_data.affinity, &affinity->mask);
434
435
436 if (activated)
437 irq_domain_activate_irq(&desc->irq_data, false);
438
439out_unlock:
440 irq_put_desc_busunlock(desc, flags);
441 return ret;
442}
443
444int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
445{
446 struct irq_desc *desc = irq_to_desc(irq);
447 unsigned long flags;
448 int ret;
449
450 if (!desc)
451 return -EINVAL;
452
453 raw_spin_lock_irqsave(&desc->lock, flags);
454 ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
455 raw_spin_unlock_irqrestore(&desc->lock, flags);
456 return ret;
457}
458
459int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
460{
461 unsigned long flags;
462 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
463
464 if (!desc)
465 return -EINVAL;
466 desc->affinity_hint = m;
467 irq_put_desc_unlock(desc, flags);
468
469 if (m)
470 __irq_set_affinity(irq, m, false);
471 return 0;
472}
473EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
474
475static void irq_affinity_notify(struct work_struct *work)
476{
477 struct irq_affinity_notify *notify =
478 container_of(work, struct irq_affinity_notify, work);
479 struct irq_desc *desc = irq_to_desc(notify->irq);
480 cpumask_var_t cpumask;
481 unsigned long flags;
482
483 if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
484 goto out;
485
486 raw_spin_lock_irqsave(&desc->lock, flags);
487 if (irq_move_pending(&desc->irq_data))
488 irq_get_pending(cpumask, desc);
489 else
490 cpumask_copy(cpumask, desc->irq_common_data.affinity);
491 raw_spin_unlock_irqrestore(&desc->lock, flags);
492
493 notify->notify(notify, cpumask);
494
495 free_cpumask_var(cpumask);
496out:
497 kref_put(¬ify->kref, notify->release);
498}
499
500
501
502
503
504
505
506
507
508
509
510
511int
512irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
513{
514 struct irq_desc *desc = irq_to_desc(irq);
515 struct irq_affinity_notify *old_notify;
516 unsigned long flags;
517
518
519 might_sleep();
520
521 if (!desc || desc->istate & IRQS_NMI)
522 return -EINVAL;
523
524
525 if (notify) {
526 notify->irq = irq;
527 kref_init(¬ify->kref);
528 INIT_WORK(¬ify->work, irq_affinity_notify);
529 }
530
531 raw_spin_lock_irqsave(&desc->lock, flags);
532 old_notify = desc->affinity_notify;
533 desc->affinity_notify = notify;
534 raw_spin_unlock_irqrestore(&desc->lock, flags);
535
536 if (old_notify) {
537 if (cancel_work_sync(&old_notify->work)) {
538
539 kref_put(&old_notify->kref, old_notify->release);
540 }
541 kref_put(&old_notify->kref, old_notify->release);
542 }
543
544 return 0;
545}
546EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
547
548#ifndef CONFIG_AUTO_IRQ_AFFINITY
549
550
551
552int irq_setup_affinity(struct irq_desc *desc)
553{
554 struct cpumask *set = irq_default_affinity;
555 int ret, node = irq_desc_get_node(desc);
556 static DEFINE_RAW_SPINLOCK(mask_lock);
557 static struct cpumask mask;
558
559
560 if (!__irq_can_set_affinity(desc))
561 return 0;
562
563 raw_spin_lock(&mask_lock);
564
565
566
567
568 if (irqd_affinity_is_managed(&desc->irq_data) ||
569 irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
570 if (cpumask_intersects(desc->irq_common_data.affinity,
571 cpu_online_mask))
572 set = desc->irq_common_data.affinity;
573 else
574 irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
575 }
576
577 cpumask_and(&mask, cpu_online_mask, set);
578 if (cpumask_empty(&mask))
579 cpumask_copy(&mask, cpu_online_mask);
580
581 if (node != NUMA_NO_NODE) {
582 const struct cpumask *nodemask = cpumask_of_node(node);
583
584
585 if (cpumask_intersects(&mask, nodemask))
586 cpumask_and(&mask, &mask, nodemask);
587 }
588 ret = irq_do_set_affinity(&desc->irq_data, &mask, false);
589 raw_spin_unlock(&mask_lock);
590 return ret;
591}
592#else
593
594int irq_setup_affinity(struct irq_desc *desc)
595{
596 return irq_select_affinity(irq_desc_get_irq(desc));
597}
598#endif
599#endif
600
601
602
603
604
605
606
607
608
609
610
611
612
613int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
614{
615 unsigned long flags;
616 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
617 struct irq_data *data;
618 struct irq_chip *chip;
619 int ret = -ENOSYS;
620
621 if (!desc)
622 return -EINVAL;
623
624 data = irq_desc_get_irq_data(desc);
625 do {
626 chip = irq_data_get_irq_chip(data);
627 if (chip && chip->irq_set_vcpu_affinity)
628 break;
629#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
630 data = data->parent_data;
631#else
632 data = NULL;
633#endif
634 } while (data);
635
636 if (data)
637 ret = chip->irq_set_vcpu_affinity(data, vcpu_info);
638 irq_put_desc_unlock(desc, flags);
639
640 return ret;
641}
642EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
643
644void __disable_irq(struct irq_desc *desc)
645{
646 if (!desc->depth++)
647 irq_disable(desc);
648}
649
650static int __disable_irq_nosync(unsigned int irq)
651{
652 unsigned long flags;
653 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
654
655 if (!desc)
656 return -EINVAL;
657 __disable_irq(desc);
658 irq_put_desc_busunlock(desc, flags);
659 return 0;
660}
661
662
663
664
665
666
667
668
669
670
671
672
673void disable_irq_nosync(unsigned int irq)
674{
675 __disable_irq_nosync(irq);
676}
677EXPORT_SYMBOL(disable_irq_nosync);
678
679
680
681
682
683
684
685
686
687
688
689
690
691void disable_irq(unsigned int irq)
692{
693 if (!__disable_irq_nosync(irq))
694 synchronize_irq(irq);
695}
696EXPORT_SYMBOL(disable_irq);
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715bool disable_hardirq(unsigned int irq)
716{
717 if (!__disable_irq_nosync(irq))
718 return synchronize_hardirq(irq);
719
720 return false;
721}
722EXPORT_SYMBOL_GPL(disable_hardirq);
723
724
725
726
727
728
729
730
731
732
733
734void disable_nmi_nosync(unsigned int irq)
735{
736 disable_irq_nosync(irq);
737}
738
739void __enable_irq(struct irq_desc *desc)
740{
741 switch (desc->depth) {
742 case 0:
743 err_out:
744 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
745 irq_desc_get_irq(desc));
746 break;
747 case 1: {
748 if (desc->istate & IRQS_SUSPENDED)
749 goto err_out;
750
751 irq_settings_set_noprobe(desc);
752
753
754
755
756
757
758
759 irq_startup(desc, IRQ_RESEND, IRQ_START_FORCE);
760 break;
761 }
762 default:
763 desc->depth--;
764 }
765}
766
767
768
769
770
771
772
773
774
775
776
777
778void enable_irq(unsigned int irq)
779{
780 unsigned long flags;
781 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
782
783 if (!desc)
784 return;
785 if (WARN(!desc->irq_data.chip,
786 KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
787 goto out;
788
789 __enable_irq(desc);
790out:
791 irq_put_desc_busunlock(desc, flags);
792}
793EXPORT_SYMBOL(enable_irq);
794
795
796
797
798
799
800
801
802
803
804void enable_nmi(unsigned int irq)
805{
806 enable_irq(irq);
807}
808
809static int set_irq_wake_real(unsigned int irq, unsigned int on)
810{
811 struct irq_desc *desc = irq_to_desc(irq);
812 int ret = -ENXIO;
813
814 if (irq_desc_get_chip(desc)->flags & IRQCHIP_SKIP_SET_WAKE)
815 return 0;
816
817 if (desc->irq_data.chip->irq_set_wake)
818 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
819
820 return ret;
821}
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842int irq_set_irq_wake(unsigned int irq, unsigned int on)
843{
844 unsigned long flags;
845 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
846 int ret = 0;
847
848 if (!desc)
849 return -EINVAL;
850
851
852 if (desc->istate & IRQS_NMI) {
853 ret = -EINVAL;
854 goto out_unlock;
855 }
856
857
858
859
860 if (on) {
861 if (desc->wake_depth++ == 0) {
862 ret = set_irq_wake_real(irq, on);
863 if (ret)
864 desc->wake_depth = 0;
865 else
866 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
867 }
868 } else {
869 if (desc->wake_depth == 0) {
870 WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
871 } else if (--desc->wake_depth == 0) {
872 ret = set_irq_wake_real(irq, on);
873 if (ret)
874 desc->wake_depth = 1;
875 else
876 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
877 }
878 }
879
880out_unlock:
881 irq_put_desc_busunlock(desc, flags);
882 return ret;
883}
884EXPORT_SYMBOL(irq_set_irq_wake);
885
886
887
888
889
890
891int can_request_irq(unsigned int irq, unsigned long irqflags)
892{
893 unsigned long flags;
894 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
895 int canrequest = 0;
896
897 if (!desc)
898 return 0;
899
900 if (irq_settings_can_request(desc)) {
901 if (!desc->action ||
902 irqflags & desc->action->flags & IRQF_SHARED)
903 canrequest = 1;
904 }
905 irq_put_desc_unlock(desc, flags);
906 return canrequest;
907}
908
909int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
910{
911 struct irq_chip *chip = desc->irq_data.chip;
912 int ret, unmask = 0;
913
914 if (!chip || !chip->irq_set_type) {
915
916
917
918
919 pr_debug("No set_type function for IRQ %d (%s)\n",
920 irq_desc_get_irq(desc),
921 chip ? (chip->name ? : "unknown") : "unknown");
922 return 0;
923 }
924
925 if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
926 if (!irqd_irq_masked(&desc->irq_data))
927 mask_irq(desc);
928 if (!irqd_irq_disabled(&desc->irq_data))
929 unmask = 1;
930 }
931
932
933 flags &= IRQ_TYPE_SENSE_MASK;
934 ret = chip->irq_set_type(&desc->irq_data, flags);
935
936 switch (ret) {
937 case IRQ_SET_MASK_OK:
938 case IRQ_SET_MASK_OK_DONE:
939 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
940 irqd_set(&desc->irq_data, flags);
941 fallthrough;
942
943 case IRQ_SET_MASK_OK_NOCOPY:
944 flags = irqd_get_trigger_type(&desc->irq_data);
945 irq_settings_set_trigger_mask(desc, flags);
946 irqd_clear(&desc->irq_data, IRQD_LEVEL);
947 irq_settings_clr_level(desc);
948 if (flags & IRQ_TYPE_LEVEL_MASK) {
949 irq_settings_set_level(desc);
950 irqd_set(&desc->irq_data, IRQD_LEVEL);
951 }
952
953 ret = 0;
954 break;
955 default:
956 pr_err("Setting trigger mode %lu for irq %u failed (%pS)\n",
957 flags, irq_desc_get_irq(desc), chip->irq_set_type);
958 }
959 if (unmask)
960 unmask_irq(desc);
961 return ret;
962}
963
964#ifdef CONFIG_HARDIRQS_SW_RESEND
965int irq_set_parent(int irq, int parent_irq)
966{
967 unsigned long flags;
968 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
969
970 if (!desc)
971 return -EINVAL;
972
973 desc->parent_irq = parent_irq;
974
975 irq_put_desc_unlock(desc, flags);
976 return 0;
977}
978EXPORT_SYMBOL_GPL(irq_set_parent);
979#endif
980
981
982
983
984
985
986static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
987{
988 return IRQ_WAKE_THREAD;
989}
990
991
992
993
994
995static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
996{
997 WARN(1, "Primary handler called for nested irq %d\n", irq);
998 return IRQ_NONE;
999}
1000
1001static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
1002{
1003 WARN(1, "Secondary action handler called for irq %d\n", irq);
1004 return IRQ_NONE;
1005}
1006
1007static int irq_wait_for_interrupt(struct irqaction *action)
1008{
1009 for (;;) {
1010 set_current_state(TASK_INTERRUPTIBLE);
1011
1012 if (kthread_should_stop()) {
1013
1014 if (test_and_clear_bit(IRQTF_RUNTHREAD,
1015 &action->thread_flags)) {
1016 __set_current_state(TASK_RUNNING);
1017 return 0;
1018 }
1019 __set_current_state(TASK_RUNNING);
1020 return -1;
1021 }
1022
1023 if (test_and_clear_bit(IRQTF_RUNTHREAD,
1024 &action->thread_flags)) {
1025 __set_current_state(TASK_RUNNING);
1026 return 0;
1027 }
1028 schedule();
1029 }
1030}
1031
1032
1033
1034
1035
1036
1037static void irq_finalize_oneshot(struct irq_desc *desc,
1038 struct irqaction *action)
1039{
1040 if (!(desc->istate & IRQS_ONESHOT) ||
1041 action->handler == irq_forced_secondary_handler)
1042 return;
1043again:
1044 chip_bus_lock(desc);
1045 raw_spin_lock_irq(&desc->lock);
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061 if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
1062 raw_spin_unlock_irq(&desc->lock);
1063 chip_bus_sync_unlock(desc);
1064 cpu_relax();
1065 goto again;
1066 }
1067
1068
1069
1070
1071
1072
1073 if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
1074 goto out_unlock;
1075
1076 desc->threads_oneshot &= ~action->thread_mask;
1077
1078 if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
1079 irqd_irq_masked(&desc->irq_data))
1080 unmask_threaded_irq(desc);
1081
1082out_unlock:
1083 raw_spin_unlock_irq(&desc->lock);
1084 chip_bus_sync_unlock(desc);
1085}
1086
1087#ifdef CONFIG_SMP
1088
1089
1090
1091static void
1092irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
1093{
1094 cpumask_var_t mask;
1095 bool valid = true;
1096
1097 if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
1098 return;
1099
1100
1101
1102
1103
1104 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1105 set_bit(IRQTF_AFFINITY, &action->thread_flags);
1106 return;
1107 }
1108
1109 raw_spin_lock_irq(&desc->lock);
1110
1111
1112
1113
1114 if (cpumask_available(desc->irq_common_data.affinity)) {
1115 const struct cpumask *m;
1116
1117 m = irq_data_get_effective_affinity_mask(&desc->irq_data);
1118 cpumask_copy(mask, m);
1119 } else {
1120 valid = false;
1121 }
1122 raw_spin_unlock_irq(&desc->lock);
1123
1124 if (valid)
1125 set_cpus_allowed_ptr(current, mask);
1126 free_cpumask_var(mask);
1127}
1128#else
1129static inline void
1130irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
1131#endif
1132
1133
1134
1135
1136
1137
1138
1139static irqreturn_t
1140irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
1141{
1142 irqreturn_t ret;
1143
1144 local_bh_disable();
1145 ret = action->thread_fn(action->irq, action->dev_id);
1146 if (ret == IRQ_HANDLED)
1147 atomic_inc(&desc->threads_handled);
1148
1149 irq_finalize_oneshot(desc, action);
1150 local_bh_enable();
1151 return ret;
1152}
1153
1154
1155
1156
1157
1158
1159static irqreturn_t irq_thread_fn(struct irq_desc *desc,
1160 struct irqaction *action)
1161{
1162 irqreturn_t ret;
1163
1164 ret = action->thread_fn(action->irq, action->dev_id);
1165 if (ret == IRQ_HANDLED)
1166 atomic_inc(&desc->threads_handled);
1167
1168 irq_finalize_oneshot(desc, action);
1169 return ret;
1170}
1171
1172static void wake_threads_waitq(struct irq_desc *desc)
1173{
1174 if (atomic_dec_and_test(&desc->threads_active))
1175 wake_up(&desc->wait_for_threads);
1176}
1177
1178static void irq_thread_dtor(struct callback_head *unused)
1179{
1180 struct task_struct *tsk = current;
1181 struct irq_desc *desc;
1182 struct irqaction *action;
1183
1184 if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
1185 return;
1186
1187 action = kthread_data(tsk);
1188
1189 pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
1190 tsk->comm, tsk->pid, action->irq);
1191
1192
1193 desc = irq_to_desc(action->irq);
1194
1195
1196
1197
1198 if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
1199 wake_threads_waitq(desc);
1200
1201
1202 irq_finalize_oneshot(desc, action);
1203}
1204
1205static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
1206{
1207 struct irqaction *secondary = action->secondary;
1208
1209 if (WARN_ON_ONCE(!secondary))
1210 return;
1211
1212 raw_spin_lock_irq(&desc->lock);
1213 __irq_wake_thread(desc, secondary);
1214 raw_spin_unlock_irq(&desc->lock);
1215}
1216
1217
1218
1219
1220static int irq_thread(void *data)
1221{
1222 struct callback_head on_exit_work;
1223 struct irqaction *action = data;
1224 struct irq_desc *desc = irq_to_desc(action->irq);
1225 irqreturn_t (*handler_fn)(struct irq_desc *desc,
1226 struct irqaction *action);
1227
1228 if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
1229 &action->thread_flags))
1230 handler_fn = irq_forced_thread_fn;
1231 else
1232 handler_fn = irq_thread_fn;
1233
1234 init_task_work(&on_exit_work, irq_thread_dtor);
1235 task_work_add(current, &on_exit_work, TWA_NONE);
1236
1237 irq_thread_check_affinity(desc, action);
1238
1239 while (!irq_wait_for_interrupt(action)) {
1240 irqreturn_t action_ret;
1241
1242 irq_thread_check_affinity(desc, action);
1243
1244 action_ret = handler_fn(desc, action);
1245 if (action_ret == IRQ_WAKE_THREAD)
1246 irq_wake_secondary(desc, action);
1247
1248 wake_threads_waitq(desc);
1249 }
1250
1251
1252
1253
1254
1255
1256
1257 task_work_cancel(current, irq_thread_dtor);
1258 return 0;
1259}
1260
1261
1262
1263
1264
1265
1266
1267void irq_wake_thread(unsigned int irq, void *dev_id)
1268{
1269 struct irq_desc *desc = irq_to_desc(irq);
1270 struct irqaction *action;
1271 unsigned long flags;
1272
1273 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1274 return;
1275
1276 raw_spin_lock_irqsave(&desc->lock, flags);
1277 for_each_action_of_desc(desc, action) {
1278 if (action->dev_id == dev_id) {
1279 if (action->thread)
1280 __irq_wake_thread(desc, action);
1281 break;
1282 }
1283 }
1284 raw_spin_unlock_irqrestore(&desc->lock, flags);
1285}
1286EXPORT_SYMBOL_GPL(irq_wake_thread);
1287
1288static int irq_setup_forced_threading(struct irqaction *new)
1289{
1290 if (!force_irqthreads)
1291 return 0;
1292 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1293 return 0;
1294
1295
1296
1297
1298
1299 if (new->handler == irq_default_primary_handler)
1300 return 0;
1301
1302 new->flags |= IRQF_ONESHOT;
1303
1304
1305
1306
1307
1308
1309 if (new->handler && new->thread_fn) {
1310
1311 new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1312 if (!new->secondary)
1313 return -ENOMEM;
1314 new->secondary->handler = irq_forced_secondary_handler;
1315 new->secondary->thread_fn = new->thread_fn;
1316 new->secondary->dev_id = new->dev_id;
1317 new->secondary->irq = new->irq;
1318 new->secondary->name = new->name;
1319 }
1320
1321 set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
1322 new->thread_fn = new->handler;
1323 new->handler = irq_default_primary_handler;
1324 return 0;
1325}
1326
1327static int irq_request_resources(struct irq_desc *desc)
1328{
1329 struct irq_data *d = &desc->irq_data;
1330 struct irq_chip *c = d->chip;
1331
1332 return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1333}
1334
1335static void irq_release_resources(struct irq_desc *desc)
1336{
1337 struct irq_data *d = &desc->irq_data;
1338 struct irq_chip *c = d->chip;
1339
1340 if (c->irq_release_resources)
1341 c->irq_release_resources(d);
1342}
1343
1344static bool irq_supports_nmi(struct irq_desc *desc)
1345{
1346 struct irq_data *d = irq_desc_get_irq_data(desc);
1347
1348#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1349
1350 if (d->parent_data)
1351 return false;
1352#endif
1353
1354 if (d->chip->irq_bus_lock || d->chip->irq_bus_sync_unlock)
1355 return false;
1356
1357 return d->chip->flags & IRQCHIP_SUPPORTS_NMI;
1358}
1359
1360static int irq_nmi_setup(struct irq_desc *desc)
1361{
1362 struct irq_data *d = irq_desc_get_irq_data(desc);
1363 struct irq_chip *c = d->chip;
1364
1365 return c->irq_nmi_setup ? c->irq_nmi_setup(d) : -EINVAL;
1366}
1367
1368static void irq_nmi_teardown(struct irq_desc *desc)
1369{
1370 struct irq_data *d = irq_desc_get_irq_data(desc);
1371 struct irq_chip *c = d->chip;
1372
1373 if (c->irq_nmi_teardown)
1374 c->irq_nmi_teardown(d);
1375}
1376
1377static int
1378setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
1379{
1380 struct task_struct *t;
1381
1382 if (!secondary) {
1383 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1384 new->name);
1385 } else {
1386 t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
1387 new->name);
1388 }
1389
1390 if (IS_ERR(t))
1391 return PTR_ERR(t);
1392
1393 sched_set_fifo(t);
1394
1395
1396
1397
1398
1399
1400 new->thread = get_task_struct(t);
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410 set_bit(IRQTF_AFFINITY, &new->thread_flags);
1411 return 0;
1412}
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428static int
1429__setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1430{
1431 struct irqaction *old, **old_ptr;
1432 unsigned long flags, thread_mask = 0;
1433 int ret, nested, shared = 0;
1434
1435 if (!desc)
1436 return -EINVAL;
1437
1438 if (desc->irq_data.chip == &no_irq_chip)
1439 return -ENOSYS;
1440 if (!try_module_get(desc->owner))
1441 return -ENODEV;
1442
1443 new->irq = irq;
1444
1445
1446
1447
1448
1449 if (!(new->flags & IRQF_TRIGGER_MASK))
1450 new->flags |= irqd_get_trigger_type(&desc->irq_data);
1451
1452
1453
1454
1455
1456 nested = irq_settings_is_nested_thread(desc);
1457 if (nested) {
1458 if (!new->thread_fn) {
1459 ret = -EINVAL;
1460 goto out_mput;
1461 }
1462
1463
1464
1465
1466
1467 new->handler = irq_nested_primary_handler;
1468 } else {
1469 if (irq_settings_can_thread(desc)) {
1470 ret = irq_setup_forced_threading(new);
1471 if (ret)
1472 goto out_mput;
1473 }
1474 }
1475
1476
1477
1478
1479
1480
1481 if (new->thread_fn && !nested) {
1482 ret = setup_irq_thread(new, irq, false);
1483 if (ret)
1484 goto out_mput;
1485 if (new->secondary) {
1486 ret = setup_irq_thread(new->secondary, irq, true);
1487 if (ret)
1488 goto out_thread;
1489 }
1490 }
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501 if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1502 new->flags &= ~IRQF_ONESHOT;
1503
1504
1505
1506
1507
1508
1509
1510
1511 mutex_lock(&desc->request_mutex);
1512
1513
1514
1515
1516
1517
1518 chip_bus_lock(desc);
1519
1520
1521 if (!desc->action) {
1522 ret = irq_request_resources(desc);
1523 if (ret) {
1524 pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1525 new->name, irq, desc->irq_data.chip->name);
1526 goto out_bus_unlock;
1527 }
1528 }
1529
1530
1531
1532
1533
1534
1535
1536 raw_spin_lock_irqsave(&desc->lock, flags);
1537 old_ptr = &desc->action;
1538 old = *old_ptr;
1539 if (old) {
1540
1541
1542
1543
1544
1545
1546
1547
1548 unsigned int oldtype;
1549
1550 if (desc->istate & IRQS_NMI) {
1551 pr_err("Invalid attempt to share NMI for %s (irq %d) on irqchip %s.\n",
1552 new->name, irq, desc->irq_data.chip->name);
1553 ret = -EINVAL;
1554 goto out_unlock;
1555 }
1556
1557
1558
1559
1560
1561 if (irqd_trigger_type_was_set(&desc->irq_data)) {
1562 oldtype = irqd_get_trigger_type(&desc->irq_data);
1563 } else {
1564 oldtype = new->flags & IRQF_TRIGGER_MASK;
1565 irqd_set_trigger_type(&desc->irq_data, oldtype);
1566 }
1567
1568 if (!((old->flags & new->flags) & IRQF_SHARED) ||
1569 (oldtype != (new->flags & IRQF_TRIGGER_MASK)) ||
1570 ((old->flags ^ new->flags) & IRQF_ONESHOT))
1571 goto mismatch;
1572
1573
1574 if ((old->flags & IRQF_PERCPU) !=
1575 (new->flags & IRQF_PERCPU))
1576 goto mismatch;
1577
1578
1579 do {
1580
1581
1582
1583
1584
1585 thread_mask |= old->thread_mask;
1586 old_ptr = &old->next;
1587 old = *old_ptr;
1588 } while (old);
1589 shared = 1;
1590 }
1591
1592
1593
1594
1595
1596
1597 if (new->flags & IRQF_ONESHOT) {
1598
1599
1600
1601
1602 if (thread_mask == ~0UL) {
1603 ret = -EBUSY;
1604 goto out_unlock;
1605 }
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626 new->thread_mask = 1UL << ffz(thread_mask);
1627
1628 } else if (new->handler == irq_default_primary_handler &&
1629 !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for %s (irq %d)\n",
1646 new->name, irq);
1647 ret = -EINVAL;
1648 goto out_unlock;
1649 }
1650
1651 if (!shared) {
1652 init_waitqueue_head(&desc->wait_for_threads);
1653
1654
1655 if (new->flags & IRQF_TRIGGER_MASK) {
1656 ret = __irq_set_trigger(desc,
1657 new->flags & IRQF_TRIGGER_MASK);
1658
1659 if (ret)
1660 goto out_unlock;
1661 }
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674 ret = irq_activate(desc);
1675 if (ret)
1676 goto out_unlock;
1677
1678 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1679 IRQS_ONESHOT | IRQS_WAITING);
1680 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1681
1682 if (new->flags & IRQF_PERCPU) {
1683 irqd_set(&desc->irq_data, IRQD_PER_CPU);
1684 irq_settings_set_per_cpu(desc);
1685 }
1686
1687 if (new->flags & IRQF_ONESHOT)
1688 desc->istate |= IRQS_ONESHOT;
1689
1690
1691 if (new->flags & IRQF_NOBALANCING) {
1692 irq_settings_set_no_balancing(desc);
1693 irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1694 }
1695
1696 if (irq_settings_can_autoenable(desc)) {
1697 irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
1698 } else {
1699
1700
1701
1702
1703
1704
1705 WARN_ON_ONCE(new->flags & IRQF_SHARED);
1706
1707 desc->depth = 1;
1708 }
1709
1710 } else if (new->flags & IRQF_TRIGGER_MASK) {
1711 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1712 unsigned int omsk = irqd_get_trigger_type(&desc->irq_data);
1713
1714 if (nmsk != omsk)
1715
1716 pr_warn("irq %d uses trigger mode %u; requested %u\n",
1717 irq, omsk, nmsk);
1718 }
1719
1720 *old_ptr = new;
1721
1722 irq_pm_install_action(desc, new);
1723
1724
1725 desc->irq_count = 0;
1726 desc->irqs_unhandled = 0;
1727
1728
1729
1730
1731
1732 if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1733 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1734 __enable_irq(desc);
1735 }
1736
1737 raw_spin_unlock_irqrestore(&desc->lock, flags);
1738 chip_bus_sync_unlock(desc);
1739 mutex_unlock(&desc->request_mutex);
1740
1741 irq_setup_timings(desc, new);
1742
1743
1744
1745
1746
1747 if (new->thread)
1748 wake_up_process(new->thread);
1749 if (new->secondary)
1750 wake_up_process(new->secondary->thread);
1751
1752 register_irq_proc(irq, desc);
1753 new->dir = NULL;
1754 register_handler_proc(irq, new);
1755 return 0;
1756
1757mismatch:
1758 if (!(new->flags & IRQF_PROBE_SHARED)) {
1759 pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1760 irq, new->flags, new->name, old->flags, old->name);
1761#ifdef CONFIG_DEBUG_SHIRQ
1762 dump_stack();
1763#endif
1764 }
1765 ret = -EBUSY;
1766
1767out_unlock:
1768 raw_spin_unlock_irqrestore(&desc->lock, flags);
1769
1770 if (!desc->action)
1771 irq_release_resources(desc);
1772out_bus_unlock:
1773 chip_bus_sync_unlock(desc);
1774 mutex_unlock(&desc->request_mutex);
1775
1776out_thread:
1777 if (new->thread) {
1778 struct task_struct *t = new->thread;
1779
1780 new->thread = NULL;
1781 kthread_stop(t);
1782 put_task_struct(t);
1783 }
1784 if (new->secondary && new->secondary->thread) {
1785 struct task_struct *t = new->secondary->thread;
1786
1787 new->secondary->thread = NULL;
1788 kthread_stop(t);
1789 put_task_struct(t);
1790 }
1791out_mput:
1792 module_put(desc->owner);
1793 return ret;
1794}
1795
1796
1797
1798
1799
1800static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
1801{
1802 unsigned irq = desc->irq_data.irq;
1803 struct irqaction *action, **action_ptr;
1804 unsigned long flags;
1805
1806 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1807
1808 mutex_lock(&desc->request_mutex);
1809 chip_bus_lock(desc);
1810 raw_spin_lock_irqsave(&desc->lock, flags);
1811
1812
1813
1814
1815
1816 action_ptr = &desc->action;
1817 for (;;) {
1818 action = *action_ptr;
1819
1820 if (!action) {
1821 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1822 raw_spin_unlock_irqrestore(&desc->lock, flags);
1823 chip_bus_sync_unlock(desc);
1824 mutex_unlock(&desc->request_mutex);
1825 return NULL;
1826 }
1827
1828 if (action->dev_id == dev_id)
1829 break;
1830 action_ptr = &action->next;
1831 }
1832
1833
1834 *action_ptr = action->next;
1835
1836 irq_pm_remove_action(desc, action);
1837
1838
1839 if (!desc->action) {
1840 irq_settings_clr_disable_unlazy(desc);
1841
1842 irq_shutdown(desc);
1843 }
1844
1845#ifdef CONFIG_SMP
1846
1847 if (WARN_ON_ONCE(desc->affinity_hint))
1848 desc->affinity_hint = NULL;
1849#endif
1850
1851 raw_spin_unlock_irqrestore(&desc->lock, flags);
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866 chip_bus_sync_unlock(desc);
1867
1868 unregister_handler_proc(irq, action);
1869
1870
1871
1872
1873
1874
1875 __synchronize_hardirq(desc, true);
1876
1877#ifdef CONFIG_DEBUG_SHIRQ
1878
1879
1880
1881
1882
1883
1884
1885
1886 if (action->flags & IRQF_SHARED) {
1887 local_irq_save(flags);
1888 action->handler(irq, dev_id);
1889 local_irq_restore(flags);
1890 }
1891#endif
1892
1893
1894
1895
1896
1897
1898
1899 if (action->thread) {
1900 kthread_stop(action->thread);
1901 put_task_struct(action->thread);
1902 if (action->secondary && action->secondary->thread) {
1903 kthread_stop(action->secondary->thread);
1904 put_task_struct(action->secondary->thread);
1905 }
1906 }
1907
1908
1909 if (!desc->action) {
1910
1911
1912
1913
1914 chip_bus_lock(desc);
1915
1916
1917
1918
1919 raw_spin_lock_irqsave(&desc->lock, flags);
1920 irq_domain_deactivate_irq(&desc->irq_data);
1921 raw_spin_unlock_irqrestore(&desc->lock, flags);
1922
1923 irq_release_resources(desc);
1924 chip_bus_sync_unlock(desc);
1925 irq_remove_timings(desc);
1926 }
1927
1928 mutex_unlock(&desc->request_mutex);
1929
1930 irq_chip_pm_put(&desc->irq_data);
1931 module_put(desc->owner);
1932 kfree(action->secondary);
1933 return action;
1934}
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952const void *free_irq(unsigned int irq, void *dev_id)
1953{
1954 struct irq_desc *desc = irq_to_desc(irq);
1955 struct irqaction *action;
1956 const char *devname;
1957
1958 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1959 return NULL;
1960
1961#ifdef CONFIG_SMP
1962 if (WARN_ON(desc->affinity_notify))
1963 desc->affinity_notify = NULL;
1964#endif
1965
1966 action = __free_irq(desc, dev_id);
1967
1968 if (!action)
1969 return NULL;
1970
1971 devname = action->name;
1972 kfree(action);
1973 return devname;
1974}
1975EXPORT_SYMBOL(free_irq);
1976
1977
1978static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc)
1979{
1980 const char *devname = NULL;
1981
1982 desc->istate &= ~IRQS_NMI;
1983
1984 if (!WARN_ON(desc->action == NULL)) {
1985 irq_pm_remove_action(desc, desc->action);
1986 devname = desc->action->name;
1987 unregister_handler_proc(irq, desc->action);
1988
1989 kfree(desc->action);
1990 desc->action = NULL;
1991 }
1992
1993 irq_settings_clr_disable_unlazy(desc);
1994 irq_shutdown_and_deactivate(desc);
1995
1996 irq_release_resources(desc);
1997
1998 irq_chip_pm_put(&desc->irq_data);
1999 module_put(desc->owner);
2000
2001 return devname;
2002}
2003
2004const void *free_nmi(unsigned int irq, void *dev_id)
2005{
2006 struct irq_desc *desc = irq_to_desc(irq);
2007 unsigned long flags;
2008 const void *devname;
2009
2010 if (!desc || WARN_ON(!(desc->istate & IRQS_NMI)))
2011 return NULL;
2012
2013 if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
2014 return NULL;
2015
2016
2017 if (WARN_ON(desc->depth == 0))
2018 disable_nmi_nosync(irq);
2019
2020 raw_spin_lock_irqsave(&desc->lock, flags);
2021
2022 irq_nmi_teardown(desc);
2023 devname = __cleanup_nmi(irq, desc);
2024
2025 raw_spin_unlock_irqrestore(&desc->lock, flags);
2026
2027 return devname;
2028}
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072int request_threaded_irq(unsigned int irq, irq_handler_t handler,
2073 irq_handler_t thread_fn, unsigned long irqflags,
2074 const char *devname, void *dev_id)
2075{
2076 struct irqaction *action;
2077 struct irq_desc *desc;
2078 int retval;
2079
2080 if (irq == IRQ_NOTCONNECTED)
2081 return -ENOTCONN;
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092 if (((irqflags & IRQF_SHARED) && !dev_id) ||
2093 (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
2094 ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
2095 return -EINVAL;
2096
2097 desc = irq_to_desc(irq);
2098 if (!desc)
2099 return -EINVAL;
2100
2101 if (!irq_settings_can_request(desc) ||
2102 WARN_ON(irq_settings_is_per_cpu_devid(desc)))
2103 return -EINVAL;
2104
2105 if (!handler) {
2106 if (!thread_fn)
2107 return -EINVAL;
2108 handler = irq_default_primary_handler;
2109 }
2110
2111 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2112 if (!action)
2113 return -ENOMEM;
2114
2115 action->handler = handler;
2116 action->thread_fn = thread_fn;
2117 action->flags = irqflags;
2118 action->name = devname;
2119 action->dev_id = dev_id;
2120
2121 retval = irq_chip_pm_get(&desc->irq_data);
2122 if (retval < 0) {
2123 kfree(action);
2124 return retval;
2125 }
2126
2127 retval = __setup_irq(irq, desc, action);
2128
2129 if (retval) {
2130 irq_chip_pm_put(&desc->irq_data);
2131 kfree(action->secondary);
2132 kfree(action);
2133 }
2134
2135#ifdef CONFIG_DEBUG_SHIRQ_FIXME
2136 if (!retval && (irqflags & IRQF_SHARED)) {
2137
2138
2139
2140
2141
2142
2143 unsigned long flags;
2144
2145 disable_irq(irq);
2146 local_irq_save(flags);
2147
2148 handler(irq, dev_id);
2149
2150 local_irq_restore(flags);
2151 enable_irq(irq);
2152 }
2153#endif
2154 return retval;
2155}
2156EXPORT_SYMBOL(request_threaded_irq);
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175int request_any_context_irq(unsigned int irq, irq_handler_t handler,
2176 unsigned long flags, const char *name, void *dev_id)
2177{
2178 struct irq_desc *desc;
2179 int ret;
2180
2181 if (irq == IRQ_NOTCONNECTED)
2182 return -ENOTCONN;
2183
2184 desc = irq_to_desc(irq);
2185 if (!desc)
2186 return -EINVAL;
2187
2188 if (irq_settings_is_nested_thread(desc)) {
2189 ret = request_threaded_irq(irq, NULL, handler,
2190 flags, name, dev_id);
2191 return !ret ? IRQC_IS_NESTED : ret;
2192 }
2193
2194 ret = request_irq(irq, handler, flags, name, dev_id);
2195 return !ret ? IRQC_IS_HARDIRQ : ret;
2196}
2197EXPORT_SYMBOL_GPL(request_any_context_irq);
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225int request_nmi(unsigned int irq, irq_handler_t handler,
2226 unsigned long irqflags, const char *name, void *dev_id)
2227{
2228 struct irqaction *action;
2229 struct irq_desc *desc;
2230 unsigned long flags;
2231 int retval;
2232
2233 if (irq == IRQ_NOTCONNECTED)
2234 return -ENOTCONN;
2235
2236
2237 if (irqflags & (IRQF_SHARED | IRQF_COND_SUSPEND | IRQF_IRQPOLL))
2238 return -EINVAL;
2239
2240 if (!(irqflags & IRQF_PERCPU))
2241 return -EINVAL;
2242
2243 if (!handler)
2244 return -EINVAL;
2245
2246 desc = irq_to_desc(irq);
2247
2248 if (!desc || irq_settings_can_autoenable(desc) ||
2249 !irq_settings_can_request(desc) ||
2250 WARN_ON(irq_settings_is_per_cpu_devid(desc)) ||
2251 !irq_supports_nmi(desc))
2252 return -EINVAL;
2253
2254 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2255 if (!action)
2256 return -ENOMEM;
2257
2258 action->handler = handler;
2259 action->flags = irqflags | IRQF_NO_THREAD | IRQF_NOBALANCING;
2260 action->name = name;
2261 action->dev_id = dev_id;
2262
2263 retval = irq_chip_pm_get(&desc->irq_data);
2264 if (retval < 0)
2265 goto err_out;
2266
2267 retval = __setup_irq(irq, desc, action);
2268 if (retval)
2269 goto err_irq_setup;
2270
2271 raw_spin_lock_irqsave(&desc->lock, flags);
2272
2273
2274 desc->istate |= IRQS_NMI;
2275 retval = irq_nmi_setup(desc);
2276 if (retval) {
2277 __cleanup_nmi(irq, desc);
2278 raw_spin_unlock_irqrestore(&desc->lock, flags);
2279 return -EINVAL;
2280 }
2281
2282 raw_spin_unlock_irqrestore(&desc->lock, flags);
2283
2284 return 0;
2285
2286err_irq_setup:
2287 irq_chip_pm_put(&desc->irq_data);
2288err_out:
2289 kfree(action);
2290
2291 return retval;
2292}
2293
2294void enable_percpu_irq(unsigned int irq, unsigned int type)
2295{
2296 unsigned int cpu = smp_processor_id();
2297 unsigned long flags;
2298 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2299
2300 if (!desc)
2301 return;
2302
2303
2304
2305
2306
2307 type &= IRQ_TYPE_SENSE_MASK;
2308 if (type == IRQ_TYPE_NONE)
2309 type = irqd_get_trigger_type(&desc->irq_data);
2310
2311 if (type != IRQ_TYPE_NONE) {
2312 int ret;
2313
2314 ret = __irq_set_trigger(desc, type);
2315
2316 if (ret) {
2317 WARN(1, "failed to set type for IRQ%d\n", irq);
2318 goto out;
2319 }
2320 }
2321
2322 irq_percpu_enable(desc, cpu);
2323out:
2324 irq_put_desc_unlock(desc, flags);
2325}
2326EXPORT_SYMBOL_GPL(enable_percpu_irq);
2327
2328void enable_percpu_nmi(unsigned int irq, unsigned int type)
2329{
2330 enable_percpu_irq(irq, type);
2331}
2332
2333
2334
2335
2336
2337
2338
2339
2340bool irq_percpu_is_enabled(unsigned int irq)
2341{
2342 unsigned int cpu = smp_processor_id();
2343 struct irq_desc *desc;
2344 unsigned long flags;
2345 bool is_enabled;
2346
2347 desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2348 if (!desc)
2349 return false;
2350
2351 is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
2352 irq_put_desc_unlock(desc, flags);
2353
2354 return is_enabled;
2355}
2356EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
2357
2358void disable_percpu_irq(unsigned int irq)
2359{
2360 unsigned int cpu = smp_processor_id();
2361 unsigned long flags;
2362 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2363
2364 if (!desc)
2365 return;
2366
2367 irq_percpu_disable(desc, cpu);
2368 irq_put_desc_unlock(desc, flags);
2369}
2370EXPORT_SYMBOL_GPL(disable_percpu_irq);
2371
2372void disable_percpu_nmi(unsigned int irq)
2373{
2374 disable_percpu_irq(irq);
2375}
2376
2377
2378
2379
2380static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
2381{
2382 struct irq_desc *desc = irq_to_desc(irq);
2383 struct irqaction *action;
2384 unsigned long flags;
2385
2386 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
2387
2388 if (!desc)
2389 return NULL;
2390
2391 raw_spin_lock_irqsave(&desc->lock, flags);
2392
2393 action = desc->action;
2394 if (!action || action->percpu_dev_id != dev_id) {
2395 WARN(1, "Trying to free already-free IRQ %d\n", irq);
2396 goto bad;
2397 }
2398
2399 if (!cpumask_empty(desc->percpu_enabled)) {
2400 WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
2401 irq, cpumask_first(desc->percpu_enabled));
2402 goto bad;
2403 }
2404
2405
2406 desc->action = NULL;
2407
2408 desc->istate &= ~IRQS_NMI;
2409
2410 raw_spin_unlock_irqrestore(&desc->lock, flags);
2411
2412 unregister_handler_proc(irq, action);
2413
2414 irq_chip_pm_put(&desc->irq_data);
2415 module_put(desc->owner);
2416 return action;
2417
2418bad:
2419 raw_spin_unlock_irqrestore(&desc->lock, flags);
2420 return NULL;
2421}
2422
2423
2424
2425
2426
2427
2428
2429
2430void remove_percpu_irq(unsigned int irq, struct irqaction *act)
2431{
2432 struct irq_desc *desc = irq_to_desc(irq);
2433
2434 if (desc && irq_settings_is_per_cpu_devid(desc))
2435 __free_percpu_irq(irq, act->percpu_dev_id);
2436}
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
2451{
2452 struct irq_desc *desc = irq_to_desc(irq);
2453
2454 if (!desc || !irq_settings_is_per_cpu_devid(desc))
2455 return;
2456
2457 chip_bus_lock(desc);
2458 kfree(__free_percpu_irq(irq, dev_id));
2459 chip_bus_sync_unlock(desc);
2460}
2461EXPORT_SYMBOL_GPL(free_percpu_irq);
2462
2463void free_percpu_nmi(unsigned int irq, void __percpu *dev_id)
2464{
2465 struct irq_desc *desc = irq_to_desc(irq);
2466
2467 if (!desc || !irq_settings_is_per_cpu_devid(desc))
2468 return;
2469
2470 if (WARN_ON(!(desc->istate & IRQS_NMI)))
2471 return;
2472
2473 kfree(__free_percpu_irq(irq, dev_id));
2474}
2475
2476
2477
2478
2479
2480
2481
2482
2483int setup_percpu_irq(unsigned int irq, struct irqaction *act)
2484{
2485 struct irq_desc *desc = irq_to_desc(irq);
2486 int retval;
2487
2488 if (!desc || !irq_settings_is_per_cpu_devid(desc))
2489 return -EINVAL;
2490
2491 retval = irq_chip_pm_get(&desc->irq_data);
2492 if (retval < 0)
2493 return retval;
2494
2495 retval = __setup_irq(irq, desc, act);
2496
2497 if (retval)
2498 irq_chip_pm_put(&desc->irq_data);
2499
2500 return retval;
2501}
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
2521 unsigned long flags, const char *devname,
2522 void __percpu *dev_id)
2523{
2524 struct irqaction *action;
2525 struct irq_desc *desc;
2526 int retval;
2527
2528 if (!dev_id)
2529 return -EINVAL;
2530
2531 desc = irq_to_desc(irq);
2532 if (!desc || !irq_settings_can_request(desc) ||
2533 !irq_settings_is_per_cpu_devid(desc))
2534 return -EINVAL;
2535
2536 if (flags && flags != IRQF_TIMER)
2537 return -EINVAL;
2538
2539 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2540 if (!action)
2541 return -ENOMEM;
2542
2543 action->handler = handler;
2544 action->flags = flags | IRQF_PERCPU | IRQF_NO_SUSPEND;
2545 action->name = devname;
2546 action->percpu_dev_id = dev_id;
2547
2548 retval = irq_chip_pm_get(&desc->irq_data);
2549 if (retval < 0) {
2550 kfree(action);
2551 return retval;
2552 }
2553
2554 retval = __setup_irq(irq, desc, action);
2555
2556 if (retval) {
2557 irq_chip_pm_put(&desc->irq_data);
2558 kfree(action);
2559 }
2560
2561 return retval;
2562}
2563EXPORT_SYMBOL_GPL(__request_percpu_irq);
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586int request_percpu_nmi(unsigned int irq, irq_handler_t handler,
2587 const char *name, void __percpu *dev_id)
2588{
2589 struct irqaction *action;
2590 struct irq_desc *desc;
2591 unsigned long flags;
2592 int retval;
2593
2594 if (!handler)
2595 return -EINVAL;
2596
2597 desc = irq_to_desc(irq);
2598
2599 if (!desc || !irq_settings_can_request(desc) ||
2600 !irq_settings_is_per_cpu_devid(desc) ||
2601 irq_settings_can_autoenable(desc) ||
2602 !irq_supports_nmi(desc))
2603 return -EINVAL;
2604
2605
2606 if (desc->istate & IRQS_NMI)
2607 return -EINVAL;
2608
2609 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2610 if (!action)
2611 return -ENOMEM;
2612
2613 action->handler = handler;
2614 action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND | IRQF_NO_THREAD
2615 | IRQF_NOBALANCING;
2616 action->name = name;
2617 action->percpu_dev_id = dev_id;
2618
2619 retval = irq_chip_pm_get(&desc->irq_data);
2620 if (retval < 0)
2621 goto err_out;
2622
2623 retval = __setup_irq(irq, desc, action);
2624 if (retval)
2625 goto err_irq_setup;
2626
2627 raw_spin_lock_irqsave(&desc->lock, flags);
2628 desc->istate |= IRQS_NMI;
2629 raw_spin_unlock_irqrestore(&desc->lock, flags);
2630
2631 return 0;
2632
2633err_irq_setup:
2634 irq_chip_pm_put(&desc->irq_data);
2635err_out:
2636 kfree(action);
2637
2638 return retval;
2639}
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654int prepare_percpu_nmi(unsigned int irq)
2655{
2656 unsigned long flags;
2657 struct irq_desc *desc;
2658 int ret = 0;
2659
2660 WARN_ON(preemptible());
2661
2662 desc = irq_get_desc_lock(irq, &flags,
2663 IRQ_GET_DESC_CHECK_PERCPU);
2664 if (!desc)
2665 return -EINVAL;
2666
2667 if (WARN(!(desc->istate & IRQS_NMI),
2668 KERN_ERR "prepare_percpu_nmi called for a non-NMI interrupt: irq %u\n",
2669 irq)) {
2670 ret = -EINVAL;
2671 goto out;
2672 }
2673
2674 ret = irq_nmi_setup(desc);
2675 if (ret) {
2676 pr_err("Failed to setup NMI delivery: irq %u\n", irq);
2677 goto out;
2678 }
2679
2680out:
2681 irq_put_desc_unlock(desc, flags);
2682 return ret;
2683}
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697void teardown_percpu_nmi(unsigned int irq)
2698{
2699 unsigned long flags;
2700 struct irq_desc *desc;
2701
2702 WARN_ON(preemptible());
2703
2704 desc = irq_get_desc_lock(irq, &flags,
2705 IRQ_GET_DESC_CHECK_PERCPU);
2706 if (!desc)
2707 return;
2708
2709 if (WARN_ON(!(desc->istate & IRQS_NMI)))
2710 goto out;
2711
2712 irq_nmi_teardown(desc);
2713out:
2714 irq_put_desc_unlock(desc, flags);
2715}
2716
2717int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which,
2718 bool *state)
2719{
2720 struct irq_chip *chip;
2721 int err = -EINVAL;
2722
2723 do {
2724 chip = irq_data_get_irq_chip(data);
2725 if (WARN_ON_ONCE(!chip))
2726 return -ENODEV;
2727 if (chip->irq_get_irqchip_state)
2728 break;
2729#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2730 data = data->parent_data;
2731#else
2732 data = NULL;
2733#endif
2734 } while (data);
2735
2736 if (data)
2737 err = chip->irq_get_irqchip_state(data, which, state);
2738 return err;
2739}
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2755 bool *state)
2756{
2757 struct irq_desc *desc;
2758 struct irq_data *data;
2759 unsigned long flags;
2760 int err = -EINVAL;
2761
2762 desc = irq_get_desc_buslock(irq, &flags, 0);
2763 if (!desc)
2764 return err;
2765
2766 data = irq_desc_get_irq_data(desc);
2767
2768 err = __irq_get_irqchip_state(data, which, state);
2769
2770 irq_put_desc_busunlock(desc, flags);
2771 return err;
2772}
2773EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2788 bool val)
2789{
2790 struct irq_desc *desc;
2791 struct irq_data *data;
2792 struct irq_chip *chip;
2793 unsigned long flags;
2794 int err = -EINVAL;
2795
2796 desc = irq_get_desc_buslock(irq, &flags, 0);
2797 if (!desc)
2798 return err;
2799
2800 data = irq_desc_get_irq_data(desc);
2801
2802 do {
2803 chip = irq_data_get_irq_chip(data);
2804 if (WARN_ON_ONCE(!chip)) {
2805 err = -ENODEV;
2806 goto out_unlock;
2807 }
2808 if (chip->irq_set_irqchip_state)
2809 break;
2810#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2811 data = data->parent_data;
2812#else
2813 data = NULL;
2814#endif
2815 } while (data);
2816
2817 if (data)
2818 err = chip->irq_set_irqchip_state(data, which, val);
2819
2820out_unlock:
2821 irq_put_desc_busunlock(desc, flags);
2822 return err;
2823}
2824EXPORT_SYMBOL_GPL(irq_set_irqchip_state);
2825
2826
2827
2828
2829
2830
2831
2832bool irq_has_action(unsigned int irq)
2833{
2834 bool res;
2835
2836 rcu_read_lock();
2837 res = irq_desc_has_action(irq_to_desc(irq));
2838 rcu_read_unlock();
2839 return res;
2840}
2841EXPORT_SYMBOL_GPL(irq_has_action);
2842
2843
2844
2845
2846
2847
2848
2849
2850bool irq_check_status_bit(unsigned int irq, unsigned int bitmask)
2851{
2852 struct irq_desc *desc;
2853 bool res = false;
2854
2855 rcu_read_lock();
2856 desc = irq_to_desc(irq);
2857 if (desc)
2858 res = !!(desc->status_use_accessors & bitmask);
2859 rcu_read_unlock();
2860 return res;
2861}
2862EXPORT_SYMBOL_GPL(irq_check_status_bit);
2863