1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/export.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
21#include <linux/freezer.h>
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
25#include <linux/smp.h>
26#include <linux/smpboot.h>
27#include <linux/tick.h>
28
29#define CREATE_TRACE_POINTS
30#include <trace/events/irq.h>
31
32#include <asm/irq.h>
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51#ifndef __ARCH_IRQ_STAT
52irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
53EXPORT_SYMBOL(irq_stat);
54#endif
55
56static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
57
58DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
59
60char *softirq_to_name[NR_SOFTIRQS] = {
61 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
62 "TASKLET", "SCHED", "HRTIMER", "RCU"
63};
64
65
66
67
68
69
70
71static void wakeup_softirqd(void)
72{
73
74 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
75
76 if (tsk && tsk->state != TASK_RUNNING)
77 wake_up_process(tsk);
78}
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94#ifdef CONFIG_TRACE_IRQFLAGS
95static void __local_bh_disable(unsigned long ip, unsigned int cnt)
96{
97 unsigned long flags;
98
99 WARN_ON_ONCE(in_irq());
100
101 raw_local_irq_save(flags);
102
103
104
105
106
107
108
109 preempt_count() += cnt;
110
111
112
113 if (softirq_count() == cnt)
114 trace_softirqs_off(ip);
115 raw_local_irq_restore(flags);
116
117 if (preempt_count() == cnt)
118 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
119}
120#else
121static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
122{
123 add_preempt_count(cnt);
124 barrier();
125}
126#endif
127
128void local_bh_disable(void)
129{
130 __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
131}
132
133EXPORT_SYMBOL(local_bh_disable);
134
135static void __local_bh_enable(unsigned int cnt)
136{
137 WARN_ON_ONCE(in_irq());
138 WARN_ON_ONCE(!irqs_disabled());
139
140 if (softirq_count() == cnt)
141 trace_softirqs_on(_RET_IP_);
142 sub_preempt_count(cnt);
143}
144
145
146
147
148
149
150void _local_bh_enable(void)
151{
152 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
153}
154
155EXPORT_SYMBOL(_local_bh_enable);
156
157static inline void _local_bh_enable_ip(unsigned long ip)
158{
159 WARN_ON_ONCE(in_irq() || irqs_disabled());
160#ifdef CONFIG_TRACE_IRQFLAGS
161 local_irq_disable();
162#endif
163
164
165
166 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
167 trace_softirqs_on(ip);
168
169
170
171
172 sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
173
174 if (unlikely(!in_interrupt() && local_softirq_pending()))
175 do_softirq();
176
177 dec_preempt_count();
178#ifdef CONFIG_TRACE_IRQFLAGS
179 local_irq_enable();
180#endif
181 preempt_check_resched();
182}
183
184void local_bh_enable(void)
185{
186 _local_bh_enable_ip(_RET_IP_);
187}
188EXPORT_SYMBOL(local_bh_enable);
189
190void local_bh_enable_ip(unsigned long ip)
191{
192 _local_bh_enable_ip(ip);
193}
194EXPORT_SYMBOL(local_bh_enable_ip);
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
210#define MAX_SOFTIRQ_RESTART 10
211
212asmlinkage void __do_softirq(void)
213{
214 struct softirq_action *h;
215 __u32 pending;
216 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
217 int cpu;
218 unsigned long old_flags = current->flags;
219 int max_restart = MAX_SOFTIRQ_RESTART;
220
221
222
223
224
225
226 current->flags &= ~PF_MEMALLOC;
227
228 pending = local_softirq_pending();
229 account_irq_enter_time(current);
230
231 __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
232 lockdep_softirq_enter();
233
234 cpu = smp_processor_id();
235restart:
236
237 set_softirq_pending(0);
238
239 local_irq_enable();
240
241 h = softirq_vec;
242
243 do {
244 if (pending & 1) {
245 unsigned int vec_nr = h - softirq_vec;
246 int prev_count = preempt_count();
247
248 kstat_incr_softirqs_this_cpu(vec_nr);
249
250 trace_softirq_entry(vec_nr);
251 h->action(h);
252 trace_softirq_exit(vec_nr);
253 if (unlikely(prev_count != preempt_count())) {
254 printk(KERN_ERR "huh, entered softirq %u %s %p"
255 "with preempt_count %08x,"
256 " exited with %08x?\n", vec_nr,
257 softirq_to_name[vec_nr], h->action,
258 prev_count, preempt_count());
259 preempt_count() = prev_count;
260 }
261
262 rcu_bh_qs(cpu);
263 }
264 h++;
265 pending >>= 1;
266 } while (pending);
267
268 local_irq_disable();
269
270 pending = local_softirq_pending();
271 if (pending) {
272 if (time_before(jiffies, end) && !need_resched() &&
273 --max_restart)
274 goto restart;
275
276 wakeup_softirqd();
277 }
278
279 lockdep_softirq_exit();
280
281 account_irq_exit_time(current);
282 __local_bh_enable(SOFTIRQ_OFFSET);
283 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
284}
285
286#ifndef __ARCH_HAS_DO_SOFTIRQ
287
288asmlinkage void do_softirq(void)
289{
290 __u32 pending;
291 unsigned long flags;
292
293 if (in_interrupt())
294 return;
295
296 local_irq_save(flags);
297
298 pending = local_softirq_pending();
299
300 if (pending)
301 __do_softirq();
302
303 local_irq_restore(flags);
304}
305
306#endif
307
308
309
310
311void irq_enter(void)
312{
313 int cpu = smp_processor_id();
314
315 rcu_irq_enter();
316 if (is_idle_task(current) && !in_interrupt()) {
317
318
319
320
321 local_bh_disable();
322 tick_check_idle(cpu);
323 _local_bh_enable();
324 }
325
326 __irq_enter();
327}
328
329static inline void invoke_softirq(void)
330{
331 if (!force_irqthreads)
332 __do_softirq();
333 else
334 wakeup_softirqd();
335}
336
337static inline void tick_irq_exit(void)
338{
339#ifdef CONFIG_NO_HZ_COMMON
340 int cpu = smp_processor_id();
341
342
343 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
344 if (!in_interrupt())
345 tick_nohz_irq_exit();
346 }
347#endif
348}
349
350
351
352
353void irq_exit(void)
354{
355#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
356 local_irq_disable();
357#else
358 WARN_ON_ONCE(!irqs_disabled());
359#endif
360
361 account_irq_exit_time(current);
362 trace_hardirq_exit();
363 sub_preempt_count(HARDIRQ_OFFSET);
364 if (!in_interrupt() && local_softirq_pending())
365 invoke_softirq();
366
367 tick_irq_exit();
368 rcu_irq_exit();
369}
370
371
372
373
374inline void raise_softirq_irqoff(unsigned int nr)
375{
376 __raise_softirq_irqoff(nr);
377
378
379
380
381
382
383
384
385
386
387 if (!in_interrupt())
388 wakeup_softirqd();
389}
390
391void raise_softirq(unsigned int nr)
392{
393 unsigned long flags;
394
395 local_irq_save(flags);
396 raise_softirq_irqoff(nr);
397 local_irq_restore(flags);
398}
399
400void __raise_softirq_irqoff(unsigned int nr)
401{
402 trace_softirq_raise(nr);
403 or_softirq_pending(1UL << nr);
404}
405
406void open_softirq(int nr, void (*action)(struct softirq_action *))
407{
408 softirq_vec[nr].action = action;
409}
410
411
412
413
414struct tasklet_head
415{
416 struct tasklet_struct *head;
417 struct tasklet_struct **tail;
418};
419
420static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
421static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
422
423void __tasklet_schedule(struct tasklet_struct *t)
424{
425 unsigned long flags;
426
427 local_irq_save(flags);
428 t->next = NULL;
429 *__this_cpu_read(tasklet_vec.tail) = t;
430 __this_cpu_write(tasklet_vec.tail, &(t->next));
431 raise_softirq_irqoff(TASKLET_SOFTIRQ);
432 local_irq_restore(flags);
433}
434
435EXPORT_SYMBOL(__tasklet_schedule);
436
437void __tasklet_hi_schedule(struct tasklet_struct *t)
438{
439 unsigned long flags;
440
441 local_irq_save(flags);
442 t->next = NULL;
443 *__this_cpu_read(tasklet_hi_vec.tail) = t;
444 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
445 raise_softirq_irqoff(HI_SOFTIRQ);
446 local_irq_restore(flags);
447}
448
449EXPORT_SYMBOL(__tasklet_hi_schedule);
450
451void __tasklet_hi_schedule_first(struct tasklet_struct *t)
452{
453 BUG_ON(!irqs_disabled());
454
455 t->next = __this_cpu_read(tasklet_hi_vec.head);
456 __this_cpu_write(tasklet_hi_vec.head, t);
457 __raise_softirq_irqoff(HI_SOFTIRQ);
458}
459
460EXPORT_SYMBOL(__tasklet_hi_schedule_first);
461
462static void tasklet_action(struct softirq_action *a)
463{
464 struct tasklet_struct *list;
465
466 local_irq_disable();
467 list = __this_cpu_read(tasklet_vec.head);
468 __this_cpu_write(tasklet_vec.head, NULL);
469 __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
470 local_irq_enable();
471
472 while (list) {
473 struct tasklet_struct *t = list;
474
475 list = list->next;
476
477 if (tasklet_trylock(t)) {
478 if (!atomic_read(&t->count)) {
479 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
480 BUG();
481 t->func(t->data);
482 tasklet_unlock(t);
483 continue;
484 }
485 tasklet_unlock(t);
486 }
487
488 local_irq_disable();
489 t->next = NULL;
490 *__this_cpu_read(tasklet_vec.tail) = t;
491 __this_cpu_write(tasklet_vec.tail, &(t->next));
492 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
493 local_irq_enable();
494 }
495}
496
497static void tasklet_hi_action(struct softirq_action *a)
498{
499 struct tasklet_struct *list;
500
501 local_irq_disable();
502 list = __this_cpu_read(tasklet_hi_vec.head);
503 __this_cpu_write(tasklet_hi_vec.head, NULL);
504 __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
505 local_irq_enable();
506
507 while (list) {
508 struct tasklet_struct *t = list;
509
510 list = list->next;
511
512 if (tasklet_trylock(t)) {
513 if (!atomic_read(&t->count)) {
514 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
515 BUG();
516 t->func(t->data);
517 tasklet_unlock(t);
518 continue;
519 }
520 tasklet_unlock(t);
521 }
522
523 local_irq_disable();
524 t->next = NULL;
525 *__this_cpu_read(tasklet_hi_vec.tail) = t;
526 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
527 __raise_softirq_irqoff(HI_SOFTIRQ);
528 local_irq_enable();
529 }
530}
531
532
533void tasklet_init(struct tasklet_struct *t,
534 void (*func)(unsigned long), unsigned long data)
535{
536 t->next = NULL;
537 t->state = 0;
538 atomic_set(&t->count, 0);
539 t->func = func;
540 t->data = data;
541}
542
543EXPORT_SYMBOL(tasklet_init);
544
545void tasklet_kill(struct tasklet_struct *t)
546{
547 if (in_interrupt())
548 printk("Attempt to kill tasklet from interrupt\n");
549
550 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
551 do {
552 yield();
553 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
554 }
555 tasklet_unlock_wait(t);
556 clear_bit(TASKLET_STATE_SCHED, &t->state);
557}
558
559EXPORT_SYMBOL(tasklet_kill);
560
561
562
563
564
565
566
567
568
569
570static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
571{
572 struct tasklet_hrtimer *ttimer =
573 container_of(timer, struct tasklet_hrtimer, timer);
574
575 tasklet_hi_schedule(&ttimer->tasklet);
576 return HRTIMER_NORESTART;
577}
578
579
580
581
582
583static void __tasklet_hrtimer_trampoline(unsigned long data)
584{
585 struct tasklet_hrtimer *ttimer = (void *)data;
586 enum hrtimer_restart restart;
587
588 restart = ttimer->function(&ttimer->timer);
589 if (restart != HRTIMER_NORESTART)
590 hrtimer_restart(&ttimer->timer);
591}
592
593
594
595
596
597
598
599
600void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
601 enum hrtimer_restart (*function)(struct hrtimer *),
602 clockid_t which_clock, enum hrtimer_mode mode)
603{
604 hrtimer_init(&ttimer->timer, which_clock, mode);
605 ttimer->timer.function = __hrtimer_tasklet_trampoline;
606 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
607 (unsigned long)ttimer);
608 ttimer->function = function;
609}
610EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
611
612
613
614
615
616DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
617EXPORT_PER_CPU_SYMBOL(softirq_work_list);
618
619static void __local_trigger(struct call_single_data *cp, int softirq)
620{
621 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
622
623 list_add_tail(&cp->list, head);
624
625
626 if (head->next == &cp->list)
627 raise_softirq_irqoff(softirq);
628}
629
630#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
631static void remote_softirq_receive(void *data)
632{
633 struct call_single_data *cp = data;
634 unsigned long flags;
635 int softirq;
636
637 softirq = *(int *)cp->info;
638 local_irq_save(flags);
639 __local_trigger(cp, softirq);
640 local_irq_restore(flags);
641}
642
643static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
644{
645 if (cpu_online(cpu)) {
646 cp->func = remote_softirq_receive;
647 cp->info = &softirq;
648 cp->flags = 0;
649
650 __smp_call_function_single(cpu, cp, 0);
651 return 0;
652 }
653 return 1;
654}
655#else
656static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
657{
658 return 1;
659}
660#endif
661
662
663
664
665
666
667
668
669
670
671
672
673
674void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
675{
676 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
677 __local_trigger(cp, softirq);
678}
679EXPORT_SYMBOL(__send_remote_softirq);
680
681
682
683
684
685
686
687
688
689
690void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
691{
692 unsigned long flags;
693 int this_cpu;
694
695 local_irq_save(flags);
696 this_cpu = smp_processor_id();
697 __send_remote_softirq(cp, cpu, this_cpu, softirq);
698 local_irq_restore(flags);
699}
700EXPORT_SYMBOL(send_remote_softirq);
701
702static int remote_softirq_cpu_notify(struct notifier_block *self,
703 unsigned long action, void *hcpu)
704{
705
706
707
708
709 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
710 int cpu = (unsigned long) hcpu;
711 int i;
712
713 local_irq_disable();
714 for (i = 0; i < NR_SOFTIRQS; i++) {
715 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
716 struct list_head *local_head;
717
718 if (list_empty(head))
719 continue;
720
721 local_head = &__get_cpu_var(softirq_work_list[i]);
722 list_splice_init(head, local_head);
723 raise_softirq_irqoff(i);
724 }
725 local_irq_enable();
726 }
727
728 return NOTIFY_OK;
729}
730
731static struct notifier_block remote_softirq_cpu_notifier = {
732 .notifier_call = remote_softirq_cpu_notify,
733};
734
735void __init softirq_init(void)
736{
737 int cpu;
738
739 for_each_possible_cpu(cpu) {
740 int i;
741
742 per_cpu(tasklet_vec, cpu).tail =
743 &per_cpu(tasklet_vec, cpu).head;
744 per_cpu(tasklet_hi_vec, cpu).tail =
745 &per_cpu(tasklet_hi_vec, cpu).head;
746 for (i = 0; i < NR_SOFTIRQS; i++)
747 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
748 }
749
750 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
751
752 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
753 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
754}
755
756static int ksoftirqd_should_run(unsigned int cpu)
757{
758 return local_softirq_pending();
759}
760
761static void run_ksoftirqd(unsigned int cpu)
762{
763 local_irq_disable();
764 if (local_softirq_pending()) {
765 __do_softirq();
766 rcu_note_context_switch(cpu);
767 local_irq_enable();
768 cond_resched();
769 return;
770 }
771 local_irq_enable();
772}
773
774#ifdef CONFIG_HOTPLUG_CPU
775
776
777
778
779
780
781
782
783
784void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
785{
786 struct tasklet_struct **i;
787
788 BUG_ON(cpu_online(cpu));
789 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
790
791 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
792 return;
793
794
795 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
796 if (*i == t) {
797 *i = t->next;
798
799 if (*i == NULL)
800 per_cpu(tasklet_vec, cpu).tail = i;
801 return;
802 }
803 }
804 BUG();
805}
806
807static void takeover_tasklets(unsigned int cpu)
808{
809
810 local_irq_disable();
811
812
813 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
814 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
815 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
816 per_cpu(tasklet_vec, cpu).head = NULL;
817 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
818 }
819 raise_softirq_irqoff(TASKLET_SOFTIRQ);
820
821 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
822 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
823 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
824 per_cpu(tasklet_hi_vec, cpu).head = NULL;
825 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
826 }
827 raise_softirq_irqoff(HI_SOFTIRQ);
828
829 local_irq_enable();
830}
831#endif
832
833static int cpu_callback(struct notifier_block *nfb,
834 unsigned long action,
835 void *hcpu)
836{
837 switch (action) {
838#ifdef CONFIG_HOTPLUG_CPU
839 case CPU_DEAD:
840 case CPU_DEAD_FROZEN:
841 takeover_tasklets((unsigned long)hcpu);
842 break;
843#endif
844 }
845 return NOTIFY_OK;
846}
847
848static struct notifier_block cpu_nfb = {
849 .notifier_call = cpu_callback
850};
851
852static struct smp_hotplug_thread softirq_threads = {
853 .store = &ksoftirqd,
854 .thread_should_run = ksoftirqd_should_run,
855 .thread_fn = run_ksoftirqd,
856 .thread_comm = "ksoftirqd/%u",
857};
858
859static __init int spawn_ksoftirqd(void)
860{
861 register_cpu_notifier(&cpu_nfb);
862
863 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
864
865 return 0;
866}
867early_initcall(spawn_ksoftirqd);
868
869
870
871
872
873
874int __init __weak early_irq_init(void)
875{
876 return 0;
877}
878
879#ifdef CONFIG_GENERIC_HARDIRQS
880int __init __weak arch_probe_nr_irqs(void)
881{
882 return NR_IRQS_LEGACY;
883}
884
885int __init __weak arch_early_irq_init(void)
886{
887 return 0;
888}
889#endif
890