1
2
3
4
5
6
7
8
9#include <linux/module.h>
10#include <linux/kernel_stat.h>
11#include <linux/interrupt.h>
12#include <linux/init.h>
13#include <linux/mm.h>
14#include <linux/notifier.h>
15#include <linux/percpu.h>
16#include <linux/cpu.h>
17#include <linux/freezer.h>
18#include <linux/kthread.h>
19#include <linux/rcupdate.h>
20#include <linux/smp.h>
21#include <linux/tick.h>
22
23#include <asm/irq.h>
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42#ifndef __ARCH_IRQ_STAT
43irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
44EXPORT_SYMBOL(irq_stat);
45#endif
46
47static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
48
49static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
50
51
52
53
54
55
56
57static inline void wakeup_softirqd(void)
58{
59
60 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
61
62 if (tsk && tsk->state != TASK_RUNNING)
63 wake_up_process(tsk);
64}
65
66
67
68
69
70#ifdef CONFIG_TRACE_IRQFLAGS
71static void __local_bh_disable(unsigned long ip)
72{
73 unsigned long flags;
74
75 WARN_ON_ONCE(in_irq());
76
77 raw_local_irq_save(flags);
78 add_preempt_count(SOFTIRQ_OFFSET);
79
80
81
82 if (softirq_count() == SOFTIRQ_OFFSET)
83 trace_softirqs_off(ip);
84 raw_local_irq_restore(flags);
85}
86#else
87static inline void __local_bh_disable(unsigned long ip)
88{
89 add_preempt_count(SOFTIRQ_OFFSET);
90 barrier();
91}
92#endif
93
94void local_bh_disable(void)
95{
96 __local_bh_disable((unsigned long)__builtin_return_address(0));
97}
98
99EXPORT_SYMBOL(local_bh_disable);
100
101void __local_bh_enable(void)
102{
103 WARN_ON_ONCE(in_irq());
104
105
106
107
108
109 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
110
111 sub_preempt_count(SOFTIRQ_OFFSET);
112}
113EXPORT_SYMBOL_GPL(__local_bh_enable);
114
115
116
117
118
119
120void _local_bh_enable(void)
121{
122 WARN_ON_ONCE(in_irq());
123 WARN_ON_ONCE(!irqs_disabled());
124
125 if (softirq_count() == SOFTIRQ_OFFSET)
126 trace_softirqs_on((unsigned long)__builtin_return_address(0));
127 sub_preempt_count(SOFTIRQ_OFFSET);
128}
129
130EXPORT_SYMBOL(_local_bh_enable);
131
132void local_bh_enable(void)
133{
134#ifdef CONFIG_TRACE_IRQFLAGS
135 unsigned long flags;
136
137 WARN_ON_ONCE(in_irq());
138#endif
139 WARN_ON_ONCE(irqs_disabled());
140
141#ifdef CONFIG_TRACE_IRQFLAGS
142 local_irq_save(flags);
143#endif
144
145
146
147 if (softirq_count() == SOFTIRQ_OFFSET)
148 trace_softirqs_on((unsigned long)__builtin_return_address(0));
149
150
151
152
153 sub_preempt_count(SOFTIRQ_OFFSET - 1);
154
155 if (unlikely(!in_interrupt() && local_softirq_pending()))
156 do_softirq();
157
158 dec_preempt_count();
159#ifdef CONFIG_TRACE_IRQFLAGS
160 local_irq_restore(flags);
161#endif
162 preempt_check_resched();
163}
164EXPORT_SYMBOL(local_bh_enable);
165
166void local_bh_enable_ip(unsigned long ip)
167{
168#ifdef CONFIG_TRACE_IRQFLAGS
169 unsigned long flags;
170
171 WARN_ON_ONCE(in_irq());
172
173 local_irq_save(flags);
174#endif
175
176
177
178 if (softirq_count() == SOFTIRQ_OFFSET)
179 trace_softirqs_on(ip);
180
181
182
183
184 sub_preempt_count(SOFTIRQ_OFFSET - 1);
185
186 if (unlikely(!in_interrupt() && local_softirq_pending()))
187 do_softirq();
188
189 dec_preempt_count();
190#ifdef CONFIG_TRACE_IRQFLAGS
191 local_irq_restore(flags);
192#endif
193 preempt_check_resched();
194}
195EXPORT_SYMBOL(local_bh_enable_ip);
196
197
198
199
200
201
202
203
204
205
206#define MAX_SOFTIRQ_RESTART 10
207
208asmlinkage void __do_softirq(void)
209{
210 struct softirq_action *h;
211 __u32 pending;
212 int max_restart = MAX_SOFTIRQ_RESTART;
213 int cpu;
214
215 pending = local_softirq_pending();
216 account_system_vtime(current);
217
218 __local_bh_disable((unsigned long)__builtin_return_address(0));
219 trace_softirq_enter();
220
221 cpu = smp_processor_id();
222restart:
223
224 set_softirq_pending(0);
225
226 local_irq_enable();
227
228 h = softirq_vec;
229
230 do {
231 if (pending & 1) {
232 h->action(h);
233 rcu_bh_qsctr_inc(cpu);
234 }
235 h++;
236 pending >>= 1;
237 } while (pending);
238
239 local_irq_disable();
240
241 pending = local_softirq_pending();
242 if (pending && --max_restart)
243 goto restart;
244
245 if (pending)
246 wakeup_softirqd();
247
248 trace_softirq_exit();
249
250 account_system_vtime(current);
251 _local_bh_enable();
252}
253
254#ifndef __ARCH_HAS_DO_SOFTIRQ
255
256asmlinkage void do_softirq(void)
257{
258 __u32 pending;
259 unsigned long flags;
260
261 if (in_interrupt())
262 return;
263
264 local_irq_save(flags);
265
266 pending = local_softirq_pending();
267
268 if (pending)
269 __do_softirq();
270
271 local_irq_restore(flags);
272}
273
274#endif
275
276
277
278
279void irq_enter(void)
280{
281 __irq_enter();
282#ifdef CONFIG_NO_HZ
283 if (idle_cpu(smp_processor_id()))
284 tick_nohz_update_jiffies();
285#endif
286}
287
288#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
289# define invoke_softirq() __do_softirq()
290#else
291# define invoke_softirq() do_softirq()
292#endif
293
294
295
296
297void irq_exit(void)
298{
299 account_system_vtime(current);
300 trace_hardirq_exit();
301 sub_preempt_count(IRQ_EXIT_OFFSET);
302 if (!in_interrupt() && local_softirq_pending())
303 invoke_softirq();
304
305#ifdef CONFIG_NO_HZ
306
307 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
308 tick_nohz_stop_sched_tick();
309#endif
310 preempt_enable_no_resched();
311}
312
313
314
315
316inline fastcall void raise_softirq_irqoff(unsigned int nr)
317{
318 __raise_softirq_irqoff(nr);
319
320
321
322
323
324
325
326
327
328
329 if (!in_interrupt())
330 wakeup_softirqd();
331}
332
333void fastcall raise_softirq(unsigned int nr)
334{
335 unsigned long flags;
336
337 local_irq_save(flags);
338 raise_softirq_irqoff(nr);
339 local_irq_restore(flags);
340}
341
342void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
343{
344 softirq_vec[nr].data = data;
345 softirq_vec[nr].action = action;
346}
347
348
349struct tasklet_head
350{
351 struct tasklet_struct *list;
352};
353
354
355
356static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
357static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
358
359void fastcall __tasklet_schedule(struct tasklet_struct *t)
360{
361 unsigned long flags;
362
363 local_irq_save(flags);
364 t->next = __get_cpu_var(tasklet_vec).list;
365 __get_cpu_var(tasklet_vec).list = t;
366 raise_softirq_irqoff(TASKLET_SOFTIRQ);
367 local_irq_restore(flags);
368}
369
370EXPORT_SYMBOL(__tasklet_schedule);
371
372void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
373{
374 unsigned long flags;
375
376 local_irq_save(flags);
377 t->next = __get_cpu_var(tasklet_hi_vec).list;
378 __get_cpu_var(tasklet_hi_vec).list = t;
379 raise_softirq_irqoff(HI_SOFTIRQ);
380 local_irq_restore(flags);
381}
382
383EXPORT_SYMBOL(__tasklet_hi_schedule);
384
385static void tasklet_action(struct softirq_action *a)
386{
387 struct tasklet_struct *list;
388
389 local_irq_disable();
390 list = __get_cpu_var(tasklet_vec).list;
391 __get_cpu_var(tasklet_vec).list = NULL;
392 local_irq_enable();
393
394 while (list) {
395 struct tasklet_struct *t = list;
396
397 list = list->next;
398
399 if (tasklet_trylock(t)) {
400 if (!atomic_read(&t->count)) {
401 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
402 BUG();
403 t->func(t->data);
404 tasklet_unlock(t);
405 continue;
406 }
407 tasklet_unlock(t);
408 }
409
410 local_irq_disable();
411 t->next = __get_cpu_var(tasklet_vec).list;
412 __get_cpu_var(tasklet_vec).list = t;
413 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
414 local_irq_enable();
415 }
416}
417
418static void tasklet_hi_action(struct softirq_action *a)
419{
420 struct tasklet_struct *list;
421
422 local_irq_disable();
423 list = __get_cpu_var(tasklet_hi_vec).list;
424 __get_cpu_var(tasklet_hi_vec).list = NULL;
425 local_irq_enable();
426
427 while (list) {
428 struct tasklet_struct *t = list;
429
430 list = list->next;
431
432 if (tasklet_trylock(t)) {
433 if (!atomic_read(&t->count)) {
434 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
435 BUG();
436 t->func(t->data);
437 tasklet_unlock(t);
438 continue;
439 }
440 tasklet_unlock(t);
441 }
442
443 local_irq_disable();
444 t->next = __get_cpu_var(tasklet_hi_vec).list;
445 __get_cpu_var(tasklet_hi_vec).list = t;
446 __raise_softirq_irqoff(HI_SOFTIRQ);
447 local_irq_enable();
448 }
449}
450
451
452void tasklet_init(struct tasklet_struct *t,
453 void (*func)(unsigned long), unsigned long data)
454{
455 t->next = NULL;
456 t->state = 0;
457 atomic_set(&t->count, 0);
458 t->func = func;
459 t->data = data;
460}
461
462EXPORT_SYMBOL(tasklet_init);
463
464void tasklet_kill(struct tasklet_struct *t)
465{
466 if (in_interrupt())
467 printk("Attempt to kill tasklet from interrupt\n");
468
469 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
470 do
471 yield();
472 while (test_bit(TASKLET_STATE_SCHED, &t->state));
473 }
474 tasklet_unlock_wait(t);
475 clear_bit(TASKLET_STATE_SCHED, &t->state);
476}
477
478EXPORT_SYMBOL(tasklet_kill);
479
480void __init softirq_init(void)
481{
482 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
483 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
484}
485
486static int ksoftirqd(void * __bind_cpu)
487{
488 set_current_state(TASK_INTERRUPTIBLE);
489
490 while (!kthread_should_stop()) {
491 preempt_disable();
492 if (!local_softirq_pending()) {
493 preempt_enable_no_resched();
494 schedule();
495 preempt_disable();
496 }
497
498 __set_current_state(TASK_RUNNING);
499
500 while (local_softirq_pending()) {
501
502
503
504 if (cpu_is_offline((long)__bind_cpu))
505 goto wait_to_die;
506 do_softirq();
507 preempt_enable_no_resched();
508 cond_resched();
509 preempt_disable();
510 }
511 preempt_enable();
512 set_current_state(TASK_INTERRUPTIBLE);
513 }
514 __set_current_state(TASK_RUNNING);
515 return 0;
516
517wait_to_die:
518 preempt_enable();
519
520 set_current_state(TASK_INTERRUPTIBLE);
521 while (!kthread_should_stop()) {
522 schedule();
523 set_current_state(TASK_INTERRUPTIBLE);
524 }
525 __set_current_state(TASK_RUNNING);
526 return 0;
527}
528
529#ifdef CONFIG_HOTPLUG_CPU
530
531
532
533
534
535
536
537
538
539void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
540{
541 struct tasklet_struct **i;
542
543 BUG_ON(cpu_online(cpu));
544 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
545
546 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
547 return;
548
549
550 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
551 if (*i == t) {
552 *i = t->next;
553 return;
554 }
555 }
556 BUG();
557}
558
559static void takeover_tasklets(unsigned int cpu)
560{
561 struct tasklet_struct **i;
562
563
564 local_irq_disable();
565
566
567 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
568 *i = per_cpu(tasklet_vec, cpu).list;
569 per_cpu(tasklet_vec, cpu).list = NULL;
570 raise_softirq_irqoff(TASKLET_SOFTIRQ);
571
572 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
573 *i = per_cpu(tasklet_hi_vec, cpu).list;
574 per_cpu(tasklet_hi_vec, cpu).list = NULL;
575 raise_softirq_irqoff(HI_SOFTIRQ);
576
577 local_irq_enable();
578}
579#endif
580
581static int __cpuinit cpu_callback(struct notifier_block *nfb,
582 unsigned long action,
583 void *hcpu)
584{
585 int hotcpu = (unsigned long)hcpu;
586 struct task_struct *p;
587
588 switch (action) {
589 case CPU_UP_PREPARE:
590 case CPU_UP_PREPARE_FROZEN:
591 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
592 if (IS_ERR(p)) {
593 printk("ksoftirqd for %i failed\n", hotcpu);
594 return NOTIFY_BAD;
595 }
596 kthread_bind(p, hotcpu);
597 per_cpu(ksoftirqd, hotcpu) = p;
598 break;
599 case CPU_ONLINE:
600 case CPU_ONLINE_FROZEN:
601 wake_up_process(per_cpu(ksoftirqd, hotcpu));
602 break;
603#ifdef CONFIG_HOTPLUG_CPU
604 case CPU_UP_CANCELED:
605 case CPU_UP_CANCELED_FROZEN:
606 if (!per_cpu(ksoftirqd, hotcpu))
607 break;
608
609 kthread_bind(per_cpu(ksoftirqd, hotcpu),
610 any_online_cpu(cpu_online_map));
611 case CPU_DEAD:
612 case CPU_DEAD_FROZEN: {
613 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
614
615 p = per_cpu(ksoftirqd, hotcpu);
616 per_cpu(ksoftirqd, hotcpu) = NULL;
617 sched_setscheduler(p, SCHED_FIFO, ¶m);
618 kthread_stop(p);
619 takeover_tasklets(hotcpu);
620 break;
621 }
622#endif
623 }
624 return NOTIFY_OK;
625}
626
627static struct notifier_block __cpuinitdata cpu_nfb = {
628 .notifier_call = cpu_callback
629};
630
631__init int spawn_ksoftirqd(void)
632{
633 void *cpu = (void *)(long)smp_processor_id();
634 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
635
636 BUG_ON(err == NOTIFY_BAD);
637 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
638 register_cpu_notifier(&cpu_nfb);
639 return 0;
640}
641
642#ifdef CONFIG_SMP
643
644
645
646int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
647{
648 int ret = 0;
649
650 preempt_disable();
651 ret = smp_call_function(func, info, retry, wait);
652 local_irq_disable();
653 func(info);
654 local_irq_enable();
655 preempt_enable();
656 return ret;
657}
658EXPORT_SYMBOL(on_each_cpu);
659#endif
660