1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/kernel.h>
25#include <linux/sched.h>
26#include <linux/module.h>
27#include <linux/interrupt.h>
28#include <linux/percpu.h>
29#include <linux/bitops.h>
30#include <linux/errno.h>
31#include <linux/kthread.h>
32#include <linux/unistd.h>
33#include <linux/io.h>
34#include <linux/atomic.h>
35#include <asm/irq_handler.h>
36
37DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
38
39asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
40
41static void __ipipe_no_irqtail(void);
42
43unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail;
44EXPORT_SYMBOL(__ipipe_irq_tail_hook);
45
46unsigned long __ipipe_core_clock;
47EXPORT_SYMBOL(__ipipe_core_clock);
48
49unsigned long __ipipe_freq_scale;
50EXPORT_SYMBOL(__ipipe_freq_scale);
51
52atomic_t __ipipe_irq_lvdepth[IVG15 + 1];
53
54unsigned long __ipipe_irq_lvmask = bfin_no_irqs;
55EXPORT_SYMBOL(__ipipe_irq_lvmask);
56
57static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
58{
59 desc->ipipe_ack(irq, desc);
60}
61
62
63
64
65
66void __ipipe_enable_pipeline(void)
67{
68 unsigned irq;
69
70 __ipipe_core_clock = get_cclk();
71 __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock;
72
73 for (irq = 0; irq < NR_IRQS; ++irq)
74 ipipe_virtualize_irq(ipipe_root_domain,
75 irq,
76 (ipipe_irq_handler_t)&asm_do_IRQ,
77 NULL,
78 &__ipipe_ack_irq,
79 IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
80}
81
82
83
84
85
86
87void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
88{
89 struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
90 struct ipipe_domain *this_domain, *next_domain;
91 struct list_head *head, *pos;
92 struct ipipe_irqdesc *idesc;
93 int m_ack, s = -1;
94
95
96
97
98
99
100
101 m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR);
102 this_domain = __ipipe_current_domain;
103 idesc = &this_domain->irqs[irq];
104
105 if (unlikely(test_bit(IPIPE_STICKY_FLAG, &idesc->control)))
106 head = &this_domain->p_link;
107 else {
108 head = __ipipe_pipeline.next;
109 next_domain = list_entry(head, struct ipipe_domain, p_link);
110 idesc = &next_domain->irqs[irq];
111 if (likely(test_bit(IPIPE_WIRED_FLAG, &idesc->control))) {
112 if (!m_ack && idesc->acknowledge != NULL)
113 idesc->acknowledge(irq, irq_to_desc(irq));
114 if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
115 s = __test_and_set_bit(IPIPE_STALL_FLAG,
116 &p->status);
117 __ipipe_dispatch_wired(next_domain, irq);
118 goto out;
119 }
120 }
121
122
123
124 pos = head;
125 while (pos != &__ipipe_pipeline) {
126 next_domain = list_entry(pos, struct ipipe_domain, p_link);
127 idesc = &next_domain->irqs[irq];
128 if (test_bit(IPIPE_HANDLE_FLAG, &idesc->control)) {
129 __ipipe_set_irq_pending(next_domain, irq);
130 if (!m_ack && idesc->acknowledge != NULL) {
131 idesc->acknowledge(irq, irq_to_desc(irq));
132 m_ack = 1;
133 }
134 }
135 if (!test_bit(IPIPE_PASS_FLAG, &idesc->control))
136 break;
137 pos = next_domain->p_link.next;
138 }
139
140
141
142
143
144
145
146
147
148 if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
149 s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
150
151
152
153
154
155
156 if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
157 !__ipipe_ipending_p(ipipe_head_cpudom_ptr()))
158 goto out;
159
160 __ipipe_walk_pipeline(head);
161out:
162 if (!s)
163 __clear_bit(IPIPE_STALL_FLAG, &p->status);
164}
165
166void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
167{
168 struct irq_desc *desc = irq_to_desc(irq);
169 int prio = __ipipe_get_irq_priority(irq);
170
171 desc->depth = 0;
172 if (ipd != &ipipe_root &&
173 atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1)
174 __set_bit(prio, &__ipipe_irq_lvmask);
175}
176EXPORT_SYMBOL(__ipipe_enable_irqdesc);
177
178void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
179{
180 int prio = __ipipe_get_irq_priority(irq);
181
182 if (ipd != &ipipe_root &&
183 atomic_dec_and_test(&__ipipe_irq_lvdepth[prio]))
184 __clear_bit(prio, &__ipipe_irq_lvmask);
185}
186EXPORT_SYMBOL(__ipipe_disable_irqdesc);
187
188asmlinkage int __ipipe_syscall_root(struct pt_regs *regs)
189{
190 struct ipipe_percpu_domain_data *p;
191 void (*hook)(void);
192 int ret;
193
194 WARN_ON_ONCE(irqs_disabled_hw());
195
196
197
198
199
200
201 hook = (__typeof__(hook))__ipipe_irq_tail_hook;
202 hook();
203
204
205
206
207
208
209
210
211
212
213 if (!__ipipe_syscall_watched_p(current, regs->orig_p0) ||
214 !__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
215 return 0;
216
217 ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
218
219 hard_local_irq_disable();
220
221
222
223
224
225 if (current->ipipe_flags & PF_EVTRET) {
226 current->ipipe_flags &= ~PF_EVTRET;
227 __ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
228 }
229
230 if (!__ipipe_root_domain_p)
231 ret = -1;
232 else {
233 p = ipipe_root_cpudom_ptr();
234 if (__ipipe_ipending_p(p))
235 __ipipe_sync_pipeline();
236 }
237
238 hard_local_irq_enable();
239
240 return -ret;
241}
242
243static void __ipipe_no_irqtail(void)
244{
245}
246
247int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
248{
249 info->sys_nr_cpus = num_online_cpus();
250 info->sys_cpu_freq = ipipe_cpu_freq();
251 info->sys_hrtimer_irq = IPIPE_TIMER_IRQ;
252 info->sys_hrtimer_freq = __ipipe_core_clock;
253 info->sys_hrclock_freq = __ipipe_core_clock;
254
255 return 0;
256}
257
258
259
260
261
262
263int ipipe_trigger_irq(unsigned irq)
264{
265 unsigned long flags;
266
267#ifdef CONFIG_IPIPE_DEBUG
268 if (irq >= IPIPE_NR_IRQS ||
269 (ipipe_virtual_irq_p(irq)
270 && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
271 return -EINVAL;
272#endif
273
274 flags = hard_local_irq_save();
275 __ipipe_handle_irq(irq, NULL);
276 hard_local_irq_restore(flags);
277
278 return 1;
279}
280
281asmlinkage void __ipipe_sync_root(void)
282{
283 void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
284 struct ipipe_percpu_domain_data *p;
285 unsigned long flags;
286
287 BUG_ON(irqs_disabled());
288
289 flags = hard_local_irq_save();
290
291 if (irq_tail_hook)
292 irq_tail_hook();
293
294 clear_thread_flag(TIF_IRQ_SYNC);
295
296 p = ipipe_root_cpudom_ptr();
297 if (__ipipe_ipending_p(p))
298 __ipipe_sync_pipeline();
299
300 hard_local_irq_restore(flags);
301}
302
303void ___ipipe_sync_pipeline(void)
304{
305 if (__ipipe_root_domain_p &&
306 test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status)))
307 return;
308
309 __ipipe_sync_stage();
310}
311
312void __ipipe_disable_root_irqs_hw(void)
313{
314
315
316
317
318
319
320
321
322 bfin_sti(__ipipe_irq_lvmask);
323 __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
324}
325
326void __ipipe_enable_root_irqs_hw(void)
327{
328 __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
329 bfin_sti(bfin_irq_flags);
330}
331
332
333
334
335
336
337void __ipipe_stall_root(void)
338{
339 unsigned long *p, flags;
340
341 flags = hard_local_irq_save();
342 p = &__ipipe_root_status;
343 __set_bit(IPIPE_STALL_FLAG, p);
344 hard_local_irq_restore(flags);
345}
346EXPORT_SYMBOL(__ipipe_stall_root);
347
348unsigned long __ipipe_test_and_stall_root(void)
349{
350 unsigned long *p, flags;
351 int x;
352
353 flags = hard_local_irq_save();
354 p = &__ipipe_root_status;
355 x = __test_and_set_bit(IPIPE_STALL_FLAG, p);
356 hard_local_irq_restore(flags);
357
358 return x;
359}
360EXPORT_SYMBOL(__ipipe_test_and_stall_root);
361
362unsigned long __ipipe_test_root(void)
363{
364 const unsigned long *p;
365 unsigned long flags;
366 int x;
367
368 flags = hard_local_irq_save_smp();
369 p = &__ipipe_root_status;
370 x = test_bit(IPIPE_STALL_FLAG, p);
371 hard_local_irq_restore_smp(flags);
372
373 return x;
374}
375EXPORT_SYMBOL(__ipipe_test_root);
376
377void __ipipe_lock_root(void)
378{
379 unsigned long *p, flags;
380
381 flags = hard_local_irq_save();
382 p = &__ipipe_root_status;
383 __set_bit(IPIPE_SYNCDEFER_FLAG, p);
384 hard_local_irq_restore(flags);
385}
386EXPORT_SYMBOL(__ipipe_lock_root);
387
388void __ipipe_unlock_root(void)
389{
390 unsigned long *p, flags;
391
392 flags = hard_local_irq_save();
393 p = &__ipipe_root_status;
394 __clear_bit(IPIPE_SYNCDEFER_FLAG, p);
395 hard_local_irq_restore(flags);
396}
397EXPORT_SYMBOL(__ipipe_unlock_root);
398