1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/sched.h>
16#include <linux/preempt.h>
17#include <linux/module.h>
18#include <linux/fs.h>
19#include <linux/kprobes.h>
20#include <linux/elfcore.h>
21#include <linux/tick.h>
22#include <linux/init.h>
23#include <linux/mm.h>
24#include <linux/compat.h>
25#include <linux/hardirq.h>
26#include <linux/syscalls.h>
27#include <linux/kernel.h>
28#include <linux/tracehook.h>
29#include <linux/signal.h>
30#include <linux/delay.h>
31#include <linux/context_tracking.h>
32#include <asm/stack.h>
33#include <asm/switch_to.h>
34#include <asm/homecache.h>
35#include <asm/syscalls.h>
36#include <asm/traps.h>
37#include <asm/setup.h>
38#include <asm/uaccess.h>
39#ifdef CONFIG_HARDWALL
40#include <asm/hardwall.h>
41#endif
42#include <arch/chip.h>
43#include <arch/abi.h>
44#include <arch/sim_def.h>
45
46
47
48
49
50
51static int __init idle_setup(char *str)
52{
53 if (!str)
54 return -EINVAL;
55
56 if (!strcmp(str, "poll")) {
57 pr_info("using polling idle threads\n");
58 cpu_idle_poll_ctrl(true);
59 return 0;
60 } else if (!strcmp(str, "halt")) {
61 return 0;
62 }
63 return -1;
64}
65early_param("idle", idle_setup);
66
67void arch_cpu_idle(void)
68{
69 __this_cpu_write(irq_stat.idle_timestamp, jiffies);
70 _cpu_idle();
71}
72
73
74
75
76void arch_release_thread_info(struct thread_info *info)
77{
78 struct single_step_state *step_state = info->step_state;
79
80 if (step_state) {
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96 kfree(step_state);
97 }
98}
99
100static void save_arch_state(struct thread_struct *t);
101
102int copy_thread(unsigned long clone_flags, unsigned long sp,
103 unsigned long arg, struct task_struct *p)
104{
105 struct pt_regs *childregs = task_pt_regs(p);
106 unsigned long ksp;
107 unsigned long *callee_regs;
108
109
110
111
112
113
114
115
116
117 ksp = (unsigned long) childregs;
118 ksp -= C_ABI_SAVE_AREA_SIZE;
119 ((long *)ksp)[0] = ((long *)ksp)[1] = 0;
120 ksp -= CALLEE_SAVED_REGS_COUNT * sizeof(unsigned long);
121 callee_regs = (unsigned long *)ksp;
122 ksp -= C_ABI_SAVE_AREA_SIZE;
123 ((long *)ksp)[0] = ((long *)ksp)[1] = 0;
124 p->thread.ksp = ksp;
125
126
127 p->thread.creator_pid = current->pid;
128
129 if (unlikely(p->flags & PF_KTHREAD)) {
130
131 memset(childregs, 0, sizeof(struct pt_regs));
132 memset(&callee_regs[2], 0,
133 (CALLEE_SAVED_REGS_COUNT - 2) * sizeof(unsigned long));
134 callee_regs[0] = sp;
135 callee_regs[1] = arg;
136 p->thread.pc = (unsigned long) ret_from_kernel_thread;
137 return 0;
138 }
139
140
141
142
143
144 p->thread.pc = (unsigned long) ret_from_fork;
145
146
147
148
149
150 task_thread_info(p)->step_state = NULL;
151
152#ifdef __tilegx__
153
154
155
156
157 task_thread_info(p)->unalign_jit_base = NULL;
158#endif
159
160
161
162
163
164 *childregs = *current_pt_regs();
165 childregs->regs[0] = 0;
166 if (sp)
167 childregs->sp = sp;
168 memcpy(callee_regs, &childregs->regs[CALLEE_SAVED_FIRST_REG],
169 CALLEE_SAVED_REGS_COUNT * sizeof(unsigned long));
170
171
172 p->thread.usp0 = childregs->sp;
173
174
175
176
177
178 if (clone_flags & CLONE_SETTLS)
179 childregs->tp = childregs->regs[4];
180
181
182#if CHIP_HAS_TILE_DMA()
183
184
185
186
187 memset(&p->thread.tile_dma_state, 0, sizeof(struct tile_dma_state));
188 memset(&p->thread.dma_async_tlb, 0, sizeof(struct async_tlb));
189#endif
190
191
192 p->thread.proc_status = 0;
193
194#ifdef CONFIG_HARDWALL
195
196 memset(&p->thread.hardwall[0], 0,
197 sizeof(struct hardwall_task) * HARDWALL_TYPES);
198#endif
199
200
201
202
203
204
205 save_arch_state(&p->thread);
206
207 return 0;
208}
209
210int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
211{
212 task_thread_info(tsk)->align_ctl = val;
213 return 0;
214}
215
216int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
217{
218 return put_user(task_thread_info(tsk)->align_ctl,
219 (unsigned int __user *)adr);
220}
221
222static struct task_struct corrupt_current = { .comm = "<corrupt>" };
223
224
225
226
227
228struct task_struct *validate_current(void)
229{
230 struct task_struct *tsk = current;
231 if (unlikely((unsigned long)tsk < PAGE_OFFSET ||
232 (high_memory && (void *)tsk > high_memory) ||
233 ((unsigned long)tsk & (__alignof__(*tsk) - 1)) != 0)) {
234 pr_err("Corrupt 'current' %p (sp %#lx)\n", tsk, stack_pointer);
235 tsk = &corrupt_current;
236 }
237 return tsk;
238}
239
240
241struct task_struct *sim_notify_fork(struct task_struct *prev)
242{
243 struct task_struct *tsk = current;
244 __insn_mtspr(SPR_SIM_CONTROL, SIM_CONTROL_OS_FORK_PARENT |
245 (tsk->thread.creator_pid << _SIM_CONTROL_OPERATOR_BITS));
246 __insn_mtspr(SPR_SIM_CONTROL, SIM_CONTROL_OS_FORK |
247 (tsk->pid << _SIM_CONTROL_OPERATOR_BITS));
248 return prev;
249}
250
251int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
252{
253 struct pt_regs *ptregs = task_pt_regs(tsk);
254 elf_core_copy_regs(regs, ptregs);
255 return 1;
256}
257
258#if CHIP_HAS_TILE_DMA()
259
260
261void grant_dma_mpls(void)
262{
263#if CONFIG_KERNEL_PL == 2
264 __insn_mtspr(SPR_MPL_DMA_CPL_SET_1, 1);
265 __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_1, 1);
266#else
267 __insn_mtspr(SPR_MPL_DMA_CPL_SET_0, 1);
268 __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_0, 1);
269#endif
270}
271
272
273void restrict_dma_mpls(void)
274{
275#if CONFIG_KERNEL_PL == 2
276 __insn_mtspr(SPR_MPL_DMA_CPL_SET_2, 1);
277 __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_2, 1);
278#else
279 __insn_mtspr(SPR_MPL_DMA_CPL_SET_1, 1);
280 __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_1, 1);
281#endif
282}
283
284
285static void save_tile_dma_state(struct tile_dma_state *dma)
286{
287 unsigned long state = __insn_mfspr(SPR_DMA_USER_STATUS);
288 unsigned long post_suspend_state;
289
290
291 if ((state & DMA_STATUS_MASK) == SPR_DMA_STATUS__RUNNING_MASK)
292 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
293
294
295
296
297
298
299
300
301
302 do {
303 post_suspend_state = __insn_mfspr(SPR_DMA_USER_STATUS);
304 } while (post_suspend_state & SPR_DMA_STATUS__BUSY_MASK);
305
306 dma->src = __insn_mfspr(SPR_DMA_SRC_ADDR);
307 dma->src_chunk = __insn_mfspr(SPR_DMA_SRC_CHUNK_ADDR);
308 dma->dest = __insn_mfspr(SPR_DMA_DST_ADDR);
309 dma->dest_chunk = __insn_mfspr(SPR_DMA_DST_CHUNK_ADDR);
310 dma->strides = __insn_mfspr(SPR_DMA_STRIDE);
311 dma->chunk_size = __insn_mfspr(SPR_DMA_CHUNK_SIZE);
312 dma->byte = __insn_mfspr(SPR_DMA_BYTE);
313 dma->status = (state & SPR_DMA_STATUS__RUNNING_MASK) |
314 (post_suspend_state & SPR_DMA_STATUS__DONE_MASK);
315}
316
317
318static void restore_tile_dma_state(struct thread_struct *t)
319{
320 const struct tile_dma_state *dma = &t->tile_dma_state;
321
322
323
324
325
326 if ((dma->status & SPR_DMA_STATUS__DONE_MASK) &&
327 !(__insn_mfspr(SPR_DMA_USER_STATUS) & SPR_DMA_STATUS__DONE_MASK)) {
328 __insn_mtspr(SPR_DMA_BYTE, 0);
329 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
330 while (__insn_mfspr(SPR_DMA_USER_STATUS) &
331 SPR_DMA_STATUS__BUSY_MASK)
332 ;
333 }
334
335 __insn_mtspr(SPR_DMA_SRC_ADDR, dma->src);
336 __insn_mtspr(SPR_DMA_SRC_CHUNK_ADDR, dma->src_chunk);
337 __insn_mtspr(SPR_DMA_DST_ADDR, dma->dest);
338 __insn_mtspr(SPR_DMA_DST_CHUNK_ADDR, dma->dest_chunk);
339 __insn_mtspr(SPR_DMA_STRIDE, dma->strides);
340 __insn_mtspr(SPR_DMA_CHUNK_SIZE, dma->chunk_size);
341 __insn_mtspr(SPR_DMA_BYTE, dma->byte);
342
343
344
345
346
347
348
349
350
351 if ((dma->status & DMA_STATUS_MASK) == SPR_DMA_STATUS__RUNNING_MASK) {
352 t->dma_async_tlb.fault_num = 0;
353 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
354 }
355}
356
357#endif
358
359static void save_arch_state(struct thread_struct *t)
360{
361#if CHIP_HAS_SPLIT_INTR_MASK()
362 t->interrupt_mask = __insn_mfspr(SPR_INTERRUPT_MASK_0_0) |
363 ((u64)__insn_mfspr(SPR_INTERRUPT_MASK_0_1) << 32);
364#else
365 t->interrupt_mask = __insn_mfspr(SPR_INTERRUPT_MASK_0);
366#endif
367 t->ex_context[0] = __insn_mfspr(SPR_EX_CONTEXT_0_0);
368 t->ex_context[1] = __insn_mfspr(SPR_EX_CONTEXT_0_1);
369 t->system_save[0] = __insn_mfspr(SPR_SYSTEM_SAVE_0_0);
370 t->system_save[1] = __insn_mfspr(SPR_SYSTEM_SAVE_0_1);
371 t->system_save[2] = __insn_mfspr(SPR_SYSTEM_SAVE_0_2);
372 t->system_save[3] = __insn_mfspr(SPR_SYSTEM_SAVE_0_3);
373 t->intctrl_0 = __insn_mfspr(SPR_INTCTRL_0_STATUS);
374 t->proc_status = __insn_mfspr(SPR_PROC_STATUS);
375#if !CHIP_HAS_FIXED_INTVEC_BASE()
376 t->interrupt_vector_base = __insn_mfspr(SPR_INTERRUPT_VECTOR_BASE_0);
377#endif
378 t->tile_rtf_hwm = __insn_mfspr(SPR_TILE_RTF_HWM);
379#if CHIP_HAS_DSTREAM_PF()
380 t->dstream_pf = __insn_mfspr(SPR_DSTREAM_PF);
381#endif
382}
383
384static void restore_arch_state(const struct thread_struct *t)
385{
386#if CHIP_HAS_SPLIT_INTR_MASK()
387 __insn_mtspr(SPR_INTERRUPT_MASK_0_0, (u32) t->interrupt_mask);
388 __insn_mtspr(SPR_INTERRUPT_MASK_0_1, t->interrupt_mask >> 32);
389#else
390 __insn_mtspr(SPR_INTERRUPT_MASK_0, t->interrupt_mask);
391#endif
392 __insn_mtspr(SPR_EX_CONTEXT_0_0, t->ex_context[0]);
393 __insn_mtspr(SPR_EX_CONTEXT_0_1, t->ex_context[1]);
394 __insn_mtspr(SPR_SYSTEM_SAVE_0_0, t->system_save[0]);
395 __insn_mtspr(SPR_SYSTEM_SAVE_0_1, t->system_save[1]);
396 __insn_mtspr(SPR_SYSTEM_SAVE_0_2, t->system_save[2]);
397 __insn_mtspr(SPR_SYSTEM_SAVE_0_3, t->system_save[3]);
398 __insn_mtspr(SPR_INTCTRL_0_STATUS, t->intctrl_0);
399 __insn_mtspr(SPR_PROC_STATUS, t->proc_status);
400#if !CHIP_HAS_FIXED_INTVEC_BASE()
401 __insn_mtspr(SPR_INTERRUPT_VECTOR_BASE_0, t->interrupt_vector_base);
402#endif
403 __insn_mtspr(SPR_TILE_RTF_HWM, t->tile_rtf_hwm);
404#if CHIP_HAS_DSTREAM_PF()
405 __insn_mtspr(SPR_DSTREAM_PF, t->dstream_pf);
406#endif
407}
408
409
410void _prepare_arch_switch(struct task_struct *next)
411{
412#if CHIP_HAS_TILE_DMA()
413 struct tile_dma_state *dma = ¤t->thread.tile_dma_state;
414 if (dma->enabled)
415 save_tile_dma_state(dma);
416#endif
417}
418
419
420struct task_struct *__sched _switch_to(struct task_struct *prev,
421 struct task_struct *next)
422{
423
424 save_arch_state(&prev->thread);
425
426#if CHIP_HAS_TILE_DMA()
427
428
429
430
431
432
433 if (next->thread.tile_dma_state.enabled) {
434 restore_tile_dma_state(&next->thread);
435 grant_dma_mpls();
436 } else {
437 restrict_dma_mpls();
438 }
439#endif
440
441
442 restore_arch_state(&next->thread);
443
444#ifdef CONFIG_HARDWALL
445
446 hardwall_switch_tasks(prev, next);
447#endif
448
449
450 if (unlikely(prev->state == TASK_DEAD))
451 __insn_mtspr(SPR_SIM_CONTROL, SIM_CONTROL_OS_EXIT |
452 (prev->pid << _SIM_CONTROL_OPERATOR_BITS));
453
454
455
456
457
458
459
460 return __switch_to(prev, next, next_current_ksp0(next));
461}
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477int do_work_pending(struct pt_regs *regs, u32 thread_info_flags)
478{
479
480 if (!user_mode(regs))
481 return 0;
482
483 user_exit();
484
485
486 local_irq_enable();
487
488 if (thread_info_flags & _TIF_NEED_RESCHED) {
489 schedule();
490 return 1;
491 }
492#if CHIP_HAS_TILE_DMA()
493 if (thread_info_flags & _TIF_ASYNC_TLB) {
494 do_async_page_fault(regs);
495 return 1;
496 }
497#endif
498 if (thread_info_flags & _TIF_SIGPENDING) {
499 do_signal(regs);
500 return 1;
501 }
502 if (thread_info_flags & _TIF_NOTIFY_RESUME) {
503 clear_thread_flag(TIF_NOTIFY_RESUME);
504 tracehook_notify_resume(regs);
505 return 1;
506 }
507 if (thread_info_flags & _TIF_SINGLESTEP)
508 single_step_once(regs);
509
510 user_enter();
511
512 return 0;
513}
514
515unsigned long get_wchan(struct task_struct *p)
516{
517 struct KBacktraceIterator kbt;
518
519 if (!p || p == current || p->state == TASK_RUNNING)
520 return 0;
521
522 for (KBacktraceIterator_init(&kbt, p, NULL);
523 !KBacktraceIterator_end(&kbt);
524 KBacktraceIterator_next(&kbt)) {
525 if (!in_sched_functions(kbt.it.pc))
526 return kbt.it.pc;
527 }
528
529 return 0;
530}
531
532
533void flush_thread(void)
534{
535
536}
537
538
539
540
541void exit_thread(void)
542{
543#ifdef CONFIG_HARDWALL
544
545
546
547
548
549
550 hardwall_deactivate_all(current);
551#endif
552}
553
554void tile_show_regs(struct pt_regs *regs)
555{
556 int i;
557#ifdef __tilegx__
558 for (i = 0; i < 17; i++)
559 pr_err(" r%-2d: "REGFMT" r%-2d: "REGFMT" r%-2d: "REGFMT"\n",
560 i, regs->regs[i], i+18, regs->regs[i+18],
561 i+36, regs->regs[i+36]);
562 pr_err(" r17: "REGFMT" r35: "REGFMT" tp : "REGFMT"\n",
563 regs->regs[17], regs->regs[35], regs->tp);
564 pr_err(" sp : "REGFMT" lr : "REGFMT"\n", regs->sp, regs->lr);
565#else
566 for (i = 0; i < 13; i++)
567 pr_err(" r%-2d: "REGFMT" r%-2d: "REGFMT
568 " r%-2d: "REGFMT" r%-2d: "REGFMT"\n",
569 i, regs->regs[i], i+14, regs->regs[i+14],
570 i+27, regs->regs[i+27], i+40, regs->regs[i+40]);
571 pr_err(" r13: "REGFMT" tp : "REGFMT" sp : "REGFMT" lr : "REGFMT"\n",
572 regs->regs[13], regs->tp, regs->sp, regs->lr);
573#endif
574 pr_err(" pc : "REGFMT" ex1: %ld faultnum: %ld flags:%s%s%s%s\n",
575 regs->pc, regs->ex1, regs->faultnum,
576 is_compat_task() ? " compat" : "",
577 (regs->flags & PT_FLAGS_DISABLE_IRQ) ? " noirq" : "",
578 !(regs->flags & PT_FLAGS_CALLER_SAVES) ? " nocallersave" : "",
579 (regs->flags & PT_FLAGS_RESTORE_REGS) ? " restoreregs" : "");
580}
581
582void show_regs(struct pt_regs *regs)
583{
584 struct KBacktraceIterator kbt;
585
586 show_regs_print_info(KERN_DEFAULT);
587 tile_show_regs(regs);
588
589 KBacktraceIterator_init(&kbt, NULL, regs);
590 tile_show_stack(&kbt);
591}
592
593
594static DEFINE_SPINLOCK(backtrace_lock);
595
596static atomic_t backtrace_cpus;
597
598static struct cpumask backtrace_mask;
599
600void do_nmi_dump_stack(struct pt_regs *regs)
601{
602 int is_idle = is_idle_task(current) && !in_interrupt();
603 int cpu;
604
605 nmi_enter();
606 cpu = smp_processor_id();
607 if (WARN_ON_ONCE(!cpumask_test_and_clear_cpu(cpu, &backtrace_mask)))
608 goto done;
609
610 spin_lock(&backtrace_lock);
611 if (is_idle)
612 pr_info("CPU: %d idle\n", cpu);
613 else
614 show_regs(regs);
615 spin_unlock(&backtrace_lock);
616 atomic_dec(&backtrace_cpus);
617done:
618 nmi_exit();
619}
620
621#ifdef __tilegx__
622void arch_trigger_all_cpu_backtrace(bool self)
623{
624 struct cpumask mask;
625 HV_Coord tile;
626 unsigned int timeout;
627 int cpu;
628 int ongoing;
629 HV_NMI_Info info[NR_CPUS];
630
631 ongoing = atomic_cmpxchg(&backtrace_cpus, 0, num_online_cpus() - 1);
632 if (ongoing != 0) {
633 pr_err("Trying to do all-cpu backtrace.\n");
634 pr_err("But another all-cpu backtrace is ongoing (%d cpus left)\n",
635 ongoing);
636 if (self) {
637 pr_err("Reporting the stack on this cpu only.\n");
638 dump_stack();
639 }
640 return;
641 }
642
643 cpumask_copy(&mask, cpu_online_mask);
644 cpumask_clear_cpu(smp_processor_id(), &mask);
645 cpumask_copy(&backtrace_mask, &mask);
646
647
648 if (self)
649 dump_stack();
650
651
652 timeout = 100;
653 while (!cpumask_empty(&mask) && timeout) {
654 for_each_cpu(cpu, &mask) {
655 tile.x = cpu_x(cpu);
656 tile.y = cpu_y(cpu);
657 info[cpu] = hv_send_nmi(tile, TILE_NMI_DUMP_STACK, 0);
658 if (info[cpu].result == HV_NMI_RESULT_OK)
659 cpumask_clear_cpu(cpu, &mask);
660 }
661
662 mdelay(10);
663 timeout--;
664 }
665
666
667 if (!cpumask_empty(&mask)) {
668 for_each_cpu(cpu, &mask) {
669 switch (info[cpu].result) {
670 case HV_NMI_RESULT_FAIL_ICS:
671 pr_warn("Skipping stack dump of cpu %d in ICS at pc %#llx\n",
672 cpu, info[cpu].pc);
673 break;
674 case HV_NMI_RESULT_FAIL_HV:
675 pr_warn("Skipping stack dump of cpu %d in hypervisor\n",
676 cpu);
677 break;
678 case HV_ENOSYS:
679 pr_warn("Hypervisor too old to allow remote stack dumps.\n");
680 goto skip_for_each;
681 default:
682 pr_warn("Skipping stack dump of cpu %d [%d,%#llx]\n",
683 cpu, info[cpu].result, info[cpu].pc);
684 break;
685 }
686 }
687skip_for_each:
688 atomic_sub(cpumask_weight(&mask), &backtrace_cpus);
689 }
690}
691#endif
692