1
2
3
4
5
6
7
8
9
10
11
12#include <linux/errno.h>
13#include <linux/export.h>
14#include <linux/sched.h>
15#include <linux/sched/debug.h>
16#include <linux/sched/task.h>
17#include <linux/sched/task_stack.h>
18#include <linux/kernel.h>
19#include <linux/mm.h>
20#include <linux/fs.h>
21#include <linux/smp.h>
22#include <linux/stddef.h>
23#include <linux/ptrace.h>
24#include <linux/slab.h>
25#include <linux/user.h>
26#include <linux/delay.h>
27#include <linux/compat.h>
28#include <linux/tick.h>
29#include <linux/init.h>
30#include <linux/cpu.h>
31#include <linux/perf_event.h>
32#include <linux/elfcore.h>
33#include <linux/sysrq.h>
34#include <linux/nmi.h>
35#include <linux/context_tracking.h>
36#include <linux/signal.h>
37
38#include <linux/uaccess.h>
39#include <asm/page.h>
40#include <asm/pgalloc.h>
41#include <asm/processor.h>
42#include <asm/pstate.h>
43#include <asm/elf.h>
44#include <asm/fpumacro.h>
45#include <asm/head.h>
46#include <asm/cpudata.h>
47#include <asm/mmu_context.h>
48#include <asm/unistd.h>
49#include <asm/hypervisor.h>
50#include <asm/syscalls.h>
51#include <asm/irq_regs.h>
52#include <asm/smp.h>
53#include <asm/pcr.h>
54
55#include "kstack.h"
56
57
58void arch_cpu_idle(void)
59{
60 if (tlb_type != hypervisor) {
61 touch_nmi_watchdog();
62 raw_local_irq_enable();
63 } else {
64 unsigned long pstate;
65
66 raw_local_irq_enable();
67
68
69
70
71 __asm__ __volatile__(
72 "rdpr %%pstate, %0\n\t"
73 "andn %0, %1, %0\n\t"
74 "wrpr %0, %%g0, %%pstate"
75 : "=&r" (pstate)
76 : "i" (PSTATE_IE));
77
78 if (!need_resched() && !cpu_is_offline(smp_processor_id())) {
79 sun4v_cpu_yield();
80
81
82
83 scheduler_poke();
84 }
85
86
87 __asm__ __volatile__(
88 "rdpr %%pstate, %0\n\t"
89 "or %0, %1, %0\n\t"
90 "wrpr %0, %%g0, %%pstate"
91 : "=&r" (pstate)
92 : "i" (PSTATE_IE));
93 }
94}
95
96#ifdef CONFIG_HOTPLUG_CPU
97void arch_cpu_idle_dead(void)
98{
99 sched_preempt_enable_no_resched();
100 cpu_play_dead();
101}
102#endif
103
104#ifdef CONFIG_COMPAT
105static void show_regwindow32(struct pt_regs *regs)
106{
107 struct reg_window32 __user *rw;
108 struct reg_window32 r_w;
109 mm_segment_t old_fs;
110
111 __asm__ __volatile__ ("flushw");
112 rw = compat_ptr((unsigned int)regs->u_regs[14]);
113 old_fs = get_fs();
114 set_fs (USER_DS);
115 if (copy_from_user (&r_w, rw, sizeof(r_w))) {
116 set_fs (old_fs);
117 return;
118 }
119
120 set_fs (old_fs);
121 printk("l0: %08x l1: %08x l2: %08x l3: %08x "
122 "l4: %08x l5: %08x l6: %08x l7: %08x\n",
123 r_w.locals[0], r_w.locals[1], r_w.locals[2], r_w.locals[3],
124 r_w.locals[4], r_w.locals[5], r_w.locals[6], r_w.locals[7]);
125 printk("i0: %08x i1: %08x i2: %08x i3: %08x "
126 "i4: %08x i5: %08x i6: %08x i7: %08x\n",
127 r_w.ins[0], r_w.ins[1], r_w.ins[2], r_w.ins[3],
128 r_w.ins[4], r_w.ins[5], r_w.ins[6], r_w.ins[7]);
129}
130#else
131#define show_regwindow32(regs) do { } while (0)
132#endif
133
134static void show_regwindow(struct pt_regs *regs)
135{
136 struct reg_window __user *rw;
137 struct reg_window *rwk;
138 struct reg_window r_w;
139 mm_segment_t old_fs;
140
141 if ((regs->tstate & TSTATE_PRIV) || !(test_thread_flag(TIF_32BIT))) {
142 __asm__ __volatile__ ("flushw");
143 rw = (struct reg_window __user *)
144 (regs->u_regs[14] + STACK_BIAS);
145 rwk = (struct reg_window *)
146 (regs->u_regs[14] + STACK_BIAS);
147 if (!(regs->tstate & TSTATE_PRIV)) {
148 old_fs = get_fs();
149 set_fs (USER_DS);
150 if (copy_from_user (&r_w, rw, sizeof(r_w))) {
151 set_fs (old_fs);
152 return;
153 }
154 rwk = &r_w;
155 set_fs (old_fs);
156 }
157 } else {
158 show_regwindow32(regs);
159 return;
160 }
161 printk("l0: %016lx l1: %016lx l2: %016lx l3: %016lx\n",
162 rwk->locals[0], rwk->locals[1], rwk->locals[2], rwk->locals[3]);
163 printk("l4: %016lx l5: %016lx l6: %016lx l7: %016lx\n",
164 rwk->locals[4], rwk->locals[5], rwk->locals[6], rwk->locals[7]);
165 printk("i0: %016lx i1: %016lx i2: %016lx i3: %016lx\n",
166 rwk->ins[0], rwk->ins[1], rwk->ins[2], rwk->ins[3]);
167 printk("i4: %016lx i5: %016lx i6: %016lx i7: %016lx\n",
168 rwk->ins[4], rwk->ins[5], rwk->ins[6], rwk->ins[7]);
169 if (regs->tstate & TSTATE_PRIV)
170 printk("I7: <%pS>\n", (void *) rwk->ins[7]);
171}
172
173void show_regs(struct pt_regs *regs)
174{
175 show_regs_print_info(KERN_DEFAULT);
176
177 printk("TSTATE: %016lx TPC: %016lx TNPC: %016lx Y: %08x %s\n", regs->tstate,
178 regs->tpc, regs->tnpc, regs->y, print_tainted());
179 printk("TPC: <%pS>\n", (void *) regs->tpc);
180 printk("g0: %016lx g1: %016lx g2: %016lx g3: %016lx\n",
181 regs->u_regs[0], regs->u_regs[1], regs->u_regs[2],
182 regs->u_regs[3]);
183 printk("g4: %016lx g5: %016lx g6: %016lx g7: %016lx\n",
184 regs->u_regs[4], regs->u_regs[5], regs->u_regs[6],
185 regs->u_regs[7]);
186 printk("o0: %016lx o1: %016lx o2: %016lx o3: %016lx\n",
187 regs->u_regs[8], regs->u_regs[9], regs->u_regs[10],
188 regs->u_regs[11]);
189 printk("o4: %016lx o5: %016lx sp: %016lx ret_pc: %016lx\n",
190 regs->u_regs[12], regs->u_regs[13], regs->u_regs[14],
191 regs->u_regs[15]);
192 printk("RPC: <%pS>\n", (void *) regs->u_regs[15]);
193 show_regwindow(regs);
194 show_stack(current, (unsigned long *)regs->u_regs[UREG_FP], KERN_DEFAULT);
195}
196
197union global_cpu_snapshot global_cpu_snapshot[NR_CPUS];
198static DEFINE_SPINLOCK(global_cpu_snapshot_lock);
199
200static void __global_reg_self(struct thread_info *tp, struct pt_regs *regs,
201 int this_cpu)
202{
203 struct global_reg_snapshot *rp;
204
205 flushw_all();
206
207 rp = &global_cpu_snapshot[this_cpu].reg;
208
209 rp->tstate = regs->tstate;
210 rp->tpc = regs->tpc;
211 rp->tnpc = regs->tnpc;
212 rp->o7 = regs->u_regs[UREG_I7];
213
214 if (regs->tstate & TSTATE_PRIV) {
215 struct reg_window *rw;
216
217 rw = (struct reg_window *)
218 (regs->u_regs[UREG_FP] + STACK_BIAS);
219 if (kstack_valid(tp, (unsigned long) rw)) {
220 rp->i7 = rw->ins[7];
221 rw = (struct reg_window *)
222 (rw->ins[6] + STACK_BIAS);
223 if (kstack_valid(tp, (unsigned long) rw))
224 rp->rpc = rw->ins[7];
225 }
226 } else {
227 rp->i7 = 0;
228 rp->rpc = 0;
229 }
230 rp->thread = tp;
231}
232
233
234
235
236
237
238static void __global_reg_poll(struct global_reg_snapshot *gp)
239{
240 int limit = 0;
241
242 while (!gp->thread && ++limit < 100) {
243 barrier();
244 udelay(1);
245 }
246}
247
248void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self)
249{
250 struct thread_info *tp = current_thread_info();
251 struct pt_regs *regs = get_irq_regs();
252 unsigned long flags;
253 int this_cpu, cpu;
254
255 if (!regs)
256 regs = tp->kregs;
257
258 spin_lock_irqsave(&global_cpu_snapshot_lock, flags);
259
260 this_cpu = raw_smp_processor_id();
261
262 memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot));
263
264 if (cpumask_test_cpu(this_cpu, mask) && !exclude_self)
265 __global_reg_self(tp, regs, this_cpu);
266
267 smp_fetch_global_regs();
268
269 for_each_cpu(cpu, mask) {
270 struct global_reg_snapshot *gp;
271
272 if (exclude_self && cpu == this_cpu)
273 continue;
274
275 gp = &global_cpu_snapshot[cpu].reg;
276
277 __global_reg_poll(gp);
278
279 tp = gp->thread;
280 printk("%c CPU[%3d]: TSTATE[%016lx] TPC[%016lx] TNPC[%016lx] TASK[%s:%d]\n",
281 (cpu == this_cpu ? '*' : ' '), cpu,
282 gp->tstate, gp->tpc, gp->tnpc,
283 ((tp && tp->task) ? tp->task->comm : "NULL"),
284 ((tp && tp->task) ? tp->task->pid : -1));
285
286 if (gp->tstate & TSTATE_PRIV) {
287 printk(" TPC[%pS] O7[%pS] I7[%pS] RPC[%pS]\n",
288 (void *) gp->tpc,
289 (void *) gp->o7,
290 (void *) gp->i7,
291 (void *) gp->rpc);
292 } else {
293 printk(" TPC[%lx] O7[%lx] I7[%lx] RPC[%lx]\n",
294 gp->tpc, gp->o7, gp->i7, gp->rpc);
295 }
296
297 touch_nmi_watchdog();
298 }
299
300 memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot));
301
302 spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags);
303}
304
305#ifdef CONFIG_MAGIC_SYSRQ
306
307static void sysrq_handle_globreg(int key)
308{
309 trigger_all_cpu_backtrace();
310}
311
312static const struct sysrq_key_op sparc_globalreg_op = {
313 .handler = sysrq_handle_globreg,
314 .help_msg = "global-regs(y)",
315 .action_msg = "Show Global CPU Regs",
316};
317
318static void __global_pmu_self(int this_cpu)
319{
320 struct global_pmu_snapshot *pp;
321 int i, num;
322
323 if (!pcr_ops)
324 return;
325
326 pp = &global_cpu_snapshot[this_cpu].pmu;
327
328 num = 1;
329 if (tlb_type == hypervisor &&
330 sun4v_chip_type >= SUN4V_CHIP_NIAGARA4)
331 num = 4;
332
333 for (i = 0; i < num; i++) {
334 pp->pcr[i] = pcr_ops->read_pcr(i);
335 pp->pic[i] = pcr_ops->read_pic(i);
336 }
337}
338
339static void __global_pmu_poll(struct global_pmu_snapshot *pp)
340{
341 int limit = 0;
342
343 while (!pp->pcr[0] && ++limit < 100) {
344 barrier();
345 udelay(1);
346 }
347}
348
349static void pmu_snapshot_all_cpus(void)
350{
351 unsigned long flags;
352 int this_cpu, cpu;
353
354 spin_lock_irqsave(&global_cpu_snapshot_lock, flags);
355
356 memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot));
357
358 this_cpu = raw_smp_processor_id();
359
360 __global_pmu_self(this_cpu);
361
362 smp_fetch_global_pmu();
363
364 for_each_online_cpu(cpu) {
365 struct global_pmu_snapshot *pp = &global_cpu_snapshot[cpu].pmu;
366
367 __global_pmu_poll(pp);
368
369 printk("%c CPU[%3d]: PCR[%08lx:%08lx:%08lx:%08lx] PIC[%08lx:%08lx:%08lx:%08lx]\n",
370 (cpu == this_cpu ? '*' : ' '), cpu,
371 pp->pcr[0], pp->pcr[1], pp->pcr[2], pp->pcr[3],
372 pp->pic[0], pp->pic[1], pp->pic[2], pp->pic[3]);
373
374 touch_nmi_watchdog();
375 }
376
377 memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot));
378
379 spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags);
380}
381
382static void sysrq_handle_globpmu(int key)
383{
384 pmu_snapshot_all_cpus();
385}
386
387static const struct sysrq_key_op sparc_globalpmu_op = {
388 .handler = sysrq_handle_globpmu,
389 .help_msg = "global-pmu(x)",
390 .action_msg = "Show Global PMU Regs",
391};
392
393static int __init sparc_sysrq_init(void)
394{
395 int ret = register_sysrq_key('y', &sparc_globalreg_op);
396
397 if (!ret)
398 ret = register_sysrq_key('x', &sparc_globalpmu_op);
399 return ret;
400}
401
402core_initcall(sparc_sysrq_init);
403
404#endif
405
406
407void exit_thread(struct task_struct *tsk)
408{
409 struct thread_info *t = task_thread_info(tsk);
410
411 if (t->utraps) {
412 if (t->utraps[0] < 2)
413 kfree (t->utraps);
414 else
415 t->utraps[0]--;
416 }
417}
418
419void flush_thread(void)
420{
421 struct thread_info *t = current_thread_info();
422 struct mm_struct *mm;
423
424 mm = t->task->mm;
425 if (mm)
426 tsb_context_switch(mm);
427
428 set_thread_wsaved(0);
429
430
431 t->fpsaved[0] = 0;
432}
433
434
435static unsigned long clone_stackframe(unsigned long csp, unsigned long psp)
436{
437 bool stack_64bit = test_thread_64bit_stack(psp);
438 unsigned long fp, distance, rval;
439
440 if (stack_64bit) {
441 csp += STACK_BIAS;
442 psp += STACK_BIAS;
443 __get_user(fp, &(((struct reg_window __user *)psp)->ins[6]));
444 fp += STACK_BIAS;
445 if (test_thread_flag(TIF_32BIT))
446 fp &= 0xffffffff;
447 } else
448 __get_user(fp, &(((struct reg_window32 __user *)psp)->ins[6]));
449
450
451
452
453
454 csp &= ~15UL;
455
456 distance = fp - psp;
457 rval = (csp - distance);
458 if (raw_copy_in_user((void __user *)rval, (void __user *)psp, distance))
459 rval = 0;
460 else if (!stack_64bit) {
461 if (put_user(((u32)csp),
462 &(((struct reg_window32 __user *)rval)->ins[6])))
463 rval = 0;
464 } else {
465 if (put_user(((u64)csp - STACK_BIAS),
466 &(((struct reg_window __user *)rval)->ins[6])))
467 rval = 0;
468 else
469 rval = rval - STACK_BIAS;
470 }
471
472 return rval;
473}
474
475
476static inline void shift_window_buffer(int first_win, int last_win,
477 struct thread_info *t)
478{
479 int i;
480
481 for (i = first_win; i < last_win; i++) {
482 t->rwbuf_stkptrs[i] = t->rwbuf_stkptrs[i+1];
483 memcpy(&t->reg_window[i], &t->reg_window[i+1],
484 sizeof(struct reg_window));
485 }
486}
487
488void synchronize_user_stack(void)
489{
490 struct thread_info *t = current_thread_info();
491 unsigned long window;
492
493 flush_user_windows();
494 if ((window = get_thread_wsaved()) != 0) {
495 window -= 1;
496 do {
497 struct reg_window *rwin = &t->reg_window[window];
498 int winsize = sizeof(struct reg_window);
499 unsigned long sp;
500
501 sp = t->rwbuf_stkptrs[window];
502
503 if (test_thread_64bit_stack(sp))
504 sp += STACK_BIAS;
505 else
506 winsize = sizeof(struct reg_window32);
507
508 if (!copy_to_user((char __user *)sp, rwin, winsize)) {
509 shift_window_buffer(window, get_thread_wsaved() - 1, t);
510 set_thread_wsaved(get_thread_wsaved() - 1);
511 }
512 } while (window--);
513 }
514}
515
516static void stack_unaligned(unsigned long sp)
517{
518 force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) sp);
519}
520
521static const char uwfault32[] = KERN_INFO \
522 "%s[%d]: bad register window fault: SP %08lx (orig_sp %08lx) TPC %08lx O7 %08lx\n";
523static const char uwfault64[] = KERN_INFO \
524 "%s[%d]: bad register window fault: SP %016lx (orig_sp %016lx) TPC %08lx O7 %016lx\n";
525
526void fault_in_user_windows(struct pt_regs *regs)
527{
528 struct thread_info *t = current_thread_info();
529 unsigned long window;
530
531 flush_user_windows();
532 window = get_thread_wsaved();
533
534 if (likely(window != 0)) {
535 window -= 1;
536 do {
537 struct reg_window *rwin = &t->reg_window[window];
538 int winsize = sizeof(struct reg_window);
539 unsigned long sp, orig_sp;
540
541 orig_sp = sp = t->rwbuf_stkptrs[window];
542
543 if (test_thread_64bit_stack(sp))
544 sp += STACK_BIAS;
545 else
546 winsize = sizeof(struct reg_window32);
547
548 if (unlikely(sp & 0x7UL))
549 stack_unaligned(sp);
550
551 if (unlikely(copy_to_user((char __user *)sp,
552 rwin, winsize))) {
553 if (show_unhandled_signals)
554 printk_ratelimited(is_compat_task() ?
555 uwfault32 : uwfault64,
556 current->comm, current->pid,
557 sp, orig_sp,
558 regs->tpc,
559 regs->u_regs[UREG_I7]);
560 goto barf;
561 }
562 } while (window--);
563 }
564 set_thread_wsaved(0);
565 return;
566
567barf:
568 set_thread_wsaved(window + 1);
569 force_sig(SIGSEGV);
570}
571
572
573
574
575
576
577int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg,
578 struct task_struct *p, unsigned long tls)
579{
580 struct thread_info *t = task_thread_info(p);
581 struct pt_regs *regs = current_pt_regs();
582 struct sparc_stackf *parent_sf;
583 unsigned long child_stack_sz;
584 char *child_trap_frame;
585
586
587 child_stack_sz = (STACKFRAME_SZ + TRACEREG_SZ);
588 child_trap_frame = (task_stack_page(p) +
589 (THREAD_SIZE - child_stack_sz));
590
591 t->new_child = 1;
592 t->ksp = ((unsigned long) child_trap_frame) - STACK_BIAS;
593 t->kregs = (struct pt_regs *) (child_trap_frame +
594 sizeof(struct sparc_stackf));
595 t->fpsaved[0] = 0;
596
597 if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) {
598 memset(child_trap_frame, 0, child_stack_sz);
599 __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] =
600 (current_pt_regs()->tstate + 1) & TSTATE_CWP;
601 t->current_ds = ASI_P;
602 t->kregs->u_regs[UREG_G1] = sp;
603 t->kregs->u_regs[UREG_G2] = arg;
604 return 0;
605 }
606
607 parent_sf = ((struct sparc_stackf *) regs) - 1;
608 memcpy(child_trap_frame, parent_sf, child_stack_sz);
609 if (t->flags & _TIF_32BIT) {
610 sp &= 0x00000000ffffffffUL;
611 regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL;
612 }
613 t->kregs->u_regs[UREG_FP] = sp;
614 __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] =
615 (regs->tstate + 1) & TSTATE_CWP;
616 t->current_ds = ASI_AIUS;
617 if (sp != regs->u_regs[UREG_FP]) {
618 unsigned long csp;
619
620 csp = clone_stackframe(sp, regs->u_regs[UREG_FP]);
621 if (!csp)
622 return -EFAULT;
623 t->kregs->u_regs[UREG_FP] = csp;
624 }
625 if (t->utraps)
626 t->utraps[0]++;
627
628
629 t->kregs->u_regs[UREG_I0] = current->pid;
630 t->kregs->u_regs[UREG_I1] = 1;
631
632
633 regs->u_regs[UREG_I1] = 0;
634
635 if (clone_flags & CLONE_SETTLS)
636 t->kregs->u_regs[UREG_G7] = tls;
637
638 return 0;
639}
640
641
642
643
644
645int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
646{
647 if (adi_capable()) {
648 register unsigned long tmp_mcdper;
649
650 __asm__ __volatile__(
651 ".word 0x83438000\n\t"
652 "mov %%g1, %0\n\t"
653 : "=r" (tmp_mcdper)
654 :
655 : "g1");
656 if (tmp_mcdper)
657 set_thread_flag(TIF_MCDPER);
658 else
659 clear_thread_flag(TIF_MCDPER);
660 }
661
662 *dst = *src;
663 return 0;
664}
665
666unsigned long get_wchan(struct task_struct *task)
667{
668 unsigned long pc, fp, bias = 0;
669 struct thread_info *tp;
670 struct reg_window *rw;
671 unsigned long ret = 0;
672 int count = 0;
673
674 if (!task || task == current || task_is_running(task))
675 goto out;
676
677 tp = task_thread_info(task);
678 bias = STACK_BIAS;
679 fp = task_thread_info(task)->ksp + bias;
680
681 do {
682 if (!kstack_valid(tp, fp))
683 break;
684 rw = (struct reg_window *) fp;
685 pc = rw->ins[7];
686 if (!in_sched_functions(pc)) {
687 ret = pc;
688 goto out;
689 }
690 fp = rw->ins[6] + bias;
691 } while (++count < 16);
692
693out:
694 return ret;
695}
696
