1
2
3
4
5
6
7
8
9
10
11
12#include <linux/errno.h>
13#include <linux/sched.h>
14#include <linux/sched/debug.h>
15#include <linux/sched/task.h>
16#include <linux/sched/task_stack.h>
17#include <linux/tick.h>
18#include <linux/kernel.h>
19#include <linux/mm.h>
20#include <linux/stddef.h>
21#include <linux/unistd.h>
22#include <linux/export.h>
23#include <linux/ptrace.h>
24#include <linux/mman.h>
25#include <linux/personality.h>
26#include <linux/sys.h>
27#include <linux/init.h>
28#include <linux/completion.h>
29#include <linux/kallsyms.h>
30#include <linux/random.h>
31#include <linux/prctl.h>
32#include <linux/nmi.h>
33
34#include <asm/asm.h>
35#include <asm/bootinfo.h>
36#include <asm/cpu.h>
37#include <asm/dsemul.h>
38#include <asm/dsp.h>
39#include <asm/fpu.h>
40#include <asm/irq.h>
41#include <asm/msa.h>
42#include <asm/pgtable.h>
43#include <asm/mipsregs.h>
44#include <asm/processor.h>
45#include <asm/reg.h>
46#include <linux/uaccess.h>
47#include <asm/io.h>
48#include <asm/elf.h>
49#include <asm/isadep.h>
50#include <asm/inst.h>
51#include <asm/stacktrace.h>
52#include <asm/irq_regs.h>
53
54#ifdef CONFIG_HOTPLUG_CPU
55void arch_cpu_idle_dead(void)
56{
57 play_dead();
58}
59#endif
60
61asmlinkage void ret_from_fork(void);
62asmlinkage void ret_from_kernel_thread(void);
63
64void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp)
65{
66 unsigned long status;
67
68
69 status = regs->cp0_status & ~(ST0_CU0|ST0_CU1|ST0_FR|KU_MASK);
70 status |= KU_USER;
71 regs->cp0_status = status;
72 lose_fpu(0);
73 clear_thread_flag(TIF_MSA_CTX_LIVE);
74 clear_used_math();
75 atomic_set(¤t->thread.bd_emu_frame, BD_EMUFRAME_NONE);
76 init_dsp();
77 regs->cp0_epc = pc;
78 regs->regs[29] = sp;
79}
80
81void exit_thread(struct task_struct *tsk)
82{
83
84
85
86
87 if (!(current->flags & PF_KTHREAD))
88 dsemul_thread_cleanup(tsk);
89}
90
91int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
92{
93
94
95
96
97
98
99
100 preempt_disable();
101
102 if (is_msa_enabled())
103 save_msa(current);
104 else if (is_fpu_owner())
105 _save_fp(current);
106
107 save_dsp(current);
108
109 preempt_enable();
110
111 *dst = *src;
112 return 0;
113}
114
115
116
117
118int copy_thread_tls(unsigned long clone_flags, unsigned long usp,
119 unsigned long kthread_arg, struct task_struct *p, unsigned long tls)
120{
121 struct thread_info *ti = task_thread_info(p);
122 struct pt_regs *childregs, *regs = current_pt_regs();
123 unsigned long childksp;
124
125 childksp = (unsigned long)task_stack_page(p) + THREAD_SIZE - 32;
126
127
128 childregs = (struct pt_regs *) childksp - 1;
129
130 childksp = (unsigned long) childregs;
131 p->thread.cp0_status = read_c0_status() & ~(ST0_CU2|ST0_CU1);
132 if (unlikely(p->flags & PF_KTHREAD)) {
133
134 unsigned long status = p->thread.cp0_status;
135 memset(childregs, 0, sizeof(struct pt_regs));
136 ti->addr_limit = KERNEL_DS;
137 p->thread.reg16 = usp;
138 p->thread.reg17 = kthread_arg;
139 p->thread.reg29 = childksp;
140 p->thread.reg31 = (unsigned long) ret_from_kernel_thread;
141#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
142 status = (status & ~(ST0_KUP | ST0_IEP | ST0_IEC)) |
143 ((status & (ST0_KUC | ST0_IEC)) << 2);
144#else
145 status |= ST0_EXL;
146#endif
147 childregs->cp0_status = status;
148 return 0;
149 }
150
151
152 *childregs = *regs;
153 childregs->regs[7] = 0;
154 childregs->regs[2] = 0;
155 if (usp)
156 childregs->regs[29] = usp;
157 ti->addr_limit = USER_DS;
158
159 p->thread.reg29 = (unsigned long) childregs;
160 p->thread.reg31 = (unsigned long) ret_from_fork;
161
162
163
164
165
166 childregs->cp0_status &= ~(ST0_CU2|ST0_CU1);
167
168 clear_tsk_thread_flag(p, TIF_USEDFPU);
169 clear_tsk_thread_flag(p, TIF_USEDMSA);
170 clear_tsk_thread_flag(p, TIF_MSA_CTX_LIVE);
171
172#ifdef CONFIG_MIPS_MT_FPAFF
173 clear_tsk_thread_flag(p, TIF_FPUBOUND);
174#endif
175
176 atomic_set(&p->thread.bd_emu_frame, BD_EMUFRAME_NONE);
177
178 if (clone_flags & CLONE_SETTLS)
179 ti->tp_value = tls;
180
181 return 0;
182}
183
184#ifdef CONFIG_STACKPROTECTOR
185#include <linux/stackprotector.h>
186unsigned long __stack_chk_guard __read_mostly;
187EXPORT_SYMBOL(__stack_chk_guard);
188#endif
189
190struct mips_frame_info {
191 void *func;
192 unsigned long func_size;
193 int frame_size;
194 int pc_offset;
195};
196
197#define J_TARGET(pc,target) \
198 (((unsigned long)(pc) & 0xf0000000) | ((target) << 2))
199
200static inline int is_ra_save_ins(union mips_instruction *ip, int *poff)
201{
202#ifdef CONFIG_CPU_MICROMIPS
203
204
205
206
207
208
209
210
211
212 if (mm_insn_16bit(ip->word >> 16)) {
213 switch (ip->mm16_r5_format.opcode) {
214 case mm_swsp16_op:
215 if (ip->mm16_r5_format.rt != 31)
216 return 0;
217
218 *poff = ip->mm16_r5_format.imm;
219 *poff = (*poff << 2) / sizeof(ulong);
220 return 1;
221
222 case mm_pool16c_op:
223 switch (ip->mm16_m_format.func) {
224 case mm_swm16_op:
225 *poff = ip->mm16_m_format.imm;
226 *poff += 1 + ip->mm16_m_format.rlist;
227 *poff = (*poff << 2) / sizeof(ulong);
228 return 1;
229
230 default:
231 return 0;
232 }
233
234 default:
235 return 0;
236 }
237 }
238
239 switch (ip->i_format.opcode) {
240 case mm_sw32_op:
241 if (ip->i_format.rs != 29)
242 return 0;
243 if (ip->i_format.rt != 31)
244 return 0;
245
246 *poff = ip->i_format.simmediate / sizeof(ulong);
247 return 1;
248
249 case mm_pool32b_op:
250 switch (ip->mm_m_format.func) {
251 case mm_swm32_func:
252 if (ip->mm_m_format.rd < 0x10)
253 return 0;
254 if (ip->mm_m_format.base != 29)
255 return 0;
256
257 *poff = ip->mm_m_format.simmediate;
258 *poff += (ip->mm_m_format.rd & 0xf) * sizeof(u32);
259 *poff /= sizeof(ulong);
260 return 1;
261 default:
262 return 0;
263 }
264
265 default:
266 return 0;
267 }
268#else
269
270 if ((ip->i_format.opcode == sw_op || ip->i_format.opcode == sd_op) &&
271 ip->i_format.rs == 29 && ip->i_format.rt == 31) {
272 *poff = ip->i_format.simmediate / sizeof(ulong);
273 return 1;
274 }
275
276 return 0;
277#endif
278}
279
280static inline int is_jump_ins(union mips_instruction *ip)
281{
282#ifdef CONFIG_CPU_MICROMIPS
283
284
285
286
287
288
289
290
291 if (mm_insn_16bit(ip->word >> 16)) {
292 if ((ip->mm16_r5_format.opcode == mm_pool16c_op &&
293 (ip->mm16_r5_format.rt & mm_jr16_op) == mm_jr16_op))
294 return 1;
295 return 0;
296 }
297
298 if (ip->j_format.opcode == mm_j32_op)
299 return 1;
300 if (ip->j_format.opcode == mm_jal32_op)
301 return 1;
302 if (ip->r_format.opcode != mm_pool32a_op ||
303 ip->r_format.func != mm_pool32axf_op)
304 return 0;
305 return ((ip->u_format.uimmediate >> 6) & mm_jalr_op) == mm_jalr_op;
306#else
307 if (ip->j_format.opcode == j_op)
308 return 1;
309 if (ip->j_format.opcode == jal_op)
310 return 1;
311 if (ip->r_format.opcode != spec_op)
312 return 0;
313 return ip->r_format.func == jalr_op || ip->r_format.func == jr_op;
314#endif
315}
316
317static inline int is_sp_move_ins(union mips_instruction *ip, int *frame_size)
318{
319#ifdef CONFIG_CPU_MICROMIPS
320 unsigned short tmp;
321
322
323
324
325
326
327
328
329
330 if (mm_insn_16bit(ip->word >> 16)) {
331 if (ip->mm16_r3_format.opcode == mm_pool16d_op &&
332 ip->mm16_r3_format.simmediate & mm_addiusp_func) {
333 tmp = ip->mm_b0_format.simmediate >> 1;
334 tmp = ((tmp & 0x1ff) ^ 0x100) - 0x100;
335 if ((tmp + 2) < 4)
336 tmp ^= 0x100;
337 *frame_size = -(signed short)(tmp << 2);
338 return 1;
339 }
340 if (ip->mm16_r5_format.opcode == mm_pool16d_op &&
341 ip->mm16_r5_format.rt == 29) {
342 tmp = ip->mm16_r5_format.imm >> 1;
343 *frame_size = -(signed short)(tmp & 0xf);
344 return 1;
345 }
346 return 0;
347 }
348
349 if (ip->mm_i_format.opcode == mm_addiu32_op &&
350 ip->mm_i_format.rt == 29 && ip->mm_i_format.rs == 29) {
351 *frame_size = -ip->i_format.simmediate;
352 return 1;
353 }
354#else
355
356 if (ip->i_format.rs != 29 || ip->i_format.rt != 29)
357 return 0;
358
359 if (ip->i_format.opcode == addiu_op ||
360 ip->i_format.opcode == daddiu_op) {
361 *frame_size = -ip->i_format.simmediate;
362 return 1;
363 }
364#endif
365 return 0;
366}
367
368static int get_frame_info(struct mips_frame_info *info)
369{
370 bool is_mmips = IS_ENABLED(CONFIG_CPU_MICROMIPS);
371 union mips_instruction insn, *ip, *ip_end;
372 const unsigned int max_insns = 128;
373 unsigned int last_insn_size = 0;
374 unsigned int i;
375 bool saw_jump = false;
376
377 info->pc_offset = -1;
378 info->frame_size = 0;
379
380 ip = (void *)msk_isa16_mode((ulong)info->func);
381 if (!ip)
382 goto err;
383
384 ip_end = (void *)ip + info->func_size;
385
386 for (i = 0; i < max_insns && ip < ip_end; i++) {
387 ip = (void *)ip + last_insn_size;
388 if (is_mmips && mm_insn_16bit(ip->halfword[0])) {
389 insn.word = ip->halfword[0] << 16;
390 last_insn_size = 2;
391 } else if (is_mmips) {
392 insn.word = ip->halfword[0] << 16 | ip->halfword[1];
393 last_insn_size = 4;
394 } else {
395 insn.word = ip->word;
396 last_insn_size = 4;
397 }
398
399 if (!info->frame_size) {
400 is_sp_move_ins(&insn, &info->frame_size);
401 continue;
402 } else if (!saw_jump && is_jump_ins(ip)) {
403
404
405
406
407
408
409
410
411
412
413
414
415
416 saw_jump = true;
417 continue;
418 }
419 if (info->pc_offset == -1 &&
420 is_ra_save_ins(&insn, &info->pc_offset))
421 break;
422 if (saw_jump)
423 break;
424 }
425 if (info->frame_size && info->pc_offset >= 0)
426 return 0;
427 if (info->pc_offset < 0)
428 return 1;
429
430err:
431 return -1;
432}
433
434static struct mips_frame_info schedule_mfi __read_mostly;
435
436#ifdef CONFIG_KALLSYMS
437static unsigned long get___schedule_addr(void)
438{
439 return kallsyms_lookup_name("__schedule");
440}
441#else
442static unsigned long get___schedule_addr(void)
443{
444 union mips_instruction *ip = (void *)schedule;
445 int max_insns = 8;
446 int i;
447
448 for (i = 0; i < max_insns; i++, ip++) {
449 if (ip->j_format.opcode == j_op)
450 return J_TARGET(ip, ip->j_format.target);
451 }
452 return 0;
453}
454#endif
455
456static int __init frame_info_init(void)
457{
458 unsigned long size = 0;
459#ifdef CONFIG_KALLSYMS
460 unsigned long ofs;
461#endif
462 unsigned long addr;
463
464 addr = get___schedule_addr();
465 if (!addr)
466 addr = (unsigned long)schedule;
467
468#ifdef CONFIG_KALLSYMS
469 kallsyms_lookup_size_offset(addr, &size, &ofs);
470#endif
471 schedule_mfi.func = (void *)addr;
472 schedule_mfi.func_size = size;
473
474 get_frame_info(&schedule_mfi);
475
476
477
478
479
480 if (schedule_mfi.pc_offset < 0)
481 printk("Can't analyze schedule() prologue at %p\n", schedule);
482
483 return 0;
484}
485
486arch_initcall(frame_info_init);
487
488
489
490
491static unsigned long thread_saved_pc(struct task_struct *tsk)
492{
493 struct thread_struct *t = &tsk->thread;
494
495
496 if (t->reg31 == (unsigned long) ret_from_fork)
497 return t->reg31;
498 if (schedule_mfi.pc_offset < 0)
499 return 0;
500 return ((unsigned long *)t->reg29)[schedule_mfi.pc_offset];
501}
502
503
504#ifdef CONFIG_KALLSYMS
505
506unsigned long notrace unwind_stack_by_address(unsigned long stack_page,
507 unsigned long *sp,
508 unsigned long pc,
509 unsigned long *ra)
510{
511 unsigned long low, high, irq_stack_high;
512 struct mips_frame_info info;
513 unsigned long size, ofs;
514 struct pt_regs *regs;
515 int leaf;
516
517 if (!stack_page)
518 return 0;
519
520
521
522
523
524 low = stack_page;
525 if (!preemptible() && on_irq_stack(raw_smp_processor_id(), *sp)) {
526 high = stack_page + IRQ_STACK_START;
527 irq_stack_high = high;
528 } else {
529 high = stack_page + THREAD_SIZE - 32;
530 irq_stack_high = 0;
531 }
532
533
534
535
536
537 if (unlikely(*sp == irq_stack_high)) {
538 unsigned long task_sp = *(unsigned long *)*sp;
539
540
541
542
543
544 if (!object_is_on_stack((void *)task_sp))
545 return 0;
546
547
548
549
550
551 regs = (struct pt_regs *)task_sp;
552 pc = regs->cp0_epc;
553 if (!user_mode(regs) && __kernel_text_address(pc)) {
554 *sp = regs->regs[29];
555 *ra = regs->regs[31];
556 return pc;
557 }
558 return 0;
559 }
560 if (!kallsyms_lookup_size_offset(pc, &size, &ofs))
561 return 0;
562
563
564
565 if (unlikely(ofs == 0)) {
566 pc = *ra;
567 *ra = 0;
568 return pc;
569 }
570
571 info.func = (void *)(pc - ofs);
572 info.func_size = ofs;
573 leaf = get_frame_info(&info);
574 if (leaf < 0)
575 return 0;
576
577 if (*sp < low || *sp + info.frame_size > high)
578 return 0;
579
580 if (leaf)
581
582
583
584
585
586
587 pc = pc != *ra ? *ra : 0;
588 else
589 pc = ((unsigned long *)(*sp))[info.pc_offset];
590
591 *sp += info.frame_size;
592 *ra = 0;
593 return __kernel_text_address(pc) ? pc : 0;
594}
595EXPORT_SYMBOL(unwind_stack_by_address);
596
597
598unsigned long unwind_stack(struct task_struct *task, unsigned long *sp,
599 unsigned long pc, unsigned long *ra)
600{
601 unsigned long stack_page = 0;
602 int cpu;
603
604 for_each_possible_cpu(cpu) {
605 if (on_irq_stack(cpu, *sp)) {
606 stack_page = (unsigned long)irq_stack[cpu];
607 break;
608 }
609 }
610
611 if (!stack_page)
612 stack_page = (unsigned long)task_stack_page(task);
613
614 return unwind_stack_by_address(stack_page, sp, pc, ra);
615}
616#endif
617
618
619
620
621unsigned long get_wchan(struct task_struct *task)
622{
623 unsigned long pc = 0;
624#ifdef CONFIG_KALLSYMS
625 unsigned long sp;
626 unsigned long ra = 0;
627#endif
628
629 if (!task || task == current || task->state == TASK_RUNNING)
630 goto out;
631 if (!task_stack_page(task))
632 goto out;
633
634 pc = thread_saved_pc(task);
635
636#ifdef CONFIG_KALLSYMS
637 sp = task->thread.reg29 + schedule_mfi.frame_size;
638
639 while (in_sched_functions(pc))
640 pc = unwind_stack(task, &sp, pc, &ra);
641#endif
642
643out:
644 return pc;
645}
646
647
648
649
650
651unsigned long arch_align_stack(unsigned long sp)
652{
653 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
654 sp -= get_random_int() & ~PAGE_MASK;
655
656 return sp & ALMASK;
657}
658
659static DEFINE_PER_CPU(call_single_data_t, backtrace_csd);
660static struct cpumask backtrace_csd_busy;
661
662static void handle_backtrace(void *info)
663{
664 nmi_cpu_backtrace(get_irq_regs());
665 cpumask_clear_cpu(smp_processor_id(), &backtrace_csd_busy);
666}
667
668static void raise_backtrace(cpumask_t *mask)
669{
670 call_single_data_t *csd;
671 int cpu;
672
673 for_each_cpu(cpu, mask) {
674
675
676
677
678
679
680 if (cpumask_test_and_set_cpu(cpu, &backtrace_csd_busy)) {
681 pr_warn("Unable to send backtrace IPI to CPU%u - perhaps it hung?\n",
682 cpu);
683 continue;
684 }
685
686 csd = &per_cpu(backtrace_csd, cpu);
687 csd->func = handle_backtrace;
688 smp_call_function_single_async(cpu, csd);
689 }
690}
691
692void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self)
693{
694 nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace);
695}
696
697int mips_get_process_fp_mode(struct task_struct *task)
698{
699 int value = 0;
700
701 if (!test_tsk_thread_flag(task, TIF_32BIT_FPREGS))
702 value |= PR_FP_MODE_FR;
703 if (test_tsk_thread_flag(task, TIF_HYBRID_FPREGS))
704 value |= PR_FP_MODE_FRE;
705
706 return value;
707}
708
709static void prepare_for_fp_mode_switch(void *info)
710{
711 struct mm_struct *mm = info;
712
713 if (current->mm == mm)
714 lose_fpu(1);
715}
716
717int mips_set_process_fp_mode(struct task_struct *task, unsigned int value)
718{
719 const unsigned int known_bits = PR_FP_MODE_FR | PR_FP_MODE_FRE;
720 struct task_struct *t;
721 int max_users;
722
723
724 if (value == mips_get_process_fp_mode(task))
725 return 0;
726
727
728 if (!IS_ENABLED(CONFIG_MIPS_O32_FP64_SUPPORT))
729 return -EOPNOTSUPP;
730
731
732 if (IS_ENABLED(CONFIG_64BIT) && !test_thread_flag(TIF_32BIT_REGS))
733 return -EOPNOTSUPP;
734
735
736 if (value & ~known_bits)
737 return -EOPNOTSUPP;
738
739
740 if ((value & (PR_FP_MODE_FR | PR_FP_MODE_FRE)) == PR_FP_MODE_FRE)
741 return -EOPNOTSUPP;
742
743
744 if ((value & PR_FP_MODE_FR) && raw_cpu_has_fpu &&
745 !(raw_current_cpu_data.fpu_id & MIPS_FPIR_F64))
746 return -EOPNOTSUPP;
747 if ((value & PR_FP_MODE_FRE) && raw_cpu_has_fpu && !cpu_has_fre)
748 return -EOPNOTSUPP;
749
750
751 if (!(value & PR_FP_MODE_FR) && raw_cpu_has_fpu && cpu_has_mips_r6)
752 return -EOPNOTSUPP;
753
754
755 preempt_disable();
756
757
758 if (task->signal == current->signal)
759 lose_fpu(1);
760
761
762 atomic_set(&task->mm->context.fp_mode_switching, 1);
763 smp_mb__after_atomic();
764
765
766
767
768
769
770 if (num_online_cpus() > 1) {
771
772 max_users = (task->mm == current->mm) ? 1 : 0;
773
774 if (atomic_read(¤t->mm->mm_users) > max_users)
775 smp_call_function(prepare_for_fp_mode_switch,
776 (void *)current->mm, 1);
777 }
778
779
780
781
782
783 for_each_thread(task, t) {
784
785 if (value & PR_FP_MODE_FR) {
786 clear_tsk_thread_flag(t, TIF_32BIT_FPREGS);
787 } else {
788 set_tsk_thread_flag(t, TIF_32BIT_FPREGS);
789 clear_tsk_thread_flag(t, TIF_MSA_CTX_LIVE);
790 }
791
792
793 if (value & PR_FP_MODE_FRE)
794 set_tsk_thread_flag(t, TIF_HYBRID_FPREGS);
795 else
796 clear_tsk_thread_flag(t, TIF_HYBRID_FPREGS);
797 }
798
799
800 atomic_set(&task->mm->context.fp_mode_switching, 0);
801 preempt_enable();
802
803 wake_up_var(&task->mm->context.fp_mode_switching);
804
805 return 0;
806}
807
808#if defined(CONFIG_32BIT) || defined(CONFIG_MIPS32_O32)
809void mips_dump_regs32(u32 *uregs, const struct pt_regs *regs)
810{
811 unsigned int i;
812
813 for (i = MIPS32_EF_R1; i <= MIPS32_EF_R31; i++) {
814
815 if (i == MIPS32_EF_R26 || i == MIPS32_EF_R27)
816 uregs[i] = 0;
817 else
818 uregs[i] = regs->regs[i - MIPS32_EF_R0];
819 }
820
821 uregs[MIPS32_EF_LO] = regs->lo;
822 uregs[MIPS32_EF_HI] = regs->hi;
823 uregs[MIPS32_EF_CP0_EPC] = regs->cp0_epc;
824 uregs[MIPS32_EF_CP0_BADVADDR] = regs->cp0_badvaddr;
825 uregs[MIPS32_EF_CP0_STATUS] = regs->cp0_status;
826 uregs[MIPS32_EF_CP0_CAUSE] = regs->cp0_cause;
827}
828#endif
829
830#ifdef CONFIG_64BIT
831void mips_dump_regs64(u64 *uregs, const struct pt_regs *regs)
832{
833 unsigned int i;
834
835 for (i = MIPS64_EF_R1; i <= MIPS64_EF_R31; i++) {
836
837 if (i == MIPS64_EF_R26 || i == MIPS64_EF_R27)
838 uregs[i] = 0;
839 else
840 uregs[i] = regs->regs[i - MIPS64_EF_R0];
841 }
842
843 uregs[MIPS64_EF_LO] = regs->lo;
844 uregs[MIPS64_EF_HI] = regs->hi;
845 uregs[MIPS64_EF_CP0_EPC] = regs->cp0_epc;
846 uregs[MIPS64_EF_CP0_BADVADDR] = regs->cp0_badvaddr;
847 uregs[MIPS64_EF_CP0_STATUS] = regs->cp0_status;
848 uregs[MIPS64_EF_CP0_CAUSE] = regs->cp0_cause;
849}
850#endif
851