1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/cpu.h>
18#include <linux/errno.h>
19#include <linux/sched.h>
20#include <linux/sched/task.h>
21#include <linux/sched/task_stack.h>
22#include <linux/fs.h>
23#include <linux/kernel.h>
24#include <linux/mm.h>
25#include <linux/elfcore.h>
26#include <linux/smp.h>
27#include <linux/slab.h>
28#include <linux/user.h>
29#include <linux/interrupt.h>
30#include <linux/delay.h>
31#include <linux/export.h>
32#include <linux/ptrace.h>
33#include <linux/notifier.h>
34#include <linux/kprobes.h>
35#include <linux/kdebug.h>
36#include <linux/prctl.h>
37#include <linux/uaccess.h>
38#include <linux/io.h>
39#include <linux/ftrace.h>
40#include <linux/syscalls.h>
41
42#include <asm/pgtable.h>
43#include <asm/processor.h>
44#include <asm/fpu/internal.h>
45#include <asm/mmu_context.h>
46#include <asm/prctl.h>
47#include <asm/desc.h>
48#include <asm/proto.h>
49#include <asm/ia32.h>
50#include <asm/syscalls.h>
51#include <asm/debugreg.h>
52#include <asm/switch_to.h>
53#include <asm/xen/hypervisor.h>
54#include <asm/vdso.h>
55#include <asm/intel_rdt.h>
56#include <asm/unistd.h>
57#ifdef CONFIG_IA32_EMULATION
58
59#include <asm/unistd_32_ia32.h>
60#endif
61
62__visible DEFINE_PER_CPU(unsigned long, rsp_scratch);
63
64
65void __show_regs(struct pt_regs *regs, int all)
66{
67 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
68 unsigned long d0, d1, d2, d3, d6, d7;
69 unsigned int fsindex, gsindex;
70 unsigned int ds, cs, es;
71
72 printk(KERN_DEFAULT "RIP: %04lx:%pS\n", regs->cs & 0xffff,
73 (void *)regs->ip);
74 printk(KERN_DEFAULT "RSP: %04lx:%016lx EFLAGS: %08lx", regs->ss,
75 regs->sp, regs->flags);
76 if (regs->orig_ax != -1)
77 pr_cont(" ORIG_RAX: %016lx\n", regs->orig_ax);
78 else
79 pr_cont("\n");
80
81 printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
82 regs->ax, regs->bx, regs->cx);
83 printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
84 regs->dx, regs->si, regs->di);
85 printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
86 regs->bp, regs->r8, regs->r9);
87 printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
88 regs->r10, regs->r11, regs->r12);
89 printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
90 regs->r13, regs->r14, regs->r15);
91
92 asm("movl %%ds,%0" : "=r" (ds));
93 asm("movl %%cs,%0" : "=r" (cs));
94 asm("movl %%es,%0" : "=r" (es));
95 asm("movl %%fs,%0" : "=r" (fsindex));
96 asm("movl %%gs,%0" : "=r" (gsindex));
97
98 rdmsrl(MSR_FS_BASE, fs);
99 rdmsrl(MSR_GS_BASE, gs);
100 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
101
102 if (!all)
103 return;
104
105 cr0 = read_cr0();
106 cr2 = read_cr2();
107 cr3 = __read_cr3();
108 cr4 = __read_cr4();
109
110 printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
111 fs, fsindex, gs, gsindex, shadowgs);
112 printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
113 es, cr0);
114 printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
115 cr4);
116
117 get_debugreg(d0, 0);
118 get_debugreg(d1, 1);
119 get_debugreg(d2, 2);
120 get_debugreg(d3, 3);
121 get_debugreg(d6, 6);
122 get_debugreg(d7, 7);
123
124
125 if (!((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
126 (d6 == DR6_RESERVED) && (d7 == 0x400))) {
127 printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n",
128 d0, d1, d2);
129 printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n",
130 d3, d6, d7);
131 }
132
133 if (boot_cpu_has(X86_FEATURE_OSPKE))
134 printk(KERN_DEFAULT "PKRU: %08x\n", read_pkru());
135}
136
137void release_thread(struct task_struct *dead_task)
138{
139 if (dead_task->mm) {
140#ifdef CONFIG_MODIFY_LDT_SYSCALL
141 if (dead_task->mm->context.ldt) {
142 pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
143 dead_task->comm,
144 dead_task->mm->context.ldt->entries,
145 dead_task->mm->context.ldt->nr_entries);
146 BUG();
147 }
148#endif
149 }
150}
151
152int copy_thread_tls(unsigned long clone_flags, unsigned long sp,
153 unsigned long arg, struct task_struct *p, unsigned long tls)
154{
155 int err;
156 struct pt_regs *childregs;
157 struct fork_frame *fork_frame;
158 struct inactive_task_frame *frame;
159 struct task_struct *me = current;
160
161 p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
162 childregs = task_pt_regs(p);
163 fork_frame = container_of(childregs, struct fork_frame, regs);
164 frame = &fork_frame->frame;
165 frame->bp = 0;
166 frame->ret_addr = (unsigned long) ret_from_fork;
167 p->thread.sp = (unsigned long) fork_frame;
168 p->thread.io_bitmap_ptr = NULL;
169
170 savesegment(gs, p->thread.gsindex);
171 p->thread.gsbase = p->thread.gsindex ? 0 : me->thread.gsbase;
172 savesegment(fs, p->thread.fsindex);
173 p->thread.fsbase = p->thread.fsindex ? 0 : me->thread.fsbase;
174 savesegment(es, p->thread.es);
175 savesegment(ds, p->thread.ds);
176 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
177
178 if (unlikely(p->flags & PF_KTHREAD)) {
179
180 memset(childregs, 0, sizeof(struct pt_regs));
181 frame->bx = sp;
182 frame->r12 = arg;
183 return 0;
184 }
185 frame->bx = 0;
186 *childregs = *current_pt_regs();
187
188 childregs->ax = 0;
189 if (sp)
190 childregs->sp = sp;
191
192 err = -ENOMEM;
193 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
194 p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
195 IO_BITMAP_BYTES, GFP_KERNEL);
196 if (!p->thread.io_bitmap_ptr) {
197 p->thread.io_bitmap_max = 0;
198 return -ENOMEM;
199 }
200 set_tsk_thread_flag(p, TIF_IO_BITMAP);
201 }
202
203
204
205
206 if (clone_flags & CLONE_SETTLS) {
207#ifdef CONFIG_IA32_EMULATION
208 if (in_ia32_syscall())
209 err = do_set_thread_area(p, -1,
210 (struct user_desc __user *)tls, 0);
211 else
212#endif
213 err = do_arch_prctl_64(p, ARCH_SET_FS, tls);
214 if (err)
215 goto out;
216 }
217 err = 0;
218out:
219 if (err && p->thread.io_bitmap_ptr) {
220 kfree(p->thread.io_bitmap_ptr);
221 p->thread.io_bitmap_max = 0;
222 }
223
224 return err;
225}
226
227static void
228start_thread_common(struct pt_regs *regs, unsigned long new_ip,
229 unsigned long new_sp,
230 unsigned int _cs, unsigned int _ss, unsigned int _ds)
231{
232 loadsegment(fs, 0);
233 loadsegment(es, _ds);
234 loadsegment(ds, _ds);
235 load_gs_index(0);
236 regs->ip = new_ip;
237 regs->sp = new_sp;
238 regs->cs = _cs;
239 regs->ss = _ss;
240 regs->flags = X86_EFLAGS_IF;
241 force_iret();
242}
243
244void
245start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
246{
247 start_thread_common(regs, new_ip, new_sp,
248 __USER_CS, __USER_DS, 0);
249}
250
251#ifdef CONFIG_COMPAT
252void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp)
253{
254 start_thread_common(regs, new_ip, new_sp,
255 test_thread_flag(TIF_X32)
256 ? __USER_CS : __USER32_CS,
257 __USER_DS, __USER_DS);
258}
259#endif
260
261
262
263
264
265
266
267
268
269
270
271__visible __notrace_funcgraph struct task_struct *
272__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
273{
274 struct thread_struct *prev = &prev_p->thread;
275 struct thread_struct *next = &next_p->thread;
276 struct fpu *prev_fpu = &prev->fpu;
277 struct fpu *next_fpu = &next->fpu;
278 int cpu = smp_processor_id();
279 struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
280 unsigned prev_fsindex, prev_gsindex;
281
282 switch_fpu_prepare(prev_fpu, cpu);
283
284
285
286
287
288
289 savesegment(fs, prev_fsindex);
290 savesegment(gs, prev_gsindex);
291
292
293
294
295
296 load_TLS(next, cpu);
297
298
299
300
301
302
303
304
305 arch_end_context_switch(next_p);
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321 savesegment(es, prev->es);
322 if (unlikely(next->es | prev->es))
323 loadsegment(es, next->es);
324
325 savesegment(ds, prev->ds);
326 if (unlikely(next->ds | prev->ds))
327 loadsegment(ds, next->ds);
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348 if (next->fsindex) {
349
350 loadsegment(fs, next->fsindex);
351 } else {
352 if (next->fsbase) {
353
354 if (prev_fsindex)
355 loadsegment(fs, 0);
356 wrmsrl(MSR_FS_BASE, next->fsbase);
357 } else {
358
359 if (static_cpu_has_bug(X86_BUG_NULL_SEG)) {
360
361
362
363
364 loadsegment(fs, __USER_DS);
365 loadsegment(fs, 0);
366 } else {
367
368
369
370
371
372 if (prev->fsbase || prev_fsindex)
373 loadsegment(fs, 0);
374 }
375 }
376 }
377
378
379
380
381
382
383
384 if (prev_fsindex)
385 prev->fsbase = 0;
386 prev->fsindex = prev_fsindex;
387
388 if (next->gsindex) {
389
390 load_gs_index(next->gsindex);
391 } else {
392 if (next->gsbase) {
393
394 if (prev_gsindex)
395 load_gs_index(0);
396 wrmsrl(MSR_KERNEL_GS_BASE, next->gsbase);
397 } else {
398
399 if (static_cpu_has_bug(X86_BUG_NULL_SEG)) {
400
401
402
403
404
405
406
407
408 load_gs_index(__USER_DS);
409 load_gs_index(0);
410 } else {
411
412
413
414
415
416 if (prev->gsbase || prev_gsindex)
417 load_gs_index(0);
418 }
419 }
420 }
421
422
423
424
425
426
427
428 if (prev_gsindex)
429 prev->gsbase = 0;
430 prev->gsindex = prev_gsindex;
431
432 switch_fpu_finish(next_fpu, cpu);
433
434
435
436
437 this_cpu_write(current_task, next_p);
438
439
440 load_sp0(tss, next);
441
442
443
444
445 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
446 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
447 __switch_to_xtra(prev_p, next_p, tss);
448
449#ifdef CONFIG_XEN_PV
450
451
452
453
454
455 if (unlikely(static_cpu_has(X86_FEATURE_XENPV) &&
456 prev->iopl != next->iopl))
457 xen_set_iopl_mask(next->iopl);
458#endif
459
460 if (static_cpu_has_bug(X86_BUG_SYSRET_SS_ATTRS)) {
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482 unsigned short ss_sel;
483 savesegment(ss, ss_sel);
484 if (ss_sel != __KERNEL_DS)
485 loadsegment(ss, __KERNEL_DS);
486 }
487
488
489 intel_rdt_sched_in();
490
491 return prev_p;
492}
493
494void set_personality_64bit(void)
495{
496
497
498
499 clear_thread_flag(TIF_IA32);
500 clear_thread_flag(TIF_ADDR32);
501 clear_thread_flag(TIF_X32);
502
503 task_pt_regs(current)->orig_ax = __NR_execve;
504
505
506 if (current->mm)
507 current->mm->context.ia32_compat = 0;
508
509
510
511
512
513 current->personality &= ~READ_IMPLIES_EXEC;
514}
515
516static void __set_personality_x32(void)
517{
518#ifdef CONFIG_X86_X32
519 clear_thread_flag(TIF_IA32);
520 set_thread_flag(TIF_X32);
521 if (current->mm)
522 current->mm->context.ia32_compat = TIF_X32;
523 current->personality &= ~READ_IMPLIES_EXEC;
524
525
526
527
528
529
530
531
532 task_pt_regs(current)->orig_ax = __NR_x32_execve | __X32_SYSCALL_BIT;
533 current->thread.status &= ~TS_COMPAT;
534#endif
535}
536
537static void __set_personality_ia32(void)
538{
539#ifdef CONFIG_IA32_EMULATION
540 set_thread_flag(TIF_IA32);
541 clear_thread_flag(TIF_X32);
542 if (current->mm)
543 current->mm->context.ia32_compat = TIF_IA32;
544 current->personality |= force_personality32;
545
546 task_pt_regs(current)->orig_ax = __NR_ia32_execve;
547 current->thread.status |= TS_COMPAT;
548#endif
549}
550
551void set_personality_ia32(bool x32)
552{
553
554 set_thread_flag(TIF_ADDR32);
555
556 if (x32)
557 __set_personality_x32();
558 else
559 __set_personality_ia32();
560}
561EXPORT_SYMBOL_GPL(set_personality_ia32);
562
563#ifdef CONFIG_CHECKPOINT_RESTORE
564static long prctl_map_vdso(const struct vdso_image *image, unsigned long addr)
565{
566 int ret;
567
568 ret = map_vdso_once(image, addr);
569 if (ret)
570 return ret;
571
572 return (long)image->size;
573}
574#endif
575
576long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
577{
578 int ret = 0;
579 int doit = task == current;
580 int cpu;
581
582 switch (option) {
583 case ARCH_SET_GS:
584 if (arg2 >= TASK_SIZE_MAX)
585 return -EPERM;
586 cpu = get_cpu();
587 task->thread.gsindex = 0;
588 task->thread.gsbase = arg2;
589 if (doit) {
590 load_gs_index(0);
591 ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, arg2);
592 }
593 put_cpu();
594 break;
595 case ARCH_SET_FS:
596
597
598 if (arg2 >= TASK_SIZE_MAX)
599 return -EPERM;
600 cpu = get_cpu();
601 task->thread.fsindex = 0;
602 task->thread.fsbase = arg2;
603 if (doit) {
604
605 loadsegment(fs, 0);
606 ret = wrmsrl_safe(MSR_FS_BASE, arg2);
607 }
608 put_cpu();
609 break;
610 case ARCH_GET_FS: {
611 unsigned long base;
612
613 if (doit)
614 rdmsrl(MSR_FS_BASE, base);
615 else
616 base = task->thread.fsbase;
617 ret = put_user(base, (unsigned long __user *)arg2);
618 break;
619 }
620 case ARCH_GET_GS: {
621 unsigned long base;
622
623 if (doit)
624 rdmsrl(MSR_KERNEL_GS_BASE, base);
625 else
626 base = task->thread.gsbase;
627 ret = put_user(base, (unsigned long __user *)arg2);
628 break;
629 }
630
631#ifdef CONFIG_CHECKPOINT_RESTORE
632# ifdef CONFIG_X86_X32_ABI
633 case ARCH_MAP_VDSO_X32:
634 return prctl_map_vdso(&vdso_image_x32, arg2);
635# endif
636# if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
637 case ARCH_MAP_VDSO_32:
638 return prctl_map_vdso(&vdso_image_32, arg2);
639# endif
640 case ARCH_MAP_VDSO_64:
641 return prctl_map_vdso(&vdso_image_64, arg2);
642#endif
643
644 default:
645 ret = -EINVAL;
646 break;
647 }
648
649 return ret;
650}
651
652SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
653{
654 long ret;
655
656 ret = do_arch_prctl_64(current, option, arg2);
657 if (ret == -EINVAL)
658 ret = do_arch_prctl_common(current, option, arg2);
659
660 return ret;
661}
662
663#ifdef CONFIG_IA32_EMULATION
664COMPAT_SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
665{
666 return do_arch_prctl_common(current, option, arg2);
667}
668#endif
669
670unsigned long KSTK_ESP(struct task_struct *task)
671{
672 return task_pt_regs(task)->sp;
673}
674