1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/cpu.h>
18#include <linux/errno.h>
19#include <linux/sched.h>
20#include <linux/fs.h>
21#include <linux/kernel.h>
22#include <linux/mm.h>
23#include <linux/elfcore.h>
24#include <linux/smp.h>
25#include <linux/slab.h>
26#include <linux/user.h>
27#include <linux/interrupt.h>
28#include <linux/delay.h>
29#include <linux/module.h>
30#include <linux/ptrace.h>
31#include <linux/notifier.h>
32#include <linux/kprobes.h>
33#include <linux/kdebug.h>
34#include <linux/prctl.h>
35#include <linux/uaccess.h>
36#include <linux/io.h>
37#include <linux/ftrace.h>
38
39#include <asm/pgtable.h>
40#include <asm/processor.h>
41#include <asm/i387.h>
42#include <asm/fpu-internal.h>
43#include <asm/mmu_context.h>
44#include <asm/prctl.h>
45#include <asm/desc.h>
46#include <asm/proto.h>
47#include <asm/ia32.h>
48#include <asm/idle.h>
49#include <asm/syscalls.h>
50#include <asm/debugreg.h>
51#include <asm/switch_to.h>
52
53asmlinkage extern void ret_from_fork(void);
54
55DEFINE_PER_CPU(unsigned long, old_rsp);
56
57
58void __show_regs(struct pt_regs *regs, int all)
59{
60 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
61 unsigned long d0, d1, d2, d3, d6, d7;
62 unsigned int fsindex, gsindex;
63 unsigned int ds, cs, es;
64
65 printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
66 printk_address(regs->ip, 1);
67 printk(KERN_DEFAULT "RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss,
68 regs->sp, regs->flags);
69 printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
70 regs->ax, regs->bx, regs->cx);
71 printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
72 regs->dx, regs->si, regs->di);
73 printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
74 regs->bp, regs->r8, regs->r9);
75 printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
76 regs->r10, regs->r11, regs->r12);
77 printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
78 regs->r13, regs->r14, regs->r15);
79
80 asm("movl %%ds,%0" : "=r" (ds));
81 asm("movl %%cs,%0" : "=r" (cs));
82 asm("movl %%es,%0" : "=r" (es));
83 asm("movl %%fs,%0" : "=r" (fsindex));
84 asm("movl %%gs,%0" : "=r" (gsindex));
85
86 rdmsrl(MSR_FS_BASE, fs);
87 rdmsrl(MSR_GS_BASE, gs);
88 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
89
90 if (!all)
91 return;
92
93 cr0 = read_cr0();
94 cr2 = read_cr2();
95 cr3 = read_cr3();
96 cr4 = read_cr4();
97
98 printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
99 fs, fsindex, gs, gsindex, shadowgs);
100 printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
101 es, cr0);
102 printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
103 cr4);
104
105 get_debugreg(d0, 0);
106 get_debugreg(d1, 1);
107 get_debugreg(d2, 2);
108 printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
109 get_debugreg(d3, 3);
110 get_debugreg(d6, 6);
111 get_debugreg(d7, 7);
112 printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
113}
114
115void release_thread(struct task_struct *dead_task)
116{
117 if (dead_task->mm) {
118 if (dead_task->mm->context.size) {
119 pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
120 dead_task->comm,
121 dead_task->mm->context.ldt,
122 dead_task->mm->context.size);
123 BUG();
124 }
125 }
126}
127
128static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
129{
130 struct user_desc ud = {
131 .base_addr = addr,
132 .limit = 0xfffff,
133 .seg_32bit = 1,
134 .limit_in_pages = 1,
135 .useable = 1,
136 };
137 struct desc_struct *desc = t->thread.tls_array;
138 desc += tls;
139 fill_ldt(desc, &ud);
140}
141
142static inline u32 read_32bit_tls(struct task_struct *t, int tls)
143{
144 return get_desc_base(&t->thread.tls_array[tls]);
145}
146
147int copy_thread(unsigned long clone_flags, unsigned long sp,
148 unsigned long arg, struct task_struct *p)
149{
150 int err;
151 struct pt_regs *childregs;
152 struct task_struct *me = current;
153
154 p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
155 childregs = task_pt_regs(p);
156 p->thread.sp = (unsigned long) childregs;
157 p->thread.usersp = me->thread.usersp;
158 set_tsk_thread_flag(p, TIF_FORK);
159 p->fpu_counter = 0;
160 p->thread.io_bitmap_ptr = NULL;
161
162 savesegment(gs, p->thread.gsindex);
163 p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
164 savesegment(fs, p->thread.fsindex);
165 p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
166 savesegment(es, p->thread.es);
167 savesegment(ds, p->thread.ds);
168 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
169
170 if (unlikely(p->flags & PF_KTHREAD)) {
171
172 memset(childregs, 0, sizeof(struct pt_regs));
173 childregs->sp = (unsigned long)childregs;
174 childregs->ss = __KERNEL_DS;
175 childregs->bx = sp;
176 childregs->bp = arg;
177 childregs->orig_ax = -1;
178 childregs->cs = __KERNEL_CS | get_kernel_rpl();
179 childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_BIT1;
180 return 0;
181 }
182 *childregs = *current_pt_regs();
183
184 childregs->ax = 0;
185 if (sp)
186 childregs->sp = sp;
187
188 err = -ENOMEM;
189 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
190
191 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
192 p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
193 IO_BITMAP_BYTES, GFP_KERNEL);
194 if (!p->thread.io_bitmap_ptr) {
195 p->thread.io_bitmap_max = 0;
196 return -ENOMEM;
197 }
198 set_tsk_thread_flag(p, TIF_IO_BITMAP);
199 }
200
201
202
203
204 if (clone_flags & CLONE_SETTLS) {
205#ifdef CONFIG_IA32_EMULATION
206 if (test_thread_flag(TIF_IA32))
207 err = do_set_thread_area(p, -1,
208 (struct user_desc __user *)childregs->si, 0);
209 else
210#endif
211 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
212 if (err)
213 goto out;
214 }
215 err = 0;
216out:
217 if (err && p->thread.io_bitmap_ptr) {
218 kfree(p->thread.io_bitmap_ptr);
219 p->thread.io_bitmap_max = 0;
220 }
221
222 return err;
223}
224
225static void
226start_thread_common(struct pt_regs *regs, unsigned long new_ip,
227 unsigned long new_sp,
228 unsigned int _cs, unsigned int _ss, unsigned int _ds)
229{
230 loadsegment(fs, 0);
231 loadsegment(es, _ds);
232 loadsegment(ds, _ds);
233 load_gs_index(0);
234 current->thread.usersp = new_sp;
235 regs->ip = new_ip;
236 regs->sp = new_sp;
237 this_cpu_write(old_rsp, new_sp);
238 regs->cs = _cs;
239 regs->ss = _ss;
240 regs->flags = X86_EFLAGS_IF;
241}
242
243void
244start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
245{
246 start_thread_common(regs, new_ip, new_sp,
247 __USER_CS, __USER_DS, 0);
248}
249
250#ifdef CONFIG_IA32_EMULATION
251void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp)
252{
253 start_thread_common(regs, new_ip, new_sp,
254 test_thread_flag(TIF_X32)
255 ? __USER_CS : __USER32_CS,
256 __USER_DS, __USER_DS);
257}
258#endif
259
260
261
262
263
264
265
266
267
268
269
270__notrace_funcgraph struct task_struct *
271__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
272{
273 struct thread_struct *prev = &prev_p->thread;
274 struct thread_struct *next = &next_p->thread;
275 int cpu = smp_processor_id();
276 struct tss_struct *tss = &per_cpu(init_tss, cpu);
277 unsigned fsindex, gsindex;
278 fpu_switch_t fpu;
279
280 fpu = switch_fpu_prepare(prev_p, next_p, cpu);
281
282
283
284
285 load_sp0(tss, next);
286
287
288
289
290
291 savesegment(es, prev->es);
292 if (unlikely(next->es | prev->es))
293 loadsegment(es, next->es);
294
295 savesegment(ds, prev->ds);
296 if (unlikely(next->ds | prev->ds))
297 loadsegment(ds, next->ds);
298
299
300
301
302
303
304
305 savesegment(fs, fsindex);
306 savesegment(gs, gsindex);
307
308 load_TLS(next, cpu);
309
310
311
312
313
314
315
316
317 arch_end_context_switch(next_p);
318
319
320
321
322
323
324
325
326 if (unlikely(fsindex | next->fsindex | prev->fs)) {
327 loadsegment(fs, next->fsindex);
328
329
330
331
332
333 if (fsindex)
334 prev->fs = 0;
335 }
336
337 if (next->fs)
338 wrmsrl(MSR_FS_BASE, next->fs);
339 prev->fsindex = fsindex;
340
341 if (unlikely(gsindex | next->gsindex | prev->gs)) {
342 load_gs_index(next->gsindex);
343 if (gsindex)
344 prev->gs = 0;
345 }
346 if (next->gs)
347 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
348 prev->gsindex = gsindex;
349
350 switch_fpu_finish(next_p, fpu);
351
352
353
354
355 prev->usersp = this_cpu_read(old_rsp);
356 this_cpu_write(old_rsp, next->usersp);
357 this_cpu_write(current_task, next_p);
358
359 this_cpu_write(kernel_stack,
360 (unsigned long)task_stack_page(next_p) +
361 THREAD_SIZE - KERNEL_STACK_OFFSET);
362
363
364
365
366 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
367 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
368 __switch_to_xtra(prev_p, next_p, tss);
369
370 return prev_p;
371}
372
373void set_personality_64bit(void)
374{
375
376
377
378 clear_thread_flag(TIF_IA32);
379 clear_thread_flag(TIF_ADDR32);
380 clear_thread_flag(TIF_X32);
381
382
383 if (current->mm)
384 current->mm->context.ia32_compat = 0;
385
386
387
388
389
390 current->personality &= ~READ_IMPLIES_EXEC;
391}
392
393void set_personality_ia32(bool x32)
394{
395
396
397
398 set_thread_flag(TIF_ADDR32);
399
400
401 if (current->mm)
402 current->mm->context.ia32_compat = 1;
403
404 if (x32) {
405 clear_thread_flag(TIF_IA32);
406 set_thread_flag(TIF_X32);
407 current->personality &= ~READ_IMPLIES_EXEC;
408
409
410 current_thread_info()->status &= ~TS_COMPAT;
411 } else {
412 set_thread_flag(TIF_IA32);
413 clear_thread_flag(TIF_X32);
414 current->personality |= force_personality32;
415
416 current_thread_info()->status |= TS_COMPAT;
417 }
418}
419EXPORT_SYMBOL_GPL(set_personality_ia32);
420
421unsigned long get_wchan(struct task_struct *p)
422{
423 unsigned long stack;
424 u64 fp, ip;
425 int count = 0;
426
427 if (!p || p == current || p->state == TASK_RUNNING)
428 return 0;
429 stack = (unsigned long)task_stack_page(p);
430 if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
431 return 0;
432 fp = *(u64 *)(p->thread.sp);
433 do {
434 if (fp < (unsigned long)stack ||
435 fp >= (unsigned long)stack+THREAD_SIZE)
436 return 0;
437 ip = *(u64 *)(fp+8);
438 if (!in_sched_functions(ip))
439 return ip;
440 fp = *(u64 *)fp;
441 } while (count++ < 16);
442 return 0;
443}
444
445long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
446{
447 int ret = 0;
448 int doit = task == current;
449 int cpu;
450
451 switch (code) {
452 case ARCH_SET_GS:
453 if (addr >= TASK_SIZE_OF(task))
454 return -EPERM;
455 cpu = get_cpu();
456
457
458 if (addr <= 0xffffffff) {
459 set_32bit_tls(task, GS_TLS, addr);
460 if (doit) {
461 load_TLS(&task->thread, cpu);
462 load_gs_index(GS_TLS_SEL);
463 }
464 task->thread.gsindex = GS_TLS_SEL;
465 task->thread.gs = 0;
466 } else {
467 task->thread.gsindex = 0;
468 task->thread.gs = addr;
469 if (doit) {
470 load_gs_index(0);
471 ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr);
472 }
473 }
474 put_cpu();
475 break;
476 case ARCH_SET_FS:
477
478
479 if (addr >= TASK_SIZE_OF(task))
480 return -EPERM;
481 cpu = get_cpu();
482
483
484 if (addr <= 0xffffffff) {
485 set_32bit_tls(task, FS_TLS, addr);
486 if (doit) {
487 load_TLS(&task->thread, cpu);
488 loadsegment(fs, FS_TLS_SEL);
489 }
490 task->thread.fsindex = FS_TLS_SEL;
491 task->thread.fs = 0;
492 } else {
493 task->thread.fsindex = 0;
494 task->thread.fs = addr;
495 if (doit) {
496
497
498 loadsegment(fs, 0);
499 ret = wrmsrl_safe(MSR_FS_BASE, addr);
500 }
501 }
502 put_cpu();
503 break;
504 case ARCH_GET_FS: {
505 unsigned long base;
506 if (task->thread.fsindex == FS_TLS_SEL)
507 base = read_32bit_tls(task, FS_TLS);
508 else if (doit)
509 rdmsrl(MSR_FS_BASE, base);
510 else
511 base = task->thread.fs;
512 ret = put_user(base, (unsigned long __user *)addr);
513 break;
514 }
515 case ARCH_GET_GS: {
516 unsigned long base;
517 unsigned gsindex;
518 if (task->thread.gsindex == GS_TLS_SEL)
519 base = read_32bit_tls(task, GS_TLS);
520 else if (doit) {
521 savesegment(gs, gsindex);
522 if (gsindex)
523 rdmsrl(MSR_KERNEL_GS_BASE, base);
524 else
525 base = task->thread.gs;
526 } else
527 base = task->thread.gs;
528 ret = put_user(base, (unsigned long __user *)addr);
529 break;
530 }
531
532 default:
533 ret = -EINVAL;
534 break;
535 }
536
537 return ret;
538}
539
540long sys_arch_prctl(int code, unsigned long addr)
541{
542 return do_arch_prctl(current, code, addr);
543}
544
545unsigned long KSTK_ESP(struct task_struct *task)
546{
547 return (test_tsk_thread_flag(task, TIF_IA32)) ?
548 (task_pt_regs(task)->sp) : ((task)->thread.usersp);
549}
550