1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/anon_inodes.h>
16#include <linux/slab.h>
17#include <linux/sched/autogroup.h>
18#include <linux/sched/mm.h>
19#include <linux/sched/coredump.h>
20#include <linux/sched/user.h>
21#include <linux/sched/numa_balancing.h>
22#include <linux/sched/stat.h>
23#include <linux/sched/task.h>
24#include <linux/sched/task_stack.h>
25#include <linux/sched/cputime.h>
26#include <linux/seq_file.h>
27#include <linux/rtmutex.h>
28#include <linux/init.h>
29#include <linux/unistd.h>
30#include <linux/module.h>
31#include <linux/vmalloc.h>
32#include <linux/completion.h>
33#include <linux/personality.h>
34#include <linux/mempolicy.h>
35#include <linux/sem.h>
36#include <linux/file.h>
37#include <linux/fdtable.h>
38#include <linux/iocontext.h>
39#include <linux/key.h>
40#include <linux/binfmts.h>
41#include <linux/mman.h>
42#include <linux/mmu_notifier.h>
43#include <linux/fs.h>
44#include <linux/mm.h>
45#include <linux/vmacache.h>
46#include <linux/nsproxy.h>
47#include <linux/capability.h>
48#include <linux/cpu.h>
49#include <linux/cgroup.h>
50#include <linux/security.h>
51#include <linux/hugetlb.h>
52#include <linux/seccomp.h>
53#include <linux/swap.h>
54#include <linux/syscalls.h>
55#include <linux/jiffies.h>
56#include <linux/futex.h>
57#include <linux/compat.h>
58#include <linux/kthread.h>
59#include <linux/task_io_accounting_ops.h>
60#include <linux/rcupdate.h>
61#include <linux/ptrace.h>
62#include <linux/mount.h>
63#include <linux/audit.h>
64#include <linux/memcontrol.h>
65#include <linux/ftrace.h>
66#include <linux/proc_fs.h>
67#include <linux/profile.h>
68#include <linux/rmap.h>
69#include <linux/ksm.h>
70#include <linux/acct.h>
71#include <linux/userfaultfd_k.h>
72#include <linux/tsacct_kern.h>
73#include <linux/cn_proc.h>
74#include <linux/freezer.h>
75#include <linux/delayacct.h>
76#include <linux/taskstats_kern.h>
77#include <linux/random.h>
78#include <linux/tty.h>
79#include <linux/blkdev.h>
80#include <linux/fs_struct.h>
81#include <linux/magic.h>
82#include <linux/perf_event.h>
83#include <linux/posix-timers.h>
84#include <linux/user-return-notifier.h>
85#include <linux/oom.h>
86#include <linux/khugepaged.h>
87#include <linux/signalfd.h>
88#include <linux/uprobes.h>
89#include <linux/aio.h>
90#include <linux/compiler.h>
91#include <linux/sysctl.h>
92#include <linux/kcov.h>
93#include <linux/livepatch.h>
94#include <linux/thread_info.h>
95#include <linux/stackleak.h>
96#include <linux/kasan.h>
97#include <linux/scs.h>
98
99#include <asm/pgalloc.h>
100#include <linux/uaccess.h>
101#include <asm/mmu_context.h>
102#include <asm/cacheflush.h>
103#include <asm/tlbflush.h>
104
105#include <trace/events/sched.h>
106
107#define CREATE_TRACE_POINTS
108#include <trace/events/task.h>
109
110
111
112
113#define MIN_THREADS 20
114
115
116
117
118#define MAX_THREADS FUTEX_TID_MASK
119
120
121
122
123unsigned long total_forks;
124int nr_threads;
125
126static int max_threads;
127
128#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)
129
130static const char * const resident_page_types[] = {
131 NAMED_ARRAY_INDEX(MM_FILEPAGES),
132 NAMED_ARRAY_INDEX(MM_ANONPAGES),
133 NAMED_ARRAY_INDEX(MM_SWAPENTS),
134 NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
135};
136
137DEFINE_PER_CPU(unsigned long, process_counts) = 0;
138
139__cacheline_aligned DEFINE_RWLOCK(tasklist_lock);
140
141#ifdef CONFIG_PROVE_RCU
142int lockdep_tasklist_lock_is_held(void)
143{
144 return lockdep_is_held(&tasklist_lock);
145}
146EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
147#endif
148
149int nr_processes(void)
150{
151 int cpu;
152 int total = 0;
153
154 for_each_possible_cpu(cpu)
155 total += per_cpu(process_counts, cpu);
156
157 return total;
158}
159
160void __weak arch_release_task_struct(struct task_struct *tsk)
161{
162}
163
164#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
165static struct kmem_cache *task_struct_cachep;
166
167static inline struct task_struct *alloc_task_struct_node(int node)
168{
169 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
170}
171
172static inline void free_task_struct(struct task_struct *tsk)
173{
174 kmem_cache_free(task_struct_cachep, tsk);
175}
176#endif
177
178#ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
179
180
181
182
183
184# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
185
186#ifdef CONFIG_VMAP_STACK
187
188
189
190
191#define NR_CACHED_STACKS 2
192static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
193
194static int free_vm_stack_cache(unsigned int cpu)
195{
196 struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
197 int i;
198
199 for (i = 0; i < NR_CACHED_STACKS; i++) {
200 struct vm_struct *vm_stack = cached_vm_stacks[i];
201
202 if (!vm_stack)
203 continue;
204
205 vfree(vm_stack->addr);
206 cached_vm_stacks[i] = NULL;
207 }
208
209 return 0;
210}
211#endif
212
213static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
214{
215#ifdef CONFIG_VMAP_STACK
216 void *stack;
217 int i;
218
219 for (i = 0; i < NR_CACHED_STACKS; i++) {
220 struct vm_struct *s;
221
222 s = this_cpu_xchg(cached_stacks[i], NULL);
223
224 if (!s)
225 continue;
226
227
228 kasan_unpoison_shadow(s->addr, THREAD_SIZE);
229
230
231 memset(s->addr, 0, THREAD_SIZE);
232
233 tsk->stack_vm_area = s;
234 tsk->stack = s->addr;
235 return s->addr;
236 }
237
238
239
240
241
242
243 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
244 VMALLOC_START, VMALLOC_END,
245 THREADINFO_GFP & ~__GFP_ACCOUNT,
246 PAGE_KERNEL,
247 0, node, __builtin_return_address(0));
248
249
250
251
252
253
254 if (stack) {
255 tsk->stack_vm_area = find_vm_area(stack);
256 tsk->stack = stack;
257 }
258 return stack;
259#else
260 struct page *page = alloc_pages_node(node, THREADINFO_GFP,
261 THREAD_SIZE_ORDER);
262
263 if (likely(page)) {
264 tsk->stack = kasan_reset_tag(page_address(page));
265 return tsk->stack;
266 }
267 return NULL;
268#endif
269}
270
271static inline void free_thread_stack(struct task_struct *tsk)
272{
273#ifdef CONFIG_VMAP_STACK
274 struct vm_struct *vm = task_stack_vm_area(tsk);
275
276 if (vm) {
277 int i;
278
279 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
280 memcg_kmem_uncharge_page(vm->pages[i], 0);
281
282 for (i = 0; i < NR_CACHED_STACKS; i++) {
283 if (this_cpu_cmpxchg(cached_stacks[i],
284 NULL, tsk->stack_vm_area) != NULL)
285 continue;
286
287 return;
288 }
289
290 vfree_atomic(tsk->stack);
291 return;
292 }
293#endif
294
295 __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
296}
297# else
298static struct kmem_cache *thread_stack_cache;
299
300static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
301 int node)
302{
303 unsigned long *stack;
304 stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
305 stack = kasan_reset_tag(stack);
306 tsk->stack = stack;
307 return stack;
308}
309
310static void free_thread_stack(struct task_struct *tsk)
311{
312 kmem_cache_free(thread_stack_cache, tsk->stack);
313}
314
315void thread_stack_cache_init(void)
316{
317 thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
318 THREAD_SIZE, THREAD_SIZE, 0, 0,
319 THREAD_SIZE, NULL);
320 BUG_ON(thread_stack_cache == NULL);
321}
322# endif
323#endif
324
325
326static struct kmem_cache *signal_cachep;
327
328
329struct kmem_cache *sighand_cachep;
330
331
332struct kmem_cache *files_cachep;
333
334
335struct kmem_cache *fs_cachep;
336
337
338static struct kmem_cache *vm_area_cachep;
339
340
341static struct kmem_cache *mm_cachep;
342
343struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
344{
345 struct vm_area_struct *vma;
346
347 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
348 if (vma)
349 vma_init(vma, mm);
350 return vma;
351}
352
353struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
354{
355 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
356
357 if (new) {
358 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
359 ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
360
361
362
363
364 *new = data_race(*orig);
365 INIT_LIST_HEAD(&new->anon_vma_chain);
366 new->vm_next = new->vm_prev = NULL;
367 }
368 return new;
369}
370
371void vm_area_free(struct vm_area_struct *vma)
372{
373 kmem_cache_free(vm_area_cachep, vma);
374}
375
376static void account_kernel_stack(struct task_struct *tsk, int account)
377{
378 void *stack = task_stack_page(tsk);
379 struct vm_struct *vm = task_stack_vm_area(tsk);
380
381
382
383 if (vm)
384 mod_lruvec_page_state(vm->pages[0], NR_KERNEL_STACK_KB,
385 account * (THREAD_SIZE / 1024));
386 else
387 mod_lruvec_slab_state(stack, NR_KERNEL_STACK_KB,
388 account * (THREAD_SIZE / 1024));
389}
390
391static int memcg_charge_kernel_stack(struct task_struct *tsk)
392{
393#ifdef CONFIG_VMAP_STACK
394 struct vm_struct *vm = task_stack_vm_area(tsk);
395 int ret;
396
397 BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
398
399 if (vm) {
400 int i;
401
402 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
403
404 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
405
406
407
408
409
410 ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL,
411 0);
412 if (ret)
413 return ret;
414 }
415 }
416#endif
417 return 0;
418}
419
420static void release_task_stack(struct task_struct *tsk)
421{
422 if (WARN_ON(tsk->state != TASK_DEAD))
423 return;
424
425 account_kernel_stack(tsk, -1);
426 free_thread_stack(tsk);
427 tsk->stack = NULL;
428#ifdef CONFIG_VMAP_STACK
429 tsk->stack_vm_area = NULL;
430#endif
431}
432
433#ifdef CONFIG_THREAD_INFO_IN_TASK
434void put_task_stack(struct task_struct *tsk)
435{
436 if (refcount_dec_and_test(&tsk->stack_refcount))
437 release_task_stack(tsk);
438}
439#endif
440
441void free_task(struct task_struct *tsk)
442{
443 scs_release(tsk);
444
445#ifndef CONFIG_THREAD_INFO_IN_TASK
446
447
448
449
450 release_task_stack(tsk);
451#else
452
453
454
455
456 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
457#endif
458 rt_mutex_debug_task_free(tsk);
459 ftrace_graph_exit_task(tsk);
460 arch_release_task_struct(tsk);
461 if (tsk->flags & PF_KTHREAD)
462 free_kthread_struct(tsk);
463 free_task_struct(tsk);
464}
465EXPORT_SYMBOL(free_task);
466
467#ifdef CONFIG_MMU
468static __latent_entropy int dup_mmap(struct mm_struct *mm,
469 struct mm_struct *oldmm)
470{
471 struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
472 struct rb_node **rb_link, *rb_parent;
473 int retval;
474 unsigned long charge;
475 LIST_HEAD(uf);
476
477 uprobe_start_dup_mmap();
478 if (mmap_write_lock_killable(oldmm)) {
479 retval = -EINTR;
480 goto fail_uprobe_end;
481 }
482 flush_cache_dup_mm(oldmm);
483 uprobe_dup_mmap(oldmm, mm);
484
485
486
487 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING);
488
489
490 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
491
492 mm->total_vm = oldmm->total_vm;
493 mm->data_vm = oldmm->data_vm;
494 mm->exec_vm = oldmm->exec_vm;
495 mm->stack_vm = oldmm->stack_vm;
496
497 rb_link = &mm->mm_rb.rb_node;
498 rb_parent = NULL;
499 pprev = &mm->mmap;
500 retval = ksm_fork(mm, oldmm);
501 if (retval)
502 goto out;
503 retval = khugepaged_fork(mm, oldmm);
504 if (retval)
505 goto out;
506
507 prev = NULL;
508 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
509 struct file *file;
510
511 if (mpnt->vm_flags & VM_DONTCOPY) {
512 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
513 continue;
514 }
515 charge = 0;
516
517
518
519
520 if (fatal_signal_pending(current)) {
521 retval = -EINTR;
522 goto out;
523 }
524 if (mpnt->vm_flags & VM_ACCOUNT) {
525 unsigned long len = vma_pages(mpnt);
526
527 if (security_vm_enough_memory_mm(oldmm, len))
528 goto fail_nomem;
529 charge = len;
530 }
531 tmp = vm_area_dup(mpnt);
532 if (!tmp)
533 goto fail_nomem;
534 retval = vma_dup_policy(mpnt, tmp);
535 if (retval)
536 goto fail_nomem_policy;
537 tmp->vm_mm = mm;
538 retval = dup_userfaultfd(tmp, &uf);
539 if (retval)
540 goto fail_nomem_anon_vma_fork;
541 if (tmp->vm_flags & VM_WIPEONFORK) {
542
543
544
545
546
547 tmp->anon_vma = NULL;
548 } else if (anon_vma_fork(tmp, mpnt))
549 goto fail_nomem_anon_vma_fork;
550 tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
551 file = tmp->vm_file;
552 if (file) {
553 struct inode *inode = file_inode(file);
554 struct address_space *mapping = file->f_mapping;
555
556 get_file(file);
557 if (tmp->vm_flags & VM_DENYWRITE)
558 atomic_dec(&inode->i_writecount);
559 i_mmap_lock_write(mapping);
560 if (tmp->vm_flags & VM_SHARED)
561 atomic_inc(&mapping->i_mmap_writable);
562 flush_dcache_mmap_lock(mapping);
563
564 vma_interval_tree_insert_after(tmp, mpnt,
565 &mapping->i_mmap);
566 flush_dcache_mmap_unlock(mapping);
567 i_mmap_unlock_write(mapping);
568 }
569
570
571
572
573
574
575 if (is_vm_hugetlb_page(tmp))
576 reset_vma_resv_huge_pages(tmp);
577
578
579
580
581 *pprev = tmp;
582 pprev = &tmp->vm_next;
583 tmp->vm_prev = prev;
584 prev = tmp;
585
586 __vma_link_rb(mm, tmp, rb_link, rb_parent);
587 rb_link = &tmp->vm_rb.rb_right;
588 rb_parent = &tmp->vm_rb;
589
590 mm->map_count++;
591 if (!(tmp->vm_flags & VM_WIPEONFORK))
592 retval = copy_page_range(mm, oldmm, mpnt, tmp);
593
594 if (tmp->vm_ops && tmp->vm_ops->open)
595 tmp->vm_ops->open(tmp);
596
597 if (retval)
598 goto out;
599 }
600
601 retval = arch_dup_mmap(oldmm, mm);
602out:
603 mmap_write_unlock(mm);
604 flush_tlb_mm(oldmm);
605 mmap_write_unlock(oldmm);
606 dup_userfaultfd_complete(&uf);
607fail_uprobe_end:
608 uprobe_end_dup_mmap();
609 return retval;
610fail_nomem_anon_vma_fork:
611 mpol_put(vma_policy(tmp));
612fail_nomem_policy:
613 vm_area_free(tmp);
614fail_nomem:
615 retval = -ENOMEM;
616 vm_unacct_memory(charge);
617 goto out;
618}
619
620static inline int mm_alloc_pgd(struct mm_struct *mm)
621{
622 mm->pgd = pgd_alloc(mm);
623 if (unlikely(!mm->pgd))
624 return -ENOMEM;
625 return 0;
626}
627
628static inline void mm_free_pgd(struct mm_struct *mm)
629{
630 pgd_free(mm, mm->pgd);
631}
632#else
633static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
634{
635 mmap_write_lock(oldmm);
636 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
637 mmap_write_unlock(oldmm);
638 return 0;
639}
640#define mm_alloc_pgd(mm) (0)
641#define mm_free_pgd(mm)
642#endif
643
644static void check_mm(struct mm_struct *mm)
645{
646 int i;
647
648 BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
649 "Please make sure 'struct resident_page_types[]' is updated as well");
650
651 for (i = 0; i < NR_MM_COUNTERS; i++) {
652 long x = atomic_long_read(&mm->rss_stat.count[i]);
653
654 if (unlikely(x))
655 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",
656 mm, resident_page_types[i], x);
657 }
658
659 if (mm_pgtables_bytes(mm))
660 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
661 mm_pgtables_bytes(mm));
662
663#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
664 VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
665#endif
666}
667
668#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
669#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
670
671
672
673
674
675
676void __mmdrop(struct mm_struct *mm)
677{
678 BUG_ON(mm == &init_mm);
679 WARN_ON_ONCE(mm == current->mm);
680 WARN_ON_ONCE(mm == current->active_mm);
681 mm_free_pgd(mm);
682 destroy_context(mm);
683 mmu_notifier_subscriptions_destroy(mm);
684 check_mm(mm);
685 put_user_ns(mm->user_ns);
686 free_mm(mm);
687}
688EXPORT_SYMBOL_GPL(__mmdrop);
689
690static void mmdrop_async_fn(struct work_struct *work)
691{
692 struct mm_struct *mm;
693
694 mm = container_of(work, struct mm_struct, async_put_work);
695 __mmdrop(mm);
696}
697
698static void mmdrop_async(struct mm_struct *mm)
699{
700 if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
701 INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
702 schedule_work(&mm->async_put_work);
703 }
704}
705
706static inline void free_signal_struct(struct signal_struct *sig)
707{
708 taskstats_tgid_free(sig);
709 sched_autogroup_exit(sig);
710
711
712
713
714 if (sig->oom_mm)
715 mmdrop_async(sig->oom_mm);
716 kmem_cache_free(signal_cachep, sig);
717}
718
719static inline void put_signal_struct(struct signal_struct *sig)
720{
721 if (refcount_dec_and_test(&sig->sigcnt))
722 free_signal_struct(sig);
723}
724
725void __put_task_struct(struct task_struct *tsk)
726{
727 WARN_ON(!tsk->exit_state);
728 WARN_ON(refcount_read(&tsk->usage));
729 WARN_ON(tsk == current);
730
731 cgroup_free(tsk);
732 task_numa_free(tsk, true);
733 security_task_free(tsk);
734 exit_creds(tsk);
735 delayacct_tsk_free(tsk);
736 put_signal_struct(tsk->signal);
737
738 if (!profile_handoff_task(tsk))
739 free_task(tsk);
740}
741EXPORT_SYMBOL_GPL(__put_task_struct);
742
743void __init __weak arch_task_cache_init(void) { }
744
745
746
747
748static void set_max_threads(unsigned int max_threads_suggested)
749{
750 u64 threads;
751 unsigned long nr_pages = totalram_pages();
752
753
754
755
756
757 if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
758 threads = MAX_THREADS;
759 else
760 threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
761 (u64) THREAD_SIZE * 8UL);
762
763 if (threads > max_threads_suggested)
764 threads = max_threads_suggested;
765
766 max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
767}
768
769#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
770
771int arch_task_struct_size __read_mostly;
772#endif
773
774#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
775static void task_struct_whitelist(unsigned long *offset, unsigned long *size)
776{
777
778 arch_thread_struct_whitelist(offset, size);
779
780
781
782
783
784 if (unlikely(*size == 0))
785 *offset = 0;
786 else
787 *offset += offsetof(struct task_struct, thread);
788}
789#endif
790
791void __init fork_init(void)
792{
793 int i;
794#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
795#ifndef ARCH_MIN_TASKALIGN
796#define ARCH_MIN_TASKALIGN 0
797#endif
798 int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
799 unsigned long useroffset, usersize;
800
801
802 task_struct_whitelist(&useroffset, &usersize);
803 task_struct_cachep = kmem_cache_create_usercopy("task_struct",
804 arch_task_struct_size, align,
805 SLAB_PANIC|SLAB_ACCOUNT,
806 useroffset, usersize, NULL);
807#endif
808
809
810 arch_task_cache_init();
811
812 set_max_threads(MAX_THREADS);
813
814 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
815 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
816 init_task.signal->rlim[RLIMIT_SIGPENDING] =
817 init_task.signal->rlim[RLIMIT_NPROC];
818
819 for (i = 0; i < UCOUNT_COUNTS; i++) {
820 init_user_ns.ucount_max[i] = max_threads/2;
821 }
822
823#ifdef CONFIG_VMAP_STACK
824 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
825 NULL, free_vm_stack_cache);
826#endif
827
828 scs_init();
829
830 lockdep_init_task(&init_task);
831 uprobes_init();
832}
833
834int __weak arch_dup_task_struct(struct task_struct *dst,
835 struct task_struct *src)
836{
837 *dst = *src;
838 return 0;
839}
840
841void set_task_stack_end_magic(struct task_struct *tsk)
842{
843 unsigned long *stackend;
844
845 stackend = end_of_stack(tsk);
846 *stackend = STACK_END_MAGIC;
847}
848
849static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
850{
851 struct task_struct *tsk;
852 unsigned long *stack;
853 struct vm_struct *stack_vm_area __maybe_unused;
854 int err;
855
856 if (node == NUMA_NO_NODE)
857 node = tsk_fork_get_node(orig);
858 tsk = alloc_task_struct_node(node);
859 if (!tsk)
860 return NULL;
861
862 stack = alloc_thread_stack_node(tsk, node);
863 if (!stack)
864 goto free_tsk;
865
866 if (memcg_charge_kernel_stack(tsk))
867 goto free_stack;
868
869 stack_vm_area = task_stack_vm_area(tsk);
870
871 err = arch_dup_task_struct(tsk, orig);
872
873
874
875
876
877
878 tsk->stack = stack;
879#ifdef CONFIG_VMAP_STACK
880 tsk->stack_vm_area = stack_vm_area;
881#endif
882#ifdef CONFIG_THREAD_INFO_IN_TASK
883 refcount_set(&tsk->stack_refcount, 1);
884#endif
885
886 if (err)
887 goto free_stack;
888
889 err = scs_prepare(tsk, node);
890 if (err)
891 goto free_stack;
892
893#ifdef CONFIG_SECCOMP
894
895
896
897
898
899
900 tsk->seccomp.filter = NULL;
901#endif
902
903 setup_thread_stack(tsk, orig);
904 clear_user_return_notifier(tsk);
905 clear_tsk_need_resched(tsk);
906 set_task_stack_end_magic(tsk);
907
908#ifdef CONFIG_STACKPROTECTOR
909 tsk->stack_canary = get_random_canary();
910#endif
911 if (orig->cpus_ptr == &orig->cpus_mask)
912 tsk->cpus_ptr = &tsk->cpus_mask;
913
914
915
916
917
918 refcount_set(&tsk->rcu_users, 2);
919
920 refcount_set(&tsk->usage, 1);
921#ifdef CONFIG_BLK_DEV_IO_TRACE
922 tsk->btrace_seq = 0;
923#endif
924 tsk->splice_pipe = NULL;
925 tsk->task_frag.page = NULL;
926 tsk->wake_q.next = NULL;
927
928 account_kernel_stack(tsk, 1);
929
930 kcov_task_init(tsk);
931
932#ifdef CONFIG_FAULT_INJECTION
933 tsk->fail_nth = 0;
934#endif
935
936#ifdef CONFIG_BLK_CGROUP
937 tsk->throttle_queue = NULL;
938 tsk->use_memdelay = 0;
939#endif
940
941#ifdef CONFIG_MEMCG
942 tsk->active_memcg = NULL;
943#endif
944 return tsk;
945
946free_stack:
947 free_thread_stack(tsk);
948free_tsk:
949 free_task_struct(tsk);
950 return NULL;
951}
952
953__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
954
955static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
956
957static int __init coredump_filter_setup(char *s)
958{
959 default_dump_filter =
960 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
961 MMF_DUMP_FILTER_MASK;
962 return 1;
963}
964
965__setup("coredump_filter=", coredump_filter_setup);
966
967#include <linux/init_task.h>
968
969static void mm_init_aio(struct mm_struct *mm)
970{
971#ifdef CONFIG_AIO
972 spin_lock_init(&mm->ioctx_lock);
973 mm->ioctx_table = NULL;
974#endif
975}
976
977static __always_inline void mm_clear_owner(struct mm_struct *mm,
978 struct task_struct *p)
979{
980#ifdef CONFIG_MEMCG
981 if (mm->owner == p)
982 WRITE_ONCE(mm->owner, NULL);
983#endif
984}
985
986static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
987{
988#ifdef CONFIG_MEMCG
989 mm->owner = p;
990#endif
991}
992
993static void mm_init_uprobes_state(struct mm_struct *mm)
994{
995#ifdef CONFIG_UPROBES
996 mm->uprobes_state.xol_area = NULL;
997#endif
998}
999
1000static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
1001 struct user_namespace *user_ns)
1002{
1003 mm->mmap = NULL;
1004 mm->mm_rb = RB_ROOT;
1005 mm->vmacache_seqnum = 0;
1006 atomic_set(&mm->mm_users, 1);
1007 atomic_set(&mm->mm_count, 1);
1008 mmap_init_lock(mm);
1009 INIT_LIST_HEAD(&mm->mmlist);
1010 mm->core_state = NULL;
1011 mm_pgtables_bytes_init(mm);
1012 mm->map_count = 0;
1013 mm->locked_vm = 0;
1014 atomic_set(&mm->has_pinned, 0);
1015 atomic64_set(&mm->pinned_vm, 0);
1016 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
1017 spin_lock_init(&mm->page_table_lock);
1018 spin_lock_init(&mm->arg_lock);
1019 mm_init_cpumask(mm);
1020 mm_init_aio(mm);
1021 mm_init_owner(mm, p);
1022 RCU_INIT_POINTER(mm->exe_file, NULL);
1023 mmu_notifier_subscriptions_init(mm);
1024 init_tlb_flush_pending(mm);
1025#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
1026 mm->pmd_huge_pte = NULL;
1027#endif
1028 mm_init_uprobes_state(mm);
1029
1030 if (current->mm) {
1031 mm->flags = current->mm->flags & MMF_INIT_MASK;
1032 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
1033 } else {
1034 mm->flags = default_dump_filter;
1035 mm->def_flags = 0;
1036 }
1037
1038 if (mm_alloc_pgd(mm))
1039 goto fail_nopgd;
1040
1041 if (init_new_context(p, mm))
1042 goto fail_nocontext;
1043
1044 mm->user_ns = get_user_ns(user_ns);
1045 return mm;
1046
1047fail_nocontext:
1048 mm_free_pgd(mm);
1049fail_nopgd:
1050 free_mm(mm);
1051 return NULL;
1052}
1053
1054
1055
1056
1057struct mm_struct *mm_alloc(void)
1058{
1059 struct mm_struct *mm;
1060
1061 mm = allocate_mm();
1062 if (!mm)
1063 return NULL;
1064
1065 memset(mm, 0, sizeof(*mm));
1066 return mm_init(mm, current, current_user_ns());
1067}
1068
1069static inline void __mmput(struct mm_struct *mm)
1070{
1071 VM_BUG_ON(atomic_read(&mm->mm_users));
1072
1073 uprobe_clear_state(mm);
1074 exit_aio(mm);
1075 ksm_exit(mm);
1076 khugepaged_exit(mm);
1077 exit_mmap(mm);
1078 mm_put_huge_zero_page(mm);
1079 set_mm_exe_file(mm, NULL);
1080 if (!list_empty(&mm->mmlist)) {
1081 spin_lock(&mmlist_lock);
1082 list_del(&mm->mmlist);
1083 spin_unlock(&mmlist_lock);
1084 }
1085 if (mm->binfmt)
1086 module_put(mm->binfmt->module);
1087 mmdrop(mm);
1088}
1089
1090
1091
1092
1093void mmput(struct mm_struct *mm)
1094{
1095 might_sleep();
1096
1097 if (atomic_dec_and_test(&mm->mm_users))
1098 __mmput(mm);
1099}
1100EXPORT_SYMBOL_GPL(mmput);
1101
1102#ifdef CONFIG_MMU
1103static void mmput_async_fn(struct work_struct *work)
1104{
1105 struct mm_struct *mm = container_of(work, struct mm_struct,
1106 async_put_work);
1107
1108 __mmput(mm);
1109}
1110
1111void mmput_async(struct mm_struct *mm)
1112{
1113 if (atomic_dec_and_test(&mm->mm_users)) {
1114 INIT_WORK(&mm->async_put_work, mmput_async_fn);
1115 schedule_work(&mm->async_put_work);
1116 }
1117}
1118#endif
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
1132{
1133 struct file *old_exe_file;
1134
1135
1136
1137
1138
1139
1140 old_exe_file = rcu_dereference_raw(mm->exe_file);
1141
1142 if (new_exe_file)
1143 get_file(new_exe_file);
1144 rcu_assign_pointer(mm->exe_file, new_exe_file);
1145 if (old_exe_file)
1146 fput(old_exe_file);
1147}
1148
1149
1150
1151
1152
1153
1154
1155struct file *get_mm_exe_file(struct mm_struct *mm)
1156{
1157 struct file *exe_file;
1158
1159 rcu_read_lock();
1160 exe_file = rcu_dereference(mm->exe_file);
1161 if (exe_file && !get_file_rcu(exe_file))
1162 exe_file = NULL;
1163 rcu_read_unlock();
1164 return exe_file;
1165}
1166EXPORT_SYMBOL(get_mm_exe_file);
1167
1168
1169
1170
1171
1172
1173
1174
1175struct file *get_task_exe_file(struct task_struct *task)
1176{
1177 struct file *exe_file = NULL;
1178 struct mm_struct *mm;
1179
1180 task_lock(task);
1181 mm = task->mm;
1182 if (mm) {
1183 if (!(task->flags & PF_KTHREAD))
1184 exe_file = get_mm_exe_file(mm);
1185 }
1186 task_unlock(task);
1187 return exe_file;
1188}
1189EXPORT_SYMBOL(get_task_exe_file);
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200struct mm_struct *get_task_mm(struct task_struct *task)
1201{
1202 struct mm_struct *mm;
1203
1204 task_lock(task);
1205 mm = task->mm;
1206 if (mm) {
1207 if (task->flags & PF_KTHREAD)
1208 mm = NULL;
1209 else
1210 mmget(mm);
1211 }
1212 task_unlock(task);
1213 return mm;
1214}
1215EXPORT_SYMBOL_GPL(get_task_mm);
1216
1217struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
1218{
1219 struct mm_struct *mm;
1220 int err;
1221
1222 err = mutex_lock_killable(&task->signal->exec_update_mutex);
1223 if (err)
1224 return ERR_PTR(err);
1225
1226 mm = get_task_mm(task);
1227 if (mm && mm != current->mm &&
1228 !ptrace_may_access(task, mode)) {
1229 mmput(mm);
1230 mm = ERR_PTR(-EACCES);
1231 }
1232 mutex_unlock(&task->signal->exec_update_mutex);
1233
1234 return mm;
1235}
1236
1237static void complete_vfork_done(struct task_struct *tsk)
1238{
1239 struct completion *vfork;
1240
1241 task_lock(tsk);
1242 vfork = tsk->vfork_done;
1243 if (likely(vfork)) {
1244 tsk->vfork_done = NULL;
1245 complete(vfork);
1246 }
1247 task_unlock(tsk);
1248}
1249
1250static int wait_for_vfork_done(struct task_struct *child,
1251 struct completion *vfork)
1252{
1253 int killed;
1254
1255 freezer_do_not_count();
1256 cgroup_enter_frozen();
1257 killed = wait_for_completion_killable(vfork);
1258 cgroup_leave_frozen(false);
1259 freezer_count();
1260
1261 if (killed) {
1262 task_lock(child);
1263 child->vfork_done = NULL;
1264 task_unlock(child);
1265 }
1266
1267 put_task_struct(child);
1268 return killed;
1269}
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
1285{
1286 uprobe_free_utask(tsk);
1287
1288
1289 deactivate_mm(tsk, mm);
1290
1291
1292
1293
1294
1295
1296 if (tsk->clear_child_tid) {
1297 if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) &&
1298 atomic_read(&mm->mm_users) > 1) {
1299
1300
1301
1302
1303 put_user(0, tsk->clear_child_tid);
1304 do_futex(tsk->clear_child_tid, FUTEX_WAKE,
1305 1, NULL, NULL, 0, 0);
1306 }
1307 tsk->clear_child_tid = NULL;
1308 }
1309
1310
1311
1312
1313
1314 if (tsk->vfork_done)
1315 complete_vfork_done(tsk);
1316}
1317
1318void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1319{
1320 futex_exit_release(tsk);
1321 mm_release(tsk, mm);
1322}
1323
1324void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1325{
1326 futex_exec_release(tsk);
1327 mm_release(tsk, mm);
1328}
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340static struct mm_struct *dup_mm(struct task_struct *tsk,
1341 struct mm_struct *oldmm)
1342{
1343 struct mm_struct *mm;
1344 int err;
1345
1346 mm = allocate_mm();
1347 if (!mm)
1348 goto fail_nomem;
1349
1350 memcpy(mm, oldmm, sizeof(*mm));
1351
1352 if (!mm_init(mm, tsk, mm->user_ns))
1353 goto fail_nomem;
1354
1355 err = dup_mmap(mm, oldmm);
1356 if (err)
1357 goto free_pt;
1358
1359 mm->hiwater_rss = get_mm_rss(mm);
1360 mm->hiwater_vm = mm->total_vm;
1361
1362 if (mm->binfmt && !try_module_get(mm->binfmt->module))
1363 goto free_pt;
1364
1365 return mm;
1366
1367free_pt:
1368
1369 mm->binfmt = NULL;
1370 mm_init_owner(mm, NULL);
1371 mmput(mm);
1372
1373fail_nomem:
1374 return NULL;
1375}
1376
1377static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
1378{
1379 struct mm_struct *mm, *oldmm;
1380 int retval;
1381
1382 tsk->min_flt = tsk->maj_flt = 0;
1383 tsk->nvcsw = tsk->nivcsw = 0;
1384#ifdef CONFIG_DETECT_HUNG_TASK
1385 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1386 tsk->last_switch_time = 0;
1387#endif
1388
1389 tsk->mm = NULL;
1390 tsk->active_mm = NULL;
1391
1392
1393
1394
1395
1396
1397 oldmm = current->mm;
1398 if (!oldmm)
1399 return 0;
1400
1401
1402 vmacache_flush(tsk);
1403
1404 if (clone_flags & CLONE_VM) {
1405 mmget(oldmm);
1406 mm = oldmm;
1407 goto good_mm;
1408 }
1409
1410 retval = -ENOMEM;
1411 mm = dup_mm(tsk, current->mm);
1412 if (!mm)
1413 goto fail_nomem;
1414
1415good_mm:
1416 tsk->mm = mm;
1417 tsk->active_mm = mm;
1418 return 0;
1419
1420fail_nomem:
1421 return retval;
1422}
1423
1424static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
1425{
1426 struct fs_struct *fs = current->fs;
1427 if (clone_flags & CLONE_FS) {
1428
1429 spin_lock(&fs->lock);
1430 if (fs->in_exec) {
1431 spin_unlock(&fs->lock);
1432 return -EAGAIN;
1433 }
1434 fs->users++;
1435 spin_unlock(&fs->lock);
1436 return 0;
1437 }
1438 tsk->fs = copy_fs_struct(fs);
1439 if (!tsk->fs)
1440 return -ENOMEM;
1441 return 0;
1442}
1443
1444static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
1445{
1446 struct files_struct *oldf, *newf;
1447 int error = 0;
1448
1449
1450
1451
1452 oldf = current->files;
1453 if (!oldf)
1454 goto out;
1455
1456 if (clone_flags & CLONE_FILES) {
1457 atomic_inc(&oldf->count);
1458 goto out;
1459 }
1460
1461 newf = dup_fd(oldf, NR_OPEN_MAX, &error);
1462 if (!newf)
1463 goto out;
1464
1465 tsk->files = newf;
1466 error = 0;
1467out:
1468 return error;
1469}
1470
1471static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
1472{
1473#ifdef CONFIG_BLOCK
1474 struct io_context *ioc = current->io_context;
1475 struct io_context *new_ioc;
1476
1477 if (!ioc)
1478 return 0;
1479
1480
1481
1482 if (clone_flags & CLONE_IO) {
1483 ioc_task_link(ioc);
1484 tsk->io_context = ioc;
1485 } else if (ioprio_valid(ioc->ioprio)) {
1486 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
1487 if (unlikely(!new_ioc))
1488 return -ENOMEM;
1489
1490 new_ioc->ioprio = ioc->ioprio;
1491 put_io_context(new_ioc);
1492 }
1493#endif
1494 return 0;
1495}
1496
1497static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
1498{
1499 struct sighand_struct *sig;
1500
1501 if (clone_flags & CLONE_SIGHAND) {
1502 refcount_inc(¤t->sighand->count);
1503 return 0;
1504 }
1505 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1506 RCU_INIT_POINTER(tsk->sighand, sig);
1507 if (!sig)
1508 return -ENOMEM;
1509
1510 refcount_set(&sig->count, 1);
1511 spin_lock_irq(¤t->sighand->siglock);
1512 memcpy(sig->action, current->sighand->action, sizeof(sig->action));
1513 spin_unlock_irq(¤t->sighand->siglock);
1514
1515
1516 if (clone_flags & CLONE_CLEAR_SIGHAND)
1517 flush_signal_handlers(tsk, 0);
1518
1519 return 0;
1520}
1521
1522void __cleanup_sighand(struct sighand_struct *sighand)
1523{
1524 if (refcount_dec_and_test(&sighand->count)) {
1525 signalfd_cleanup(sighand);
1526
1527
1528
1529
1530 kmem_cache_free(sighand_cachep, sighand);
1531 }
1532}
1533
1534
1535
1536
1537static void posix_cpu_timers_init_group(struct signal_struct *sig)
1538{
1539 struct posix_cputimers *pct = &sig->posix_cputimers;
1540 unsigned long cpu_limit;
1541
1542 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1543 posix_cputimers_group_init(pct, cpu_limit);
1544}
1545
1546static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
1547{
1548 struct signal_struct *sig;
1549
1550 if (clone_flags & CLONE_THREAD)
1551 return 0;
1552
1553 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
1554 tsk->signal = sig;
1555 if (!sig)
1556 return -ENOMEM;
1557
1558 sig->nr_threads = 1;
1559 atomic_set(&sig->live, 1);
1560 refcount_set(&sig->sigcnt, 1);
1561
1562
1563 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
1564 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
1565
1566 init_waitqueue_head(&sig->wait_chldexit);
1567 sig->curr_target = tsk;
1568 init_sigpending(&sig->shared_pending);
1569 INIT_HLIST_HEAD(&sig->multiprocess);
1570 seqlock_init(&sig->stats_lock);
1571 prev_cputime_init(&sig->prev_cputime);
1572
1573#ifdef CONFIG_POSIX_TIMERS
1574 INIT_LIST_HEAD(&sig->posix_timers);
1575 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1576 sig->real_timer.function = it_real_fn;
1577#endif
1578
1579 task_lock(current->group_leader);
1580 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1581 task_unlock(current->group_leader);
1582
1583 posix_cpu_timers_init_group(sig);
1584
1585 tty_audit_fork(sig);
1586 sched_autogroup_fork(sig);
1587
1588 sig->oom_score_adj = current->signal->oom_score_adj;
1589 sig->oom_score_adj_min = current->signal->oom_score_adj_min;
1590
1591 mutex_init(&sig->cred_guard_mutex);
1592 mutex_init(&sig->exec_update_mutex);
1593
1594 return 0;
1595}
1596
1597static void copy_seccomp(struct task_struct *p)
1598{
1599#ifdef CONFIG_SECCOMP
1600
1601
1602
1603
1604
1605
1606 assert_spin_locked(¤t->sighand->siglock);
1607
1608
1609 get_seccomp_filter(current);
1610 p->seccomp = current->seccomp;
1611
1612
1613
1614
1615
1616
1617 if (task_no_new_privs(current))
1618 task_set_no_new_privs(p);
1619
1620
1621
1622
1623
1624
1625 if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1626 set_tsk_thread_flag(p, TIF_SECCOMP);
1627#endif
1628}
1629
1630SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
1631{
1632 current->clear_child_tid = tidptr;
1633
1634 return task_pid_vnr(current);
1635}
1636
1637static void rt_mutex_init_task(struct task_struct *p)
1638{
1639 raw_spin_lock_init(&p->pi_lock);
1640#ifdef CONFIG_RT_MUTEXES
1641 p->pi_waiters = RB_ROOT_CACHED;
1642 p->pi_top_task = NULL;
1643 p->pi_blocked_on = NULL;
1644#endif
1645}
1646
1647static inline void init_task_pid_links(struct task_struct *task)
1648{
1649 enum pid_type type;
1650
1651 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
1652 INIT_HLIST_NODE(&task->pid_links[type]);
1653 }
1654}
1655
1656static inline void
1657init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
1658{
1659 if (type == PIDTYPE_PID)
1660 task->thread_pid = pid;
1661 else
1662 task->signal->pids[type] = pid;
1663}
1664
1665static inline void rcu_copy_process(struct task_struct *p)
1666{
1667#ifdef CONFIG_PREEMPT_RCU
1668 p->rcu_read_lock_nesting = 0;
1669 p->rcu_read_unlock_special.s = 0;
1670 p->rcu_blocked_node = NULL;
1671 INIT_LIST_HEAD(&p->rcu_node_entry);
1672#endif
1673#ifdef CONFIG_TASKS_RCU
1674 p->rcu_tasks_holdout = false;
1675 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
1676 p->rcu_tasks_idle_cpu = -1;
1677#endif
1678#ifdef CONFIG_TASKS_TRACE_RCU
1679 p->trc_reader_nesting = 0;
1680 p->trc_reader_special.s = 0;
1681 INIT_LIST_HEAD(&p->trc_holdout_list);
1682#endif
1683}
1684
1685struct pid *pidfd_pid(const struct file *file)
1686{
1687 if (file->f_op == &pidfd_fops)
1688 return file->private_data;
1689
1690 return ERR_PTR(-EBADF);
1691}
1692
1693static int pidfd_release(struct inode *inode, struct file *file)
1694{
1695 struct pid *pid = file->private_data;
1696
1697 file->private_data = NULL;
1698 put_pid(pid);
1699 return 0;
1700}
1701
1702#ifdef CONFIG_PROC_FS
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
1739{
1740 struct pid *pid = f->private_data;
1741 struct pid_namespace *ns;
1742 pid_t nr = -1;
1743
1744 if (likely(pid_has_task(pid, PIDTYPE_PID))) {
1745 ns = proc_pid_ns(file_inode(m->file)->i_sb);
1746 nr = pid_nr_ns(pid, ns);
1747 }
1748
1749 seq_put_decimal_ll(m, "Pid:\t", nr);
1750
1751#ifdef CONFIG_PID_NS
1752 seq_put_decimal_ll(m, "\nNSpid:\t", nr);
1753 if (nr > 0) {
1754 int i;
1755
1756
1757
1758
1759
1760
1761 for (i = ns->level + 1; i <= pid->level; i++)
1762 seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
1763 }
1764#endif
1765 seq_putc(m, '\n');
1766}
1767#endif
1768
1769
1770
1771
1772static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts)
1773{
1774 struct pid *pid = file->private_data;
1775 __poll_t poll_flags = 0;
1776
1777 poll_wait(file, &pid->wait_pidfd, pts);
1778
1779
1780
1781
1782
1783
1784 if (thread_group_exited(pid))
1785 poll_flags = EPOLLIN | EPOLLRDNORM;
1786
1787 return poll_flags;
1788}
1789
1790const struct file_operations pidfd_fops = {
1791 .release = pidfd_release,
1792 .poll = pidfd_poll,
1793#ifdef CONFIG_PROC_FS
1794 .show_fdinfo = pidfd_show_fdinfo,
1795#endif
1796};
1797
1798static void __delayed_free_task(struct rcu_head *rhp)
1799{
1800 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1801
1802 free_task(tsk);
1803}
1804
1805static __always_inline void delayed_free_task(struct task_struct *tsk)
1806{
1807 if (IS_ENABLED(CONFIG_MEMCG))
1808 call_rcu(&tsk->rcu, __delayed_free_task);
1809 else
1810 free_task(tsk);
1811}
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821static __latent_entropy struct task_struct *copy_process(
1822 struct pid *pid,
1823 int trace,
1824 int node,
1825 struct kernel_clone_args *args)
1826{
1827 int pidfd = -1, retval;
1828 struct task_struct *p;
1829 struct multiprocess_signals delayed;
1830 struct file *pidfile = NULL;
1831 u64 clone_flags = args->flags;
1832 struct nsproxy *nsp = current->nsproxy;
1833
1834
1835
1836
1837
1838 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
1839 return ERR_PTR(-EINVAL);
1840
1841 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
1842 return ERR_PTR(-EINVAL);
1843
1844
1845
1846
1847
1848 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
1849 return ERR_PTR(-EINVAL);
1850
1851
1852
1853
1854
1855
1856 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
1857 return ERR_PTR(-EINVAL);
1858
1859
1860
1861
1862
1863
1864
1865 if ((clone_flags & CLONE_PARENT) &&
1866 current->signal->flags & SIGNAL_UNKILLABLE)
1867 return ERR_PTR(-EINVAL);
1868
1869
1870
1871
1872
1873 if (clone_flags & CLONE_THREAD) {
1874 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
1875 (task_active_pid_ns(current) != nsp->pid_ns_for_children))
1876 return ERR_PTR(-EINVAL);
1877 }
1878
1879
1880
1881
1882
1883 if (clone_flags & (CLONE_THREAD | CLONE_VM)) {
1884 if (nsp->time_ns != nsp->time_ns_for_children)
1885 return ERR_PTR(-EINVAL);
1886 }
1887
1888 if (clone_flags & CLONE_PIDFD) {
1889
1890
1891
1892
1893
1894 if (clone_flags & (CLONE_DETACHED | CLONE_THREAD))
1895 return ERR_PTR(-EINVAL);
1896 }
1897
1898
1899
1900
1901
1902
1903
1904 sigemptyset(&delayed.signal);
1905 INIT_HLIST_NODE(&delayed.node);
1906
1907 spin_lock_irq(¤t->sighand->siglock);
1908 if (!(clone_flags & CLONE_THREAD))
1909 hlist_add_head(&delayed.node, ¤t->signal->multiprocess);
1910 recalc_sigpending();
1911 spin_unlock_irq(¤t->sighand->siglock);
1912 retval = -ERESTARTNOINTR;
1913 if (signal_pending(current))
1914 goto fork_out;
1915
1916 retval = -ENOMEM;
1917 p = dup_task_struct(current, node);
1918 if (!p)
1919 goto fork_out;
1920
1921
1922
1923
1924
1925
1926
1927 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
1928
1929
1930
1931 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
1932
1933 ftrace_graph_init_task(p);
1934
1935 rt_mutex_init_task(p);
1936
1937 lockdep_assert_irqs_enabled();
1938#ifdef CONFIG_PROVE_LOCKING
1939 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1940#endif
1941 retval = -EAGAIN;
1942 if (atomic_read(&p->real_cred->user->processes) >=
1943 task_rlimit(p, RLIMIT_NPROC)) {
1944 if (p->real_cred->user != INIT_USER &&
1945 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
1946 goto bad_fork_free;
1947 }
1948 current->flags &= ~PF_NPROC_EXCEEDED;
1949
1950 retval = copy_creds(p, clone_flags);
1951 if (retval < 0)
1952 goto bad_fork_free;
1953
1954
1955
1956
1957
1958
1959 retval = -EAGAIN;
1960 if (data_race(nr_threads >= max_threads))
1961 goto bad_fork_cleanup_count;
1962
1963 delayacct_tsk_init(p);
1964 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE);
1965 p->flags |= PF_FORKNOEXEC;
1966 INIT_LIST_HEAD(&p->children);
1967 INIT_LIST_HEAD(&p->sibling);
1968 rcu_copy_process(p);
1969 p->vfork_done = NULL;
1970 spin_lock_init(&p->alloc_lock);
1971
1972 init_sigpending(&p->pending);
1973
1974 p->utime = p->stime = p->gtime = 0;
1975#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
1976 p->utimescaled = p->stimescaled = 0;
1977#endif
1978 prev_cputime_init(&p->prev_cputime);
1979
1980#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1981 seqcount_init(&p->vtime.seqcount);
1982 p->vtime.starttime = 0;
1983 p->vtime.state = VTIME_INACTIVE;
1984#endif
1985
1986#if defined(SPLIT_RSS_COUNTING)
1987 memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1988#endif
1989
1990 p->default_timer_slack_ns = current->timer_slack_ns;
1991
1992#ifdef CONFIG_PSI
1993 p->psi_flags = 0;
1994#endif
1995
1996 task_io_accounting_init(&p->ioac);
1997 acct_clear_integrals(p);
1998
1999 posix_cputimers_init(&p->posix_cputimers);
2000
2001 p->io_context = NULL;
2002 audit_set_context(p, NULL);
2003 cgroup_fork(p);
2004#ifdef CONFIG_NUMA
2005 p->mempolicy = mpol_dup(p->mempolicy);
2006 if (IS_ERR(p->mempolicy)) {
2007 retval = PTR_ERR(p->mempolicy);
2008 p->mempolicy = NULL;
2009 goto bad_fork_cleanup_threadgroup_lock;
2010 }
2011#endif
2012#ifdef CONFIG_CPUSETS
2013 p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
2014 p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
2015 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock);
2016#endif
2017#ifdef CONFIG_TRACE_IRQFLAGS
2018 memset(&p->irqtrace, 0, sizeof(p->irqtrace));
2019 p->irqtrace.hardirq_disable_ip = _THIS_IP_;
2020 p->irqtrace.softirq_enable_ip = _THIS_IP_;
2021 p->softirqs_enabled = 1;
2022 p->softirq_context = 0;
2023#endif
2024
2025 p->pagefault_disabled = 0;
2026
2027#ifdef CONFIG_LOCKDEP
2028 lockdep_init_task(p);
2029#endif
2030
2031#ifdef CONFIG_DEBUG_MUTEXES
2032 p->blocked_on = NULL;
2033#endif
2034#ifdef CONFIG_BCACHE
2035 p->sequential_io = 0;
2036 p->sequential_io_avg = 0;
2037#endif
2038
2039
2040 retval = sched_fork(clone_flags, p);
2041 if (retval)
2042 goto bad_fork_cleanup_policy;
2043
2044 retval = perf_event_init_task(p);
2045 if (retval)
2046 goto bad_fork_cleanup_policy;
2047 retval = audit_alloc(p);
2048 if (retval)
2049 goto bad_fork_cleanup_perf;
2050
2051 shm_init_task(p);
2052 retval = security_task_alloc(p, clone_flags);
2053 if (retval)
2054 goto bad_fork_cleanup_audit;
2055 retval = copy_semundo(clone_flags, p);
2056 if (retval)
2057 goto bad_fork_cleanup_security;
2058 retval = copy_files(clone_flags, p);
2059 if (retval)
2060 goto bad_fork_cleanup_semundo;
2061 retval = copy_fs(clone_flags, p);
2062 if (retval)
2063 goto bad_fork_cleanup_files;
2064 retval = copy_sighand(clone_flags, p);
2065 if (retval)
2066 goto bad_fork_cleanup_fs;
2067 retval = copy_signal(clone_flags, p);
2068 if (retval)
2069 goto bad_fork_cleanup_sighand;
2070 retval = copy_mm(clone_flags, p);
2071 if (retval)
2072 goto bad_fork_cleanup_signal;
2073 retval = copy_namespaces(clone_flags, p);
2074 if (retval)
2075 goto bad_fork_cleanup_mm;
2076 retval = copy_io(clone_flags, p);
2077 if (retval)
2078 goto bad_fork_cleanup_namespaces;
2079 retval = copy_thread(clone_flags, args->stack, args->stack_size, p, args->tls);
2080 if (retval)
2081 goto bad_fork_cleanup_io;
2082
2083 stackleak_task_init(p);
2084
2085 if (pid != &init_struct_pid) {
2086 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid,
2087 args->set_tid_size);
2088 if (IS_ERR(pid)) {
2089 retval = PTR_ERR(pid);
2090 goto bad_fork_cleanup_thread;
2091 }
2092 }
2093
2094
2095
2096
2097
2098
2099 if (clone_flags & CLONE_PIDFD) {
2100 retval = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
2101 if (retval < 0)
2102 goto bad_fork_free_pid;
2103
2104 pidfd = retval;
2105
2106 pidfile = anon_inode_getfile("[pidfd]", &pidfd_fops, pid,
2107 O_RDWR | O_CLOEXEC);
2108 if (IS_ERR(pidfile)) {
2109 put_unused_fd(pidfd);
2110 retval = PTR_ERR(pidfile);
2111 goto bad_fork_free_pid;
2112 }
2113 get_pid(pid);
2114
2115 retval = put_user(pidfd, args->pidfd);
2116 if (retval)
2117 goto bad_fork_put_pidfd;
2118 }
2119
2120#ifdef CONFIG_BLOCK
2121 p->plug = NULL;
2122#endif
2123 futex_init_task(p);
2124
2125
2126
2127
2128 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
2129 sas_ss_reset(p);
2130
2131
2132
2133
2134
2135 user_disable_single_step(p);
2136 clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
2137#ifdef TIF_SYSCALL_EMU
2138 clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
2139#endif
2140 clear_tsk_latency_tracing(p);
2141
2142
2143 p->pid = pid_nr(pid);
2144 if (clone_flags & CLONE_THREAD) {
2145 p->exit_signal = -1;
2146 p->group_leader = current->group_leader;
2147 p->tgid = current->tgid;
2148 } else {
2149 if (clone_flags & CLONE_PARENT)
2150 p->exit_signal = current->group_leader->exit_signal;
2151 else
2152 p->exit_signal = args->exit_signal;
2153 p->group_leader = p;
2154 p->tgid = p->pid;
2155 }
2156
2157 p->nr_dirtied = 0;
2158 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
2159 p->dirty_paused_when = 0;
2160
2161 p->pdeath_signal = 0;
2162 INIT_LIST_HEAD(&p->thread_group);
2163 p->task_works = NULL;
2164
2165
2166
2167
2168
2169
2170
2171 retval = cgroup_can_fork(p, args);
2172 if (retval)
2173 goto bad_fork_put_pidfd;
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183 p->start_time = ktime_get_ns();
2184 p->start_boottime = ktime_get_boottime_ns();
2185
2186
2187
2188
2189
2190 write_lock_irq(&tasklist_lock);
2191
2192
2193 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
2194 p->real_parent = current->real_parent;
2195 p->parent_exec_id = current->parent_exec_id;
2196 } else {
2197 p->real_parent = current;
2198 p->parent_exec_id = current->self_exec_id;
2199 }
2200
2201 klp_copy_process(p);
2202
2203 spin_lock(¤t->sighand->siglock);
2204
2205
2206
2207
2208
2209 copy_seccomp(p);
2210
2211 rseq_fork(p, clone_flags);
2212
2213
2214 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
2215 retval = -ENOMEM;
2216 goto bad_fork_cancel_cgroup;
2217 }
2218
2219
2220 if (fatal_signal_pending(current)) {
2221 retval = -EINTR;
2222 goto bad_fork_cancel_cgroup;
2223 }
2224
2225
2226 if (pidfile)
2227 fd_install(pidfd, pidfile);
2228
2229 init_task_pid_links(p);
2230 if (likely(p->pid)) {
2231 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
2232
2233 init_task_pid(p, PIDTYPE_PID, pid);
2234 if (thread_group_leader(p)) {
2235 init_task_pid(p, PIDTYPE_TGID, pid);
2236 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
2237 init_task_pid(p, PIDTYPE_SID, task_session(current));
2238
2239 if (is_child_reaper(pid)) {
2240 ns_of_pid(pid)->child_reaper = p;
2241 p->signal->flags |= SIGNAL_UNKILLABLE;
2242 }
2243 p->signal->shared_pending.signal = delayed.signal;
2244 p->signal->tty = tty_kref_get(current->signal->tty);
2245
2246
2247
2248
2249
2250 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
2251 p->real_parent->signal->is_child_subreaper;
2252 list_add_tail(&p->sibling, &p->real_parent->children);
2253 list_add_tail_rcu(&p->tasks, &init_task.tasks);
2254 attach_pid(p, PIDTYPE_TGID);
2255 attach_pid(p, PIDTYPE_PGID);
2256 attach_pid(p, PIDTYPE_SID);
2257 __this_cpu_inc(process_counts);
2258 } else {
2259 current->signal->nr_threads++;
2260 atomic_inc(¤t->signal->live);
2261 refcount_inc(¤t->signal->sigcnt);
2262 task_join_group_stop(p);
2263 list_add_tail_rcu(&p->thread_group,
2264 &p->group_leader->thread_group);
2265 list_add_tail_rcu(&p->thread_node,
2266 &p->signal->thread_head);
2267 }
2268 attach_pid(p, PIDTYPE_PID);
2269 nr_threads++;
2270 }
2271 total_forks++;
2272 hlist_del_init(&delayed.node);
2273 spin_unlock(¤t->sighand->siglock);
2274 syscall_tracepoint_update(p);
2275 write_unlock_irq(&tasklist_lock);
2276
2277 proc_fork_connector(p);
2278 sched_post_fork(p);
2279 cgroup_post_fork(p, args);
2280 perf_event_fork(p);
2281
2282 trace_task_newtask(p, clone_flags);
2283 uprobe_copy_process(p, clone_flags);
2284
2285 return p;
2286
2287bad_fork_cancel_cgroup:
2288 spin_unlock(¤t->sighand->siglock);
2289 write_unlock_irq(&tasklist_lock);
2290 cgroup_cancel_fork(p, args);
2291bad_fork_put_pidfd:
2292 if (clone_flags & CLONE_PIDFD) {
2293 fput(pidfile);
2294 put_unused_fd(pidfd);
2295 }
2296bad_fork_free_pid:
2297 if (pid != &init_struct_pid)
2298 free_pid(pid);
2299bad_fork_cleanup_thread:
2300 exit_thread(p);
2301bad_fork_cleanup_io:
2302 if (p->io_context)
2303 exit_io_context(p);
2304bad_fork_cleanup_namespaces:
2305 exit_task_namespaces(p);
2306bad_fork_cleanup_mm:
2307 if (p->mm) {
2308 mm_clear_owner(p->mm, p);
2309 mmput(p->mm);
2310 }
2311bad_fork_cleanup_signal:
2312 if (!(clone_flags & CLONE_THREAD))
2313 free_signal_struct(p->signal);
2314bad_fork_cleanup_sighand:
2315 __cleanup_sighand(p->sighand);
2316bad_fork_cleanup_fs:
2317 exit_fs(p);
2318bad_fork_cleanup_files:
2319 exit_files(p);
2320bad_fork_cleanup_semundo:
2321 exit_sem(p);
2322bad_fork_cleanup_security:
2323 security_task_free(p);
2324bad_fork_cleanup_audit:
2325 audit_free(p);
2326bad_fork_cleanup_perf:
2327 perf_event_free_task(p);
2328bad_fork_cleanup_policy:
2329 lockdep_free_task(p);
2330#ifdef CONFIG_NUMA
2331 mpol_put(p->mempolicy);
2332bad_fork_cleanup_threadgroup_lock:
2333#endif
2334 delayacct_tsk_free(p);
2335bad_fork_cleanup_count:
2336 atomic_dec(&p->cred->user->processes);
2337 exit_creds(p);
2338bad_fork_free:
2339 p->state = TASK_DEAD;
2340 put_task_stack(p);
2341 delayed_free_task(p);
2342fork_out:
2343 spin_lock_irq(¤t->sighand->siglock);
2344 hlist_del_init(&delayed.node);
2345 spin_unlock_irq(¤t->sighand->siglock);
2346 return ERR_PTR(retval);
2347}
2348
2349static inline void init_idle_pids(struct task_struct *idle)
2350{
2351 enum pid_type type;
2352
2353 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
2354 INIT_HLIST_NODE(&idle->pid_links[type]);
2355 init_task_pid(idle, type, &init_struct_pid);
2356 }
2357}
2358
2359struct task_struct *fork_idle(int cpu)
2360{
2361 struct task_struct *task;
2362 struct kernel_clone_args args = {
2363 .flags = CLONE_VM,
2364 };
2365
2366 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);
2367 if (!IS_ERR(task)) {
2368 init_idle_pids(task);
2369 init_idle(task, cpu);
2370 }
2371
2372 return task;
2373}
2374
2375struct mm_struct *copy_init_mm(void)
2376{
2377 return dup_mm(NULL, &init_mm);
2378}
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388long _do_fork(struct kernel_clone_args *args)
2389{
2390 u64 clone_flags = args->flags;
2391 struct completion vfork;
2392 struct pid *pid;
2393 struct task_struct *p;
2394 int trace = 0;
2395 long nr;
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406 if ((args->flags & CLONE_PIDFD) &&
2407 (args->flags & CLONE_PARENT_SETTID) &&
2408 (args->pidfd == args->parent_tid))
2409 return -EINVAL;
2410
2411
2412
2413
2414
2415
2416
2417 if (!(clone_flags & CLONE_UNTRACED)) {
2418 if (clone_flags & CLONE_VFORK)
2419 trace = PTRACE_EVENT_VFORK;
2420 else if (args->exit_signal != SIGCHLD)
2421 trace = PTRACE_EVENT_CLONE;
2422 else
2423 trace = PTRACE_EVENT_FORK;
2424
2425 if (likely(!ptrace_event_enabled(current, trace)))
2426 trace = 0;
2427 }
2428
2429 p = copy_process(NULL, trace, NUMA_NO_NODE, args);
2430 add_latent_entropy();
2431
2432 if (IS_ERR(p))
2433 return PTR_ERR(p);
2434
2435
2436
2437
2438
2439 trace_sched_process_fork(current, p);
2440
2441 pid = get_task_pid(p, PIDTYPE_PID);
2442 nr = pid_vnr(pid);
2443
2444 if (clone_flags & CLONE_PARENT_SETTID)
2445 put_user(nr, args->parent_tid);
2446
2447 if (clone_flags & CLONE_VFORK) {
2448 p->vfork_done = &vfork;
2449 init_completion(&vfork);
2450 get_task_struct(p);
2451 }
2452
2453 wake_up_new_task(p);
2454
2455
2456 if (unlikely(trace))
2457 ptrace_event_pid(trace, pid);
2458
2459 if (clone_flags & CLONE_VFORK) {
2460 if (!wait_for_vfork_done(p, &vfork))
2461 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
2462 }
2463
2464 put_pid(pid);
2465 return nr;
2466}
2467
2468
2469
2470
2471pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
2472{
2473 struct kernel_clone_args args = {
2474 .flags = ((lower_32_bits(flags) | CLONE_VM |
2475 CLONE_UNTRACED) & ~CSIGNAL),
2476 .exit_signal = (lower_32_bits(flags) & CSIGNAL),
2477 .stack = (unsigned long)fn,
2478 .stack_size = (unsigned long)arg,
2479 };
2480
2481 return _do_fork(&args);
2482}
2483
2484#ifdef __ARCH_WANT_SYS_FORK
2485SYSCALL_DEFINE0(fork)
2486{
2487#ifdef CONFIG_MMU
2488 struct kernel_clone_args args = {
2489 .exit_signal = SIGCHLD,
2490 };
2491
2492 return _do_fork(&args);
2493#else
2494
2495 return -EINVAL;
2496#endif
2497}
2498#endif
2499
2500#ifdef __ARCH_WANT_SYS_VFORK
2501SYSCALL_DEFINE0(vfork)
2502{
2503 struct kernel_clone_args args = {
2504 .flags = CLONE_VFORK | CLONE_VM,
2505 .exit_signal = SIGCHLD,
2506 };
2507
2508 return _do_fork(&args);
2509}
2510#endif
2511
2512#ifdef __ARCH_WANT_SYS_CLONE
2513#ifdef CONFIG_CLONE_BACKWARDS
2514SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2515 int __user *, parent_tidptr,
2516 unsigned long, tls,
2517 int __user *, child_tidptr)
2518#elif defined(CONFIG_CLONE_BACKWARDS2)
2519SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
2520 int __user *, parent_tidptr,
2521 int __user *, child_tidptr,
2522 unsigned long, tls)
2523#elif defined(CONFIG_CLONE_BACKWARDS3)
2524SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
2525 int, stack_size,
2526 int __user *, parent_tidptr,
2527 int __user *, child_tidptr,
2528 unsigned long, tls)
2529#else
2530SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2531 int __user *, parent_tidptr,
2532 int __user *, child_tidptr,
2533 unsigned long, tls)
2534#endif
2535{
2536 struct kernel_clone_args args = {
2537 .flags = (lower_32_bits(clone_flags) & ~CSIGNAL),
2538 .pidfd = parent_tidptr,
2539 .child_tid = child_tidptr,
2540 .parent_tid = parent_tidptr,
2541 .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL),
2542 .stack = newsp,
2543 .tls = tls,
2544 };
2545
2546 return _do_fork(&args);
2547}
2548#endif
2549
2550#ifdef __ARCH_WANT_SYS_CLONE3
2551
2552noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs,
2553 struct clone_args __user *uargs,
2554 size_t usize)
2555{
2556 int err;
2557 struct clone_args args;
2558 pid_t *kset_tid = kargs->set_tid;
2559
2560 BUILD_BUG_ON(offsetofend(struct clone_args, tls) !=
2561 CLONE_ARGS_SIZE_VER0);
2562 BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) !=
2563 CLONE_ARGS_SIZE_VER1);
2564 BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) !=
2565 CLONE_ARGS_SIZE_VER2);
2566 BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2);
2567
2568 if (unlikely(usize > PAGE_SIZE))
2569 return -E2BIG;
2570 if (unlikely(usize < CLONE_ARGS_SIZE_VER0))
2571 return -EINVAL;
2572
2573 err = copy_struct_from_user(&args, sizeof(args), uargs, usize);
2574 if (err)
2575 return err;
2576
2577 if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))
2578 return -EINVAL;
2579
2580 if (unlikely(!args.set_tid && args.set_tid_size > 0))
2581 return -EINVAL;
2582
2583 if (unlikely(args.set_tid && args.set_tid_size == 0))
2584 return -EINVAL;
2585
2586
2587
2588
2589
2590 if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) ||
2591 !valid_signal(args.exit_signal)))
2592 return -EINVAL;
2593
2594 if ((args.flags & CLONE_INTO_CGROUP) &&
2595 (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2))
2596 return -EINVAL;
2597
2598 *kargs = (struct kernel_clone_args){
2599 .flags = args.flags,
2600 .pidfd = u64_to_user_ptr(args.pidfd),
2601 .child_tid = u64_to_user_ptr(args.child_tid),
2602 .parent_tid = u64_to_user_ptr(args.parent_tid),
2603 .exit_signal = args.exit_signal,
2604 .stack = args.stack,
2605 .stack_size = args.stack_size,
2606 .tls = args.tls,
2607 .set_tid_size = args.set_tid_size,
2608 .cgroup = args.cgroup,
2609 };
2610
2611 if (args.set_tid &&
2612 copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
2613 (kargs->set_tid_size * sizeof(pid_t))))
2614 return -EFAULT;
2615
2616 kargs->set_tid = kset_tid;
2617
2618 return 0;
2619}
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
2630{
2631 if (kargs->stack == 0) {
2632 if (kargs->stack_size > 0)
2633 return false;
2634 } else {
2635 if (kargs->stack_size == 0)
2636 return false;
2637
2638 if (!access_ok((void __user *)kargs->stack, kargs->stack_size))
2639 return false;
2640
2641#if !defined(CONFIG_STACK_GROWSUP) && !defined(CONFIG_IA64)
2642 kargs->stack += kargs->stack_size;
2643#endif
2644 }
2645
2646 return true;
2647}
2648
2649static bool clone3_args_valid(struct kernel_clone_args *kargs)
2650{
2651
2652 if (kargs->flags &
2653 ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | CLONE_INTO_CGROUP))
2654 return false;
2655
2656
2657
2658
2659
2660 if (kargs->flags & (CLONE_DETACHED | CSIGNAL))
2661 return false;
2662
2663 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) ==
2664 (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND))
2665 return false;
2666
2667 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
2668 kargs->exit_signal)
2669 return false;
2670
2671 if (!clone3_stack_valid(kargs))
2672 return false;
2673
2674 return true;
2675}
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
2689{
2690 int err;
2691
2692 struct kernel_clone_args kargs;
2693 pid_t set_tid[MAX_PID_NS_LEVEL];
2694
2695 kargs.set_tid = set_tid;
2696
2697 err = copy_clone_args_from_user(&kargs, uargs, size);
2698 if (err)
2699 return err;
2700
2701 if (!clone3_args_valid(&kargs))
2702 return -EINVAL;
2703
2704 return _do_fork(&kargs);
2705}
2706#endif
2707
2708void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
2709{
2710 struct task_struct *leader, *parent, *child;
2711 int res;
2712
2713 read_lock(&tasklist_lock);
2714 leader = top = top->group_leader;
2715down:
2716 for_each_thread(leader, parent) {
2717 list_for_each_entry(child, &parent->children, sibling) {
2718 res = visitor(child, data);
2719 if (res) {
2720 if (res < 0)
2721 goto out;
2722 leader = child;
2723 goto down;
2724 }
2725up:
2726 ;
2727 }
2728 }
2729
2730 if (leader != top) {
2731 child = leader;
2732 parent = child->real_parent;
2733 leader = parent->group_leader;
2734 goto up;
2735 }
2736out:
2737 read_unlock(&tasklist_lock);
2738}
2739
2740#ifndef ARCH_MIN_MMSTRUCT_ALIGN
2741#define ARCH_MIN_MMSTRUCT_ALIGN 0
2742#endif
2743
2744static void sighand_ctor(void *data)
2745{
2746 struct sighand_struct *sighand = data;
2747
2748 spin_lock_init(&sighand->siglock);
2749 init_waitqueue_head(&sighand->signalfd_wqh);
2750}
2751
2752void __init proc_caches_init(void)
2753{
2754 unsigned int mm_size;
2755
2756 sighand_cachep = kmem_cache_create("sighand_cache",
2757 sizeof(struct sighand_struct), 0,
2758 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
2759 SLAB_ACCOUNT, sighand_ctor);
2760 signal_cachep = kmem_cache_create("signal_cache",
2761 sizeof(struct signal_struct), 0,
2762 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
2763 NULL);
2764 files_cachep = kmem_cache_create("files_cache",
2765 sizeof(struct files_struct), 0,
2766 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
2767 NULL);
2768 fs_cachep = kmem_cache_create("fs_cache",
2769 sizeof(struct fs_struct), 0,
2770 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
2771 NULL);
2772
2773
2774
2775
2776
2777
2778 mm_size = sizeof(struct mm_struct) + cpumask_size();
2779
2780 mm_cachep = kmem_cache_create_usercopy("mm_struct",
2781 mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
2782 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
2783 offsetof(struct mm_struct, saved_auxv),
2784 sizeof_field(struct mm_struct, saved_auxv),
2785 NULL);
2786 vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT);
2787 mmap_init();
2788 nsproxy_cache_init();
2789}
2790
2791
2792
2793
2794static int check_unshare_flags(unsigned long unshare_flags)
2795{
2796 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
2797 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
2798 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
2799 CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP|
2800 CLONE_NEWTIME))
2801 return -EINVAL;
2802
2803
2804
2805
2806
2807
2808 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
2809 if (!thread_group_empty(current))
2810 return -EINVAL;
2811 }
2812 if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
2813 if (refcount_read(¤t->sighand->count) > 1)
2814 return -EINVAL;
2815 }
2816 if (unshare_flags & CLONE_VM) {
2817 if (!current_is_single_threaded())
2818 return -EINVAL;
2819 }
2820
2821 return 0;
2822}
2823
2824
2825
2826
2827static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
2828{
2829 struct fs_struct *fs = current->fs;
2830
2831 if (!(unshare_flags & CLONE_FS) || !fs)
2832 return 0;
2833
2834
2835 if (fs->users == 1)
2836 return 0;
2837
2838 *new_fsp = copy_fs_struct(fs);
2839 if (!*new_fsp)
2840 return -ENOMEM;
2841
2842 return 0;
2843}
2844
2845
2846
2847
2848int unshare_fd(unsigned long unshare_flags, unsigned int max_fds,
2849 struct files_struct **new_fdp)
2850{
2851 struct files_struct *fd = current->files;
2852 int error = 0;
2853
2854 if ((unshare_flags & CLONE_FILES) &&
2855 (fd && atomic_read(&fd->count) > 1)) {
2856 *new_fdp = dup_fd(fd, max_fds, &error);
2857 if (!*new_fdp)
2858 return error;
2859 }
2860
2861 return 0;
2862}
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872int ksys_unshare(unsigned long unshare_flags)
2873{
2874 struct fs_struct *fs, *new_fs = NULL;
2875 struct files_struct *fd, *new_fd = NULL;
2876 struct cred *new_cred = NULL;
2877 struct nsproxy *new_nsproxy = NULL;
2878 int do_sysvsem = 0;
2879 int err;
2880
2881
2882
2883
2884
2885 if (unshare_flags & CLONE_NEWUSER)
2886 unshare_flags |= CLONE_THREAD | CLONE_FS;
2887
2888
2889
2890 if (unshare_flags & CLONE_VM)
2891 unshare_flags |= CLONE_SIGHAND;
2892
2893
2894
2895 if (unshare_flags & CLONE_SIGHAND)
2896 unshare_flags |= CLONE_THREAD;
2897
2898
2899
2900 if (unshare_flags & CLONE_NEWNS)
2901 unshare_flags |= CLONE_FS;
2902
2903 err = check_unshare_flags(unshare_flags);
2904 if (err)
2905 goto bad_unshare_out;
2906
2907
2908
2909
2910
2911 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
2912 do_sysvsem = 1;
2913 err = unshare_fs(unshare_flags, &new_fs);
2914 if (err)
2915 goto bad_unshare_out;
2916 err = unshare_fd(unshare_flags, NR_OPEN_MAX, &new_fd);
2917 if (err)
2918 goto bad_unshare_cleanup_fs;
2919 err = unshare_userns(unshare_flags, &new_cred);
2920 if (err)
2921 goto bad_unshare_cleanup_fd;
2922 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
2923 new_cred, new_fs);
2924 if (err)
2925 goto bad_unshare_cleanup_cred;
2926
2927 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
2928 if (do_sysvsem) {
2929
2930
2931
2932 exit_sem(current);
2933 }
2934 if (unshare_flags & CLONE_NEWIPC) {
2935
2936 exit_shm(current);
2937 shm_init_task(current);
2938 }
2939
2940 if (new_nsproxy)
2941 switch_task_namespaces(current, new_nsproxy);
2942
2943 task_lock(current);
2944
2945 if (new_fs) {
2946 fs = current->fs;
2947 spin_lock(&fs->lock);
2948 current->fs = new_fs;
2949 if (--fs->users)
2950 new_fs = NULL;
2951 else
2952 new_fs = fs;
2953 spin_unlock(&fs->lock);
2954 }
2955
2956 if (new_fd) {
2957 fd = current->files;
2958 current->files = new_fd;
2959 new_fd = fd;
2960 }
2961
2962 task_unlock(current);
2963
2964 if (new_cred) {
2965
2966 commit_creds(new_cred);
2967 new_cred = NULL;
2968 }
2969 }
2970
2971 perf_event_namespaces(current);
2972
2973bad_unshare_cleanup_cred:
2974 if (new_cred)
2975 put_cred(new_cred);
2976bad_unshare_cleanup_fd:
2977 if (new_fd)
2978 put_files_struct(new_fd);
2979
2980bad_unshare_cleanup_fs:
2981 if (new_fs)
2982 free_fs_struct(new_fs);
2983
2984bad_unshare_out:
2985 return err;
2986}
2987
2988SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
2989{
2990 return ksys_unshare(unshare_flags);
2991}
2992
2993
2994
2995
2996
2997
2998
2999int unshare_files(struct files_struct **displaced)
3000{
3001 struct task_struct *task = current;
3002 struct files_struct *copy = NULL;
3003 int error;
3004
3005 error = unshare_fd(CLONE_FILES, NR_OPEN_MAX, ©);
3006 if (error || !copy) {
3007 *displaced = NULL;
3008 return error;
3009 }
3010 *displaced = task->files;
3011 task_lock(task);
3012 task->files = copy;
3013 task_unlock(task);
3014 return 0;
3015}
3016
3017int sysctl_max_threads(struct ctl_table *table, int write,
3018 void *buffer, size_t *lenp, loff_t *ppos)
3019{
3020 struct ctl_table t;
3021 int ret;
3022 int threads = max_threads;
3023 int min = 1;
3024 int max = MAX_THREADS;
3025
3026 t = *table;
3027 t.data = &threads;
3028 t.extra1 = &min;
3029 t.extra2 = &max;
3030
3031 ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3032 if (ret || !write)
3033 return ret;
3034
3035 max_threads = threads;
3036
3037 return 0;
3038}
3039