1
2#ifndef _LINUX_SCHED_H
3#define _LINUX_SCHED_H
4
5
6
7
8
9
10#include <uapi/linux/sched.h>
11
12#include <asm/current.h>
13
14#include <linux/pid.h>
15#include <linux/sem.h>
16#include <linux/shm.h>
17#include <linux/kcov.h>
18#include <linux/mutex.h>
19#include <linux/plist.h>
20#include <linux/hrtimer.h>
21#include <linux/seccomp.h>
22#include <linux/nodemask.h>
23#include <linux/rcupdate.h>
24#include <linux/refcount.h>
25#include <linux/resource.h>
26#include <linux/latencytop.h>
27#include <linux/sched/prio.h>
28#include <linux/signal_types.h>
29#include <linux/psi_types.h>
30#include <linux/mm_types_task.h>
31#include <linux/task_io_accounting.h>
32#include <linux/rseq.h>
33
34
35struct audit_context;
36struct backing_dev_info;
37struct bio_list;
38struct blk_plug;
39struct cfs_rq;
40struct fs_struct;
41struct futex_pi_state;
42struct io_context;
43struct mempolicy;
44struct nameidata;
45struct nsproxy;
46struct perf_event_context;
47struct pid_namespace;
48struct pipe_inode_info;
49struct rcu_node;
50struct reclaim_state;
51struct capture_control;
52struct robust_list_head;
53struct sched_attr;
54struct sched_param;
55struct seq_file;
56struct sighand_struct;
57struct signal_struct;
58struct task_delay_info;
59struct task_group;
60
61
62
63
64
65
66
67
68
69
70
71
72
73#define TASK_RUNNING 0x0000
74#define TASK_INTERRUPTIBLE 0x0001
75#define TASK_UNINTERRUPTIBLE 0x0002
76#define __TASK_STOPPED 0x0004
77#define __TASK_TRACED 0x0008
78
79#define EXIT_DEAD 0x0010
80#define EXIT_ZOMBIE 0x0020
81#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
82
83#define TASK_PARKED 0x0040
84#define TASK_DEAD 0x0080
85#define TASK_WAKEKILL 0x0100
86#define TASK_WAKING 0x0200
87#define TASK_NOLOAD 0x0400
88#define TASK_NEW 0x0800
89#define TASK_STATE_MAX 0x1000
90
91
92#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
93#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
94#define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED)
95
96#define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
97
98
99#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
100
101
102#define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
103 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
104 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
105 TASK_PARKED)
106
107#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0)
108
109#define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0)
110
111#define task_is_stopped_or_traced(task) ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
112
113#define task_contributes_to_load(task) ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
114 (task->flags & PF_FROZEN) == 0 && \
115 (task->state & TASK_NOLOAD) == 0)
116
117#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
118
119
120
121
122
123#define is_special_task_state(state) \
124 ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
125
126#define __set_current_state(state_value) \
127 do { \
128 WARN_ON_ONCE(is_special_task_state(state_value));\
129 current->task_state_change = _THIS_IP_; \
130 current->state = (state_value); \
131 } while (0)
132
133#define set_current_state(state_value) \
134 do { \
135 WARN_ON_ONCE(is_special_task_state(state_value));\
136 current->task_state_change = _THIS_IP_; \
137 smp_store_mb(current->state, (state_value)); \
138 } while (0)
139
140#define set_special_state(state_value) \
141 do { \
142 unsigned long flags; \
143 WARN_ON_ONCE(!is_special_task_state(state_value)); \
144 raw_spin_lock_irqsave(¤t->pi_lock, flags); \
145 current->task_state_change = _THIS_IP_; \
146 current->state = (state_value); \
147 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \
148 } while (0)
149#else
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187#define __set_current_state(state_value) \
188 current->state = (state_value)
189
190#define set_current_state(state_value) \
191 smp_store_mb(current->state, (state_value))
192
193
194
195
196
197
198
199#define set_special_state(state_value) \
200 do { \
201 unsigned long flags; \
202 raw_spin_lock_irqsave(¤t->pi_lock, flags); \
203 current->state = (state_value); \
204 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \
205 } while (0)
206
207#endif
208
209
210#define TASK_COMM_LEN 16
211
212extern void scheduler_tick(void);
213
214#define MAX_SCHEDULE_TIMEOUT LONG_MAX
215
216extern long schedule_timeout(long timeout);
217extern long schedule_timeout_interruptible(long timeout);
218extern long schedule_timeout_killable(long timeout);
219extern long schedule_timeout_uninterruptible(long timeout);
220extern long schedule_timeout_idle(long timeout);
221asmlinkage void schedule(void);
222extern void schedule_preempt_disabled(void);
223
224extern int __must_check io_schedule_prepare(void);
225extern void io_schedule_finish(int token);
226extern long io_schedule_timeout(long timeout);
227extern void io_schedule(void);
228
229
230
231
232
233
234
235
236
237
238struct prev_cputime {
239#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
240 u64 utime;
241 u64 stime;
242 raw_spinlock_t lock;
243#endif
244};
245
246
247
248
249
250
251
252
253
254
255
256struct task_cputime {
257 u64 utime;
258 u64 stime;
259 unsigned long long sum_exec_runtime;
260};
261
262
263#define virt_exp utime
264#define prof_exp stime
265#define sched_exp sum_exec_runtime
266
267enum vtime_state {
268
269 VTIME_INACTIVE = 0,
270
271 VTIME_USER,
272
273 VTIME_SYS,
274};
275
276struct vtime {
277 seqcount_t seqcount;
278 unsigned long long starttime;
279 enum vtime_state state;
280 u64 utime;
281 u64 stime;
282 u64 gtime;
283};
284
285struct sched_info {
286#ifdef CONFIG_SCHED_INFO
287
288
289
290 unsigned long pcount;
291
292
293 unsigned long long run_delay;
294
295
296
297
298 unsigned long long last_arrival;
299
300
301 unsigned long long last_queued;
302
303#endif
304};
305
306
307
308
309
310
311
312
313# define SCHED_FIXEDPOINT_SHIFT 10
314# define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
315
316struct load_weight {
317 unsigned long weight;
318 u32 inv_weight;
319};
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343struct util_est {
344 unsigned int enqueued;
345 unsigned int ewma;
346#define UTIL_EST_WEIGHT_SHIFT 2
347} __attribute__((__aligned__(sizeof(u64))));
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392struct sched_avg {
393 u64 last_update_time;
394 u64 load_sum;
395 u64 runnable_load_sum;
396 u32 util_sum;
397 u32 period_contrib;
398 unsigned long load_avg;
399 unsigned long runnable_load_avg;
400 unsigned long util_avg;
401 struct util_est util_est;
402} ____cacheline_aligned;
403
404struct sched_statistics {
405#ifdef CONFIG_SCHEDSTATS
406 u64 wait_start;
407 u64 wait_max;
408 u64 wait_count;
409 u64 wait_sum;
410 u64 iowait_count;
411 u64 iowait_sum;
412
413 u64 sleep_start;
414 u64 sleep_max;
415 s64 sum_sleep_runtime;
416
417 u64 block_start;
418 u64 block_max;
419 u64 exec_max;
420 u64 slice_max;
421
422 u64 nr_migrations_cold;
423 u64 nr_failed_migrations_affine;
424 u64 nr_failed_migrations_running;
425 u64 nr_failed_migrations_hot;
426 u64 nr_forced_migrations;
427
428 u64 nr_wakeups;
429 u64 nr_wakeups_sync;
430 u64 nr_wakeups_migrate;
431 u64 nr_wakeups_local;
432 u64 nr_wakeups_remote;
433 u64 nr_wakeups_affine;
434 u64 nr_wakeups_affine_attempts;
435 u64 nr_wakeups_passive;
436 u64 nr_wakeups_idle;
437#endif
438};
439
440struct sched_entity {
441
442 struct load_weight load;
443 unsigned long runnable_weight;
444 struct rb_node run_node;
445 struct list_head group_node;
446 unsigned int on_rq;
447
448 u64 exec_start;
449 u64 sum_exec_runtime;
450 u64 vruntime;
451 u64 prev_sum_exec_runtime;
452
453 u64 nr_migrations;
454
455 struct sched_statistics statistics;
456
457#ifdef CONFIG_FAIR_GROUP_SCHED
458 int depth;
459 struct sched_entity *parent;
460
461 struct cfs_rq *cfs_rq;
462
463 struct cfs_rq *my_q;
464#endif
465
466#ifdef CONFIG_SMP
467
468
469
470
471
472
473 struct sched_avg avg;
474#endif
475};
476
477struct sched_rt_entity {
478 struct list_head run_list;
479 unsigned long timeout;
480 unsigned long watchdog_stamp;
481 unsigned int time_slice;
482 unsigned short on_rq;
483 unsigned short on_list;
484
485 struct sched_rt_entity *back;
486#ifdef CONFIG_RT_GROUP_SCHED
487 struct sched_rt_entity *parent;
488
489 struct rt_rq *rt_rq;
490
491 struct rt_rq *my_q;
492#endif
493} __randomize_layout;
494
495struct sched_dl_entity {
496 struct rb_node rb_node;
497
498
499
500
501
502
503 u64 dl_runtime;
504 u64 dl_deadline;
505 u64 dl_period;
506 u64 dl_bw;
507 u64 dl_density;
508
509
510
511
512
513
514 s64 runtime;
515 u64 deadline;
516 unsigned int flags;
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542 unsigned int dl_throttled : 1;
543 unsigned int dl_boosted : 1;
544 unsigned int dl_yielded : 1;
545 unsigned int dl_non_contending : 1;
546 unsigned int dl_overrun : 1;
547
548
549
550
551
552 struct hrtimer dl_timer;
553
554
555
556
557
558
559
560
561 struct hrtimer inactive_timer;
562};
563
564union rcu_special {
565 struct {
566 u8 blocked;
567 u8 need_qs;
568 u8 exp_hint;
569 u8 pad;
570 } b;
571 u32 s;
572};
573
574enum perf_event_task_context {
575 perf_invalid_context = -1,
576 perf_hw_context = 0,
577 perf_sw_context,
578 perf_nr_task_contexts,
579};
580
581struct wake_q_node {
582 struct wake_q_node *next;
583};
584
585struct task_struct {
586#ifdef CONFIG_THREAD_INFO_IN_TASK
587
588
589
590
591 struct thread_info thread_info;
592#endif
593
594 volatile long state;
595
596
597
598
599
600 randomized_struct_fields_start
601
602 void *stack;
603 refcount_t usage;
604
605 unsigned int flags;
606 unsigned int ptrace;
607
608#ifdef CONFIG_SMP
609 struct llist_node wake_entry;
610 int on_cpu;
611#ifdef CONFIG_THREAD_INFO_IN_TASK
612
613 unsigned int cpu;
614#endif
615 unsigned int wakee_flips;
616 unsigned long wakee_flip_decay_ts;
617 struct task_struct *last_wakee;
618
619
620
621
622
623
624
625
626 int recent_used_cpu;
627 int wake_cpu;
628#endif
629 int on_rq;
630
631 int prio;
632 int static_prio;
633 int normal_prio;
634 unsigned int rt_priority;
635
636 const struct sched_class *sched_class;
637 struct sched_entity se;
638 struct sched_rt_entity rt;
639#ifdef CONFIG_CGROUP_SCHED
640 struct task_group *sched_task_group;
641#endif
642 struct sched_dl_entity dl;
643
644#ifdef CONFIG_PREEMPT_NOTIFIERS
645
646 struct hlist_head preempt_notifiers;
647#endif
648
649#ifdef CONFIG_BLK_DEV_IO_TRACE
650 unsigned int btrace_seq;
651#endif
652
653 unsigned int policy;
654 int nr_cpus_allowed;
655 cpumask_t cpus_allowed;
656
657#ifdef CONFIG_PREEMPT_RCU
658 int rcu_read_lock_nesting;
659 union rcu_special rcu_read_unlock_special;
660 struct list_head rcu_node_entry;
661 struct rcu_node *rcu_blocked_node;
662#endif
663
664#ifdef CONFIG_TASKS_RCU
665 unsigned long rcu_tasks_nvcsw;
666 u8 rcu_tasks_holdout;
667 u8 rcu_tasks_idx;
668 int rcu_tasks_idle_cpu;
669 struct list_head rcu_tasks_holdout_list;
670#endif
671
672 struct sched_info sched_info;
673
674 struct list_head tasks;
675#ifdef CONFIG_SMP
676 struct plist_node pushable_tasks;
677 struct rb_node pushable_dl_tasks;
678#endif
679
680 struct mm_struct *mm;
681 struct mm_struct *active_mm;
682
683
684 struct vmacache vmacache;
685
686#ifdef SPLIT_RSS_COUNTING
687 struct task_rss_stat rss_stat;
688#endif
689 int exit_state;
690 int exit_code;
691 int exit_signal;
692
693 int pdeath_signal;
694
695 unsigned long jobctl;
696
697
698 unsigned int personality;
699
700
701 unsigned sched_reset_on_fork:1;
702 unsigned sched_contributes_to_load:1;
703 unsigned sched_migrated:1;
704 unsigned sched_remote_wakeup:1;
705#ifdef CONFIG_PSI
706 unsigned sched_psi_wake_requeue:1;
707#endif
708
709
710 unsigned :0;
711
712
713
714
715 unsigned in_execve:1;
716 unsigned in_iowait:1;
717#ifndef TIF_RESTORE_SIGMASK
718 unsigned restore_sigmask:1;
719#endif
720#ifdef CONFIG_MEMCG
721 unsigned in_user_fault:1;
722#endif
723#ifdef CONFIG_COMPAT_BRK
724 unsigned brk_randomized:1;
725#endif
726#ifdef CONFIG_CGROUPS
727
728 unsigned no_cgroup_migration:1;
729#endif
730#ifdef CONFIG_BLK_CGROUP
731
732 unsigned use_memdelay:1;
733#endif
734
735 unsigned long atomic_flags;
736
737 struct restart_block restart_block;
738
739 pid_t pid;
740 pid_t tgid;
741
742#ifdef CONFIG_STACKPROTECTOR
743
744 unsigned long stack_canary;
745#endif
746
747
748
749
750
751
752
753 struct task_struct __rcu *real_parent;
754
755
756 struct task_struct __rcu *parent;
757
758
759
760
761 struct list_head children;
762 struct list_head sibling;
763 struct task_struct *group_leader;
764
765
766
767
768
769
770
771 struct list_head ptraced;
772 struct list_head ptrace_entry;
773
774
775 struct pid *thread_pid;
776 struct hlist_node pid_links[PIDTYPE_MAX];
777 struct list_head thread_group;
778 struct list_head thread_node;
779
780 struct completion *vfork_done;
781
782
783 int __user *set_child_tid;
784
785
786 int __user *clear_child_tid;
787
788 u64 utime;
789 u64 stime;
790#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
791 u64 utimescaled;
792 u64 stimescaled;
793#endif
794 u64 gtime;
795 struct prev_cputime prev_cputime;
796#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
797 struct vtime vtime;
798#endif
799
800#ifdef CONFIG_NO_HZ_FULL
801 atomic_t tick_dep_mask;
802#endif
803
804 unsigned long nvcsw;
805 unsigned long nivcsw;
806
807
808 u64 start_time;
809
810
811 u64 real_start_time;
812
813
814 unsigned long min_flt;
815 unsigned long maj_flt;
816
817#ifdef CONFIG_POSIX_TIMERS
818 struct task_cputime cputime_expires;
819 struct list_head cpu_timers[3];
820#endif
821
822
823
824
825 const struct cred __rcu *ptracer_cred;
826
827
828 const struct cred __rcu *real_cred;
829
830
831 const struct cred __rcu *cred;
832
833
834
835
836
837
838
839
840 char comm[TASK_COMM_LEN];
841
842 struct nameidata *nameidata;
843
844#ifdef CONFIG_SYSVIPC
845 struct sysv_sem sysvsem;
846 struct sysv_shm sysvshm;
847#endif
848#ifdef CONFIG_DETECT_HUNG_TASK
849 unsigned long last_switch_count;
850 unsigned long last_switch_time;
851#endif
852
853 struct fs_struct *fs;
854
855
856 struct files_struct *files;
857
858
859 struct nsproxy *nsproxy;
860
861
862 struct signal_struct *signal;
863 struct sighand_struct *sighand;
864 sigset_t blocked;
865 sigset_t real_blocked;
866
867 sigset_t saved_sigmask;
868 struct sigpending pending;
869 unsigned long sas_ss_sp;
870 size_t sas_ss_size;
871 unsigned int sas_ss_flags;
872
873 struct callback_head *task_works;
874
875#ifdef CONFIG_AUDIT
876#ifdef CONFIG_AUDITSYSCALL
877 struct audit_context *audit_context;
878#endif
879 kuid_t loginuid;
880 unsigned int sessionid;
881#endif
882 struct seccomp seccomp;
883
884
885 u32 parent_exec_id;
886 u32 self_exec_id;
887
888
889 spinlock_t alloc_lock;
890
891
892 raw_spinlock_t pi_lock;
893
894 struct wake_q_node wake_q;
895
896#ifdef CONFIG_RT_MUTEXES
897
898 struct rb_root_cached pi_waiters;
899
900 struct task_struct *pi_top_task;
901
902 struct rt_mutex_waiter *pi_blocked_on;
903#endif
904
905#ifdef CONFIG_DEBUG_MUTEXES
906
907 struct mutex_waiter *blocked_on;
908#endif
909
910#ifdef CONFIG_TRACE_IRQFLAGS
911 unsigned int irq_events;
912 unsigned long hardirq_enable_ip;
913 unsigned long hardirq_disable_ip;
914 unsigned int hardirq_enable_event;
915 unsigned int hardirq_disable_event;
916 int hardirqs_enabled;
917 int hardirq_context;
918 unsigned long softirq_disable_ip;
919 unsigned long softirq_enable_ip;
920 unsigned int softirq_disable_event;
921 unsigned int softirq_enable_event;
922 int softirqs_enabled;
923 int softirq_context;
924#endif
925
926#ifdef CONFIG_LOCKDEP
927# define MAX_LOCK_DEPTH 48UL
928 u64 curr_chain_key;
929 int lockdep_depth;
930 unsigned int lockdep_recursion;
931 struct held_lock held_locks[MAX_LOCK_DEPTH];
932#endif
933
934#ifdef CONFIG_UBSAN
935 unsigned int in_ubsan;
936#endif
937
938
939 void *journal_info;
940
941
942 struct bio_list *bio_list;
943
944#ifdef CONFIG_BLOCK
945
946 struct blk_plug *plug;
947#endif
948
949
950 struct reclaim_state *reclaim_state;
951
952 struct backing_dev_info *backing_dev_info;
953
954 struct io_context *io_context;
955
956#ifdef CONFIG_COMPACTION
957 struct capture_control *capture_control;
958#endif
959
960 unsigned long ptrace_message;
961 kernel_siginfo_t *last_siginfo;
962
963 struct task_io_accounting ioac;
964#ifdef CONFIG_PSI
965
966 unsigned int psi_flags;
967#endif
968#ifdef CONFIG_TASK_XACCT
969
970 u64 acct_rss_mem1;
971
972 u64 acct_vm_mem1;
973
974 u64 acct_timexpd;
975#endif
976#ifdef CONFIG_CPUSETS
977
978 nodemask_t mems_allowed;
979
980 seqcount_t mems_allowed_seq;
981 int cpuset_mem_spread_rotor;
982 int cpuset_slab_spread_rotor;
983#endif
984#ifdef CONFIG_CGROUPS
985
986 struct css_set __rcu *cgroups;
987
988 struct list_head cg_list;
989#endif
990#ifdef CONFIG_X86_CPU_RESCTRL
991 u32 closid;
992 u32 rmid;
993#endif
994#ifdef CONFIG_FUTEX
995 struct robust_list_head __user *robust_list;
996#ifdef CONFIG_COMPAT
997 struct compat_robust_list_head __user *compat_robust_list;
998#endif
999 struct list_head pi_state_list;
1000 struct futex_pi_state *pi_state_cache;
1001#endif
1002#ifdef CONFIG_PERF_EVENTS
1003 struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
1004 struct mutex perf_event_mutex;
1005 struct list_head perf_event_list;
1006#endif
1007#ifdef CONFIG_DEBUG_PREEMPT
1008 unsigned long preempt_disable_ip;
1009#endif
1010#ifdef CONFIG_NUMA
1011
1012 struct mempolicy *mempolicy;
1013 short il_prev;
1014 short pref_node_fork;
1015#endif
1016#ifdef CONFIG_NUMA_BALANCING
1017 int numa_scan_seq;
1018 unsigned int numa_scan_period;
1019 unsigned int numa_scan_period_max;
1020 int numa_preferred_nid;
1021 unsigned long numa_migrate_retry;
1022
1023 u64 node_stamp;
1024 u64 last_task_numa_placement;
1025 u64 last_sum_exec_runtime;
1026 struct callback_head numa_work;
1027
1028 struct numa_group *numa_group;
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044 unsigned long *numa_faults;
1045 unsigned long total_numa_faults;
1046
1047
1048
1049
1050
1051
1052
1053 unsigned long numa_faults_locality[3];
1054
1055 unsigned long numa_pages_migrated;
1056#endif
1057
1058#ifdef CONFIG_RSEQ
1059 struct rseq __user *rseq;
1060 u32 rseq_len;
1061 u32 rseq_sig;
1062
1063
1064
1065
1066 unsigned long rseq_event_mask;
1067#endif
1068
1069 struct tlbflush_unmap_batch tlb_ubc;
1070
1071 struct rcu_head rcu;
1072
1073
1074 struct pipe_inode_info *splice_pipe;
1075
1076 struct page_frag task_frag;
1077
1078#ifdef CONFIG_TASK_DELAY_ACCT
1079 struct task_delay_info *delays;
1080#endif
1081
1082#ifdef CONFIG_FAULT_INJECTION
1083 int make_it_fail;
1084 unsigned int fail_nth;
1085#endif
1086
1087
1088
1089
1090 int nr_dirtied;
1091 int nr_dirtied_pause;
1092
1093 unsigned long dirty_paused_when;
1094
1095#ifdef CONFIG_LATENCYTOP
1096 int latency_record_count;
1097 struct latency_record latency_record[LT_SAVECOUNT];
1098#endif
1099
1100
1101
1102
1103 u64 timer_slack_ns;
1104 u64 default_timer_slack_ns;
1105
1106#ifdef CONFIG_KASAN
1107 unsigned int kasan_depth;
1108#endif
1109
1110#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1111
1112 int curr_ret_stack;
1113 int curr_ret_depth;
1114
1115
1116 struct ftrace_ret_stack *ret_stack;
1117
1118
1119 unsigned long long ftrace_timestamp;
1120
1121
1122
1123
1124
1125 atomic_t trace_overrun;
1126
1127
1128 atomic_t tracing_graph_pause;
1129#endif
1130
1131#ifdef CONFIG_TRACING
1132
1133 unsigned long trace;
1134
1135
1136 unsigned long trace_recursion;
1137#endif
1138
1139#ifdef CONFIG_KCOV
1140
1141 unsigned int kcov_mode;
1142
1143
1144 unsigned int kcov_size;
1145
1146
1147 void *kcov_area;
1148
1149
1150 struct kcov *kcov;
1151#endif
1152
1153#ifdef CONFIG_MEMCG
1154 struct mem_cgroup *memcg_in_oom;
1155 gfp_t memcg_oom_gfp_mask;
1156 int memcg_oom_order;
1157
1158
1159 unsigned int memcg_nr_pages_over_high;
1160
1161
1162 struct mem_cgroup *active_memcg;
1163#endif
1164
1165#ifdef CONFIG_BLK_CGROUP
1166 struct request_queue *throttle_queue;
1167#endif
1168
1169#ifdef CONFIG_UPROBES
1170 struct uprobe_task *utask;
1171#endif
1172#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1173 unsigned int sequential_io;
1174 unsigned int sequential_io_avg;
1175#endif
1176#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1177 unsigned long task_state_change;
1178#endif
1179 int pagefault_disabled;
1180#ifdef CONFIG_MMU
1181 struct task_struct *oom_reaper_list;
1182#endif
1183#ifdef CONFIG_VMAP_STACK
1184 struct vm_struct *stack_vm_area;
1185#endif
1186#ifdef CONFIG_THREAD_INFO_IN_TASK
1187
1188 refcount_t stack_refcount;
1189#endif
1190#ifdef CONFIG_LIVEPATCH
1191 int patch_state;
1192#endif
1193#ifdef CONFIG_SECURITY
1194
1195 void *security;
1196#endif
1197
1198#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1199 unsigned long lowest_stack;
1200 unsigned long prev_lowest_stack;
1201#endif
1202
1203
1204
1205
1206
1207 randomized_struct_fields_end
1208
1209
1210 struct thread_struct thread;
1211
1212
1213
1214
1215
1216
1217
1218};
1219
1220static inline struct pid *task_pid(struct task_struct *task)
1221{
1222 return task->thread_pid;
1223}
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
1237
1238static inline pid_t task_pid_nr(struct task_struct *tsk)
1239{
1240 return tsk->pid;
1241}
1242
1243static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1244{
1245 return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
1246}
1247
1248static inline pid_t task_pid_vnr(struct task_struct *tsk)
1249{
1250 return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
1251}
1252
1253
1254static inline pid_t task_tgid_nr(struct task_struct *tsk)
1255{
1256 return tsk->tgid;
1257}
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269static inline int pid_alive(const struct task_struct *p)
1270{
1271 return p->thread_pid != NULL;
1272}
1273
1274static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1275{
1276 return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
1277}
1278
1279static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
1280{
1281 return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
1282}
1283
1284
1285static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1286{
1287 return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
1288}
1289
1290static inline pid_t task_session_vnr(struct task_struct *tsk)
1291{
1292 return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
1293}
1294
1295static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1296{
1297 return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
1298}
1299
1300static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1301{
1302 return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
1303}
1304
1305static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1306{
1307 pid_t pid = 0;
1308
1309 rcu_read_lock();
1310 if (pid_alive(tsk))
1311 pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1312 rcu_read_unlock();
1313
1314 return pid;
1315}
1316
1317static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1318{
1319 return task_ppid_nr_ns(tsk, &init_pid_ns);
1320}
1321
1322
1323static inline pid_t task_pgrp_nr(struct task_struct *tsk)
1324{
1325 return task_pgrp_nr_ns(tsk, &init_pid_ns);
1326}
1327
1328#define TASK_REPORT_IDLE (TASK_REPORT + 1)
1329#define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1)
1330
1331static inline unsigned int task_state_index(struct task_struct *tsk)
1332{
1333 unsigned int tsk_state = READ_ONCE(tsk->state);
1334 unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT;
1335
1336 BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
1337
1338 if (tsk_state == TASK_IDLE)
1339 state = TASK_REPORT_IDLE;
1340
1341 return fls(state);
1342}
1343
1344static inline char task_index_to_char(unsigned int state)
1345{
1346 static const char state_char[] = "RSDTtXZPI";
1347
1348 BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
1349
1350 return state_char[state];
1351}
1352
1353static inline char task_state_to_char(struct task_struct *tsk)
1354{
1355 return task_index_to_char(task_state_index(tsk));
1356}
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367static inline int is_global_init(struct task_struct *tsk)
1368{
1369 return task_tgid_nr(tsk) == 1;
1370}
1371
1372extern struct pid *cad_pid;
1373
1374
1375
1376
1377#define PF_IDLE 0x00000002
1378#define PF_EXITING 0x00000004
1379#define PF_EXITPIDONE 0x00000008
1380#define PF_VCPU 0x00000010
1381#define PF_WQ_WORKER 0x00000020
1382#define PF_FORKNOEXEC 0x00000040
1383#define PF_MCE_PROCESS 0x00000080
1384#define PF_SUPERPRIV 0x00000100
1385#define PF_DUMPCORE 0x00000200
1386#define PF_SIGNALED 0x00000400
1387#define PF_MEMALLOC 0x00000800
1388#define PF_NPROC_EXCEEDED 0x00001000
1389#define PF_USED_MATH 0x00002000
1390#define PF_USED_ASYNC 0x00004000
1391#define PF_NOFREEZE 0x00008000
1392#define PF_FROZEN 0x00010000
1393#define PF_KSWAPD 0x00020000
1394#define PF_MEMALLOC_NOFS 0x00040000
1395#define PF_MEMALLOC_NOIO 0x00080000
1396#define PF_LESS_THROTTLE 0x00100000
1397#define PF_KTHREAD 0x00200000
1398#define PF_RANDOMIZE 0x00400000
1399#define PF_SWAPWRITE 0x00800000
1400#define PF_MEMSTALL 0x01000000
1401#define PF_UMH 0x02000000
1402#define PF_NO_SETAFFINITY 0x04000000
1403#define PF_MCE_EARLY 0x08000000
1404#define PF_MEMALLOC_NOCMA 0x10000000
1405#define PF_FREEZER_SKIP 0x40000000
1406#define PF_SUSPEND_TASK 0x80000000
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419#define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
1420#define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
1421#define clear_used_math() clear_stopped_child_used_math(current)
1422#define set_used_math() set_stopped_child_used_math(current)
1423
1424#define conditional_stopped_child_used_math(condition, child) \
1425 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
1426
1427#define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current)
1428
1429#define copy_to_stopped_child_used_math(child) \
1430 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
1431
1432
1433#define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
1434#define used_math() tsk_used_math(current)
1435
1436static inline bool is_percpu_thread(void)
1437{
1438#ifdef CONFIG_SMP
1439 return (current->flags & PF_NO_SETAFFINITY) &&
1440 (current->nr_cpus_allowed == 1);
1441#else
1442 return true;
1443#endif
1444}
1445
1446
1447#define PFA_NO_NEW_PRIVS 0
1448#define PFA_SPREAD_PAGE 1
1449#define PFA_SPREAD_SLAB 2
1450#define PFA_SPEC_SSB_DISABLE 3
1451#define PFA_SPEC_SSB_FORCE_DISABLE 4
1452#define PFA_SPEC_IB_DISABLE 5
1453#define PFA_SPEC_IB_FORCE_DISABLE 6
1454#define PFA_SPEC_SSB_NOEXEC 7
1455
1456#define TASK_PFA_TEST(name, func) \
1457 static inline bool task_##func(struct task_struct *p) \
1458 { return test_bit(PFA_##name, &p->atomic_flags); }
1459
1460#define TASK_PFA_SET(name, func) \
1461 static inline void task_set_##func(struct task_struct *p) \
1462 { set_bit(PFA_##name, &p->atomic_flags); }
1463
1464#define TASK_PFA_CLEAR(name, func) \
1465 static inline void task_clear_##func(struct task_struct *p) \
1466 { clear_bit(PFA_##name, &p->atomic_flags); }
1467
1468TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
1469TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
1470
1471TASK_PFA_TEST(SPREAD_PAGE, spread_page)
1472TASK_PFA_SET(SPREAD_PAGE, spread_page)
1473TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
1474
1475TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
1476TASK_PFA_SET(SPREAD_SLAB, spread_slab)
1477TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1478
1479TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
1480TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
1481TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
1482
1483TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1484TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1485TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1486
1487TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1488TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1489
1490TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
1491TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
1492TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
1493
1494TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1495TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1496
1497static inline void
1498current_restore_flags(unsigned long orig_flags, unsigned long flags)
1499{
1500 current->flags &= ~flags;
1501 current->flags |= orig_flags & flags;
1502}
1503
1504extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
1505extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
1506#ifdef CONFIG_SMP
1507extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
1508extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
1509#else
1510static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1511{
1512}
1513static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1514{
1515 if (!cpumask_test_cpu(0, new_mask))
1516 return -EINVAL;
1517 return 0;
1518}
1519#endif
1520
1521#ifndef cpu_relax_yield
1522#define cpu_relax_yield() cpu_relax()
1523#endif
1524
1525extern int yield_to(struct task_struct *p, bool preempt);
1526extern void set_user_nice(struct task_struct *p, long nice);
1527extern int task_prio(const struct task_struct *p);
1528
1529
1530
1531
1532
1533
1534
1535static inline int task_nice(const struct task_struct *p)
1536{
1537 return PRIO_TO_NICE((p)->static_prio);
1538}
1539
1540extern int can_nice(const struct task_struct *p, const int nice);
1541extern int task_curr(const struct task_struct *p);
1542extern int idle_cpu(int cpu);
1543extern int available_idle_cpu(int cpu);
1544extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
1545extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
1546extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1547extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
1548extern struct task_struct *idle_task(int cpu);
1549
1550
1551
1552
1553
1554
1555
1556static inline bool is_idle_task(const struct task_struct *p)
1557{
1558 return !!(p->flags & PF_IDLE);
1559}
1560
1561extern struct task_struct *curr_task(int cpu);
1562extern void ia64_set_curr_task(int cpu, struct task_struct *p);
1563
1564void yield(void);
1565
1566union thread_union {
1567#ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
1568 struct task_struct task;
1569#endif
1570#ifndef CONFIG_THREAD_INFO_IN_TASK
1571 struct thread_info thread_info;
1572#endif
1573 unsigned long stack[THREAD_SIZE/sizeof(long)];
1574};
1575
1576#ifndef CONFIG_THREAD_INFO_IN_TASK
1577extern struct thread_info init_thread_info;
1578#endif
1579
1580extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
1581
1582#ifdef CONFIG_THREAD_INFO_IN_TASK
1583static inline struct thread_info *task_thread_info(struct task_struct *task)
1584{
1585 return &task->thread_info;
1586}
1587#elif !defined(__HAVE_THREAD_FUNCTIONS)
1588# define task_thread_info(task) ((struct thread_info *)(task)->stack)
1589#endif
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602extern struct task_struct *find_task_by_vpid(pid_t nr);
1603extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1604
1605
1606
1607
1608extern struct task_struct *find_get_task_by_vpid(pid_t nr);
1609
1610extern int wake_up_state(struct task_struct *tsk, unsigned int state);
1611extern int wake_up_process(struct task_struct *tsk);
1612extern void wake_up_new_task(struct task_struct *tsk);
1613
1614#ifdef CONFIG_SMP
1615extern void kick_process(struct task_struct *tsk);
1616#else
1617static inline void kick_process(struct task_struct *tsk) { }
1618#endif
1619
1620extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
1621
1622static inline void set_task_comm(struct task_struct *tsk, const char *from)
1623{
1624 __set_task_comm(tsk, from, false);
1625}
1626
1627extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
1628#define get_task_comm(buf, tsk) ({ \
1629 BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN); \
1630 __get_task_comm(buf, sizeof(buf), tsk); \
1631})
1632
1633#ifdef CONFIG_SMP
1634void scheduler_ipi(void);
1635extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
1636#else
1637static inline void scheduler_ipi(void) { }
1638static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1639{
1640 return 1;
1641}
1642#endif
1643
1644
1645
1646
1647
1648static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
1649{
1650 set_ti_thread_flag(task_thread_info(tsk), flag);
1651}
1652
1653static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1654{
1655 clear_ti_thread_flag(task_thread_info(tsk), flag);
1656}
1657
1658static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
1659 bool value)
1660{
1661 update_ti_thread_flag(task_thread_info(tsk), flag, value);
1662}
1663
1664static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
1665{
1666 return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
1667}
1668
1669static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1670{
1671 return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
1672}
1673
1674static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
1675{
1676 return test_ti_thread_flag(task_thread_info(tsk), flag);
1677}
1678
1679static inline void set_tsk_need_resched(struct task_struct *tsk)
1680{
1681 set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
1682}
1683
1684static inline void clear_tsk_need_resched(struct task_struct *tsk)
1685{
1686 clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
1687}
1688
1689static inline int test_tsk_need_resched(struct task_struct *tsk)
1690{
1691 return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
1692}
1693
1694
1695
1696
1697
1698
1699
1700#ifndef CONFIG_PREEMPT
1701extern int _cond_resched(void);
1702#else
1703static inline int _cond_resched(void) { return 0; }
1704#endif
1705
1706#define cond_resched() ({ \
1707 ___might_sleep(__FILE__, __LINE__, 0); \
1708 _cond_resched(); \
1709})
1710
1711extern int __cond_resched_lock(spinlock_t *lock);
1712
1713#define cond_resched_lock(lock) ({ \
1714 ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1715 __cond_resched_lock(lock); \
1716})
1717
1718static inline void cond_resched_rcu(void)
1719{
1720#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
1721 rcu_read_unlock();
1722 cond_resched();
1723 rcu_read_lock();
1724#endif
1725}
1726
1727
1728
1729
1730
1731
1732static inline int spin_needbreak(spinlock_t *lock)
1733{
1734#ifdef CONFIG_PREEMPT
1735 return spin_is_contended(lock);
1736#else
1737 return 0;
1738#endif
1739}
1740
1741static __always_inline bool need_resched(void)
1742{
1743 return unlikely(tif_need_resched());
1744}
1745
1746
1747
1748
1749#ifdef CONFIG_SMP
1750
1751static inline unsigned int task_cpu(const struct task_struct *p)
1752{
1753#ifdef CONFIG_THREAD_INFO_IN_TASK
1754 return READ_ONCE(p->cpu);
1755#else
1756 return READ_ONCE(task_thread_info(p)->cpu);
1757#endif
1758}
1759
1760extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
1761
1762#else
1763
1764static inline unsigned int task_cpu(const struct task_struct *p)
1765{
1766 return 0;
1767}
1768
1769static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
1770{
1771}
1772
1773#endif
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783#ifndef vcpu_is_preempted
1784# define vcpu_is_preempted(cpu) false
1785#endif
1786
1787extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
1788extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
1789
1790#ifndef TASK_SIZE_OF
1791#define TASK_SIZE_OF(tsk) TASK_SIZE
1792#endif
1793
1794#ifdef CONFIG_RSEQ
1795
1796
1797
1798
1799
1800enum rseq_event_mask_bits {
1801 RSEQ_EVENT_PREEMPT_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
1802 RSEQ_EVENT_SIGNAL_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
1803 RSEQ_EVENT_MIGRATE_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
1804};
1805
1806enum rseq_event_mask {
1807 RSEQ_EVENT_PREEMPT = (1U << RSEQ_EVENT_PREEMPT_BIT),
1808 RSEQ_EVENT_SIGNAL = (1U << RSEQ_EVENT_SIGNAL_BIT),
1809 RSEQ_EVENT_MIGRATE = (1U << RSEQ_EVENT_MIGRATE_BIT),
1810};
1811
1812static inline void rseq_set_notify_resume(struct task_struct *t)
1813{
1814 if (t->rseq)
1815 set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
1816}
1817
1818void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
1819
1820static inline void rseq_handle_notify_resume(struct ksignal *ksig,
1821 struct pt_regs *regs)
1822{
1823 if (current->rseq)
1824 __rseq_handle_notify_resume(ksig, regs);
1825}
1826
1827static inline void rseq_signal_deliver(struct ksignal *ksig,
1828 struct pt_regs *regs)
1829{
1830 preempt_disable();
1831 __set_bit(RSEQ_EVENT_SIGNAL_BIT, ¤t->rseq_event_mask);
1832 preempt_enable();
1833 rseq_handle_notify_resume(ksig, regs);
1834}
1835
1836
1837static inline void rseq_preempt(struct task_struct *t)
1838{
1839 __set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
1840 rseq_set_notify_resume(t);
1841}
1842
1843
1844static inline void rseq_migrate(struct task_struct *t)
1845{
1846 __set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
1847 rseq_set_notify_resume(t);
1848}
1849
1850
1851
1852
1853
1854static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
1855{
1856 if (clone_flags & CLONE_THREAD) {
1857 t->rseq = NULL;
1858 t->rseq_len = 0;
1859 t->rseq_sig = 0;
1860 t->rseq_event_mask = 0;
1861 } else {
1862 t->rseq = current->rseq;
1863 t->rseq_len = current->rseq_len;
1864 t->rseq_sig = current->rseq_sig;
1865 t->rseq_event_mask = current->rseq_event_mask;
1866 }
1867}
1868
1869static inline void rseq_execve(struct task_struct *t)
1870{
1871 t->rseq = NULL;
1872 t->rseq_len = 0;
1873 t->rseq_sig = 0;
1874 t->rseq_event_mask = 0;
1875}
1876
1877#else
1878
1879static inline void rseq_set_notify_resume(struct task_struct *t)
1880{
1881}
1882static inline void rseq_handle_notify_resume(struct ksignal *ksig,
1883 struct pt_regs *regs)
1884{
1885}
1886static inline void rseq_signal_deliver(struct ksignal *ksig,
1887 struct pt_regs *regs)
1888{
1889}
1890static inline void rseq_preempt(struct task_struct *t)
1891{
1892}
1893static inline void rseq_migrate(struct task_struct *t)
1894{
1895}
1896static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
1897{
1898}
1899static inline void rseq_execve(struct task_struct *t)
1900{
1901}
1902
1903#endif
1904
1905void __exit_umh(struct task_struct *tsk);
1906
1907static inline void exit_umh(struct task_struct *tsk)
1908{
1909 if (unlikely(tsk->flags & PF_UMH))
1910 __exit_umh(tsk);
1911}
1912
1913#ifdef CONFIG_DEBUG_RSEQ
1914
1915void rseq_syscall(struct pt_regs *regs);
1916
1917#else
1918
1919static inline void rseq_syscall(struct pt_regs *regs)
1920{
1921}
1922
1923#endif
1924
1925#endif
1926