1
2#ifndef _LINUX_SCHED_H
3#define _LINUX_SCHED_H
4
5
6
7
8
9
10#include <uapi/linux/sched.h>
11
12#include <asm/current.h>
13
14#include <linux/pid.h>
15#include <linux/sem.h>
16#include <linux/shm.h>
17#include <linux/kcov.h>
18#include <linux/mutex.h>
19#include <linux/plist.h>
20#include <linux/hrtimer.h>
21#include <linux/seccomp.h>
22#include <linux/nodemask.h>
23#include <linux/rcupdate.h>
24#include <linux/resource.h>
25#include <linux/latencytop.h>
26#include <linux/sched/prio.h>
27#include <linux/signal_types.h>
28#include <linux/mm_types_task.h>
29#include <linux/task_io_accounting.h>
30
31
32struct audit_context;
33struct backing_dev_info;
34struct bio_list;
35struct blk_plug;
36struct cfs_rq;
37struct fs_struct;
38struct futex_pi_state;
39struct io_context;
40struct mempolicy;
41struct nameidata;
42struct nsproxy;
43struct perf_event_context;
44struct pid_namespace;
45struct pipe_inode_info;
46struct rcu_node;
47struct reclaim_state;
48struct robust_list_head;
49struct sched_attr;
50struct sched_param;
51struct seq_file;
52struct sighand_struct;
53struct signal_struct;
54struct task_delay_info;
55struct task_group;
56
57
58
59
60
61
62
63
64
65
66
67
68
69#define TASK_RUNNING 0x0000
70#define TASK_INTERRUPTIBLE 0x0001
71#define TASK_UNINTERRUPTIBLE 0x0002
72#define __TASK_STOPPED 0x0004
73#define __TASK_TRACED 0x0008
74
75#define EXIT_DEAD 0x0010
76#define EXIT_ZOMBIE 0x0020
77#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
78
79#define TASK_PARKED 0x0040
80#define TASK_DEAD 0x0080
81#define TASK_WAKEKILL 0x0100
82#define TASK_WAKING 0x0200
83#define TASK_NOLOAD 0x0400
84#define TASK_NEW 0x0800
85#define TASK_STATE_MAX 0x1000
86
87
88#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
89#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
90#define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED)
91
92#define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
93
94
95#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
96
97
98#define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
99 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
100 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
101 TASK_PARKED)
102
103#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0)
104
105#define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0)
106
107#define task_is_stopped_or_traced(task) ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
108
109#define task_contributes_to_load(task) ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
110 (task->flags & PF_FROZEN) == 0 && \
111 (task->state & TASK_NOLOAD) == 0)
112
113#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
114
115
116
117
118
119#define is_special_task_state(state) \
120 ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_DEAD))
121
122#define __set_current_state(state_value) \
123 do { \
124 WARN_ON_ONCE(is_special_task_state(state_value));\
125 current->task_state_change = _THIS_IP_; \
126 current->state = (state_value); \
127 } while (0)
128
129#define set_current_state(state_value) \
130 do { \
131 WARN_ON_ONCE(is_special_task_state(state_value));\
132 current->task_state_change = _THIS_IP_; \
133 smp_store_mb(current->state, (state_value)); \
134 } while (0)
135
136#define set_special_state(state_value) \
137 do { \
138 unsigned long flags; \
139 WARN_ON_ONCE(!is_special_task_state(state_value)); \
140 raw_spin_lock_irqsave(¤t->pi_lock, flags); \
141 current->task_state_change = _THIS_IP_; \
142 current->state = (state_value); \
143 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \
144 } while (0)
145#else
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183#define __set_current_state(state_value) \
184 current->state = (state_value)
185
186#define set_current_state(state_value) \
187 smp_store_mb(current->state, (state_value))
188
189
190
191
192
193
194
195#define set_special_state(state_value) \
196 do { \
197 unsigned long flags; \
198 raw_spin_lock_irqsave(¤t->pi_lock, flags); \
199 current->state = (state_value); \
200 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \
201 } while (0)
202
203#endif
204
205
206#define TASK_COMM_LEN 16
207
208extern void scheduler_tick(void);
209
210#define MAX_SCHEDULE_TIMEOUT LONG_MAX
211
212extern long schedule_timeout(long timeout);
213extern long schedule_timeout_interruptible(long timeout);
214extern long schedule_timeout_killable(long timeout);
215extern long schedule_timeout_uninterruptible(long timeout);
216extern long schedule_timeout_idle(long timeout);
217asmlinkage void schedule(void);
218extern void schedule_preempt_disabled(void);
219
220extern int __must_check io_schedule_prepare(void);
221extern void io_schedule_finish(int token);
222extern long io_schedule_timeout(long timeout);
223extern void io_schedule(void);
224
225
226
227
228
229
230
231
232
233
234struct prev_cputime {
235#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
236 u64 utime;
237 u64 stime;
238 raw_spinlock_t lock;
239#endif
240};
241
242
243
244
245
246
247
248
249
250
251
252struct task_cputime {
253 u64 utime;
254 u64 stime;
255 unsigned long long sum_exec_runtime;
256};
257
258
259#define virt_exp utime
260#define prof_exp stime
261#define sched_exp sum_exec_runtime
262
263enum vtime_state {
264
265 VTIME_INACTIVE = 0,
266
267 VTIME_USER,
268
269 VTIME_SYS,
270};
271
272struct vtime {
273 seqcount_t seqcount;
274 unsigned long long starttime;
275 enum vtime_state state;
276 u64 utime;
277 u64 stime;
278 u64 gtime;
279};
280
281struct sched_info {
282#ifdef CONFIG_SCHED_INFO
283
284
285
286 unsigned long pcount;
287
288
289 unsigned long long run_delay;
290
291
292
293
294 unsigned long long last_arrival;
295
296
297 unsigned long long last_queued;
298
299#endif
300};
301
302
303
304
305
306
307
308
309# define SCHED_FIXEDPOINT_SHIFT 10
310# define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
311
312struct load_weight {
313 unsigned long weight;
314 u32 inv_weight;
315};
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339struct util_est {
340 unsigned int enqueued;
341 unsigned int ewma;
342#define UTIL_EST_WEIGHT_SHIFT 2
343} __attribute__((__aligned__(sizeof(u64))));
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397struct sched_avg {
398 u64 last_update_time;
399 u64 load_sum;
400 u64 runnable_load_sum;
401 u32 util_sum;
402 u32 period_contrib;
403 unsigned long load_avg;
404 unsigned long runnable_load_avg;
405 unsigned long util_avg;
406 struct util_est util_est;
407} ____cacheline_aligned;
408
409struct sched_statistics {
410#ifdef CONFIG_SCHEDSTATS
411 u64 wait_start;
412 u64 wait_max;
413 u64 wait_count;
414 u64 wait_sum;
415 u64 iowait_count;
416 u64 iowait_sum;
417
418 u64 sleep_start;
419 u64 sleep_max;
420 s64 sum_sleep_runtime;
421
422 u64 block_start;
423 u64 block_max;
424 u64 exec_max;
425 u64 slice_max;
426
427 u64 nr_migrations_cold;
428 u64 nr_failed_migrations_affine;
429 u64 nr_failed_migrations_running;
430 u64 nr_failed_migrations_hot;
431 u64 nr_forced_migrations;
432
433 u64 nr_wakeups;
434 u64 nr_wakeups_sync;
435 u64 nr_wakeups_migrate;
436 u64 nr_wakeups_local;
437 u64 nr_wakeups_remote;
438 u64 nr_wakeups_affine;
439 u64 nr_wakeups_affine_attempts;
440 u64 nr_wakeups_passive;
441 u64 nr_wakeups_idle;
442#endif
443};
444
445struct sched_entity {
446
447 struct load_weight load;
448 unsigned long runnable_weight;
449 struct rb_node run_node;
450 struct list_head group_node;
451 unsigned int on_rq;
452
453 u64 exec_start;
454 u64 sum_exec_runtime;
455 u64 vruntime;
456 u64 prev_sum_exec_runtime;
457
458 u64 nr_migrations;
459
460 struct sched_statistics statistics;
461
462#ifdef CONFIG_FAIR_GROUP_SCHED
463 int depth;
464 struct sched_entity *parent;
465
466 struct cfs_rq *cfs_rq;
467
468 struct cfs_rq *my_q;
469#endif
470
471#ifdef CONFIG_SMP
472
473
474
475
476
477
478 struct sched_avg avg;
479#endif
480};
481
482struct sched_rt_entity {
483 struct list_head run_list;
484 unsigned long timeout;
485 unsigned long watchdog_stamp;
486 unsigned int time_slice;
487 unsigned short on_rq;
488 unsigned short on_list;
489
490 struct sched_rt_entity *back;
491#ifdef CONFIG_RT_GROUP_SCHED
492 struct sched_rt_entity *parent;
493
494 struct rt_rq *rt_rq;
495
496 struct rt_rq *my_q;
497#endif
498} __randomize_layout;
499
500struct sched_dl_entity {
501 struct rb_node rb_node;
502
503
504
505
506
507
508 u64 dl_runtime;
509 u64 dl_deadline;
510 u64 dl_period;
511 u64 dl_bw;
512 u64 dl_density;
513
514
515
516
517
518
519 s64 runtime;
520 u64 deadline;
521 unsigned int flags;
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547 unsigned int dl_throttled : 1;
548 unsigned int dl_boosted : 1;
549 unsigned int dl_yielded : 1;
550 unsigned int dl_non_contending : 1;
551 unsigned int dl_overrun : 1;
552
553
554
555
556
557 struct hrtimer dl_timer;
558
559
560
561
562
563
564
565
566 struct hrtimer inactive_timer;
567};
568
569union rcu_special {
570 struct {
571 u8 blocked;
572 u8 need_qs;
573 u8 exp_need_qs;
574
575
576 u8 pad;
577 } b;
578 u32 s;
579};
580
581enum perf_event_task_context {
582 perf_invalid_context = -1,
583 perf_hw_context = 0,
584 perf_sw_context,
585 perf_nr_task_contexts,
586};
587
588struct wake_q_node {
589 struct wake_q_node *next;
590};
591
592struct task_struct {
593#ifdef CONFIG_THREAD_INFO_IN_TASK
594
595
596
597
598 struct thread_info thread_info;
599#endif
600
601 volatile long state;
602
603
604
605
606
607 randomized_struct_fields_start
608
609 void *stack;
610 atomic_t usage;
611
612 unsigned int flags;
613 unsigned int ptrace;
614
615#ifdef CONFIG_SMP
616 struct llist_node wake_entry;
617 int on_cpu;
618#ifdef CONFIG_THREAD_INFO_IN_TASK
619
620 unsigned int cpu;
621#endif
622 unsigned int wakee_flips;
623 unsigned long wakee_flip_decay_ts;
624 struct task_struct *last_wakee;
625
626
627
628
629
630
631
632
633 int recent_used_cpu;
634 int wake_cpu;
635#endif
636 int on_rq;
637
638 int prio;
639 int static_prio;
640 int normal_prio;
641 unsigned int rt_priority;
642
643 const struct sched_class *sched_class;
644 struct sched_entity se;
645 struct sched_rt_entity rt;
646#ifdef CONFIG_CGROUP_SCHED
647 struct task_group *sched_task_group;
648#endif
649 struct sched_dl_entity dl;
650
651#ifdef CONFIG_PREEMPT_NOTIFIERS
652
653 struct hlist_head preempt_notifiers;
654#endif
655
656#ifdef CONFIG_BLK_DEV_IO_TRACE
657 unsigned int btrace_seq;
658#endif
659
660 unsigned int policy;
661 int nr_cpus_allowed;
662 cpumask_t cpus_allowed;
663
664#ifdef CONFIG_PREEMPT_RCU
665 int rcu_read_lock_nesting;
666 union rcu_special rcu_read_unlock_special;
667 struct list_head rcu_node_entry;
668 struct rcu_node *rcu_blocked_node;
669#endif
670
671#ifdef CONFIG_TASKS_RCU
672 unsigned long rcu_tasks_nvcsw;
673 u8 rcu_tasks_holdout;
674 u8 rcu_tasks_idx;
675 int rcu_tasks_idle_cpu;
676 struct list_head rcu_tasks_holdout_list;
677#endif
678
679 struct sched_info sched_info;
680
681 struct list_head tasks;
682#ifdef CONFIG_SMP
683 struct plist_node pushable_tasks;
684 struct rb_node pushable_dl_tasks;
685#endif
686
687 struct mm_struct *mm;
688 struct mm_struct *active_mm;
689
690
691 struct vmacache vmacache;
692
693#ifdef SPLIT_RSS_COUNTING
694 struct task_rss_stat rss_stat;
695#endif
696 int exit_state;
697 int exit_code;
698 int exit_signal;
699
700 int pdeath_signal;
701
702 unsigned long jobctl;
703
704
705 unsigned int personality;
706
707
708 unsigned sched_reset_on_fork:1;
709 unsigned sched_contributes_to_load:1;
710 unsigned sched_migrated:1;
711 unsigned sched_remote_wakeup:1;
712
713 unsigned :0;
714
715
716
717
718 unsigned in_execve:1;
719 unsigned in_iowait:1;
720#ifndef TIF_RESTORE_SIGMASK
721 unsigned restore_sigmask:1;
722#endif
723#ifdef CONFIG_MEMCG
724 unsigned memcg_may_oom:1;
725#ifndef CONFIG_SLOB
726 unsigned memcg_kmem_skip_account:1;
727#endif
728#endif
729#ifdef CONFIG_COMPAT_BRK
730 unsigned brk_randomized:1;
731#endif
732#ifdef CONFIG_CGROUPS
733
734 unsigned no_cgroup_migration:1;
735#endif
736
737 unsigned long atomic_flags;
738
739 struct restart_block restart_block;
740
741 pid_t pid;
742 pid_t tgid;
743
744#ifdef CONFIG_CC_STACKPROTECTOR
745
746 unsigned long stack_canary;
747#endif
748
749
750
751
752
753
754
755 struct task_struct __rcu *real_parent;
756
757
758 struct task_struct __rcu *parent;
759
760
761
762
763 struct list_head children;
764 struct list_head sibling;
765 struct task_struct *group_leader;
766
767
768
769
770
771
772
773 struct list_head ptraced;
774 struct list_head ptrace_entry;
775
776
777 struct pid_link pids[PIDTYPE_MAX];
778 struct list_head thread_group;
779 struct list_head thread_node;
780
781 struct completion *vfork_done;
782
783
784 int __user *set_child_tid;
785
786
787 int __user *clear_child_tid;
788
789 u64 utime;
790 u64 stime;
791#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
792 u64 utimescaled;
793 u64 stimescaled;
794#endif
795 u64 gtime;
796 struct prev_cputime prev_cputime;
797#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
798 struct vtime vtime;
799#endif
800
801#ifdef CONFIG_NO_HZ_FULL
802 atomic_t tick_dep_mask;
803#endif
804
805 unsigned long nvcsw;
806 unsigned long nivcsw;
807
808
809 u64 start_time;
810
811
812 u64 real_start_time;
813
814
815 unsigned long min_flt;
816 unsigned long maj_flt;
817
818#ifdef CONFIG_POSIX_TIMERS
819 struct task_cputime cputime_expires;
820 struct list_head cpu_timers[3];
821#endif
822
823
824
825
826 const struct cred __rcu *ptracer_cred;
827
828
829 const struct cred __rcu *real_cred;
830
831
832 const struct cred __rcu *cred;
833
834
835
836
837
838
839
840
841 char comm[TASK_COMM_LEN];
842
843 struct nameidata *nameidata;
844
845#ifdef CONFIG_SYSVIPC
846 struct sysv_sem sysvsem;
847 struct sysv_shm sysvshm;
848#endif
849#ifdef CONFIG_DETECT_HUNG_TASK
850 unsigned long last_switch_count;
851#endif
852
853 struct fs_struct *fs;
854
855
856 struct files_struct *files;
857
858
859 struct nsproxy *nsproxy;
860
861
862 struct signal_struct *signal;
863 struct sighand_struct *sighand;
864 sigset_t blocked;
865 sigset_t real_blocked;
866
867 sigset_t saved_sigmask;
868 struct sigpending pending;
869 unsigned long sas_ss_sp;
870 size_t sas_ss_size;
871 unsigned int sas_ss_flags;
872
873 struct callback_head *task_works;
874
875 struct audit_context *audit_context;
876#ifdef CONFIG_AUDITSYSCALL
877 kuid_t loginuid;
878 unsigned int sessionid;
879#endif
880 struct seccomp seccomp;
881
882
883 u32 parent_exec_id;
884 u32 self_exec_id;
885
886
887 spinlock_t alloc_lock;
888
889
890 raw_spinlock_t pi_lock;
891
892 struct wake_q_node wake_q;
893
894#ifdef CONFIG_RT_MUTEXES
895
896 struct rb_root_cached pi_waiters;
897
898 struct task_struct *pi_top_task;
899
900 struct rt_mutex_waiter *pi_blocked_on;
901#endif
902
903#ifdef CONFIG_DEBUG_MUTEXES
904
905 struct mutex_waiter *blocked_on;
906#endif
907
908#ifdef CONFIG_TRACE_IRQFLAGS
909 unsigned int irq_events;
910 unsigned long hardirq_enable_ip;
911 unsigned long hardirq_disable_ip;
912 unsigned int hardirq_enable_event;
913 unsigned int hardirq_disable_event;
914 int hardirqs_enabled;
915 int hardirq_context;
916 unsigned long softirq_disable_ip;
917 unsigned long softirq_enable_ip;
918 unsigned int softirq_disable_event;
919 unsigned int softirq_enable_event;
920 int softirqs_enabled;
921 int softirq_context;
922#endif
923
924#ifdef CONFIG_LOCKDEP
925# define MAX_LOCK_DEPTH 48UL
926 u64 curr_chain_key;
927 int lockdep_depth;
928 unsigned int lockdep_recursion;
929 struct held_lock held_locks[MAX_LOCK_DEPTH];
930#endif
931
932#ifdef CONFIG_UBSAN
933 unsigned int in_ubsan;
934#endif
935
936
937 void *journal_info;
938
939
940 struct bio_list *bio_list;
941
942#ifdef CONFIG_BLOCK
943
944 struct blk_plug *plug;
945#endif
946
947
948 struct reclaim_state *reclaim_state;
949
950 struct backing_dev_info *backing_dev_info;
951
952 struct io_context *io_context;
953
954
955 unsigned long ptrace_message;
956 siginfo_t *last_siginfo;
957
958 struct task_io_accounting ioac;
959#ifdef CONFIG_TASK_XACCT
960
961 u64 acct_rss_mem1;
962
963 u64 acct_vm_mem1;
964
965 u64 acct_timexpd;
966#endif
967#ifdef CONFIG_CPUSETS
968
969 nodemask_t mems_allowed;
970
971 seqcount_t mems_allowed_seq;
972 int cpuset_mem_spread_rotor;
973 int cpuset_slab_spread_rotor;
974#endif
975#ifdef CONFIG_CGROUPS
976
977 struct css_set __rcu *cgroups;
978
979 struct list_head cg_list;
980#endif
981#ifdef CONFIG_INTEL_RDT
982 u32 closid;
983 u32 rmid;
984#endif
985#ifdef CONFIG_FUTEX
986 struct robust_list_head __user *robust_list;
987#ifdef CONFIG_COMPAT
988 struct compat_robust_list_head __user *compat_robust_list;
989#endif
990 struct list_head pi_state_list;
991 struct futex_pi_state *pi_state_cache;
992#endif
993#ifdef CONFIG_PERF_EVENTS
994 struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
995 struct mutex perf_event_mutex;
996 struct list_head perf_event_list;
997#endif
998#ifdef CONFIG_DEBUG_PREEMPT
999 unsigned long preempt_disable_ip;
1000#endif
1001#ifdef CONFIG_NUMA
1002
1003 struct mempolicy *mempolicy;
1004 short il_prev;
1005 short pref_node_fork;
1006#endif
1007#ifdef CONFIG_NUMA_BALANCING
1008 int numa_scan_seq;
1009 unsigned int numa_scan_period;
1010 unsigned int numa_scan_period_max;
1011 int numa_preferred_nid;
1012 unsigned long numa_migrate_retry;
1013
1014 u64 node_stamp;
1015 u64 last_task_numa_placement;
1016 u64 last_sum_exec_runtime;
1017 struct callback_head numa_work;
1018
1019 struct list_head numa_entry;
1020 struct numa_group *numa_group;
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036 unsigned long *numa_faults;
1037 unsigned long total_numa_faults;
1038
1039
1040
1041
1042
1043
1044
1045 unsigned long numa_faults_locality[3];
1046
1047 unsigned long numa_pages_migrated;
1048#endif
1049
1050 struct tlbflush_unmap_batch tlb_ubc;
1051
1052 struct rcu_head rcu;
1053
1054
1055 struct pipe_inode_info *splice_pipe;
1056
1057 struct page_frag task_frag;
1058
1059#ifdef CONFIG_TASK_DELAY_ACCT
1060 struct task_delay_info *delays;
1061#endif
1062
1063#ifdef CONFIG_FAULT_INJECTION
1064 int make_it_fail;
1065 unsigned int fail_nth;
1066#endif
1067
1068
1069
1070
1071 int nr_dirtied;
1072 int nr_dirtied_pause;
1073
1074 unsigned long dirty_paused_when;
1075
1076#ifdef CONFIG_LATENCYTOP
1077 int latency_record_count;
1078 struct latency_record latency_record[LT_SAVECOUNT];
1079#endif
1080
1081
1082
1083
1084 u64 timer_slack_ns;
1085 u64 default_timer_slack_ns;
1086
1087#ifdef CONFIG_KASAN
1088 unsigned int kasan_depth;
1089#endif
1090
1091#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1092
1093 int curr_ret_stack;
1094
1095
1096 struct ftrace_ret_stack *ret_stack;
1097
1098
1099 unsigned long long ftrace_timestamp;
1100
1101
1102
1103
1104
1105 atomic_t trace_overrun;
1106
1107
1108 atomic_t tracing_graph_pause;
1109#endif
1110
1111#ifdef CONFIG_TRACING
1112
1113 unsigned long trace;
1114
1115
1116 unsigned long trace_recursion;
1117#endif
1118
1119#ifdef CONFIG_KCOV
1120
1121 enum kcov_mode kcov_mode;
1122
1123
1124 unsigned int kcov_size;
1125
1126
1127 void *kcov_area;
1128
1129
1130 struct kcov *kcov;
1131#endif
1132
1133#ifdef CONFIG_MEMCG
1134 struct mem_cgroup *memcg_in_oom;
1135 gfp_t memcg_oom_gfp_mask;
1136 int memcg_oom_order;
1137
1138
1139 unsigned int memcg_nr_pages_over_high;
1140#endif
1141
1142#ifdef CONFIG_UPROBES
1143 struct uprobe_task *utask;
1144#endif
1145#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1146 unsigned int sequential_io;
1147 unsigned int sequential_io_avg;
1148#endif
1149#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1150 unsigned long task_state_change;
1151#endif
1152 int pagefault_disabled;
1153#ifdef CONFIG_MMU
1154 struct task_struct *oom_reaper_list;
1155#endif
1156#ifdef CONFIG_VMAP_STACK
1157 struct vm_struct *stack_vm_area;
1158#endif
1159#ifdef CONFIG_THREAD_INFO_IN_TASK
1160
1161 atomic_t stack_refcount;
1162#endif
1163#ifdef CONFIG_LIVEPATCH
1164 int patch_state;
1165#endif
1166#ifdef CONFIG_SECURITY
1167
1168 void *security;
1169#endif
1170
1171
1172
1173
1174
1175 randomized_struct_fields_end
1176
1177
1178 struct thread_struct thread;
1179
1180
1181
1182
1183
1184
1185
1186};
1187
1188static inline struct pid *task_pid(struct task_struct *task)
1189{
1190 return task->pids[PIDTYPE_PID].pid;
1191}
1192
1193static inline struct pid *task_tgid(struct task_struct *task)
1194{
1195 return task->group_leader->pids[PIDTYPE_PID].pid;
1196}
1197
1198
1199
1200
1201
1202
1203static inline struct pid *task_pgrp(struct task_struct *task)
1204{
1205 return task->group_leader->pids[PIDTYPE_PGID].pid;
1206}
1207
1208static inline struct pid *task_session(struct task_struct *task)
1209{
1210 return task->group_leader->pids[PIDTYPE_SID].pid;
1211}
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
1225
1226static inline pid_t task_pid_nr(struct task_struct *tsk)
1227{
1228 return tsk->pid;
1229}
1230
1231static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1232{
1233 return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
1234}
1235
1236static inline pid_t task_pid_vnr(struct task_struct *tsk)
1237{
1238 return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
1239}
1240
1241
1242static inline pid_t task_tgid_nr(struct task_struct *tsk)
1243{
1244 return tsk->tgid;
1245}
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257static inline int pid_alive(const struct task_struct *p)
1258{
1259 return p->pids[PIDTYPE_PID].pid != NULL;
1260}
1261
1262static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1263{
1264 return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
1265}
1266
1267static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
1268{
1269 return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
1270}
1271
1272
1273static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1274{
1275 return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
1276}
1277
1278static inline pid_t task_session_vnr(struct task_struct *tsk)
1279{
1280 return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
1281}
1282
1283static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1284{
1285 return __task_pid_nr_ns(tsk, __PIDTYPE_TGID, ns);
1286}
1287
1288static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1289{
1290 return __task_pid_nr_ns(tsk, __PIDTYPE_TGID, NULL);
1291}
1292
1293static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1294{
1295 pid_t pid = 0;
1296
1297 rcu_read_lock();
1298 if (pid_alive(tsk))
1299 pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1300 rcu_read_unlock();
1301
1302 return pid;
1303}
1304
1305static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1306{
1307 return task_ppid_nr_ns(tsk, &init_pid_ns);
1308}
1309
1310
1311static inline pid_t task_pgrp_nr(struct task_struct *tsk)
1312{
1313 return task_pgrp_nr_ns(tsk, &init_pid_ns);
1314}
1315
1316#define TASK_REPORT_IDLE (TASK_REPORT + 1)
1317#define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1)
1318
1319static inline unsigned int task_state_index(struct task_struct *tsk)
1320{
1321 unsigned int tsk_state = READ_ONCE(tsk->state);
1322 unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT;
1323
1324 BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
1325
1326 if (tsk_state == TASK_IDLE)
1327 state = TASK_REPORT_IDLE;
1328
1329 return fls(state);
1330}
1331
1332static inline char task_index_to_char(unsigned int state)
1333{
1334 static const char state_char[] = "RSDTtXZPI";
1335
1336 BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
1337
1338 return state_char[state];
1339}
1340
1341static inline char task_state_to_char(struct task_struct *tsk)
1342{
1343 return task_index_to_char(task_state_index(tsk));
1344}
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355static inline int is_global_init(struct task_struct *tsk)
1356{
1357 return task_tgid_nr(tsk) == 1;
1358}
1359
1360extern struct pid *cad_pid;
1361
1362
1363
1364
1365#define PF_IDLE 0x00000002
1366#define PF_EXITING 0x00000004
1367#define PF_EXITPIDONE 0x00000008
1368#define PF_VCPU 0x00000010
1369#define PF_WQ_WORKER 0x00000020
1370#define PF_FORKNOEXEC 0x00000040
1371#define PF_MCE_PROCESS 0x00000080
1372#define PF_SUPERPRIV 0x00000100
1373#define PF_DUMPCORE 0x00000200
1374#define PF_SIGNALED 0x00000400
1375#define PF_MEMALLOC 0x00000800
1376#define PF_NPROC_EXCEEDED 0x00001000
1377#define PF_USED_MATH 0x00002000
1378#define PF_USED_ASYNC 0x00004000
1379#define PF_NOFREEZE 0x00008000
1380#define PF_FROZEN 0x00010000
1381#define PF_KSWAPD 0x00020000
1382#define PF_MEMALLOC_NOFS 0x00040000
1383#define PF_MEMALLOC_NOIO 0x00080000
1384#define PF_LESS_THROTTLE 0x00100000
1385#define PF_KTHREAD 0x00200000
1386#define PF_RANDOMIZE 0x00400000
1387#define PF_SWAPWRITE 0x00800000
1388#define PF_NO_SETAFFINITY 0x04000000
1389#define PF_MCE_EARLY 0x08000000
1390#define PF_MUTEX_TESTER 0x20000000
1391#define PF_FREEZER_SKIP 0x40000000
1392#define PF_SUSPEND_TASK 0x80000000
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405#define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
1406#define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
1407#define clear_used_math() clear_stopped_child_used_math(current)
1408#define set_used_math() set_stopped_child_used_math(current)
1409
1410#define conditional_stopped_child_used_math(condition, child) \
1411 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
1412
1413#define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current)
1414
1415#define copy_to_stopped_child_used_math(child) \
1416 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
1417
1418
1419#define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
1420#define used_math() tsk_used_math(current)
1421
1422static inline bool is_percpu_thread(void)
1423{
1424#ifdef CONFIG_SMP
1425 return (current->flags & PF_NO_SETAFFINITY) &&
1426 (current->nr_cpus_allowed == 1);
1427#else
1428 return true;
1429#endif
1430}
1431
1432
1433#define PFA_NO_NEW_PRIVS 0
1434#define PFA_SPREAD_PAGE 1
1435#define PFA_SPREAD_SLAB 2
1436#define PFA_SPEC_SSB_DISABLE 3
1437#define PFA_SPEC_SSB_FORCE_DISABLE 4
1438
1439#define TASK_PFA_TEST(name, func) \
1440 static inline bool task_##func(struct task_struct *p) \
1441 { return test_bit(PFA_##name, &p->atomic_flags); }
1442
1443#define TASK_PFA_SET(name, func) \
1444 static inline void task_set_##func(struct task_struct *p) \
1445 { set_bit(PFA_##name, &p->atomic_flags); }
1446
1447#define TASK_PFA_CLEAR(name, func) \
1448 static inline void task_clear_##func(struct task_struct *p) \
1449 { clear_bit(PFA_##name, &p->atomic_flags); }
1450
1451TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
1452TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
1453
1454TASK_PFA_TEST(SPREAD_PAGE, spread_page)
1455TASK_PFA_SET(SPREAD_PAGE, spread_page)
1456TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
1457
1458TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
1459TASK_PFA_SET(SPREAD_SLAB, spread_slab)
1460TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1461
1462TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
1463TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
1464TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
1465
1466TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1467TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1468
1469static inline void
1470current_restore_flags(unsigned long orig_flags, unsigned long flags)
1471{
1472 current->flags &= ~flags;
1473 current->flags |= orig_flags & flags;
1474}
1475
1476extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
1477extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
1478#ifdef CONFIG_SMP
1479extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
1480extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
1481#else
1482static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1483{
1484}
1485static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1486{
1487 if (!cpumask_test_cpu(0, new_mask))
1488 return -EINVAL;
1489 return 0;
1490}
1491#endif
1492
1493#ifndef cpu_relax_yield
1494#define cpu_relax_yield() cpu_relax()
1495#endif
1496
1497extern int yield_to(struct task_struct *p, bool preempt);
1498extern void set_user_nice(struct task_struct *p, long nice);
1499extern int task_prio(const struct task_struct *p);
1500
1501
1502
1503
1504
1505
1506
1507static inline int task_nice(const struct task_struct *p)
1508{
1509 return PRIO_TO_NICE((p)->static_prio);
1510}
1511
1512extern int can_nice(const struct task_struct *p, const int nice);
1513extern int task_curr(const struct task_struct *p);
1514extern int idle_cpu(int cpu);
1515extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
1516extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
1517extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1518extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
1519extern struct task_struct *idle_task(int cpu);
1520
1521
1522
1523
1524
1525
1526
1527static inline bool is_idle_task(const struct task_struct *p)
1528{
1529 return !!(p->flags & PF_IDLE);
1530}
1531
1532extern struct task_struct *curr_task(int cpu);
1533extern void ia64_set_curr_task(int cpu, struct task_struct *p);
1534
1535void yield(void);
1536
1537union thread_union {
1538#ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
1539 struct task_struct task;
1540#endif
1541#ifndef CONFIG_THREAD_INFO_IN_TASK
1542 struct thread_info thread_info;
1543#endif
1544 unsigned long stack[THREAD_SIZE/sizeof(long)];
1545};
1546
1547#ifndef CONFIG_THREAD_INFO_IN_TASK
1548extern struct thread_info init_thread_info;
1549#endif
1550
1551extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
1552
1553#ifdef CONFIG_THREAD_INFO_IN_TASK
1554static inline struct thread_info *task_thread_info(struct task_struct *task)
1555{
1556 return &task->thread_info;
1557}
1558#elif !defined(__HAVE_THREAD_FUNCTIONS)
1559# define task_thread_info(task) ((struct thread_info *)(task)->stack)
1560#endif
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573extern struct task_struct *find_task_by_vpid(pid_t nr);
1574extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1575
1576
1577
1578
1579extern struct task_struct *find_get_task_by_vpid(pid_t nr);
1580
1581extern int wake_up_state(struct task_struct *tsk, unsigned int state);
1582extern int wake_up_process(struct task_struct *tsk);
1583extern void wake_up_new_task(struct task_struct *tsk);
1584
1585#ifdef CONFIG_SMP
1586extern void kick_process(struct task_struct *tsk);
1587#else
1588static inline void kick_process(struct task_struct *tsk) { }
1589#endif
1590
1591extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
1592
1593static inline void set_task_comm(struct task_struct *tsk, const char *from)
1594{
1595 __set_task_comm(tsk, from, false);
1596}
1597
1598extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
1599#define get_task_comm(buf, tsk) ({ \
1600 BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN); \
1601 __get_task_comm(buf, sizeof(buf), tsk); \
1602})
1603
1604#ifdef CONFIG_SMP
1605void scheduler_ipi(void);
1606extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
1607#else
1608static inline void scheduler_ipi(void) { }
1609static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1610{
1611 return 1;
1612}
1613#endif
1614
1615
1616
1617
1618
1619static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
1620{
1621 set_ti_thread_flag(task_thread_info(tsk), flag);
1622}
1623
1624static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1625{
1626 clear_ti_thread_flag(task_thread_info(tsk), flag);
1627}
1628
1629static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
1630{
1631 return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
1632}
1633
1634static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1635{
1636 return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
1637}
1638
1639static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
1640{
1641 return test_ti_thread_flag(task_thread_info(tsk), flag);
1642}
1643
1644static inline void set_tsk_need_resched(struct task_struct *tsk)
1645{
1646 set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
1647}
1648
1649static inline void clear_tsk_need_resched(struct task_struct *tsk)
1650{
1651 clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
1652}
1653
1654static inline int test_tsk_need_resched(struct task_struct *tsk)
1655{
1656 return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
1657}
1658
1659
1660
1661
1662
1663
1664
1665
1666#ifndef CONFIG_PREEMPT
1667extern int _cond_resched(void);
1668#else
1669static inline int _cond_resched(void) { return 0; }
1670#endif
1671
1672#define cond_resched() ({ \
1673 ___might_sleep(__FILE__, __LINE__, 0); \
1674 _cond_resched(); \
1675})
1676
1677extern int __cond_resched_lock(spinlock_t *lock);
1678
1679#define cond_resched_lock(lock) ({ \
1680 ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1681 __cond_resched_lock(lock); \
1682})
1683
1684extern int __cond_resched_softirq(void);
1685
1686#define cond_resched_softirq() ({ \
1687 ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \
1688 __cond_resched_softirq(); \
1689})
1690
1691static inline void cond_resched_rcu(void)
1692{
1693#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
1694 rcu_read_unlock();
1695 cond_resched();
1696 rcu_read_lock();
1697#endif
1698}
1699
1700
1701
1702
1703
1704
1705static inline int spin_needbreak(spinlock_t *lock)
1706{
1707#ifdef CONFIG_PREEMPT
1708 return spin_is_contended(lock);
1709#else
1710 return 0;
1711#endif
1712}
1713
1714static __always_inline bool need_resched(void)
1715{
1716 return unlikely(tif_need_resched());
1717}
1718
1719
1720
1721
1722#ifdef CONFIG_SMP
1723
1724static inline unsigned int task_cpu(const struct task_struct *p)
1725{
1726#ifdef CONFIG_THREAD_INFO_IN_TASK
1727 return p->cpu;
1728#else
1729 return task_thread_info(p)->cpu;
1730#endif
1731}
1732
1733extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
1734
1735#else
1736
1737static inline unsigned int task_cpu(const struct task_struct *p)
1738{
1739 return 0;
1740}
1741
1742static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
1743{
1744}
1745
1746#endif
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756#ifndef vcpu_is_preempted
1757# define vcpu_is_preempted(cpu) false
1758#endif
1759
1760extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
1761extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
1762
1763#ifndef TASK_SIZE_OF
1764#define TASK_SIZE_OF(tsk) TASK_SIZE
1765#endif
1766
1767#endif
1768