1
2
3
4
5
6
7
8
9
10#include "sched.h"
11
12DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
13EXPORT_SYMBOL_GPL(housekeeping_overridden);
14static cpumask_var_t housekeeping_mask;
15static unsigned int housekeeping_flags;
16
17bool housekeeping_enabled(enum hk_flags flags)
18{
19 return !!(housekeeping_flags & flags);
20}
21EXPORT_SYMBOL_GPL(housekeeping_enabled);
22
23int housekeeping_any_cpu(enum hk_flags flags)
24{
25 if (static_branch_unlikely(&housekeeping_overridden))
26 if (housekeeping_flags & flags)
27 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
28 return smp_processor_id();
29}
30EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
31
32const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
33{
34 if (static_branch_unlikely(&housekeeping_overridden))
35 if (housekeeping_flags & flags)
36 return housekeeping_mask;
37 return cpu_possible_mask;
38}
39EXPORT_SYMBOL_GPL(housekeeping_cpumask);
40
41void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
42{
43 if (static_branch_unlikely(&housekeeping_overridden))
44 if (housekeeping_flags & flags)
45 set_cpus_allowed_ptr(t, housekeeping_mask);
46}
47EXPORT_SYMBOL_GPL(housekeeping_affine);
48
49bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
50{
51 if (static_branch_unlikely(&housekeeping_overridden))
52 if (housekeeping_flags & flags)
53 return cpumask_test_cpu(cpu, housekeeping_mask);
54 return true;
55}
56EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
57
58void __init housekeeping_init(void)
59{
60 if (!housekeeping_flags)
61 return;
62
63 static_branch_enable(&housekeeping_overridden);
64
65 if (housekeeping_flags & HK_FLAG_TICK)
66 sched_tick_offload_init();
67
68
69 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
70}
71
72static int __init housekeeping_setup(char *str, enum hk_flags flags)
73{
74 cpumask_var_t non_housekeeping_mask;
75 cpumask_var_t tmp;
76 int err;
77
78 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
79 err = cpulist_parse(str, non_housekeeping_mask);
80 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
81 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
82 free_bootmem_cpumask_var(non_housekeeping_mask);
83 return 0;
84 }
85
86 alloc_bootmem_cpumask_var(&tmp);
87 if (!housekeeping_flags) {
88 alloc_bootmem_cpumask_var(&housekeeping_mask);
89 cpumask_andnot(housekeeping_mask,
90 cpu_possible_mask, non_housekeeping_mask);
91
92 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
93 if (cpumask_empty(tmp)) {
94 pr_warn("Housekeeping: must include one present CPU, "
95 "using boot CPU:%d\n", smp_processor_id());
96 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
97 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
98 }
99 } else {
100 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
101 if (cpumask_empty(tmp))
102 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
103 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
104 if (!cpumask_equal(tmp, housekeeping_mask)) {
105 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
106 free_bootmem_cpumask_var(tmp);
107 free_bootmem_cpumask_var(non_housekeeping_mask);
108 return 0;
109 }
110 }
111 free_bootmem_cpumask_var(tmp);
112
113 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
114 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
115 tick_nohz_full_setup(non_housekeeping_mask);
116 } else {
117 pr_warn("Housekeeping: nohz unsupported."
118 " Build with CONFIG_NO_HZ_FULL\n");
119 free_bootmem_cpumask_var(non_housekeeping_mask);
120 return 0;
121 }
122 }
123
124 housekeeping_flags |= flags;
125
126 free_bootmem_cpumask_var(non_housekeeping_mask);
127
128 return 1;
129}
130
131static int __init housekeeping_nohz_full_setup(char *str)
132{
133 unsigned int flags;
134
135 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
136
137 return housekeeping_setup(str, flags);
138}
139__setup("nohz_full=", housekeeping_nohz_full_setup);
140
141static int __init housekeeping_isolcpus_setup(char *str)
142{
143 unsigned int flags = 0;
144
145 while (isalpha(*str)) {
146 if (!strncmp(str, "nohz,", 5)) {
147 str += 5;
148 flags |= HK_FLAG_TICK;
149 continue;
150 }
151
152 if (!strncmp(str, "domain,", 7)) {
153 str += 7;
154 flags |= HK_FLAG_DOMAIN;
155 continue;
156 }
157
158 pr_warn("isolcpus: Error, unknown flag\n");
159 return 0;
160 }
161
162
163 if (!flags)
164 flags |= HK_FLAG_DOMAIN;
165
166 return housekeeping_setup(str, flags);
167}
168__setup("isolcpus=", housekeeping_isolcpus_setup);
169
