1
2
3
4
5
6
7
8
9
10
11
12#include <linux/cache.h>
13#include <linux/spinlock.h>
14#include <linux/rtmutex.h>
15#include <linux/threads.h>
16#include <linux/cpumask.h>
17#include <linux/seqlock.h>
18#include <linux/swait.h>
19#include <linux/rcu_node_tree.h>
20
21#include "rcu_segcblist.h"
22
23
24struct rcu_exp_work {
25 unsigned long rew_s;
26 struct work_struct rew_work;
27};
28
29
30#define RCU_KTHREAD_STOPPED 0
31#define RCU_KTHREAD_RUNNING 1
32#define RCU_KTHREAD_WAITING 2
33#define RCU_KTHREAD_OFFCPU 3
34#define RCU_KTHREAD_YIELDING 4
35#define RCU_KTHREAD_MAX 4
36
37
38
39
40struct rcu_node {
41 raw_spinlock_t __private lock;
42
43
44 unsigned long gp_seq;
45 unsigned long gp_seq_needed;
46 unsigned long completedqs;
47 unsigned long qsmask;
48
49
50
51
52
53 unsigned long rcu_gp_init_mask;
54 unsigned long qsmaskinit;
55
56
57
58 unsigned long qsmaskinitnext;
59 unsigned long ofl_seq;
60
61 unsigned long expmask;
62
63
64 unsigned long expmaskinit;
65
66
67
68 unsigned long expmaskinitnext;
69
70
71
72 unsigned long cbovldmask;
73
74 unsigned long ffmask;
75 unsigned long grpmask;
76
77 int grplo;
78 int grphi;
79 u8 grpnum;
80 u8 level;
81 bool wait_blkd_tasks;
82
83
84
85 struct rcu_node *parent;
86 struct list_head blkd_tasks;
87
88
89
90 struct list_head *gp_tasks;
91
92
93
94 struct list_head *exp_tasks;
95
96
97
98
99
100 struct list_head *boost_tasks;
101
102
103
104
105
106
107
108 struct rt_mutex boost_mtx;
109
110
111 unsigned long boost_time;
112
113 struct task_struct *boost_kthread_task;
114
115
116 unsigned int boost_kthread_status;
117
118 unsigned long n_boosts;
119#ifdef CONFIG_RCU_NOCB_CPU
120 struct swait_queue_head nocb_gp_wq[2];
121
122#endif
123 raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp;
124
125 spinlock_t exp_lock ____cacheline_internodealigned_in_smp;
126 unsigned long exp_seq_rq;
127 wait_queue_head_t exp_wq[4];
128 struct rcu_exp_work rew;
129 bool exp_need_flush;
130} ____cacheline_internodealigned_in_smp;
131
132
133
134
135
136
137#define leaf_node_cpu_bit(rnp, cpu) (BIT((cpu) - (rnp)->grplo))
138
139
140
141
142
143union rcu_noqs {
144 struct {
145 u8 norm;
146 u8 exp;
147 } b;
148 u16 s;
149};
150
151
152struct rcu_data {
153
154 unsigned long gp_seq;
155 unsigned long gp_seq_needed;
156 union rcu_noqs cpu_no_qs;
157 bool core_needs_qs;
158 bool beenonline;
159 bool gpwrap;
160 bool exp_deferred_qs;
161 bool cpu_started;
162 struct rcu_node *mynode;
163 unsigned long grpmask;
164 unsigned long ticks_this_gp;
165
166
167
168 struct irq_work defer_qs_iw;
169 bool defer_qs_iw_pending;
170 struct work_struct strict_work;
171
172
173 struct rcu_segcblist cblist;
174
175
176 long qlen_last_fqs_check;
177
178 unsigned long n_cbs_invoked;
179 unsigned long n_force_qs_snap;
180
181 long blimit;
182
183
184 int dynticks_snap;
185 long dynticks_nesting;
186 long dynticks_nmi_nesting;
187 atomic_t dynticks;
188 bool rcu_need_heavy_qs;
189 bool rcu_urgent_qs;
190 bool rcu_forced_tick;
191 bool rcu_forced_tick_exp;
192#ifdef CONFIG_RCU_FAST_NO_HZ
193 unsigned long last_accelerate;
194 unsigned long last_advance_all;
195 int tick_nohz_enabled_snap;
196#endif
197
198
199 struct rcu_head barrier_head;
200 int exp_dynticks_snap;
201
202
203#ifdef CONFIG_RCU_NOCB_CPU
204 struct swait_queue_head nocb_cb_wq;
205 struct swait_queue_head nocb_state_wq;
206 struct task_struct *nocb_gp_kthread;
207 raw_spinlock_t nocb_lock;
208 atomic_t nocb_lock_contended;
209 int nocb_defer_wakeup;
210 struct timer_list nocb_timer;
211 unsigned long nocb_gp_adv_time;
212
213
214 raw_spinlock_t nocb_bypass_lock ____cacheline_internodealigned_in_smp;
215 struct rcu_cblist nocb_bypass;
216 unsigned long nocb_bypass_first;
217 unsigned long nocb_nobypass_last;
218 int nocb_nobypass_count;
219
220
221 raw_spinlock_t nocb_gp_lock ____cacheline_internodealigned_in_smp;
222 u8 nocb_gp_sleep;
223 u8 nocb_gp_bypass;
224 u8 nocb_gp_gp;
225 unsigned long nocb_gp_seq;
226 unsigned long nocb_gp_loops;
227 struct swait_queue_head nocb_gp_wq;
228 bool nocb_cb_sleep;
229 struct task_struct *nocb_cb_kthread;
230 struct rcu_data *nocb_next_cb_rdp;
231
232
233
234 struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp;
235
236#endif
237
238
239 struct task_struct *rcu_cpu_kthread_task;
240
241 unsigned int rcu_cpu_kthread_status;
242 char rcu_cpu_has_work;
243
244
245 unsigned int softirq_snap;
246
247 struct irq_work rcu_iw;
248 bool rcu_iw_pending;
249 unsigned long rcu_iw_gp_seq;
250 unsigned long rcu_ofl_gp_seq;
251 short rcu_ofl_gp_flags;
252 unsigned long rcu_onl_gp_seq;
253 short rcu_onl_gp_flags;
254 unsigned long last_fqs_resched;
255
256 int cpu;
257};
258
259
260#define RCU_NOCB_WAKE_NOT 0
261#define RCU_NOCB_WAKE_BYPASS 1
262#define RCU_NOCB_WAKE 2
263#define RCU_NOCB_WAKE_FORCE 3
264
265#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500))
266
267
268
269#define RCU_JIFFIES_FQS_DIV 256
270
271
272
273#define RCU_STALL_RAT_DELAY 2
274
275
276
277#define rcu_wait(cond) \
278do { \
279 for (;;) { \
280 set_current_state(TASK_INTERRUPTIBLE); \
281 if (cond) \
282 break; \
283 schedule(); \
284 } \
285 __set_current_state(TASK_RUNNING); \
286} while (0)
287
288
289
290
291
292
293
294
295
296
297
298struct rcu_state {
299 struct rcu_node node[NUM_RCU_NODES];
300 struct rcu_node *level[RCU_NUM_LVLS + 1];
301
302
303 int ncpus;
304 int n_online_cpus;
305
306
307
308 u8 boost ____cacheline_internodealigned_in_smp;
309
310 unsigned long gp_seq;
311 unsigned long gp_max;
312
313 struct task_struct *gp_kthread;
314 struct swait_queue_head gp_wq;
315 short gp_flags;
316 short gp_state;
317 unsigned long gp_wake_time;
318 unsigned long gp_wake_seq;
319
320
321
322 struct mutex barrier_mutex;
323 atomic_t barrier_cpu_count;
324 struct completion barrier_completion;
325 unsigned long barrier_sequence;
326
327
328
329 struct mutex exp_mutex;
330 struct mutex exp_wake_mutex;
331 unsigned long expedited_sequence;
332 atomic_t expedited_need_qs;
333 struct swait_queue_head expedited_wq;
334 int ncpus_snap;
335 u8 cbovld;
336 u8 cbovldnext;
337
338 unsigned long jiffies_force_qs;
339
340 unsigned long jiffies_kick_kthreads;
341
342 unsigned long n_force_qs;
343
344 unsigned long gp_start;
345
346 unsigned long gp_end;
347
348 unsigned long gp_activity;
349
350 unsigned long gp_req_activity;
351
352 unsigned long jiffies_stall;
353
354 unsigned long jiffies_resched;
355
356 unsigned long n_force_qs_gpstart;
357
358 const char *name;
359 char abbr;
360
361 raw_spinlock_t ofl_lock ____cacheline_internodealigned_in_smp;
362
363
364};
365
366
367#define RCU_GP_FLAG_INIT 0x1
368#define RCU_GP_FLAG_FQS 0x2
369#define RCU_GP_FLAG_OVLD 0x4
370
371
372#define RCU_GP_IDLE 0
373#define RCU_GP_WAIT_GPS 1
374#define RCU_GP_DONE_GPS 2
375#define RCU_GP_ONOFF 3
376#define RCU_GP_INIT 4
377#define RCU_GP_WAIT_FQS 5
378#define RCU_GP_DOING_FQS 6
379#define RCU_GP_CLEANUP 7
380#define RCU_GP_CLEANED 8
381
382
383
384
385
386
387
388
389
390#ifdef CONFIG_PREEMPT_RCU
391#define RCU_ABBR 'p'
392#define RCU_NAME_RAW "rcu_preempt"
393#else
394#define RCU_ABBR 's'
395#define RCU_NAME_RAW "rcu_sched"
396#endif
397#ifndef CONFIG_TRACING
398#define RCU_NAME RCU_NAME_RAW
399#else
400static char rcu_name[] = RCU_NAME_RAW;
401static const char *tp_rcu_varname __used __tracepoint_string = rcu_name;
402#define RCU_NAME rcu_name
403#endif
404
405
406static void rcu_bootup_announce(void);
407static void rcu_qs(void);
408static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);
409#ifdef CONFIG_HOTPLUG_CPU
410static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
411#endif
412static int rcu_print_task_exp_stall(struct rcu_node *rnp);
413static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
414static void rcu_flavor_sched_clock_irq(int user);
415static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck);
416static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
417static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
418static bool rcu_is_callbacks_kthread(void);
419static void rcu_cpu_kthread_setup(unsigned int cpu);
420static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp);
421static void __init rcu_spawn_boost_kthreads(void);
422static void rcu_cleanup_after_idle(void);
423static void rcu_prepare_for_idle(void);
424static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
425static bool rcu_preempt_need_deferred_qs(struct task_struct *t);
426static void rcu_preempt_deferred_qs(struct task_struct *t);
427static void zero_cpu_stall_ticks(struct rcu_data *rdp);
428static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);
429static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
430static void rcu_init_one_nocb(struct rcu_node *rnp);
431static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
432 unsigned long j);
433static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
434 bool *was_alldone, unsigned long flags);
435static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
436 unsigned long flags);
437static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level);
438static bool do_nocb_deferred_wakeup(struct rcu_data *rdp);
439static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
440static void rcu_spawn_cpu_nocb_kthread(int cpu);
441static void __init rcu_spawn_nocb_kthreads(void);
442static void show_rcu_nocb_state(struct rcu_data *rdp);
443static void rcu_nocb_lock(struct rcu_data *rdp);
444static void rcu_nocb_unlock(struct rcu_data *rdp);
445static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
446 unsigned long flags);
447static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp);
448#ifdef CONFIG_RCU_NOCB_CPU
449static void __init rcu_organize_nocb_kthreads(void);
450#define rcu_nocb_lock_irqsave(rdp, flags) \
451do { \
452 if (!rcu_segcblist_is_offloaded(&(rdp)->cblist)) \
453 local_irq_save(flags); \
454 else \
455 raw_spin_lock_irqsave(&(rdp)->nocb_lock, (flags)); \
456} while (0)
457#else
458#define rcu_nocb_lock_irqsave(rdp, flags) local_irq_save(flags)
459#endif
460
461static void rcu_bind_gp_kthread(void);
462static bool rcu_nohz_full_cpu(void);
463static void rcu_dynticks_task_enter(void);
464static void rcu_dynticks_task_exit(void);
465static void rcu_dynticks_task_trace_enter(void);
466static void rcu_dynticks_task_trace_exit(void);
467
468
469static void record_gp_stall_check_time(void);
470static void rcu_iw_handler(struct irq_work *iwp);
471static void check_cpu_stall(struct rcu_data *rdp);
472static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,
473 const unsigned long gpssdelay);
474