1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) fmt
14
15#include <linux/types.h>
16#include <linux/kernel.h>
17#include <linux/init.h>
18#include <linux/module.h>
19#include <linux/kthread.h>
20#include <linux/err.h>
21#include <linux/spinlock.h>
22#include <linux/smp.h>
23#include <linux/rcupdate_wait.h>
24#include <linux/interrupt.h>
25#include <linux/sched/signal.h>
26#include <uapi/linux/sched/types.h>
27#include <linux/atomic.h>
28#include <linux/bitops.h>
29#include <linux/completion.h>
30#include <linux/moduleparam.h>
31#include <linux/percpu.h>
32#include <linux/notifier.h>
33#include <linux/reboot.h>
34#include <linux/freezer.h>
35#include <linux/cpu.h>
36#include <linux/delay.h>
37#include <linux/stat.h>
38#include <linux/srcu.h>
39#include <linux/slab.h>
40#include <linux/trace_clock.h>
41#include <asm/byteorder.h>
42#include <linux/torture.h>
43#include <linux/vmalloc.h>
44#include <linux/sched/debug.h>
45#include <linux/sched/sysctl.h>
46#include <linux/oom.h>
47#include <linux/tick.h>
48#include <linux/rcupdate_trace.h>
49
50#include "rcu.h"
51
52MODULE_LICENSE("GPL");
53MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
54
55
56#define RCUTORTURE_RDR_SHIFT 8
57#define RCUTORTURE_RDR_MASK ((1 << RCUTORTURE_RDR_SHIFT) - 1)
58#define RCUTORTURE_RDR_BH 0x01
59#define RCUTORTURE_RDR_IRQ 0x02
60#define RCUTORTURE_RDR_PREEMPT 0x04
61#define RCUTORTURE_RDR_RBH 0x08
62#define RCUTORTURE_RDR_SCHED 0x10
63#define RCUTORTURE_RDR_RCU 0x20
64#define RCUTORTURE_RDR_NBITS 6
65#define RCUTORTURE_MAX_EXTEND \
66 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \
67 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED)
68#define RCUTORTURE_RDR_MAX_LOOPS 0x7
69
70#define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3)
71
72torture_param(int, extendables, RCUTORTURE_MAX_EXTEND,
73 "Extend readers by disabling bh (1), irqs (2), or preempt (4)");
74torture_param(int, fqs_duration, 0,
75 "Duration of fqs bursts (us), 0 to disable");
76torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
77torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
78torture_param(bool, fwd_progress, 1, "Test grace-period forward progress");
79torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait");
80torture_param(int, fwd_progress_holdoff, 60,
81 "Time between forward-progress tests (s)");
82torture_param(bool, fwd_progress_need_resched, 1,
83 "Hide cond_resched() behind need_resched()");
84torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
85torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
86torture_param(bool, gp_normal, false,
87 "Use normal (non-expedited) GP wait primitives");
88torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
89torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
90torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
91torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
92torture_param(int, n_barrier_cbs, 0,
93 "# of callbacks/kthreads for barrier testing");
94torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
95torture_param(int, nreaders, -1, "Number of RCU reader threads");
96torture_param(int, object_debug, 0,
97 "Enable debug-object double call_rcu() testing");
98torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
99torture_param(int, onoff_interval, 0,
100 "Time between CPU hotplugs (jiffies), 0=disable");
101torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable");
102torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)");
103torture_param(int, read_exit_delay, 13,
104 "Delay between read-then-exit episodes (s)");
105torture_param(int, read_exit_burst, 16,
106 "# of read-then-exit bursts per episode, zero to disable");
107torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
108torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
109torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
110torture_param(int, stall_cpu_holdoff, 10,
111 "Time to wait before starting stall (s).");
112torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");
113torture_param(int, stall_cpu_block, 0, "Sleep while stalling.");
114torture_param(int, stall_gp_kthread, 0,
115 "Grace-period kthread stall duration (s).");
116torture_param(int, stat_interval, 60,
117 "Number of seconds between stats printk()s");
118torture_param(int, stutter, 5, "Number of seconds to run/halt test");
119torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
120torture_param(int, test_boost_duration, 4,
121 "Duration of each boost test, seconds.");
122torture_param(int, test_boost_interval, 7,
123 "Interval between boost tests, seconds.");
124torture_param(bool, test_no_idle_hz, true,
125 "Test support for tickless idle CPUs");
126torture_param(int, verbose, 1,
127 "Enable verbose debugging printk()s");
128
129static char *torture_type = "rcu";
130module_param(torture_type, charp, 0444);
131MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)");
132
133static int nrealnocbers;
134static int nrealreaders;
135static struct task_struct *writer_task;
136static struct task_struct **fakewriter_tasks;
137static struct task_struct **reader_tasks;
138static struct task_struct **nocb_tasks;
139static struct task_struct *stats_task;
140static struct task_struct *fqs_task;
141static struct task_struct *boost_tasks[NR_CPUS];
142static struct task_struct *stall_task;
143static struct task_struct *fwd_prog_task;
144static struct task_struct **barrier_cbs_tasks;
145static struct task_struct *barrier_task;
146static struct task_struct *read_exit_task;
147
148#define RCU_TORTURE_PIPE_LEN 10
149
150
151struct rcu_torture_reader_check {
152 unsigned long rtc_myloops;
153 int rtc_chkrdr;
154 unsigned long rtc_chkloops;
155 int rtc_ready;
156 struct rcu_torture_reader_check *rtc_assigner;
157} ____cacheline_internodealigned_in_smp;
158
159
160struct rcu_torture {
161 struct rcu_head rtort_rcu;
162 int rtort_pipe_count;
163 struct list_head rtort_free;
164 int rtort_mbtest;
165 struct rcu_torture_reader_check *rtort_chkp;
166};
167
168static LIST_HEAD(rcu_torture_freelist);
169static struct rcu_torture __rcu *rcu_torture_current;
170static unsigned long rcu_torture_current_version;
171static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
172static DEFINE_SPINLOCK(rcu_torture_lock);
173static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count);
174static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch);
175static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
176static struct rcu_torture_reader_check *rcu_torture_reader_mbchk;
177static atomic_t n_rcu_torture_alloc;
178static atomic_t n_rcu_torture_alloc_fail;
179static atomic_t n_rcu_torture_free;
180static atomic_t n_rcu_torture_mberror;
181static atomic_t n_rcu_torture_mbchk_fail;
182static atomic_t n_rcu_torture_mbchk_tries;
183static atomic_t n_rcu_torture_error;
184static long n_rcu_torture_barrier_error;
185static long n_rcu_torture_boost_ktrerror;
186static long n_rcu_torture_boost_rterror;
187static long n_rcu_torture_boost_failure;
188static long n_rcu_torture_boosts;
189static atomic_long_t n_rcu_torture_timers;
190static long n_barrier_attempts;
191static long n_barrier_successes;
192static unsigned long n_read_exits;
193static struct list_head rcu_torture_removed;
194static unsigned long shutdown_jiffies;
195static unsigned long start_gp_seq;
196static atomic_long_t n_nocb_offload;
197static atomic_long_t n_nocb_deoffload;
198
199static int rcu_torture_writer_state;
200#define RTWS_FIXED_DELAY 0
201#define RTWS_DELAY 1
202#define RTWS_REPLACE 2
203#define RTWS_DEF_FREE 3
204#define RTWS_EXP_SYNC 4
205#define RTWS_COND_GET 5
206#define RTWS_COND_SYNC 6
207#define RTWS_POLL_GET 7
208#define RTWS_POLL_WAIT 8
209#define RTWS_SYNC 9
210#define RTWS_STUTTER 10
211#define RTWS_STOPPING 11
212static const char * const rcu_torture_writer_state_names[] = {
213 "RTWS_FIXED_DELAY",
214 "RTWS_DELAY",
215 "RTWS_REPLACE",
216 "RTWS_DEF_FREE",
217 "RTWS_EXP_SYNC",
218 "RTWS_COND_GET",
219 "RTWS_COND_SYNC",
220 "RTWS_POLL_GET",
221 "RTWS_POLL_WAIT",
222 "RTWS_SYNC",
223 "RTWS_STUTTER",
224 "RTWS_STOPPING",
225};
226
227
228struct rt_read_seg {
229 int rt_readstate;
230 unsigned long rt_delay_jiffies;
231 unsigned long rt_delay_ms;
232 unsigned long rt_delay_us;
233 bool rt_preempted;
234};
235static int err_segs_recorded;
236static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS];
237static int rt_read_nsegs;
238
239static const char *rcu_torture_writer_state_getname(void)
240{
241 unsigned int i = READ_ONCE(rcu_torture_writer_state);
242
243 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names))
244 return "???";
245 return rcu_torture_writer_state_names[i];
246}
247
248#if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
249#define rcu_can_boost() 1
250#else
251#define rcu_can_boost() 0
252#endif
253
254#ifdef CONFIG_RCU_TRACE
255static u64 notrace rcu_trace_clock_local(void)
256{
257 u64 ts = trace_clock_local();
258
259 (void)do_div(ts, NSEC_PER_USEC);
260 return ts;
261}
262#else
263static u64 notrace rcu_trace_clock_local(void)
264{
265 return 0ULL;
266}
267#endif
268
269
270
271
272
273static bool shutdown_time_arrived(void)
274{
275 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ);
276}
277
278static unsigned long boost_starttime;
279static DEFINE_MUTEX(boost_mutex);
280
281static atomic_t barrier_cbs_count;
282static bool barrier_phase;
283static atomic_t barrier_cbs_invoked;
284static wait_queue_head_t *barrier_cbs_wq;
285static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
286
287static bool rcu_fwd_cb_nodelay;
288
289
290
291
292static struct rcu_torture *
293rcu_torture_alloc(void)
294{
295 struct list_head *p;
296
297 spin_lock_bh(&rcu_torture_lock);
298 if (list_empty(&rcu_torture_freelist)) {
299 atomic_inc(&n_rcu_torture_alloc_fail);
300 spin_unlock_bh(&rcu_torture_lock);
301 return NULL;
302 }
303 atomic_inc(&n_rcu_torture_alloc);
304 p = rcu_torture_freelist.next;
305 list_del_init(p);
306 spin_unlock_bh(&rcu_torture_lock);
307 return container_of(p, struct rcu_torture, rtort_free);
308}
309
310
311
312
313static void
314rcu_torture_free(struct rcu_torture *p)
315{
316 atomic_inc(&n_rcu_torture_free);
317 spin_lock_bh(&rcu_torture_lock);
318 list_add_tail(&p->rtort_free, &rcu_torture_freelist);
319 spin_unlock_bh(&rcu_torture_lock);
320}
321
322
323
324
325
326struct rcu_torture_ops {
327 int ttype;
328 void (*init)(void);
329 void (*cleanup)(void);
330 int (*readlock)(void);
331 void (*read_delay)(struct torture_random_state *rrsp,
332 struct rt_read_seg *rtrsp);
333 void (*readunlock)(int idx);
334 unsigned long (*get_gp_seq)(void);
335 unsigned long (*gp_diff)(unsigned long new, unsigned long old);
336 void (*deferred_free)(struct rcu_torture *p);
337 void (*sync)(void);
338 void (*exp_sync)(void);
339 unsigned long (*get_gp_state)(void);
340 unsigned long (*start_gp_poll)(void);
341 bool (*poll_gp_state)(unsigned long oldstate);
342 void (*cond_sync)(unsigned long oldstate);
343 call_rcu_func_t call;
344 void (*cb_barrier)(void);
345 void (*fqs)(void);
346 void (*stats)(void);
347 void (*gp_kthread_dbg)(void);
348 int (*stall_dur)(void);
349 int irq_capable;
350 int can_boost;
351 int extendables;
352 int slow_gps;
353 const char *name;
354};
355
356static struct rcu_torture_ops *cur_ops;
357
358
359
360
361
362static int rcu_torture_read_lock(void) __acquires(RCU)
363{
364 rcu_read_lock();
365 return 0;
366}
367
368static void
369rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
370{
371 unsigned long started;
372 unsigned long completed;
373 const unsigned long shortdelay_us = 200;
374 unsigned long longdelay_ms = 300;
375 unsigned long long ts;
376
377
378
379
380
381 if (!READ_ONCE(rcu_fwd_cb_nodelay) &&
382 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
383 started = cur_ops->get_gp_seq();
384 ts = rcu_trace_clock_local();
385 if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK))
386 longdelay_ms = 5;
387 mdelay(longdelay_ms);
388 rtrsp->rt_delay_ms = longdelay_ms;
389 completed = cur_ops->get_gp_seq();
390 do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
391 started, completed);
392 }
393 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) {
394 udelay(shortdelay_us);
395 rtrsp->rt_delay_us = shortdelay_us;
396 }
397 if (!preempt_count() &&
398 !(torture_random(rrsp) % (nrealreaders * 500))) {
399 torture_preempt_schedule();
400 rtrsp->rt_preempted = true;
401 }
402}
403
404static void rcu_torture_read_unlock(int idx) __releases(RCU)
405{
406 rcu_read_unlock();
407}
408
409
410
411
412static bool
413rcu_torture_pipe_update_one(struct rcu_torture *rp)
414{
415 int i;
416 struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp);
417
418 if (rtrcp) {
419 WRITE_ONCE(rp->rtort_chkp, NULL);
420 smp_store_release(&rtrcp->rtc_ready, 1);
421 }
422 i = READ_ONCE(rp->rtort_pipe_count);
423 if (i > RCU_TORTURE_PIPE_LEN)
424 i = RCU_TORTURE_PIPE_LEN;
425 atomic_inc(&rcu_torture_wcount[i]);
426 WRITE_ONCE(rp->rtort_pipe_count, i + 1);
427 if (rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
428 rp->rtort_mbtest = 0;
429 return true;
430 }
431 return false;
432}
433
434
435
436
437
438static void
439rcu_torture_pipe_update(struct rcu_torture *old_rp)
440{
441 struct rcu_torture *rp;
442 struct rcu_torture *rp1;
443
444 if (old_rp)
445 list_add(&old_rp->rtort_free, &rcu_torture_removed);
446 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
447 if (rcu_torture_pipe_update_one(rp)) {
448 list_del(&rp->rtort_free);
449 rcu_torture_free(rp);
450 }
451 }
452}
453
454static void
455rcu_torture_cb(struct rcu_head *p)
456{
457 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
458
459 if (torture_must_stop_irq()) {
460
461
462 return;
463 }
464 if (rcu_torture_pipe_update_one(rp))
465 rcu_torture_free(rp);
466 else
467 cur_ops->deferred_free(rp);
468}
469
470static unsigned long rcu_no_completed(void)
471{
472 return 0;
473}
474
475static void rcu_torture_deferred_free(struct rcu_torture *p)
476{
477 call_rcu(&p->rtort_rcu, rcu_torture_cb);
478}
479
480static void rcu_sync_torture_init(void)
481{
482 INIT_LIST_HEAD(&rcu_torture_removed);
483}
484
485static struct rcu_torture_ops rcu_ops = {
486 .ttype = RCU_FLAVOR,
487 .init = rcu_sync_torture_init,
488 .readlock = rcu_torture_read_lock,
489 .read_delay = rcu_read_delay,
490 .readunlock = rcu_torture_read_unlock,
491 .get_gp_seq = rcu_get_gp_seq,
492 .gp_diff = rcu_seq_diff,
493 .deferred_free = rcu_torture_deferred_free,
494 .sync = synchronize_rcu,
495 .exp_sync = synchronize_rcu_expedited,
496 .get_gp_state = get_state_synchronize_rcu,
497 .cond_sync = cond_synchronize_rcu,
498 .call = call_rcu,
499 .cb_barrier = rcu_barrier,
500 .fqs = rcu_force_quiescent_state,
501 .stats = NULL,
502 .gp_kthread_dbg = show_rcu_gp_kthreads,
503 .stall_dur = rcu_jiffies_till_stall_check,
504 .irq_capable = 1,
505 .can_boost = rcu_can_boost(),
506 .extendables = RCUTORTURE_MAX_EXTEND,
507 .name = "rcu"
508};
509
510
511
512
513
514
515
516
517static void rcu_busted_torture_deferred_free(struct rcu_torture *p)
518{
519
520 rcu_torture_cb(&p->rtort_rcu);
521}
522
523static void synchronize_rcu_busted(void)
524{
525
526}
527
528static void
529call_rcu_busted(struct rcu_head *head, rcu_callback_t func)
530{
531
532 func(head);
533}
534
535static struct rcu_torture_ops rcu_busted_ops = {
536 .ttype = INVALID_RCU_FLAVOR,
537 .init = rcu_sync_torture_init,
538 .readlock = rcu_torture_read_lock,
539 .read_delay = rcu_read_delay,
540 .readunlock = rcu_torture_read_unlock,
541 .get_gp_seq = rcu_no_completed,
542 .deferred_free = rcu_busted_torture_deferred_free,
543 .sync = synchronize_rcu_busted,
544 .exp_sync = synchronize_rcu_busted,
545 .call = call_rcu_busted,
546 .cb_barrier = NULL,
547 .fqs = NULL,
548 .stats = NULL,
549 .irq_capable = 1,
550 .name = "busted"
551};
552
553
554
555
556
557DEFINE_STATIC_SRCU(srcu_ctl);
558static struct srcu_struct srcu_ctld;
559static struct srcu_struct *srcu_ctlp = &srcu_ctl;
560
561static int srcu_torture_read_lock(void) __acquires(srcu_ctlp)
562{
563 return srcu_read_lock(srcu_ctlp);
564}
565
566static void
567srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
568{
569 long delay;
570 const long uspertick = 1000000 / HZ;
571 const long longdelay = 10;
572
573
574
575 delay = torture_random(rrsp) %
576 (nrealreaders * 2 * longdelay * uspertick);
577 if (!delay && in_task()) {
578 schedule_timeout_interruptible(longdelay);
579 rtrsp->rt_delay_jiffies = longdelay;
580 } else {
581 rcu_read_delay(rrsp, rtrsp);
582 }
583}
584
585static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp)
586{
587 srcu_read_unlock(srcu_ctlp, idx);
588}
589
590static unsigned long srcu_torture_completed(void)
591{
592 return srcu_batches_completed(srcu_ctlp);
593}
594
595static void srcu_torture_deferred_free(struct rcu_torture *rp)
596{
597 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb);
598}
599
600static void srcu_torture_synchronize(void)
601{
602 synchronize_srcu(srcu_ctlp);
603}
604
605static unsigned long srcu_torture_get_gp_state(void)
606{
607 return get_state_synchronize_srcu(srcu_ctlp);
608}
609
610static unsigned long srcu_torture_start_gp_poll(void)
611{
612 return start_poll_synchronize_srcu(srcu_ctlp);
613}
614
615static bool srcu_torture_poll_gp_state(unsigned long oldstate)
616{
617 return poll_state_synchronize_srcu(srcu_ctlp, oldstate);
618}
619
620static void srcu_torture_call(struct rcu_head *head,
621 rcu_callback_t func)
622{
623 call_srcu(srcu_ctlp, head, func);
624}
625
626static void srcu_torture_barrier(void)
627{
628 srcu_barrier(srcu_ctlp);
629}
630
631static void srcu_torture_stats(void)
632{
633 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG);
634}
635
636static void srcu_torture_synchronize_expedited(void)
637{
638 synchronize_srcu_expedited(srcu_ctlp);
639}
640
641static struct rcu_torture_ops srcu_ops = {
642 .ttype = SRCU_FLAVOR,
643 .init = rcu_sync_torture_init,
644 .readlock = srcu_torture_read_lock,
645 .read_delay = srcu_read_delay,
646 .readunlock = srcu_torture_read_unlock,
647 .get_gp_seq = srcu_torture_completed,
648 .deferred_free = srcu_torture_deferred_free,
649 .sync = srcu_torture_synchronize,
650 .exp_sync = srcu_torture_synchronize_expedited,
651 .get_gp_state = srcu_torture_get_gp_state,
652 .start_gp_poll = srcu_torture_start_gp_poll,
653 .poll_gp_state = srcu_torture_poll_gp_state,
654 .call = srcu_torture_call,
655 .cb_barrier = srcu_torture_barrier,
656 .stats = srcu_torture_stats,
657 .irq_capable = 1,
658 .name = "srcu"
659};
660
661static void srcu_torture_init(void)
662{
663 rcu_sync_torture_init();
664 WARN_ON(init_srcu_struct(&srcu_ctld));
665 srcu_ctlp = &srcu_ctld;
666}
667
668static void srcu_torture_cleanup(void)
669{
670 cleanup_srcu_struct(&srcu_ctld);
671 srcu_ctlp = &srcu_ctl;
672}
673
674
675static struct rcu_torture_ops srcud_ops = {
676 .ttype = SRCU_FLAVOR,
677 .init = srcu_torture_init,
678 .cleanup = srcu_torture_cleanup,
679 .readlock = srcu_torture_read_lock,
680 .read_delay = srcu_read_delay,
681 .readunlock = srcu_torture_read_unlock,
682 .get_gp_seq = srcu_torture_completed,
683 .deferred_free = srcu_torture_deferred_free,
684 .sync = srcu_torture_synchronize,
685 .exp_sync = srcu_torture_synchronize_expedited,
686 .call = srcu_torture_call,
687 .cb_barrier = srcu_torture_barrier,
688 .stats = srcu_torture_stats,
689 .irq_capable = 1,
690 .name = "srcud"
691};
692
693
694static struct rcu_torture_ops busted_srcud_ops = {
695 .ttype = SRCU_FLAVOR,
696 .init = srcu_torture_init,
697 .cleanup = srcu_torture_cleanup,
698 .readlock = srcu_torture_read_lock,
699 .read_delay = rcu_read_delay,
700 .readunlock = srcu_torture_read_unlock,
701 .get_gp_seq = srcu_torture_completed,
702 .deferred_free = srcu_torture_deferred_free,
703 .sync = srcu_torture_synchronize,
704 .exp_sync = srcu_torture_synchronize_expedited,
705 .call = srcu_torture_call,
706 .cb_barrier = srcu_torture_barrier,
707 .stats = srcu_torture_stats,
708 .irq_capable = 1,
709 .extendables = RCUTORTURE_MAX_EXTEND,
710 .name = "busted_srcud"
711};
712
713
714
715
716
717static int tasks_torture_read_lock(void)
718{
719 return 0;
720}
721
722static void tasks_torture_read_unlock(int idx)
723{
724}
725
726static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
727{
728 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb);
729}
730
731static void synchronize_rcu_mult_test(void)
732{
733 synchronize_rcu_mult(call_rcu_tasks, call_rcu);
734}
735
736static struct rcu_torture_ops tasks_ops = {
737 .ttype = RCU_TASKS_FLAVOR,
738 .init = rcu_sync_torture_init,
739 .readlock = tasks_torture_read_lock,
740 .read_delay = rcu_read_delay,
741 .readunlock = tasks_torture_read_unlock,
742 .get_gp_seq = rcu_no_completed,
743 .deferred_free = rcu_tasks_torture_deferred_free,
744 .sync = synchronize_rcu_tasks,
745 .exp_sync = synchronize_rcu_mult_test,
746 .call = call_rcu_tasks,
747 .cb_barrier = rcu_barrier_tasks,
748 .gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread,
749 .fqs = NULL,
750 .stats = NULL,
751 .irq_capable = 1,
752 .slow_gps = 1,
753 .name = "tasks"
754};
755
756
757
758
759
760
761static void synchronize_rcu_trivial(void)
762{
763 int cpu;
764
765 for_each_online_cpu(cpu) {
766 rcutorture_sched_setaffinity(current->pid, cpumask_of(cpu));
767 WARN_ON_ONCE(raw_smp_processor_id() != cpu);
768 }
769}
770
771static int rcu_torture_read_lock_trivial(void) __acquires(RCU)
772{
773 preempt_disable();
774 return 0;
775}
776
777static void rcu_torture_read_unlock_trivial(int idx) __releases(RCU)
778{
779 preempt_enable();
780}
781
782static struct rcu_torture_ops trivial_ops = {
783 .ttype = RCU_TRIVIAL_FLAVOR,
784 .init = rcu_sync_torture_init,
785 .readlock = rcu_torture_read_lock_trivial,
786 .read_delay = rcu_read_delay,
787 .readunlock = rcu_torture_read_unlock_trivial,
788 .get_gp_seq = rcu_no_completed,
789 .sync = synchronize_rcu_trivial,
790 .exp_sync = synchronize_rcu_trivial,
791 .fqs = NULL,
792 .stats = NULL,
793 .irq_capable = 1,
794 .name = "trivial"
795};
796
797
798
799
800
801static void rcu_tasks_rude_torture_deferred_free(struct rcu_torture *p)
802{
803 call_rcu_tasks_rude(&p->rtort_rcu, rcu_torture_cb);
804}
805
806static struct rcu_torture_ops tasks_rude_ops = {
807 .ttype = RCU_TASKS_RUDE_FLAVOR,
808 .init = rcu_sync_torture_init,
809 .readlock = rcu_torture_read_lock_trivial,
810 .read_delay = rcu_read_delay,
811 .readunlock = rcu_torture_read_unlock_trivial,
812 .get_gp_seq = rcu_no_completed,
813 .deferred_free = rcu_tasks_rude_torture_deferred_free,
814 .sync = synchronize_rcu_tasks_rude,
815 .exp_sync = synchronize_rcu_tasks_rude,
816 .call = call_rcu_tasks_rude,
817 .cb_barrier = rcu_barrier_tasks_rude,
818 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread,
819 .fqs = NULL,
820 .stats = NULL,
821 .irq_capable = 1,
822 .name = "tasks-rude"
823};
824
825
826
827
828
829static int tasks_tracing_torture_read_lock(void)
830{
831 rcu_read_lock_trace();
832 return 0;
833}
834
835static void tasks_tracing_torture_read_unlock(int idx)
836{
837 rcu_read_unlock_trace();
838}
839
840static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p)
841{
842 call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb);
843}
844
845static struct rcu_torture_ops tasks_tracing_ops = {
846 .ttype = RCU_TASKS_TRACING_FLAVOR,
847 .init = rcu_sync_torture_init,
848 .readlock = tasks_tracing_torture_read_lock,
849 .read_delay = srcu_read_delay,
850 .readunlock = tasks_tracing_torture_read_unlock,
851 .get_gp_seq = rcu_no_completed,
852 .deferred_free = rcu_tasks_tracing_torture_deferred_free,
853 .sync = synchronize_rcu_tasks_trace,
854 .exp_sync = synchronize_rcu_tasks_trace,
855 .call = call_rcu_tasks_trace,
856 .cb_barrier = rcu_barrier_tasks_trace,
857 .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread,
858 .fqs = NULL,
859 .stats = NULL,
860 .irq_capable = 1,
861 .slow_gps = 1,
862 .name = "tasks-tracing"
863};
864
865static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old)
866{
867 if (!cur_ops->gp_diff)
868 return new - old;
869 return cur_ops->gp_diff(new, old);
870}
871
872static bool __maybe_unused torturing_tasks(void)
873{
874 return cur_ops == &tasks_ops || cur_ops == &tasks_rude_ops;
875}
876
877
878
879
880
881
882
883
884struct rcu_boost_inflight {
885 struct rcu_head rcu;
886 int inflight;
887};
888
889static void rcu_torture_boost_cb(struct rcu_head *head)
890{
891 struct rcu_boost_inflight *rbip =
892 container_of(head, struct rcu_boost_inflight, rcu);
893
894
895 smp_store_release(&rbip->inflight, 0);
896}
897
898static int old_rt_runtime = -1;
899
900static void rcu_torture_disable_rt_throttle(void)
901{
902
903
904
905
906
907
908 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1)
909 return;
910
911 old_rt_runtime = sysctl_sched_rt_runtime;
912 sysctl_sched_rt_runtime = -1;
913}
914
915static void rcu_torture_enable_rt_throttle(void)
916{
917 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1)
918 return;
919
920 sysctl_sched_rt_runtime = old_rt_runtime;
921 old_rt_runtime = -1;
922}
923
924static bool rcu_torture_boost_failed(unsigned long start, unsigned long end)
925{
926 if (end - start > test_boost_duration * HZ - HZ / 2) {
927 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
928 n_rcu_torture_boost_failure++;
929
930 return true;
931 }
932
933 return false;
934}
935
936static int rcu_torture_boost(void *arg)
937{
938 unsigned long call_rcu_time;
939 unsigned long endtime;
940 unsigned long oldstarttime;
941 struct rcu_boost_inflight rbi = { .inflight = 0 };
942 struct sched_param sp;
943
944 VERBOSE_TOROUT_STRING("rcu_torture_boost started");
945
946
947 sp.sched_priority = 1;
948 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
949 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!");
950 n_rcu_torture_boost_rterror++;
951 }
952
953 init_rcu_head_on_stack(&rbi.rcu);
954
955 do {
956
957 bool failed = false;
958
959
960 while (!kthread_should_stop()) {
961 if (mutex_trylock(&boost_mutex)) {
962 n_rcu_torture_boosts++;
963 mutex_unlock(&boost_mutex);
964 break;
965 }
966 schedule_timeout_uninterruptible(1);
967 }
968 if (kthread_should_stop())
969 goto checkwait;
970
971
972 oldstarttime = boost_starttime;
973 while (time_before(jiffies, oldstarttime)) {
974 schedule_timeout_interruptible(oldstarttime - jiffies);
975 if (stutter_wait("rcu_torture_boost"))
976 sched_set_fifo_low(current);
977 if (torture_must_stop())
978 goto checkwait;
979 }
980
981
982 endtime = oldstarttime + test_boost_duration * HZ;
983 call_rcu_time = jiffies;
984 while (time_before(jiffies, endtime)) {
985
986 if (!smp_load_acquire(&rbi.inflight)) {
987
988 smp_store_release(&rbi.inflight, 1);
989 call_rcu(&rbi.rcu, rcu_torture_boost_cb);
990
991 failed = failed ||
992 rcu_torture_boost_failed(call_rcu_time,
993 jiffies);
994 call_rcu_time = jiffies;
995 }
996 if (stutter_wait("rcu_torture_boost"))
997 sched_set_fifo_low(current);
998 if (torture_must_stop())
999 goto checkwait;
1000 }
1001
1002
1003
1004
1005
1006
1007 if (!failed && smp_load_acquire(&rbi.inflight))
1008 rcu_torture_boost_failed(call_rcu_time, jiffies);
1009
1010
1011
1012
1013
1014
1015
1016
1017 while (oldstarttime == boost_starttime &&
1018 !kthread_should_stop()) {
1019 if (mutex_trylock(&boost_mutex)) {
1020 boost_starttime = jiffies +
1021 test_boost_interval * HZ;
1022 mutex_unlock(&boost_mutex);
1023 break;
1024 }
1025 schedule_timeout_uninterruptible(1);
1026 }
1027
1028
1029checkwait: if (stutter_wait("rcu_torture_boost"))
1030 sched_set_fifo_low(current);
1031 } while (!torture_must_stop());
1032
1033
1034 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) {
1035 torture_shutdown_absorb("rcu_torture_boost");
1036 schedule_timeout_uninterruptible(1);
1037 }
1038 destroy_rcu_head_on_stack(&rbi.rcu);
1039 torture_kthread_stopping("rcu_torture_boost");
1040 return 0;
1041}
1042
1043
1044
1045
1046
1047
1048static int
1049rcu_torture_fqs(void *arg)
1050{
1051 unsigned long fqs_resume_time;
1052 int fqs_burst_remaining;
1053 int oldnice = task_nice(current);
1054
1055 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
1056 do {
1057 fqs_resume_time = jiffies + fqs_stutter * HZ;
1058 while (time_before(jiffies, fqs_resume_time) &&
1059 !kthread_should_stop()) {
1060 schedule_timeout_interruptible(1);
1061 }
1062 fqs_burst_remaining = fqs_duration;
1063 while (fqs_burst_remaining > 0 &&
1064 !kthread_should_stop()) {
1065 cur_ops->fqs();
1066 udelay(fqs_holdoff);
1067 fqs_burst_remaining -= fqs_holdoff;
1068 }
1069 if (stutter_wait("rcu_torture_fqs"))
1070 sched_set_normal(current, oldnice);
1071 } while (!torture_must_stop());
1072 torture_kthread_stopping("rcu_torture_fqs");
1073 return 0;
1074}
1075
1076
1077
1078static int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, RTWS_COND_GET, RTWS_POLL_GET, RTWS_SYNC };
1079static int nsynctypes;
1080
1081
1082
1083
1084static void rcu_torture_write_types(void)
1085{
1086 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
1087 bool gp_poll1 = gp_poll, gp_sync1 = gp_sync;
1088
1089
1090 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_poll1 && !gp_sync1)
1091 gp_cond1 = gp_exp1 = gp_normal1 = gp_poll1 = gp_sync1 = true;
1092 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
1093 synctype[nsynctypes++] = RTWS_COND_GET;
1094 pr_info("%s: Testing conditional GPs.\n", __func__);
1095 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {
1096 pr_alert("%s: gp_cond without primitives.\n", __func__);
1097 }
1098 if (gp_exp1 && cur_ops->exp_sync) {
1099 synctype[nsynctypes++] = RTWS_EXP_SYNC;
1100 pr_info("%s: Testing expedited GPs.\n", __func__);
1101 } else if (gp_exp && !cur_ops->exp_sync) {
1102 pr_alert("%s: gp_exp without primitives.\n", __func__);
1103 }
1104 if (gp_normal1 && cur_ops->deferred_free) {
1105 synctype[nsynctypes++] = RTWS_DEF_FREE;
1106 pr_info("%s: Testing asynchronous GPs.\n", __func__);
1107 } else if (gp_normal && !cur_ops->deferred_free) {
1108 pr_alert("%s: gp_normal without primitives.\n", __func__);
1109 }
1110 if (gp_poll1 && cur_ops->start_gp_poll && cur_ops->poll_gp_state) {
1111 synctype[nsynctypes++] = RTWS_POLL_GET;
1112 pr_info("%s: Testing polling GPs.\n", __func__);
1113 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
1114 pr_alert("%s: gp_poll without primitives.\n", __func__);
1115 }
1116 if (gp_sync1 && cur_ops->sync) {
1117 synctype[nsynctypes++] = RTWS_SYNC;
1118 pr_info("%s: Testing normal GPs.\n", __func__);
1119 } else if (gp_sync && !cur_ops->sync) {
1120 pr_alert("%s: gp_sync without primitives.\n", __func__);
1121 }
1122}
1123
1124
1125
1126
1127
1128
1129static int
1130rcu_torture_writer(void *arg)
1131{
1132 bool boot_ended;
1133 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal();
1134 unsigned long cookie;
1135 int expediting = 0;
1136 unsigned long gp_snap;
1137 int i;
1138 int idx;
1139 int oldnice = task_nice(current);
1140 struct rcu_torture *rp;
1141 struct rcu_torture *old_rp;
1142 static DEFINE_TORTURE_RANDOM(rand);
1143 bool stutter_waited;
1144
1145 VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
1146 if (!can_expedite)
1147 pr_alert("%s" TORTURE_FLAG
1148 " GP expediting controlled from boot/sysfs for %s.\n",
1149 torture_type, cur_ops->name);
1150 if (WARN_ONCE(nsynctypes == 0,
1151 "rcu_torture_writer: No update-side primitives.\n")) {
1152
1153
1154
1155
1156
1157 rcu_torture_writer_state = RTWS_STOPPING;
1158 torture_kthread_stopping("rcu_torture_writer");
1159 }
1160
1161 do {
1162 rcu_torture_writer_state = RTWS_FIXED_DELAY;
1163 torture_hrtimeout_us(500, 1000, &rand);
1164 rp = rcu_torture_alloc();
1165 if (rp == NULL)
1166 continue;
1167 rp->rtort_pipe_count = 0;
1168 rcu_torture_writer_state = RTWS_DELAY;
1169 udelay(torture_random(&rand) & 0x3ff);
1170 rcu_torture_writer_state = RTWS_REPLACE;
1171 old_rp = rcu_dereference_check(rcu_torture_current,
1172 current == writer_task);
1173 rp->rtort_mbtest = 1;
1174 rcu_assign_pointer(rcu_torture_current, rp);
1175 smp_wmb();
1176 if (old_rp) {
1177 i = old_rp->rtort_pipe_count;
1178 if (i > RCU_TORTURE_PIPE_LEN)
1179 i = RCU_TORTURE_PIPE_LEN;
1180 atomic_inc(&rcu_torture_wcount[i]);
1181 WRITE_ONCE(old_rp->rtort_pipe_count,
1182 old_rp->rtort_pipe_count + 1);
1183 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {
1184 idx = cur_ops->readlock();
1185 cookie = cur_ops->get_gp_state();
1186 WARN_ONCE(rcu_torture_writer_state != RTWS_DEF_FREE &&
1187 cur_ops->poll_gp_state(cookie),
1188 "%s: Cookie check 1 failed %s(%d) %lu->%lu\n",
1189 __func__,
1190 rcu_torture_writer_state_getname(),
1191 rcu_torture_writer_state,
1192 cookie, cur_ops->get_gp_state());
1193 cur_ops->readunlock(idx);
1194 }
1195 switch (synctype[torture_random(&rand) % nsynctypes]) {
1196 case RTWS_DEF_FREE:
1197 rcu_torture_writer_state = RTWS_DEF_FREE;
1198 cur_ops->deferred_free(old_rp);
1199 break;
1200 case RTWS_EXP_SYNC:
1201 rcu_torture_writer_state = RTWS_EXP_SYNC;
1202 cur_ops->exp_sync();
1203 rcu_torture_pipe_update(old_rp);
1204 break;
1205 case RTWS_COND_GET:
1206 rcu_torture_writer_state = RTWS_COND_GET;
1207 gp_snap = cur_ops->get_gp_state();
1208 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
1209 rcu_torture_writer_state = RTWS_COND_SYNC;
1210 cur_ops->cond_sync(gp_snap);
1211 rcu_torture_pipe_update(old_rp);
1212 break;
1213 case RTWS_POLL_GET:
1214 rcu_torture_writer_state = RTWS_POLL_GET;
1215 gp_snap = cur_ops->start_gp_poll();
1216 rcu_torture_writer_state = RTWS_POLL_WAIT;
1217 while (!cur_ops->poll_gp_state(gp_snap))
1218 torture_hrtimeout_jiffies(torture_random(&rand) % 16,
1219 &rand);
1220 rcu_torture_pipe_update(old_rp);
1221 break;
1222 case RTWS_SYNC:
1223 rcu_torture_writer_state = RTWS_SYNC;
1224 cur_ops->sync();
1225 rcu_torture_pipe_update(old_rp);
1226 break;
1227 default:
1228 WARN_ON_ONCE(1);
1229 break;
1230 }
1231 if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
1232 WARN_ONCE(rcu_torture_writer_state != RTWS_DEF_FREE &&
1233 !cur_ops->poll_gp_state(cookie),
1234 "%s: Cookie check 2 failed %s(%d) %lu->%lu\n",
1235 __func__,
1236 rcu_torture_writer_state_getname(),
1237 rcu_torture_writer_state,
1238 cookie, cur_ops->get_gp_state());
1239 }
1240 WRITE_ONCE(rcu_torture_current_version,
1241 rcu_torture_current_version + 1);
1242
1243 if (can_expedite &&
1244 !(torture_random(&rand) & 0xff & (!!expediting - 1))) {
1245 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
1246 if (expediting >= 0)
1247 rcu_expedite_gp();
1248 else
1249 rcu_unexpedite_gp();
1250 if (++expediting > 3)
1251 expediting = -expediting;
1252 } else if (!can_expedite) {
1253 can_expedite = !rcu_gp_is_expedited() &&
1254 !rcu_gp_is_normal();
1255 }
1256 rcu_torture_writer_state = RTWS_STUTTER;
1257 boot_ended = rcu_inkernel_boot_has_ended();
1258 stutter_waited = stutter_wait("rcu_torture_writer");
1259 if (stutter_waited &&
1260 !READ_ONCE(rcu_fwd_cb_nodelay) &&
1261 !cur_ops->slow_gps &&
1262 !torture_must_stop() &&
1263 boot_ended)
1264 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++)
1265 if (list_empty(&rcu_tortures[i].rtort_free) &&
1266 rcu_access_pointer(rcu_torture_current) !=
1267 &rcu_tortures[i]) {
1268 rcu_ftrace_dump(DUMP_ALL);
1269 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count);
1270 }
1271 if (stutter_waited)
1272 sched_set_normal(current, oldnice);
1273 } while (!torture_must_stop());
1274 rcu_torture_current = NULL;
1275
1276 if (expediting > 0)
1277 expediting = -expediting;
1278 while (can_expedite && expediting++ < 0)
1279 rcu_unexpedite_gp();
1280 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
1281 if (!can_expedite)
1282 pr_alert("%s" TORTURE_FLAG
1283 " Dynamic grace-period expediting was disabled.\n",
1284 torture_type);
1285 rcu_torture_writer_state = RTWS_STOPPING;
1286 torture_kthread_stopping("rcu_torture_writer");
1287 return 0;
1288}
1289
1290
1291
1292
1293
1294static int
1295rcu_torture_fakewriter(void *arg)
1296{
1297 unsigned long gp_snap;
1298 DEFINE_TORTURE_RANDOM(rand);
1299
1300 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1301 set_user_nice(current, MAX_NICE);
1302
1303 do {
1304 torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand);
1305 if (cur_ops->cb_barrier != NULL &&
1306 torture_random(&rand) % (nfakewriters * 8) == 0) {
1307 cur_ops->cb_barrier();
1308 } else {
1309 switch (synctype[torture_random(&rand) % nsynctypes]) {
1310 case RTWS_DEF_FREE:
1311 break;
1312 case RTWS_EXP_SYNC:
1313 cur_ops->exp_sync();
1314 break;
1315 case RTWS_COND_GET:
1316 gp_snap = cur_ops->get_gp_state();
1317 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
1318 cur_ops->cond_sync(gp_snap);
1319 break;
1320 case RTWS_POLL_GET:
1321 gp_snap = cur_ops->start_gp_poll();
1322 while (!cur_ops->poll_gp_state(gp_snap)) {
1323 torture_hrtimeout_jiffies(torture_random(&rand) % 16,
1324 &rand);
1325 }
1326 break;
1327 case RTWS_SYNC:
1328 cur_ops->sync();
1329 break;
1330 default:
1331 WARN_ON_ONCE(1);
1332 break;
1333 }
1334 }
1335 stutter_wait("rcu_torture_fakewriter");
1336 } while (!torture_must_stop());
1337
1338 torture_kthread_stopping("rcu_torture_fakewriter");
1339 return 0;
1340}
1341
1342static void rcu_torture_timer_cb(struct rcu_head *rhp)
1343{
1344 kfree(rhp);
1345}
1346
1347
1348static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp,
1349 struct torture_random_state *trsp)
1350{
1351 unsigned long loops;
1352 int noc = torture_num_online_cpus();
1353 int rdrchked;
1354 int rdrchker;
1355 struct rcu_torture_reader_check *rtrcp;
1356 struct rcu_torture_reader_check *rtrcp_assigner;
1357 struct rcu_torture_reader_check *rtrcp_chked;
1358 struct rcu_torture_reader_check *rtrcp_chker;
1359
1360 if (myid < 0)
1361 return;
1362
1363
1364 rtrcp = &rcu_torture_reader_mbchk[myid];
1365 WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1);
1366
1367
1368 rdrchked = torture_random(trsp) % nrealreaders;
1369 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked];
1370 rdrchker = torture_random(trsp) % nrealreaders;
1371 rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker];
1372 if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker &&
1373 smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 &&
1374 !READ_ONCE(rtp->rtort_chkp) &&
1375 !smp_load_acquire(&rtrcp_chker->rtc_assigner)) {
1376 rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops);
1377 WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0);
1378 rtrcp->rtc_chkrdr = rdrchked;
1379 WARN_ON_ONCE(rtrcp->rtc_ready);
1380 if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) ||
1381 cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp))
1382 (void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL);
1383 }
1384
1385
1386 rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner);
1387 if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready))
1388 return;
1389 rdrchked = rtrcp_assigner->rtc_chkrdr;
1390 if (WARN_ON_ONCE(rdrchked < 0))
1391 return;
1392 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked];
1393 loops = READ_ONCE(rtrcp_chked->rtc_myloops);
1394 atomic_inc(&n_rcu_torture_mbchk_tries);
1395 if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops))
1396 atomic_inc(&n_rcu_torture_mbchk_fail);
1397 rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2;
1398 rtrcp_assigner->rtc_ready = 0;
1399 smp_store_release(&rtrcp->rtc_assigner, NULL);
1400 smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1);
1401}
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412static void rcutorture_one_extend(int *readstate, int newstate,
1413 struct torture_random_state *trsp,
1414 struct rt_read_seg *rtrsp)
1415{
1416 unsigned long flags;
1417 int idxnew = -1;
1418 int idxold = *readstate;
1419 int statesnew = ~*readstate & newstate;
1420 int statesold = *readstate & ~newstate;
1421
1422 WARN_ON_ONCE(idxold < 0);
1423 WARN_ON_ONCE((idxold >> RCUTORTURE_RDR_SHIFT) > 1);
1424 rtrsp->rt_readstate = newstate;
1425
1426
1427 if (statesnew & RCUTORTURE_RDR_BH)
1428 local_bh_disable();
1429 if (statesnew & RCUTORTURE_RDR_IRQ)
1430 local_irq_disable();
1431 if (statesnew & RCUTORTURE_RDR_PREEMPT)
1432 preempt_disable();
1433 if (statesnew & RCUTORTURE_RDR_RBH)
1434 rcu_read_lock_bh();
1435 if (statesnew & RCUTORTURE_RDR_SCHED)
1436 rcu_read_lock_sched();
1437 if (statesnew & RCUTORTURE_RDR_RCU)
1438 idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT;
1439
1440
1441 if (statesold & RCUTORTURE_RDR_IRQ)
1442 local_irq_enable();
1443 if (statesold & RCUTORTURE_RDR_BH)
1444 local_bh_enable();
1445 if (statesold & RCUTORTURE_RDR_PREEMPT)
1446 preempt_enable();
1447 if (statesold & RCUTORTURE_RDR_RBH)
1448 rcu_read_unlock_bh();
1449 if (statesold & RCUTORTURE_RDR_SCHED)
1450 rcu_read_unlock_sched();
1451 if (statesold & RCUTORTURE_RDR_RCU) {
1452 bool lockit = !statesnew && !(torture_random(trsp) & 0xffff);
1453
1454 if (lockit)
1455 raw_spin_lock_irqsave(¤t->pi_lock, flags);
1456 cur_ops->readunlock(idxold >> RCUTORTURE_RDR_SHIFT);
1457 if (lockit)
1458 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1459 }
1460
1461
1462 if ((statesnew || statesold) && *readstate && newstate)
1463 cur_ops->read_delay(trsp, rtrsp);
1464
1465
1466 if (idxnew == -1)
1467 idxnew = idxold & ~RCUTORTURE_RDR_MASK;
1468 WARN_ON_ONCE(idxnew < 0);
1469 WARN_ON_ONCE((idxnew >> RCUTORTURE_RDR_SHIFT) > 1);
1470 *readstate = idxnew | newstate;
1471 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) < 0);
1472 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) > 1);
1473}
1474
1475
1476static int rcutorture_extend_mask_max(void)
1477{
1478 int mask;
1479
1480 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND);
1481 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;
1482 mask = mask | RCUTORTURE_RDR_RCU;
1483 return mask;
1484}
1485
1486
1487static int
1488rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp)
1489{
1490 int mask = rcutorture_extend_mask_max();
1491 unsigned long randmask1 = torture_random(trsp) >> 8;
1492 unsigned long randmask2 = randmask1 >> 3;
1493
1494 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT);
1495
1496 if (!(randmask1 & 0x7))
1497 mask = mask & randmask2;
1498 else
1499 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS));
1500
1501 if ((mask & RCUTORTURE_RDR_IRQ) &&
1502 ((!(mask & RCUTORTURE_RDR_BH) && (oldmask & RCUTORTURE_RDR_BH)) ||
1503 (!(mask & RCUTORTURE_RDR_RBH) && (oldmask & RCUTORTURE_RDR_RBH))))
1504 mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
1505 return mask ?: RCUTORTURE_RDR_RCU;
1506}
1507
1508
1509
1510
1511
1512static struct rt_read_seg *
1513rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp,
1514 struct rt_read_seg *rtrsp)
1515{
1516 int i;
1517 int j;
1518 int mask = rcutorture_extend_mask_max();
1519
1520 WARN_ON_ONCE(!*readstate);
1521 if (!((mask - 1) & mask))
1522 return rtrsp;
1523
1524 i = (torture_random(trsp) >> 3);
1525 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1;
1526 for (j = 0; j < i; j++) {
1527 mask = rcutorture_extend_mask(*readstate, trsp);
1528 rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]);
1529 }
1530 return &rtrsp[j];
1531}
1532
1533
1534
1535
1536
1537
1538static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
1539{
1540 unsigned long cookie;
1541 int i;
1542 unsigned long started;
1543 unsigned long completed;
1544 int newstate;
1545 struct rcu_torture *p;
1546 int pipe_count;
1547 int readstate = 0;
1548 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS] = { { 0 } };
1549 struct rt_read_seg *rtrsp = &rtseg[0];
1550 struct rt_read_seg *rtrsp1;
1551 unsigned long long ts;
1552
1553 WARN_ON_ONCE(!rcu_is_watching());
1554 newstate = rcutorture_extend_mask(readstate, trsp);
1555 rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++);
1556 if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
1557 cookie = cur_ops->get_gp_state();
1558 started = cur_ops->get_gp_seq();
1559 ts = rcu_trace_clock_local();
1560 p = rcu_dereference_check(rcu_torture_current,
1561 rcu_read_lock_bh_held() ||
1562 rcu_read_lock_sched_held() ||
1563 srcu_read_lock_held(srcu_ctlp) ||
1564 rcu_read_lock_trace_held() ||
1565 torturing_tasks());
1566 if (p == NULL) {
1567
1568 rcutorture_one_extend(&readstate, 0, trsp, rtrsp);
1569 return false;
1570 }
1571 if (p->rtort_mbtest == 0)
1572 atomic_inc(&n_rcu_torture_mberror);
1573 rcu_torture_reader_do_mbchk(myid, p, trsp);
1574 rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp);
1575 preempt_disable();
1576 pipe_count = READ_ONCE(p->rtort_pipe_count);
1577 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1578
1579 pipe_count = RCU_TORTURE_PIPE_LEN;
1580 }
1581 completed = cur_ops->get_gp_seq();
1582 if (pipe_count > 1) {
1583 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
1584 ts, started, completed);
1585 rcu_ftrace_dump(DUMP_ALL);
1586 }
1587 __this_cpu_inc(rcu_torture_count[pipe_count]);
1588 completed = rcutorture_seq_diff(completed, started);
1589 if (completed > RCU_TORTURE_PIPE_LEN) {
1590
1591 completed = RCU_TORTURE_PIPE_LEN;
1592 }
1593 __this_cpu_inc(rcu_torture_batch[completed]);
1594 preempt_enable();
1595 if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
1596 WARN_ONCE(cur_ops->poll_gp_state(cookie),
1597 "%s: Cookie check 3 failed %s(%d) %lu->%lu\n",
1598 __func__,
1599 rcu_torture_writer_state_getname(),
1600 rcu_torture_writer_state,
1601 cookie, cur_ops->get_gp_state());
1602 rcutorture_one_extend(&readstate, 0, trsp, rtrsp);
1603 WARN_ON_ONCE(readstate & RCUTORTURE_RDR_MASK);
1604
1605
1606 WARN_ON_ONCE(leakpointer && READ_ONCE(p->rtort_pipe_count) > 1);
1607
1608
1609 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) {
1610 i = 0;
1611 for (rtrsp1 = &rtseg[0]; rtrsp1 < rtrsp; rtrsp1++)
1612 err_segs[i++] = *rtrsp1;
1613 rt_read_nsegs = i;
1614 }
1615
1616 return true;
1617}
1618
1619static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand);
1620
1621
1622
1623
1624
1625
1626
1627static void rcu_torture_timer(struct timer_list *unused)
1628{
1629 atomic_long_inc(&n_rcu_torture_timers);
1630 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1);
1631
1632
1633 if (cur_ops->call) {
1634 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT);
1635
1636 if (rhp)
1637 cur_ops->call(rhp, rcu_torture_timer_cb);
1638 }
1639}
1640
1641
1642
1643
1644
1645
1646
1647static int
1648rcu_torture_reader(void *arg)
1649{
1650 unsigned long lastsleep = jiffies;
1651 long myid = (long)arg;
1652 int mynumonline = myid;
1653 DEFINE_TORTURE_RANDOM(rand);
1654 struct timer_list t;
1655
1656 VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
1657 set_user_nice(current, MAX_NICE);
1658 if (irqreader && cur_ops->irq_capable)
1659 timer_setup_on_stack(&t, rcu_torture_timer, 0);
1660 tick_dep_set_task(current, TICK_DEP_BIT_RCU);
1661 do {
1662 if (irqreader && cur_ops->irq_capable) {
1663 if (!timer_pending(&t))
1664 mod_timer(&t, jiffies + 1);
1665 }
1666 if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop())
1667 schedule_timeout_interruptible(HZ);
1668 if (time_after(jiffies, lastsleep) && !torture_must_stop()) {
1669 torture_hrtimeout_us(500, 1000, &rand);
1670 lastsleep = jiffies + 10;
1671 }
1672 while (torture_num_online_cpus() < mynumonline && !torture_must_stop())
1673 schedule_timeout_interruptible(HZ / 5);
1674 stutter_wait("rcu_torture_reader");
1675 } while (!torture_must_stop());
1676 if (irqreader && cur_ops->irq_capable) {
1677 del_timer_sync(&t);
1678 destroy_timer_on_stack(&t);
1679 }
1680 tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
1681 torture_kthread_stopping("rcu_torture_reader");
1682 return 0;
1683}
1684
1685
1686
1687
1688
1689static int rcu_nocb_toggle(void *arg)
1690{
1691 int cpu;
1692 int maxcpu = -1;
1693 int oldnice = task_nice(current);
1694 long r;
1695 DEFINE_TORTURE_RANDOM(rand);
1696 ktime_t toggle_delay;
1697 unsigned long toggle_fuzz;
1698 ktime_t toggle_interval = ms_to_ktime(nocbs_toggle);
1699
1700 VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started");
1701 while (!rcu_inkernel_boot_has_ended())
1702 schedule_timeout_interruptible(HZ / 10);
1703 for_each_online_cpu(cpu)
1704 maxcpu = cpu;
1705 WARN_ON(maxcpu < 0);
1706 if (toggle_interval > ULONG_MAX)
1707 toggle_fuzz = ULONG_MAX >> 3;
1708 else
1709 toggle_fuzz = toggle_interval >> 3;
1710 if (toggle_fuzz <= 0)
1711 toggle_fuzz = NSEC_PER_USEC;
1712 do {
1713 r = torture_random(&rand);
1714 cpu = (r >> 4) % (maxcpu + 1);
1715 if (r & 0x1) {
1716 rcu_nocb_cpu_offload(cpu);
1717 atomic_long_inc(&n_nocb_offload);
1718 } else {
1719 rcu_nocb_cpu_deoffload(cpu);
1720 atomic_long_inc(&n_nocb_deoffload);
1721 }
1722 toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval;
1723 set_current_state(TASK_INTERRUPTIBLE);
1724 schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL);
1725 if (stutter_wait("rcu_nocb_toggle"))
1726 sched_set_normal(current, oldnice);
1727 } while (!torture_must_stop());
1728 torture_kthread_stopping("rcu_nocb_toggle");
1729 return 0;
1730}
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740static void
1741rcu_torture_stats_print(void)
1742{
1743 int cpu;
1744 int i;
1745 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1746 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1747 struct rcu_torture *rtcp;
1748 static unsigned long rtcv_snap = ULONG_MAX;
1749 static bool splatted;
1750 struct task_struct *wtp;
1751
1752 for_each_possible_cpu(cpu) {
1753 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1754 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]);
1755 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]);
1756 }
1757 }
1758 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
1759 if (pipesummary[i] != 0)
1760 break;
1761 }
1762
1763 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1764 rtcp = rcu_access_pointer(rcu_torture_current);
1765 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1766 rtcp,
1767 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
1768 rcu_torture_current_version,
1769 list_empty(&rcu_torture_freelist),
1770 atomic_read(&n_rcu_torture_alloc),
1771 atomic_read(&n_rcu_torture_alloc_fail),
1772 atomic_read(&n_rcu_torture_free));
1773 pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld rtbre: %ld ",
1774 atomic_read(&n_rcu_torture_mberror),
1775 atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries),
1776 n_rcu_torture_barrier_error,
1777 n_rcu_torture_boost_ktrerror,
1778 n_rcu_torture_boost_rterror);
1779 pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
1780 n_rcu_torture_boost_failure,
1781 n_rcu_torture_boosts,
1782 atomic_long_read(&n_rcu_torture_timers));
1783 torture_onoff_stats();
1784 pr_cont("barrier: %ld/%ld:%ld ",
1785 data_race(n_barrier_successes),
1786 data_race(n_barrier_attempts),
1787 data_race(n_rcu_torture_barrier_error));
1788 pr_cont("read-exits: %ld ", data_race(n_read_exits));
1789 pr_cont("nocb-toggles: %ld:%ld\n",
1790 atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload));
1791
1792 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1793 if (atomic_read(&n_rcu_torture_mberror) ||
1794 atomic_read(&n_rcu_torture_mbchk_fail) ||
1795 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror ||
1796 n_rcu_torture_boost_rterror || n_rcu_torture_boost_failure ||
1797 i > 1) {
1798 pr_cont("%s", "!!! ");
1799 atomic_inc(&n_rcu_torture_error);
1800 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror));
1801 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail));
1802 WARN_ON_ONCE(n_rcu_torture_barrier_error);
1803 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror);
1804 WARN_ON_ONCE(n_rcu_torture_boost_rterror);
1805 WARN_ON_ONCE(n_rcu_torture_boost_failure);
1806 WARN_ON_ONCE(i > 1);
1807 }
1808 pr_cont("Reader Pipe: ");
1809 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1810 pr_cont(" %ld", pipesummary[i]);
1811 pr_cont("\n");
1812
1813 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1814 pr_cont("Reader Batch: ");
1815 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1816 pr_cont(" %ld", batchsummary[i]);
1817 pr_cont("\n");
1818
1819 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1820 pr_cont("Free-Block Circulation: ");
1821 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1822 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
1823 }
1824 pr_cont("\n");
1825
1826 if (cur_ops->stats)
1827 cur_ops->stats();
1828 if (rtcv_snap == rcu_torture_current_version &&
1829 rcu_access_pointer(rcu_torture_current) &&
1830 !rcu_stall_is_suppressed()) {
1831 int __maybe_unused flags = 0;
1832 unsigned long __maybe_unused gp_seq = 0;
1833
1834 rcutorture_get_gp_data(cur_ops->ttype,
1835 &flags, &gp_seq);
1836 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp,
1837 &flags, &gp_seq);
1838 wtp = READ_ONCE(writer_task);
1839 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n",
1840 rcu_torture_writer_state_getname(),
1841 rcu_torture_writer_state, gp_seq, flags,
1842 wtp == NULL ? ~0U : wtp->__state,
1843 wtp == NULL ? -1 : (int)task_cpu(wtp));
1844 if (!splatted && wtp) {
1845 sched_show_task(wtp);
1846 splatted = true;
1847 }
1848 if (cur_ops->gp_kthread_dbg)
1849 cur_ops->gp_kthread_dbg();
1850 rcu_ftrace_dump(DUMP_ALL);
1851 }
1852 rtcv_snap = rcu_torture_current_version;
1853}
1854
1855
1856
1857
1858
1859static int
1860rcu_torture_stats(void *arg)
1861{
1862 VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
1863 do {
1864 schedule_timeout_interruptible(stat_interval * HZ);
1865 rcu_torture_stats_print();
1866 torture_shutdown_absorb("rcu_torture_stats");
1867 } while (!torture_must_stop());
1868 torture_kthread_stopping("rcu_torture_stats");
1869 return 0;
1870}
1871
1872static void
1873rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
1874{
1875 pr_alert("%s" TORTURE_FLAG
1876 "--- %s: nreaders=%d nfakewriters=%d "
1877 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
1878 "shuffle_interval=%d stutter=%d irqreader=%d "
1879 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
1880 "test_boost=%d/%d test_boost_interval=%d "
1881 "test_boost_duration=%d shutdown_secs=%d "
1882 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "
1883 "stall_cpu_block=%d "
1884 "n_barrier_cbs=%d "
1885 "onoff_interval=%d onoff_holdoff=%d "
1886 "read_exit_delay=%d read_exit_burst=%d "
1887 "nocbs_nthreads=%d nocbs_toggle=%d\n",
1888 torture_type, tag, nrealreaders, nfakewriters,
1889 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
1890 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
1891 test_boost, cur_ops->can_boost,
1892 test_boost_interval, test_boost_duration, shutdown_secs,
1893 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff,
1894 stall_cpu_block,
1895 n_barrier_cbs,
1896 onoff_interval, onoff_holdoff,
1897 read_exit_delay, read_exit_burst,
1898 nocbs_nthreads, nocbs_toggle);
1899}
1900
1901static int rcutorture_booster_cleanup(unsigned int cpu)
1902{
1903 struct task_struct *t;
1904
1905 if (boost_tasks[cpu] == NULL)
1906 return 0;
1907 mutex_lock(&boost_mutex);
1908 t = boost_tasks[cpu];
1909 boost_tasks[cpu] = NULL;
1910 rcu_torture_enable_rt_throttle();
1911 mutex_unlock(&boost_mutex);
1912
1913
1914 torture_stop_kthread(rcu_torture_boost, t);
1915 return 0;
1916}
1917
1918static int rcutorture_booster_init(unsigned int cpu)
1919{
1920 int retval;
1921
1922 if (boost_tasks[cpu] != NULL)
1923 return 0;
1924
1925
1926 mutex_lock(&boost_mutex);
1927 rcu_torture_disable_rt_throttle();
1928 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
1929 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
1930 cpu_to_node(cpu),
1931 "rcu_torture_boost");
1932 if (IS_ERR(boost_tasks[cpu])) {
1933 retval = PTR_ERR(boost_tasks[cpu]);
1934 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
1935 n_rcu_torture_boost_ktrerror++;
1936 boost_tasks[cpu] = NULL;
1937 mutex_unlock(&boost_mutex);
1938 return retval;
1939 }
1940 kthread_bind(boost_tasks[cpu], cpu);
1941 wake_up_process(boost_tasks[cpu]);
1942 mutex_unlock(&boost_mutex);
1943 return 0;
1944}
1945
1946
1947
1948
1949
1950static int rcu_torture_stall(void *args)
1951{
1952 int idx;
1953 unsigned long stop_at;
1954
1955 VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
1956 if (stall_cpu_holdoff > 0) {
1957 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
1958 schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1959 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
1960 }
1961 if (!kthread_should_stop() && stall_gp_kthread > 0) {
1962 VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall");
1963 rcu_gp_set_torture_wait(stall_gp_kthread * HZ);
1964 for (idx = 0; idx < stall_gp_kthread + 2; idx++) {
1965 if (kthread_should_stop())
1966 break;
1967 schedule_timeout_uninterruptible(HZ);
1968 }
1969 }
1970 if (!kthread_should_stop() && stall_cpu > 0) {
1971 VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall");
1972 stop_at = ktime_get_seconds() + stall_cpu;
1973
1974 idx = cur_ops->readlock();
1975 if (stall_cpu_irqsoff)
1976 local_irq_disable();
1977 else if (!stall_cpu_block)
1978 preempt_disable();
1979 pr_alert("rcu_torture_stall start on CPU %d.\n",
1980 raw_smp_processor_id());
1981 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(),
1982 stop_at))
1983 if (stall_cpu_block)
1984 schedule_timeout_uninterruptible(HZ);
1985 if (stall_cpu_irqsoff)
1986 local_irq_enable();
1987 else if (!stall_cpu_block)
1988 preempt_enable();
1989 cur_ops->readunlock(idx);
1990 }
1991 pr_alert("rcu_torture_stall end.\n");
1992 torture_shutdown_absorb("rcu_torture_stall");
1993 while (!kthread_should_stop())
1994 schedule_timeout_interruptible(10 * HZ);
1995 return 0;
1996}
1997
1998
1999static int __init rcu_torture_stall_init(void)
2000{
2001 if (stall_cpu <= 0 && stall_gp_kthread <= 0)
2002 return 0;
2003 return torture_create_kthread(rcu_torture_stall, NULL, stall_task);
2004}
2005
2006
2007struct fwd_cb_state {
2008 struct rcu_head rh;
2009 int stop;
2010};
2011
2012
2013
2014
2015
2016
2017static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp)
2018{
2019 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh);
2020
2021 if (READ_ONCE(fcsp->stop)) {
2022 WRITE_ONCE(fcsp->stop, 2);
2023 return;
2024 }
2025 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb);
2026}
2027
2028
2029struct rcu_fwd_cb {
2030 struct rcu_head rh;
2031 struct rcu_fwd_cb *rfc_next;
2032 struct rcu_fwd *rfc_rfp;
2033 int rfc_gps;
2034};
2035
2036#define MAX_FWD_CB_JIFFIES (8 * HZ)
2037#define MIN_FWD_CB_LAUNDERS 3
2038#define MIN_FWD_CBS_LAUNDERED 100
2039#define FWD_CBS_HIST_DIV 10
2040#define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV))
2041
2042struct rcu_launder_hist {
2043 long n_launders;
2044 unsigned long launder_gp_seq;
2045};
2046
2047struct rcu_fwd {
2048 spinlock_t rcu_fwd_lock;
2049 struct rcu_fwd_cb *rcu_fwd_cb_head;
2050 struct rcu_fwd_cb **rcu_fwd_cb_tail;
2051 long n_launders_cb;
2052 unsigned long rcu_fwd_startat;
2053 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST];
2054 unsigned long rcu_launder_gp_seq_start;
2055};
2056
2057static DEFINE_MUTEX(rcu_fwd_mutex);
2058static struct rcu_fwd *rcu_fwds;
2059static bool rcu_fwd_emergency_stop;
2060
2061static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)
2062{
2063 unsigned long gps;
2064 unsigned long gps_old;
2065 int i;
2066 int j;
2067
2068 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)
2069 if (rfp->n_launders_hist[i].n_launders > 0)
2070 break;
2071 pr_alert("%s: Callback-invocation histogram (duration %lu jiffies):",
2072 __func__, jiffies - rfp->rcu_fwd_startat);
2073 gps_old = rfp->rcu_launder_gp_seq_start;
2074 for (j = 0; j <= i; j++) {
2075 gps = rfp->n_launders_hist[j].launder_gp_seq;
2076 pr_cont(" %ds/%d: %ld:%ld",
2077 j + 1, FWD_CBS_HIST_DIV,
2078 rfp->n_launders_hist[j].n_launders,
2079 rcutorture_seq_diff(gps, gps_old));
2080 gps_old = gps;
2081 }
2082 pr_cont("\n");
2083}
2084
2085
2086static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
2087{
2088 unsigned long flags;
2089 int i;
2090 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh);
2091 struct rcu_fwd_cb **rfcpp;
2092 struct rcu_fwd *rfp = rfcp->rfc_rfp;
2093
2094 rfcp->rfc_next = NULL;
2095 rfcp->rfc_gps++;
2096 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
2097 rfcpp = rfp->rcu_fwd_cb_tail;
2098 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;
2099 WRITE_ONCE(*rfcpp, rfcp);
2100 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);
2101 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));
2102 if (i >= ARRAY_SIZE(rfp->n_launders_hist))
2103 i = ARRAY_SIZE(rfp->n_launders_hist) - 1;
2104 rfp->n_launders_hist[i].n_launders++;
2105 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();
2106 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
2107}
2108
2109
2110static void rcu_torture_fwd_prog_cond_resched(unsigned long iter)
2111{
2112 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) {
2113
2114 if (need_resched() || (iter & 0xfff))
2115 schedule();
2116 return;
2117 }
2118
2119 cond_resched();
2120}
2121
2122
2123
2124
2125
2126static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)
2127{
2128 unsigned long flags;
2129 unsigned long freed = 0;
2130 struct rcu_fwd_cb *rfcp;
2131
2132 for (;;) {
2133 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
2134 rfcp = rfp->rcu_fwd_cb_head;
2135 if (!rfcp) {
2136 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
2137 break;
2138 }
2139 rfp->rcu_fwd_cb_head = rfcp->rfc_next;
2140 if (!rfp->rcu_fwd_cb_head)
2141 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;
2142 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
2143 kfree(rfcp);
2144 freed++;
2145 rcu_torture_fwd_prog_cond_resched(freed);
2146 if (tick_nohz_full_enabled()) {
2147 local_irq_save(flags);
2148 rcu_momentary_dyntick_idle();
2149 local_irq_restore(flags);
2150 }
2151 }
2152 return freed;
2153}
2154
2155
2156static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,
2157 int *tested, int *tested_tries)
2158{
2159 unsigned long cver;
2160 unsigned long dur;
2161 struct fwd_cb_state fcs;
2162 unsigned long gps;
2163 int idx;
2164 int sd;
2165 int sd4;
2166 bool selfpropcb = false;
2167 unsigned long stopat;
2168 static DEFINE_TORTURE_RANDOM(trs);
2169
2170 if (!cur_ops->sync)
2171 return;
2172 if (cur_ops->call && cur_ops->cb_barrier) {
2173 init_rcu_head_on_stack(&fcs.rh);
2174 selfpropcb = true;
2175 }
2176
2177
2178 WRITE_ONCE(rcu_fwd_cb_nodelay, true);
2179 cur_ops->sync();
2180 if (selfpropcb) {
2181 WRITE_ONCE(fcs.stop, 0);
2182 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
2183 }
2184 cver = READ_ONCE(rcu_torture_current_version);
2185 gps = cur_ops->get_gp_seq();
2186 sd = cur_ops->stall_dur() + 1;
2187 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div;
2188 dur = sd4 + torture_random(&trs) % (sd - sd4);
2189 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
2190 stopat = rfp->rcu_fwd_startat + dur;
2191 while (time_before(jiffies, stopat) &&
2192 !shutdown_time_arrived() &&
2193 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
2194 idx = cur_ops->readlock();
2195 udelay(10);
2196 cur_ops->readunlock(idx);
2197 if (!fwd_progress_need_resched || need_resched())
2198 cond_resched();
2199 }
2200 (*tested_tries)++;
2201 if (!time_before(jiffies, stopat) &&
2202 !shutdown_time_arrived() &&
2203 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
2204 (*tested)++;
2205 cver = READ_ONCE(rcu_torture_current_version) - cver;
2206 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
2207 WARN_ON(!cver && gps < 2);
2208 pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps);
2209 }
2210 if (selfpropcb) {
2211 WRITE_ONCE(fcs.stop, 1);
2212 cur_ops->sync();
2213 cur_ops->cb_barrier();
2214 }
2215
2216 if (selfpropcb) {
2217 WARN_ON(READ_ONCE(fcs.stop) != 2);
2218 destroy_rcu_head_on_stack(&fcs.rh);
2219 }
2220 schedule_timeout_uninterruptible(HZ / 10);
2221 WRITE_ONCE(rcu_fwd_cb_nodelay, false);
2222}
2223
2224
2225static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)
2226{
2227 unsigned long cver;
2228 unsigned long flags;
2229 unsigned long gps;
2230 int i;
2231 long n_launders;
2232 long n_launders_cb_snap;
2233 long n_launders_sa;
2234 long n_max_cbs;
2235 long n_max_gps;
2236 struct rcu_fwd_cb *rfcp;
2237 struct rcu_fwd_cb *rfcpn;
2238 unsigned long stopat;
2239 unsigned long stoppedat;
2240
2241 if (READ_ONCE(rcu_fwd_emergency_stop))
2242 return;
2243 if (!cur_ops->call)
2244 return;
2245
2246
2247 WRITE_ONCE(rcu_fwd_cb_nodelay, true);
2248 cur_ops->sync();
2249 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
2250 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;
2251 n_launders = 0;
2252 rfp->n_launders_cb = 0;
2253 n_launders_sa = 0;
2254 n_max_cbs = 0;
2255 n_max_gps = 0;
2256 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)
2257 rfp->n_launders_hist[i].n_launders = 0;
2258 cver = READ_ONCE(rcu_torture_current_version);
2259 gps = cur_ops->get_gp_seq();
2260 rfp->rcu_launder_gp_seq_start = gps;
2261 tick_dep_set_task(current, TICK_DEP_BIT_RCU);
2262 while (time_before(jiffies, stopat) &&
2263 !shutdown_time_arrived() &&
2264 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
2265 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);
2266 rfcpn = NULL;
2267 if (rfcp)
2268 rfcpn = READ_ONCE(rfcp->rfc_next);
2269 if (rfcpn) {
2270 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS &&
2271 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED)
2272 break;
2273 rfp->rcu_fwd_cb_head = rfcpn;
2274 n_launders++;
2275 n_launders_sa++;
2276 } else {
2277 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL);
2278 if (WARN_ON_ONCE(!rfcp)) {
2279 schedule_timeout_interruptible(1);
2280 continue;
2281 }
2282 n_max_cbs++;
2283 n_launders_sa = 0;
2284 rfcp->rfc_gps = 0;
2285 rfcp->rfc_rfp = rfp;
2286 }
2287 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
2288 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);
2289 if (tick_nohz_full_enabled()) {
2290 local_irq_save(flags);
2291 rcu_momentary_dyntick_idle();
2292 local_irq_restore(flags);
2293 }
2294 }
2295 stoppedat = jiffies;
2296 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);
2297 cver = READ_ONCE(rcu_torture_current_version) - cver;
2298 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
2299 cur_ops->cb_barrier();
2300 (void)rcu_torture_fwd_prog_cbfree(rfp);
2301
2302 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) &&
2303 !shutdown_time_arrived()) {
2304 WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED);
2305 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n",
2306 __func__,
2307 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,
2308 n_launders + n_max_cbs - n_launders_cb_snap,
2309 n_launders, n_launders_sa,
2310 n_max_gps, n_max_cbs, cver, gps);
2311 rcu_torture_fwd_cb_hist(rfp);
2312 }
2313 schedule_timeout_uninterruptible(HZ);
2314 tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
2315 WRITE_ONCE(rcu_fwd_cb_nodelay, false);
2316}
2317
2318
2319
2320
2321
2322
2323static int rcutorture_oom_notify(struct notifier_block *self,
2324 unsigned long notused, void *nfreed)
2325{
2326 struct rcu_fwd *rfp;
2327
2328 mutex_lock(&rcu_fwd_mutex);
2329 rfp = rcu_fwds;
2330 if (!rfp) {
2331 mutex_unlock(&rcu_fwd_mutex);
2332 return NOTIFY_OK;
2333 }
2334 WARN(1, "%s invoked upon OOM during forward-progress testing.\n",
2335 __func__);
2336 rcu_torture_fwd_cb_hist(rfp);
2337 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp->rcu_fwd_startat)) / 2);
2338 WRITE_ONCE(rcu_fwd_emergency_stop, true);
2339 smp_mb();
2340 pr_info("%s: Freed %lu RCU callbacks.\n",
2341 __func__, rcu_torture_fwd_prog_cbfree(rfp));
2342 rcu_barrier();
2343 pr_info("%s: Freed %lu RCU callbacks.\n",
2344 __func__, rcu_torture_fwd_prog_cbfree(rfp));
2345 rcu_barrier();
2346 pr_info("%s: Freed %lu RCU callbacks.\n",
2347 __func__, rcu_torture_fwd_prog_cbfree(rfp));
2348 smp_mb();
2349 (*(unsigned long *)nfreed)++;
2350 pr_info("%s returning after OOM processing.\n", __func__);
2351 mutex_unlock(&rcu_fwd_mutex);
2352 return NOTIFY_OK;
2353}
2354
2355static struct notifier_block rcutorture_oom_nb = {
2356 .notifier_call = rcutorture_oom_notify
2357};
2358
2359
2360static int rcu_torture_fwd_prog(void *args)
2361{
2362 int oldnice = task_nice(current);
2363 struct rcu_fwd *rfp = args;
2364 int tested = 0;
2365 int tested_tries = 0;
2366
2367 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started");
2368 rcu_bind_current_to_nocb();
2369 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))
2370 set_user_nice(current, MAX_NICE);
2371 do {
2372 schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
2373 WRITE_ONCE(rcu_fwd_emergency_stop, false);
2374 if (!IS_ENABLED(CONFIG_TINY_RCU) ||
2375 rcu_inkernel_boot_has_ended())
2376 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
2377 if (rcu_inkernel_boot_has_ended())
2378 rcu_torture_fwd_prog_cr(rfp);
2379
2380
2381 if (stutter_wait("rcu_torture_fwd_prog"))
2382 sched_set_normal(current, oldnice);
2383 } while (!torture_must_stop());
2384
2385 WARN_ON(!tested && tested_tries >= 5);
2386 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);
2387 torture_kthread_stopping("rcu_torture_fwd_prog");
2388 return 0;
2389}
2390
2391
2392static int __init rcu_torture_fwd_prog_init(void)
2393{
2394 struct rcu_fwd *rfp;
2395
2396 if (!fwd_progress)
2397 return 0;
2398 if ((!cur_ops->sync && !cur_ops->call) ||
2399 !cur_ops->stall_dur || cur_ops->stall_dur() <= 0 || cur_ops == &rcu_busted_ops) {
2400 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test");
2401 return 0;
2402 }
2403 if (stall_cpu > 0) {
2404 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing");
2405 if (IS_MODULE(CONFIG_RCU_TORTURE_TESTS))
2406 return -EINVAL;
2407 WARN_ON(1);
2408 return 0;
2409 }
2410 if (fwd_progress_holdoff <= 0)
2411 fwd_progress_holdoff = 1;
2412 if (fwd_progress_div <= 0)
2413 fwd_progress_div = 4;
2414 rfp = kzalloc(sizeof(*rfp), GFP_KERNEL);
2415 if (!rfp)
2416 return -ENOMEM;
2417 spin_lock_init(&rfp->rcu_fwd_lock);
2418 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;
2419 mutex_lock(&rcu_fwd_mutex);
2420 rcu_fwds = rfp;
2421 mutex_unlock(&rcu_fwd_mutex);
2422 register_oom_notifier(&rcutorture_oom_nb);
2423 return torture_create_kthread(rcu_torture_fwd_prog, rfp, fwd_prog_task);
2424}
2425
2426static void rcu_torture_fwd_prog_cleanup(void)
2427{
2428 struct rcu_fwd *rfp;
2429
2430 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_task);
2431 rfp = rcu_fwds;
2432 mutex_lock(&rcu_fwd_mutex);
2433 rcu_fwds = NULL;
2434 mutex_unlock(&rcu_fwd_mutex);
2435 unregister_oom_notifier(&rcutorture_oom_nb);
2436 kfree(rfp);
2437}
2438
2439
2440static void rcu_torture_barrier_cbf(struct rcu_head *rcu)
2441{
2442 atomic_inc(&barrier_cbs_invoked);
2443}
2444
2445
2446static void rcu_torture_barrier1cb(void *rcu_void)
2447{
2448 struct rcu_head *rhp = rcu_void;
2449
2450 cur_ops->call(rhp, rcu_torture_barrier_cbf);
2451}
2452
2453
2454static int rcu_torture_barrier_cbs(void *arg)
2455{
2456 long myid = (long)arg;
2457 bool lastphase = false;
2458 bool newphase;
2459 struct rcu_head rcu;
2460
2461 init_rcu_head_on_stack(&rcu);
2462 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
2463 set_user_nice(current, MAX_NICE);
2464 do {
2465 wait_event(barrier_cbs_wq[myid],
2466 (newphase =
2467 smp_load_acquire(&barrier_phase)) != lastphase ||
2468 torture_must_stop());
2469 lastphase = newphase;
2470 if (torture_must_stop())
2471 break;
2472
2473
2474
2475
2476 if (smp_call_function_single(myid, rcu_torture_barrier1cb,
2477 &rcu, 1)) {
2478
2479 cur_ops->call(&rcu, rcu_torture_barrier_cbf);
2480 }
2481 if (atomic_dec_and_test(&barrier_cbs_count))
2482 wake_up(&barrier_wq);
2483 } while (!torture_must_stop());
2484 if (cur_ops->cb_barrier != NULL)
2485 cur_ops->cb_barrier();
2486 destroy_rcu_head_on_stack(&rcu);
2487 torture_kthread_stopping("rcu_torture_barrier_cbs");
2488 return 0;
2489}
2490
2491
2492static int rcu_torture_barrier(void *arg)
2493{
2494 int i;
2495
2496 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
2497 do {
2498 atomic_set(&barrier_cbs_invoked, 0);
2499 atomic_set(&barrier_cbs_count, n_barrier_cbs);
2500
2501 smp_store_release(&barrier_phase, !barrier_phase);
2502 for (i = 0; i < n_barrier_cbs; i++)
2503 wake_up(&barrier_cbs_wq[i]);
2504 wait_event(barrier_wq,
2505 atomic_read(&barrier_cbs_count) == 0 ||
2506 torture_must_stop());
2507 if (torture_must_stop())
2508 break;
2509 n_barrier_attempts++;
2510 cur_ops->cb_barrier();
2511 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
2512 n_rcu_torture_barrier_error++;
2513 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
2514 atomic_read(&barrier_cbs_invoked),
2515 n_barrier_cbs);
2516 WARN_ON(1);
2517
2518 i = 0;
2519 do {
2520 if (WARN_ON(i++ > HZ))
2521 i = INT_MIN;
2522 schedule_timeout_interruptible(1);
2523 cur_ops->cb_barrier();
2524 } while (atomic_read(&barrier_cbs_invoked) !=
2525 n_barrier_cbs &&
2526 !torture_must_stop());
2527 smp_mb();
2528 if (!torture_must_stop())
2529 pr_err("Recovered: barrier_cbs_invoked = %d\n",
2530 atomic_read(&barrier_cbs_invoked));
2531 } else {
2532 n_barrier_successes++;
2533 }
2534 schedule_timeout_interruptible(HZ / 10);
2535 } while (!torture_must_stop());
2536 torture_kthread_stopping("rcu_torture_barrier");
2537 return 0;
2538}
2539
2540
2541static int rcu_torture_barrier_init(void)
2542{
2543 int i;
2544 int ret;
2545
2546 if (n_barrier_cbs <= 0)
2547 return 0;
2548 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
2549 pr_alert("%s" TORTURE_FLAG
2550 " Call or barrier ops missing for %s,\n",
2551 torture_type, cur_ops->name);
2552 pr_alert("%s" TORTURE_FLAG
2553 " RCU barrier testing omitted from run.\n",
2554 torture_type);
2555 return 0;
2556 }
2557 atomic_set(&barrier_cbs_count, 0);
2558 atomic_set(&barrier_cbs_invoked, 0);
2559 barrier_cbs_tasks =
2560 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]),
2561 GFP_KERNEL);
2562 barrier_cbs_wq =
2563 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL);
2564 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
2565 return -ENOMEM;
2566 for (i = 0; i < n_barrier_cbs; i++) {
2567 init_waitqueue_head(&barrier_cbs_wq[i]);
2568 ret = torture_create_kthread(rcu_torture_barrier_cbs,
2569 (void *)(long)i,
2570 barrier_cbs_tasks[i]);
2571 if (ret)
2572 return ret;
2573 }
2574 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task);
2575}
2576
2577
2578static void rcu_torture_barrier_cleanup(void)
2579{
2580 int i;
2581
2582 torture_stop_kthread(rcu_torture_barrier, barrier_task);
2583 if (barrier_cbs_tasks != NULL) {
2584 for (i = 0; i < n_barrier_cbs; i++)
2585 torture_stop_kthread(rcu_torture_barrier_cbs,
2586 barrier_cbs_tasks[i]);
2587 kfree(barrier_cbs_tasks);
2588 barrier_cbs_tasks = NULL;
2589 }
2590 if (barrier_cbs_wq != NULL) {
2591 kfree(barrier_cbs_wq);
2592 barrier_cbs_wq = NULL;
2593 }
2594}
2595
2596static bool rcu_torture_can_boost(void)
2597{
2598 static int boost_warn_once;
2599 int prio;
2600
2601 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2)
2602 return false;
2603
2604 prio = rcu_get_gp_kthreads_prio();
2605 if (!prio)
2606 return false;
2607
2608 if (prio < 2) {
2609 if (boost_warn_once == 1)
2610 return false;
2611
2612 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME);
2613 boost_warn_once = 1;
2614 return false;
2615 }
2616
2617 return true;
2618}
2619
2620static bool read_exit_child_stop;
2621static bool read_exit_child_stopped;
2622static wait_queue_head_t read_exit_wq;
2623
2624
2625static int rcu_torture_read_exit_child(void *trsp_in)
2626{
2627 struct torture_random_state *trsp = trsp_in;
2628
2629 set_user_nice(current, MAX_NICE);
2630
2631 while (!kthread_should_stop())
2632 schedule_timeout_uninterruptible(1);
2633 (void)rcu_torture_one_read(trsp, -1);
2634 return 0;
2635}
2636
2637
2638static int rcu_torture_read_exit(void *unused)
2639{
2640 int count = 0;
2641 bool errexit = false;
2642 int i;
2643 struct task_struct *tsp;
2644 DEFINE_TORTURE_RANDOM(trs);
2645
2646
2647 set_user_nice(current, MAX_NICE);
2648 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test");
2649
2650
2651 do {
2652 if (++count > read_exit_burst) {
2653 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode");
2654 rcu_barrier();
2655 for (i = 0; i < read_exit_delay; i++) {
2656 schedule_timeout_uninterruptible(HZ);
2657 if (READ_ONCE(read_exit_child_stop))
2658 break;
2659 }
2660 if (!READ_ONCE(read_exit_child_stop))
2661 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode");
2662 count = 0;
2663 }
2664 if (READ_ONCE(read_exit_child_stop))
2665 break;
2666
2667 tsp = kthread_run(rcu_torture_read_exit_child,
2668 &trs, "%s",
2669 "rcu_torture_read_exit_child");
2670 if (IS_ERR(tsp)) {
2671 VERBOSE_TOROUT_ERRSTRING("out of memory");
2672 errexit = true;
2673 tsp = NULL;
2674 break;
2675 }
2676 cond_resched();
2677 kthread_stop(tsp);
2678 n_read_exits ++;
2679 stutter_wait("rcu_torture_read_exit");
2680 } while (!errexit && !READ_ONCE(read_exit_child_stop));
2681
2682
2683 smp_store_release(&read_exit_child_stopped, true);
2684 smp_mb();
2685 wake_up(&read_exit_wq);
2686 while (!torture_must_stop())
2687 schedule_timeout_uninterruptible(1);
2688 torture_kthread_stopping("rcu_torture_read_exit");
2689 return 0;
2690}
2691
2692static int rcu_torture_read_exit_init(void)
2693{
2694 if (read_exit_burst <= 0)
2695 return -EINVAL;
2696 init_waitqueue_head(&read_exit_wq);
2697 read_exit_child_stop = false;
2698 read_exit_child_stopped = false;
2699 return torture_create_kthread(rcu_torture_read_exit, NULL,
2700 read_exit_task);
2701}
2702
2703static void rcu_torture_read_exit_cleanup(void)
2704{
2705 if (!read_exit_task)
2706 return;
2707 WRITE_ONCE(read_exit_child_stop, true);
2708 smp_mb();
2709 wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped));
2710 torture_stop_kthread(rcutorture_read_exit, read_exit_task);
2711}
2712
2713static enum cpuhp_state rcutor_hp;
2714
2715static void
2716rcu_torture_cleanup(void)
2717{
2718 int firsttime;
2719 int flags = 0;
2720 unsigned long gp_seq = 0;
2721 int i;
2722
2723 if (torture_cleanup_begin()) {
2724 if (cur_ops->cb_barrier != NULL)
2725 cur_ops->cb_barrier();
2726 return;
2727 }
2728 if (!cur_ops) {
2729 torture_cleanup_end();
2730 return;
2731 }
2732
2733 if (cur_ops->gp_kthread_dbg)
2734 cur_ops->gp_kthread_dbg();
2735 rcu_torture_read_exit_cleanup();
2736 rcu_torture_barrier_cleanup();
2737 rcu_torture_fwd_prog_cleanup();
2738 torture_stop_kthread(rcu_torture_stall, stall_task);
2739 torture_stop_kthread(rcu_torture_writer, writer_task);
2740
2741 if (nocb_tasks) {
2742 for (i = 0; i < nrealnocbers; i++)
2743 torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]);
2744 kfree(nocb_tasks);
2745 nocb_tasks = NULL;
2746 }
2747
2748 if (reader_tasks) {
2749 for (i = 0; i < nrealreaders; i++)
2750 torture_stop_kthread(rcu_torture_reader,
2751 reader_tasks[i]);
2752 kfree(reader_tasks);
2753 reader_tasks = NULL;
2754 }
2755 kfree(rcu_torture_reader_mbchk);
2756 rcu_torture_reader_mbchk = NULL;
2757
2758 if (fakewriter_tasks) {
2759 for (i = 0; i < nfakewriters; i++)
2760 torture_stop_kthread(rcu_torture_fakewriter,
2761 fakewriter_tasks[i]);
2762 kfree(fakewriter_tasks);
2763 fakewriter_tasks = NULL;
2764 }
2765
2766 rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq);
2767 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq);
2768 pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n",
2769 cur_ops->name, (long)gp_seq, flags,
2770 rcutorture_seq_diff(gp_seq, start_gp_seq));
2771 torture_stop_kthread(rcu_torture_stats, stats_task);
2772 torture_stop_kthread(rcu_torture_fqs, fqs_task);
2773 if (rcu_torture_can_boost())
2774 cpuhp_remove_state(rcutor_hp);
2775
2776
2777
2778
2779
2780 if (cur_ops->cb_barrier != NULL)
2781 cur_ops->cb_barrier();
2782 if (cur_ops->cleanup != NULL)
2783 cur_ops->cleanup();
2784
2785 rcu_torture_stats_print();
2786
2787 if (err_segs_recorded) {
2788 pr_alert("Failure/close-call rcutorture reader segments:\n");
2789 if (rt_read_nsegs == 0)
2790 pr_alert("\t: No segments recorded!!!\n");
2791 firsttime = 1;
2792 for (i = 0; i < rt_read_nsegs; i++) {
2793 pr_alert("\t%d: %#x ", i, err_segs[i].rt_readstate);
2794 if (err_segs[i].rt_delay_jiffies != 0) {
2795 pr_cont("%s%ldjiffies", firsttime ? "" : "+",
2796 err_segs[i].rt_delay_jiffies);
2797 firsttime = 0;
2798 }
2799 if (err_segs[i].rt_delay_ms != 0) {
2800 pr_cont("%s%ldms", firsttime ? "" : "+",
2801 err_segs[i].rt_delay_ms);
2802 firsttime = 0;
2803 }
2804 if (err_segs[i].rt_delay_us != 0) {
2805 pr_cont("%s%ldus", firsttime ? "" : "+",
2806 err_segs[i].rt_delay_us);
2807 firsttime = 0;
2808 }
2809 pr_cont("%s\n",
2810 err_segs[i].rt_preempted ? "preempted" : "");
2811
2812 }
2813 }
2814 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
2815 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
2816 else if (torture_onoff_failures())
2817 rcu_torture_print_module_parms(cur_ops,
2818 "End of test: RCU_HOTPLUG");
2819 else
2820 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
2821 torture_cleanup_end();
2822}
2823
2824#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
2825static void rcu_torture_leak_cb(struct rcu_head *rhp)
2826{
2827}
2828
2829static void rcu_torture_err_cb(struct rcu_head *rhp)
2830{
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME);
2841}
2842#endif
2843
2844
2845
2846
2847
2848
2849static void rcu_test_debug_objects(void)
2850{
2851#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
2852 struct rcu_head rh1;
2853 struct rcu_head rh2;
2854 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
2855
2856 init_rcu_head_on_stack(&rh1);
2857 init_rcu_head_on_stack(&rh2);
2858 pr_alert("%s: WARN: Duplicate call_rcu() test starting.\n", KBUILD_MODNAME);
2859
2860
2861 preempt_disable();
2862 rcu_read_lock();
2863 call_rcu(&rh1, rcu_torture_leak_cb);
2864 local_irq_disable();
2865 call_rcu(&rh2, rcu_torture_leak_cb);
2866 call_rcu(&rh2, rcu_torture_err_cb);
2867 if (rhp) {
2868 call_rcu(rhp, rcu_torture_leak_cb);
2869 call_rcu(rhp, rcu_torture_err_cb);
2870 }
2871 local_irq_enable();
2872 rcu_read_unlock();
2873 preempt_enable();
2874
2875
2876 rcu_barrier();
2877 pr_alert("%s: WARN: Duplicate call_rcu() test complete.\n", KBUILD_MODNAME);
2878 destroy_rcu_head_on_stack(&rh1);
2879 destroy_rcu_head_on_stack(&rh2);
2880#else
2881 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n", KBUILD_MODNAME);
2882#endif
2883}
2884
2885static void rcutorture_sync(void)
2886{
2887 static unsigned long n;
2888
2889 if (cur_ops->sync && !(++n & 0xfff))
2890 cur_ops->sync();
2891}
2892
2893static int __init
2894rcu_torture_init(void)
2895{
2896 long i;
2897 int cpu;
2898 int firsterr = 0;
2899 int flags = 0;
2900 unsigned long gp_seq = 0;
2901 static struct rcu_torture_ops *torture_ops[] = {
2902 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops,
2903 &busted_srcud_ops, &tasks_ops, &tasks_rude_ops,
2904 &tasks_tracing_ops, &trivial_ops,
2905 };
2906
2907 if (!torture_init_begin(torture_type, verbose))
2908 return -EBUSY;
2909
2910
2911 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
2912 cur_ops = torture_ops[i];
2913 if (strcmp(torture_type, cur_ops->name) == 0)
2914 break;
2915 }
2916 if (i == ARRAY_SIZE(torture_ops)) {
2917 pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
2918 torture_type);
2919 pr_alert("rcu-torture types:");
2920 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
2921 pr_cont(" %s", torture_ops[i]->name);
2922 pr_cont("\n");
2923 firsterr = -EINVAL;
2924 cur_ops = NULL;
2925 goto unwind;
2926 }
2927 if (cur_ops->fqs == NULL && fqs_duration != 0) {
2928 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
2929 fqs_duration = 0;
2930 }
2931 if (cur_ops->init)
2932 cur_ops->init();
2933
2934 if (nreaders >= 0) {
2935 nrealreaders = nreaders;
2936 } else {
2937 nrealreaders = num_online_cpus() - 2 - nreaders;
2938 if (nrealreaders <= 0)
2939 nrealreaders = 1;
2940 }
2941 rcu_torture_print_module_parms(cur_ops, "Start of test");
2942 rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq);
2943 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq);
2944 start_gp_seq = gp_seq;
2945 pr_alert("%s: Start-test grace-period state: g%ld f%#x\n",
2946 cur_ops->name, (long)gp_seq, flags);
2947
2948
2949
2950 INIT_LIST_HEAD(&rcu_torture_freelist);
2951 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
2952 rcu_tortures[i].rtort_mbtest = 0;
2953 list_add_tail(&rcu_tortures[i].rtort_free,
2954 &rcu_torture_freelist);
2955 }
2956
2957
2958
2959 rcu_torture_current = NULL;
2960 rcu_torture_current_version = 0;
2961 atomic_set(&n_rcu_torture_alloc, 0);
2962 atomic_set(&n_rcu_torture_alloc_fail, 0);
2963 atomic_set(&n_rcu_torture_free, 0);
2964 atomic_set(&n_rcu_torture_mberror, 0);
2965 atomic_set(&n_rcu_torture_mbchk_fail, 0);
2966 atomic_set(&n_rcu_torture_mbchk_tries, 0);
2967 atomic_set(&n_rcu_torture_error, 0);
2968 n_rcu_torture_barrier_error = 0;
2969 n_rcu_torture_boost_ktrerror = 0;
2970 n_rcu_torture_boost_rterror = 0;
2971 n_rcu_torture_boost_failure = 0;
2972 n_rcu_torture_boosts = 0;
2973 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
2974 atomic_set(&rcu_torture_wcount[i], 0);
2975 for_each_possible_cpu(cpu) {
2976 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
2977 per_cpu(rcu_torture_count, cpu)[i] = 0;
2978 per_cpu(rcu_torture_batch, cpu)[i] = 0;
2979 }
2980 }
2981 err_segs_recorded = 0;
2982 rt_read_nsegs = 0;
2983
2984
2985
2986 rcu_torture_write_types();
2987 firsterr = torture_create_kthread(rcu_torture_writer, NULL,
2988 writer_task);
2989 if (firsterr)
2990 goto unwind;
2991 if (nfakewriters > 0) {
2992 fakewriter_tasks = kcalloc(nfakewriters,
2993 sizeof(fakewriter_tasks[0]),
2994 GFP_KERNEL);
2995 if (fakewriter_tasks == NULL) {
2996 VERBOSE_TOROUT_ERRSTRING("out of memory");
2997 firsterr = -ENOMEM;
2998 goto unwind;
2999 }
3000 }
3001 for (i = 0; i < nfakewriters; i++) {
3002 firsterr = torture_create_kthread(rcu_torture_fakewriter,
3003 NULL, fakewriter_tasks[i]);
3004 if (firsterr)
3005 goto unwind;
3006 }
3007 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
3008 GFP_KERNEL);
3009 rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk),
3010 GFP_KERNEL);
3011 if (!reader_tasks || !rcu_torture_reader_mbchk) {
3012 VERBOSE_TOROUT_ERRSTRING("out of memory");
3013 firsterr = -ENOMEM;
3014 goto unwind;
3015 }
3016 for (i = 0; i < nrealreaders; i++) {
3017 rcu_torture_reader_mbchk[i].rtc_chkrdr = -1;
3018 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i,
3019 reader_tasks[i]);
3020 if (firsterr)
3021 goto unwind;
3022 }
3023 nrealnocbers = nocbs_nthreads;
3024 if (WARN_ON(nrealnocbers < 0))
3025 nrealnocbers = 1;
3026 if (WARN_ON(nocbs_toggle < 0))
3027 nocbs_toggle = HZ;
3028 if (nrealnocbers > 0) {
3029 nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL);
3030 if (nocb_tasks == NULL) {
3031 VERBOSE_TOROUT_ERRSTRING("out of memory");
3032 firsterr = -ENOMEM;
3033 goto unwind;
3034 }
3035 } else {
3036 nocb_tasks = NULL;
3037 }
3038 for (i = 0; i < nrealnocbers; i++) {
3039 firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]);
3040 if (firsterr)
3041 goto unwind;
3042 }
3043 if (stat_interval > 0) {
3044 firsterr = torture_create_kthread(rcu_torture_stats, NULL,
3045 stats_task);
3046 if (firsterr)
3047 goto unwind;
3048 }
3049 if (test_no_idle_hz && shuffle_interval > 0) {
3050 firsterr = torture_shuffle_init(shuffle_interval * HZ);
3051 if (firsterr)
3052 goto unwind;
3053 }
3054 if (stutter < 0)
3055 stutter = 0;
3056 if (stutter) {
3057 int t;
3058
3059 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;
3060 firsterr = torture_stutter_init(stutter * HZ, t);
3061 if (firsterr)
3062 goto unwind;
3063 }
3064 if (fqs_duration < 0)
3065 fqs_duration = 0;
3066 if (fqs_duration) {
3067
3068 firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
3069 fqs_task);
3070 if (firsterr)
3071 goto unwind;
3072 }
3073 if (test_boost_interval < 1)
3074 test_boost_interval = 1;
3075 if (test_boost_duration < 2)
3076 test_boost_duration = 2;
3077 if (rcu_torture_can_boost()) {
3078
3079 boost_starttime = jiffies + test_boost_interval * HZ;
3080
3081 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE",
3082 rcutorture_booster_init,
3083 rcutorture_booster_cleanup);
3084 if (firsterr < 0)
3085 goto unwind;
3086 rcutor_hp = firsterr;
3087 }
3088 shutdown_jiffies = jiffies + shutdown_secs * HZ;
3089 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
3090 if (firsterr)
3091 goto unwind;
3092 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval,
3093 rcutorture_sync);
3094 if (firsterr)
3095 goto unwind;
3096 firsterr = rcu_torture_stall_init();
3097 if (firsterr)
3098 goto unwind;
3099 firsterr = rcu_torture_fwd_prog_init();
3100 if (firsterr)
3101 goto unwind;
3102 firsterr = rcu_torture_barrier_init();
3103 if (firsterr)
3104 goto unwind;
3105 firsterr = rcu_torture_read_exit_init();
3106 if (firsterr)
3107 goto unwind;
3108 if (object_debug)
3109 rcu_test_debug_objects();
3110 torture_init_end();
3111 return 0;
3112
3113unwind:
3114 torture_init_end();
3115 rcu_torture_cleanup();
3116 if (shutdown_secs) {
3117 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST));
3118 kernel_power_off();
3119 }
3120 return firsterr;
3121}
3122
3123module_init(rcu_torture_init);
3124module_exit(rcu_torture_cleanup);
3125