1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/types.h>
24#include <linux/kernel.h>
25#include <linux/init.h>
26#include <linux/module.h>
27#include <linux/kthread.h>
28#include <linux/err.h>
29#include <linux/spinlock.h>
30#include <linux/smp.h>
31#include <linux/interrupt.h>
32#include <linux/sched.h>
33#include <linux/sched/clock.h>
34#include <linux/atomic.h>
35#include <linux/bitops.h>
36#include <linux/completion.h>
37#include <linux/moduleparam.h>
38#include <linux/percpu.h>
39#include <linux/notifier.h>
40#include <linux/reboot.h>
41#include <linux/freezer.h>
42#include <linux/cpu.h>
43#include <linux/delay.h>
44#include <linux/stat.h>
45#include <linux/slab.h>
46#include <linux/trace_clock.h>
47#include <linux/ktime.h>
48#include <asm/byteorder.h>
49#include <linux/torture.h>
50#include "rcu/rcu.h"
51
52MODULE_LICENSE("GPL");
53MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
54
55static char *torture_type;
56static bool verbose;
57
58
59#define FULLSTOP_DONTSTOP 0
60#define FULLSTOP_SHUTDOWN 1
61#define FULLSTOP_RMMOD 2
62static int fullstop = FULLSTOP_RMMOD;
63static DEFINE_MUTEX(fullstop_mutex);
64
65#ifdef CONFIG_HOTPLUG_CPU
66
67
68
69
70
71
72static struct task_struct *onoff_task;
73static long onoff_holdoff;
74static long onoff_interval;
75static long n_offline_attempts;
76static long n_offline_successes;
77static unsigned long sum_offline;
78static int min_offline = -1;
79static int max_offline;
80static long n_online_attempts;
81static long n_online_successes;
82static unsigned long sum_online;
83static int min_online = -1;
84static int max_online;
85
86
87
88
89
90
91bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
92 unsigned long *sum_offl, int *min_offl, int *max_offl)
93{
94 unsigned long delta;
95 int ret;
96 unsigned long starttime;
97
98 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
99 return false;
100
101 if (verbose)
102 pr_alert("%s" TORTURE_FLAG
103 "torture_onoff task: offlining %d\n",
104 torture_type, cpu);
105 starttime = jiffies;
106 (*n_offl_attempts)++;
107 ret = cpu_down(cpu);
108 if (ret) {
109 if (verbose)
110 pr_alert("%s" TORTURE_FLAG
111 "torture_onoff task: offline %d failed: errno %d\n",
112 torture_type, cpu, ret);
113 } else {
114 if (verbose)
115 pr_alert("%s" TORTURE_FLAG
116 "torture_onoff task: offlined %d\n",
117 torture_type, cpu);
118 (*n_offl_successes)++;
119 delta = jiffies - starttime;
120 *sum_offl += delta;
121 if (*min_offl < 0) {
122 *min_offl = delta;
123 *max_offl = delta;
124 }
125 if (*min_offl > delta)
126 *min_offl = delta;
127 if (*max_offl < delta)
128 *max_offl = delta;
129 }
130
131 return true;
132}
133EXPORT_SYMBOL_GPL(torture_offline);
134
135
136
137
138
139
140bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
141 unsigned long *sum_onl, int *min_onl, int *max_onl)
142{
143 unsigned long delta;
144 int ret;
145 unsigned long starttime;
146
147 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
148 return false;
149
150 if (verbose)
151 pr_alert("%s" TORTURE_FLAG
152 "torture_onoff task: onlining %d\n",
153 torture_type, cpu);
154 starttime = jiffies;
155 (*n_onl_attempts)++;
156 ret = cpu_up(cpu);
157 if (ret) {
158 if (verbose)
159 pr_alert("%s" TORTURE_FLAG
160 "torture_onoff task: online %d failed: errno %d\n",
161 torture_type, cpu, ret);
162 } else {
163 if (verbose)
164 pr_alert("%s" TORTURE_FLAG
165 "torture_onoff task: onlined %d\n",
166 torture_type, cpu);
167 (*n_onl_successes)++;
168 delta = jiffies - starttime;
169 *sum_onl += delta;
170 if (*min_onl < 0) {
171 *min_onl = delta;
172 *max_onl = delta;
173 }
174 if (*min_onl > delta)
175 *min_onl = delta;
176 if (*max_onl < delta)
177 *max_onl = delta;
178 }
179
180 return true;
181}
182EXPORT_SYMBOL_GPL(torture_online);
183
184
185
186
187
188static int
189torture_onoff(void *arg)
190{
191 int cpu;
192 int maxcpu = -1;
193 DEFINE_TORTURE_RANDOM(rand);
194
195 VERBOSE_TOROUT_STRING("torture_onoff task started");
196 for_each_online_cpu(cpu)
197 maxcpu = cpu;
198 WARN_ON(maxcpu < 0);
199
200 if (maxcpu == 0) {
201 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
202 goto stop;
203 }
204
205 if (onoff_holdoff > 0) {
206 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
207 schedule_timeout_interruptible(onoff_holdoff);
208 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
209 }
210 while (!torture_must_stop()) {
211 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
212 if (!torture_offline(cpu,
213 &n_offline_attempts, &n_offline_successes,
214 &sum_offline, &min_offline, &max_offline))
215 torture_online(cpu,
216 &n_online_attempts, &n_online_successes,
217 &sum_online, &min_online, &max_online);
218 schedule_timeout_interruptible(onoff_interval);
219 }
220
221stop:
222 torture_kthread_stopping("torture_onoff");
223 return 0;
224}
225
226#endif
227
228
229
230
231int torture_onoff_init(long ooholdoff, long oointerval)
232{
233 int ret = 0;
234
235#ifdef CONFIG_HOTPLUG_CPU
236 onoff_holdoff = ooholdoff;
237 onoff_interval = oointerval;
238 if (onoff_interval <= 0)
239 return 0;
240 ret = torture_create_kthread(torture_onoff, NULL, onoff_task);
241#endif
242 return ret;
243}
244EXPORT_SYMBOL_GPL(torture_onoff_init);
245
246
247
248
249static void torture_onoff_cleanup(void)
250{
251#ifdef CONFIG_HOTPLUG_CPU
252 if (onoff_task == NULL)
253 return;
254 VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
255 kthread_stop(onoff_task);
256 onoff_task = NULL;
257#endif
258}
259EXPORT_SYMBOL_GPL(torture_onoff_cleanup);
260
261
262
263
264void torture_onoff_stats(void)
265{
266#ifdef CONFIG_HOTPLUG_CPU
267 pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
268 n_online_successes, n_online_attempts,
269 n_offline_successes, n_offline_attempts,
270 min_online, max_online,
271 min_offline, max_offline,
272 sum_online, sum_offline, HZ);
273#endif
274}
275EXPORT_SYMBOL_GPL(torture_onoff_stats);
276
277
278
279
280bool torture_onoff_failures(void)
281{
282#ifdef CONFIG_HOTPLUG_CPU
283 return n_online_successes != n_online_attempts ||
284 n_offline_successes != n_offline_attempts;
285#else
286 return false;
287#endif
288}
289EXPORT_SYMBOL_GPL(torture_onoff_failures);
290
291#define TORTURE_RANDOM_MULT 39916801
292#define TORTURE_RANDOM_ADD 479001701
293#define TORTURE_RANDOM_REFRESH 10000
294
295
296
297
298
299unsigned long
300torture_random(struct torture_random_state *trsp)
301{
302 if (--trsp->trs_count < 0) {
303 trsp->trs_state += (unsigned long)local_clock();
304 trsp->trs_count = TORTURE_RANDOM_REFRESH;
305 }
306 trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
307 TORTURE_RANDOM_ADD;
308 return swahw32(trsp->trs_state);
309}
310EXPORT_SYMBOL_GPL(torture_random);
311
312
313
314
315
316
317struct shuffle_task {
318 struct list_head st_l;
319 struct task_struct *st_t;
320};
321
322static long shuffle_interval;
323static struct task_struct *shuffler_task;
324static cpumask_var_t shuffle_tmp_mask;
325static int shuffle_idle_cpu;
326static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
327static DEFINE_MUTEX(shuffle_task_mutex);
328
329
330
331
332
333void torture_shuffle_task_register(struct task_struct *tp)
334{
335 struct shuffle_task *stp;
336
337 if (WARN_ON_ONCE(tp == NULL))
338 return;
339 stp = kmalloc(sizeof(*stp), GFP_KERNEL);
340 if (WARN_ON_ONCE(stp == NULL))
341 return;
342 stp->st_t = tp;
343 mutex_lock(&shuffle_task_mutex);
344 list_add(&stp->st_l, &shuffle_task_list);
345 mutex_unlock(&shuffle_task_mutex);
346}
347EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
348
349
350
351
352static void torture_shuffle_task_unregister_all(void)
353{
354 struct shuffle_task *stp;
355 struct shuffle_task *p;
356
357 mutex_lock(&shuffle_task_mutex);
358 list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
359 list_del(&stp->st_l);
360 kfree(stp);
361 }
362 mutex_unlock(&shuffle_task_mutex);
363}
364
365
366
367
368
369static void torture_shuffle_tasks(void)
370{
371 struct shuffle_task *stp;
372
373 cpumask_setall(shuffle_tmp_mask);
374 get_online_cpus();
375
376
377 if (num_online_cpus() == 1) {
378 put_online_cpus();
379 return;
380 }
381
382
383 shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
384 if (shuffle_idle_cpu >= nr_cpu_ids)
385 shuffle_idle_cpu = -1;
386 else
387 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
388
389 mutex_lock(&shuffle_task_mutex);
390 list_for_each_entry(stp, &shuffle_task_list, st_l)
391 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
392 mutex_unlock(&shuffle_task_mutex);
393
394 put_online_cpus();
395}
396
397
398
399
400
401static int torture_shuffle(void *arg)
402{
403 VERBOSE_TOROUT_STRING("torture_shuffle task started");
404 do {
405 schedule_timeout_interruptible(shuffle_interval);
406 torture_shuffle_tasks();
407 torture_shutdown_absorb("torture_shuffle");
408 } while (!torture_must_stop());
409 torture_kthread_stopping("torture_shuffle");
410 return 0;
411}
412
413
414
415
416int torture_shuffle_init(long shuffint)
417{
418 shuffle_interval = shuffint;
419
420 shuffle_idle_cpu = -1;
421
422 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
423 VERBOSE_TOROUT_ERRSTRING("Failed to alloc mask");
424 return -ENOMEM;
425 }
426
427
428 return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
429}
430EXPORT_SYMBOL_GPL(torture_shuffle_init);
431
432
433
434
435static void torture_shuffle_cleanup(void)
436{
437 torture_shuffle_task_unregister_all();
438 if (shuffler_task) {
439 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
440 kthread_stop(shuffler_task);
441 free_cpumask_var(shuffle_tmp_mask);
442 }
443 shuffler_task = NULL;
444}
445EXPORT_SYMBOL_GPL(torture_shuffle_cleanup);
446
447
448
449
450
451static struct task_struct *shutdown_task;
452static ktime_t shutdown_time;
453static void (*torture_shutdown_hook)(void);
454
455
456
457
458
459void torture_shutdown_absorb(const char *title)
460{
461 while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
462 pr_notice("torture thread %s parking due to system shutdown\n",
463 title);
464 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
465 }
466}
467EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
468
469
470
471
472
473static int torture_shutdown(void *arg)
474{
475 ktime_t ktime_snap;
476
477 VERBOSE_TOROUT_STRING("torture_shutdown task started");
478 ktime_snap = ktime_get();
479 while (ktime_before(ktime_snap, shutdown_time) &&
480 !torture_must_stop()) {
481 if (verbose)
482 pr_alert("%s" TORTURE_FLAG
483 "torture_shutdown task: %llu ms remaining\n",
484 torture_type,
485 ktime_ms_delta(shutdown_time, ktime_snap));
486 set_current_state(TASK_INTERRUPTIBLE);
487 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
488 ktime_snap = ktime_get();
489 }
490 if (torture_must_stop()) {
491 torture_kthread_stopping("torture_shutdown");
492 return 0;
493 }
494
495
496
497 VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
498 shutdown_task = NULL;
499 if (torture_shutdown_hook)
500 torture_shutdown_hook();
501 else
502 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
503 rcu_ftrace_dump(DUMP_ALL);
504 kernel_power_off();
505 return 0;
506}
507
508
509
510
511int torture_shutdown_init(int ssecs, void (*cleanup)(void))
512{
513 int ret = 0;
514
515 torture_shutdown_hook = cleanup;
516 if (ssecs > 0) {
517 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
518 ret = torture_create_kthread(torture_shutdown, NULL,
519 shutdown_task);
520 }
521 return ret;
522}
523EXPORT_SYMBOL_GPL(torture_shutdown_init);
524
525
526
527
528static int torture_shutdown_notify(struct notifier_block *unused1,
529 unsigned long unused2, void *unused3)
530{
531 mutex_lock(&fullstop_mutex);
532 if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
533 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
534 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
535 } else {
536 pr_warn("Concurrent rmmod and shutdown illegal!\n");
537 }
538 mutex_unlock(&fullstop_mutex);
539 return NOTIFY_DONE;
540}
541
542static struct notifier_block torture_shutdown_nb = {
543 .notifier_call = torture_shutdown_notify,
544};
545
546
547
548
549
550static void torture_shutdown_cleanup(void)
551{
552 unregister_reboot_notifier(&torture_shutdown_nb);
553 if (shutdown_task != NULL) {
554 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
555 kthread_stop(shutdown_task);
556 }
557 shutdown_task = NULL;
558}
559
560
561
562
563
564
565static struct task_struct *stutter_task;
566static int stutter_pause_test;
567static int stutter;
568
569
570
571
572
573void stutter_wait(const char *title)
574{
575 int spt;
576
577 cond_resched_rcu_qs();
578 spt = READ_ONCE(stutter_pause_test);
579 for (; spt; spt = READ_ONCE(stutter_pause_test)) {
580 if (spt == 1) {
581 schedule_timeout_interruptible(1);
582 } else if (spt == 2) {
583 while (READ_ONCE(stutter_pause_test))
584 cond_resched();
585 } else {
586 schedule_timeout_interruptible(round_jiffies_relative(HZ));
587 }
588 torture_shutdown_absorb(title);
589 }
590}
591EXPORT_SYMBOL_GPL(stutter_wait);
592
593
594
595
596
597static int torture_stutter(void *arg)
598{
599 VERBOSE_TOROUT_STRING("torture_stutter task started");
600 do {
601 if (!torture_must_stop() && stutter > 1) {
602 WRITE_ONCE(stutter_pause_test, 1);
603 schedule_timeout_interruptible(stutter - 1);
604 WRITE_ONCE(stutter_pause_test, 2);
605 schedule_timeout_interruptible(1);
606 }
607 WRITE_ONCE(stutter_pause_test, 0);
608 if (!torture_must_stop())
609 schedule_timeout_interruptible(stutter);
610 torture_shutdown_absorb("torture_stutter");
611 } while (!torture_must_stop());
612 torture_kthread_stopping("torture_stutter");
613 return 0;
614}
615
616
617
618
619int torture_stutter_init(int s)
620{
621 int ret;
622
623 stutter = s;
624 ret = torture_create_kthread(torture_stutter, NULL, stutter_task);
625 return ret;
626}
627EXPORT_SYMBOL_GPL(torture_stutter_init);
628
629
630
631
632static void torture_stutter_cleanup(void)
633{
634 if (!stutter_task)
635 return;
636 VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
637 kthread_stop(stutter_task);
638 stutter_task = NULL;
639}
640
641
642
643
644
645
646
647
648
649
650bool torture_init_begin(char *ttype, bool v)
651{
652 mutex_lock(&fullstop_mutex);
653 if (torture_type != NULL) {
654 pr_alert("torture_init_begin: Refusing %s init: %s running.\n",
655 ttype, torture_type);
656 pr_alert("torture_init_begin: One torture test at a time!\n");
657 mutex_unlock(&fullstop_mutex);
658 return false;
659 }
660 torture_type = ttype;
661 verbose = v;
662 fullstop = FULLSTOP_DONTSTOP;
663 return true;
664}
665EXPORT_SYMBOL_GPL(torture_init_begin);
666
667
668
669
670void torture_init_end(void)
671{
672 mutex_unlock(&fullstop_mutex);
673 register_reboot_notifier(&torture_shutdown_nb);
674}
675EXPORT_SYMBOL_GPL(torture_init_end);
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692bool torture_cleanup_begin(void)
693{
694 mutex_lock(&fullstop_mutex);
695 if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
696 pr_warn("Concurrent rmmod and shutdown illegal!\n");
697 mutex_unlock(&fullstop_mutex);
698 schedule_timeout_uninterruptible(10);
699 return true;
700 }
701 WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
702 mutex_unlock(&fullstop_mutex);
703 torture_shutdown_cleanup();
704 torture_shuffle_cleanup();
705 torture_stutter_cleanup();
706 torture_onoff_cleanup();
707 return false;
708}
709EXPORT_SYMBOL_GPL(torture_cleanup_begin);
710
711void torture_cleanup_end(void)
712{
713 mutex_lock(&fullstop_mutex);
714 torture_type = NULL;
715 mutex_unlock(&fullstop_mutex);
716}
717EXPORT_SYMBOL_GPL(torture_cleanup_end);
718
719
720
721
722bool torture_must_stop(void)
723{
724 return torture_must_stop_irq() || kthread_should_stop();
725}
726EXPORT_SYMBOL_GPL(torture_must_stop);
727
728
729
730
731
732bool torture_must_stop_irq(void)
733{
734 return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
735}
736EXPORT_SYMBOL_GPL(torture_must_stop_irq);
737
738
739
740
741
742
743
744
745void torture_kthread_stopping(char *title)
746{
747 char buf[128];
748
749 snprintf(buf, sizeof(buf), "Stopping %s", title);
750 VERBOSE_TOROUT_STRING(buf);
751 while (!kthread_should_stop()) {
752 torture_shutdown_absorb(title);
753 schedule_timeout_uninterruptible(1);
754 }
755}
756EXPORT_SYMBOL_GPL(torture_kthread_stopping);
757
758
759
760
761
762
763int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
764 char *f, struct task_struct **tp)
765{
766 int ret = 0;
767
768 VERBOSE_TOROUT_STRING(m);
769 *tp = kthread_run(fn, arg, "%s", s);
770 if (IS_ERR(*tp)) {
771 ret = PTR_ERR(*tp);
772 VERBOSE_TOROUT_ERRSTRING(f);
773 *tp = NULL;
774 }
775 torture_shuffle_task_register(*tp);
776 return ret;
777}
778EXPORT_SYMBOL_GPL(_torture_create_kthread);
779
780
781
782
783void _torture_stop_kthread(char *m, struct task_struct **tp)
784{
785 if (*tp == NULL)
786 return;
787 VERBOSE_TOROUT_STRING(m);
788 kthread_stop(*tp);
789 *tp = NULL;
790}
791EXPORT_SYMBOL_GPL(_torture_stop_kthread);
792