1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/types.h>
24#include <linux/kernel.h>
25#include <linux/init.h>
26#include <linux/module.h>
27#include <linux/kthread.h>
28#include <linux/err.h>
29#include <linux/spinlock.h>
30#include <linux/smp.h>
31#include <linux/interrupt.h>
32#include <linux/sched.h>
33#include <linux/sched/clock.h>
34#include <linux/atomic.h>
35#include <linux/bitops.h>
36#include <linux/completion.h>
37#include <linux/moduleparam.h>
38#include <linux/percpu.h>
39#include <linux/notifier.h>
40#include <linux/reboot.h>
41#include <linux/freezer.h>
42#include <linux/cpu.h>
43#include <linux/delay.h>
44#include <linux/stat.h>
45#include <linux/slab.h>
46#include <linux/trace_clock.h>
47#include <linux/ktime.h>
48#include <asm/byteorder.h>
49#include <linux/torture.h>
50
51MODULE_LICENSE("GPL");
52MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
53
54static char *torture_type;
55static bool verbose;
56
57
58#define FULLSTOP_DONTSTOP 0
59#define FULLSTOP_SHUTDOWN 1
60#define FULLSTOP_RMMOD 2
61static int fullstop = FULLSTOP_RMMOD;
62static DEFINE_MUTEX(fullstop_mutex);
63static int *torture_runnable;
64
65#ifdef CONFIG_HOTPLUG_CPU
66
67
68
69
70
71
72static struct task_struct *onoff_task;
73static long onoff_holdoff;
74static long onoff_interval;
75static long n_offline_attempts;
76static long n_offline_successes;
77static unsigned long sum_offline;
78static int min_offline = -1;
79static int max_offline;
80static long n_online_attempts;
81static long n_online_successes;
82static unsigned long sum_online;
83static int min_online = -1;
84static int max_online;
85
86
87
88
89
90
91bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
92 unsigned long *sum_offl, int *min_offl, int *max_offl)
93{
94 unsigned long delta;
95 int ret;
96 unsigned long starttime;
97
98 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
99 return false;
100
101 if (verbose)
102 pr_alert("%s" TORTURE_FLAG
103 "torture_onoff task: offlining %d\n",
104 torture_type, cpu);
105 starttime = jiffies;
106 (*n_offl_attempts)++;
107 ret = cpu_down(cpu);
108 if (ret) {
109 if (verbose)
110 pr_alert("%s" TORTURE_FLAG
111 "torture_onoff task: offline %d failed: errno %d\n",
112 torture_type, cpu, ret);
113 } else {
114 if (verbose)
115 pr_alert("%s" TORTURE_FLAG
116 "torture_onoff task: offlined %d\n",
117 torture_type, cpu);
118 (*n_offl_successes)++;
119 delta = jiffies - starttime;
120 *sum_offl += delta;
121 if (*min_offl < 0) {
122 *min_offl = delta;
123 *max_offl = delta;
124 }
125 if (*min_offl > delta)
126 *min_offl = delta;
127 if (*max_offl < delta)
128 *max_offl = delta;
129 }
130
131 return true;
132}
133EXPORT_SYMBOL_GPL(torture_offline);
134
135
136
137
138
139
140bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
141 unsigned long *sum_onl, int *min_onl, int *max_onl)
142{
143 unsigned long delta;
144 int ret;
145 unsigned long starttime;
146
147 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
148 return false;
149
150 if (verbose)
151 pr_alert("%s" TORTURE_FLAG
152 "torture_onoff task: onlining %d\n",
153 torture_type, cpu);
154 starttime = jiffies;
155 (*n_onl_attempts)++;
156 ret = cpu_up(cpu);
157 if (ret) {
158 if (verbose)
159 pr_alert("%s" TORTURE_FLAG
160 "torture_onoff task: online %d failed: errno %d\n",
161 torture_type, cpu, ret);
162 } else {
163 if (verbose)
164 pr_alert("%s" TORTURE_FLAG
165 "torture_onoff task: onlined %d\n",
166 torture_type, cpu);
167 (*n_onl_successes)++;
168 delta = jiffies - starttime;
169 *sum_onl += delta;
170 if (*min_onl < 0) {
171 *min_onl = delta;
172 *max_onl = delta;
173 }
174 if (*min_onl > delta)
175 *min_onl = delta;
176 if (*max_onl < delta)
177 *max_onl = delta;
178 }
179
180 return true;
181}
182EXPORT_SYMBOL_GPL(torture_online);
183
184
185
186
187
188static int
189torture_onoff(void *arg)
190{
191 int cpu;
192 int maxcpu = -1;
193 DEFINE_TORTURE_RANDOM(rand);
194
195 VERBOSE_TOROUT_STRING("torture_onoff task started");
196 for_each_online_cpu(cpu)
197 maxcpu = cpu;
198 WARN_ON(maxcpu < 0);
199
200 if (maxcpu == 0) {
201 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
202 goto stop;
203 }
204
205 if (onoff_holdoff > 0) {
206 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
207 schedule_timeout_interruptible(onoff_holdoff);
208 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
209 }
210 while (!torture_must_stop()) {
211 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
212 if (!torture_offline(cpu,
213 &n_offline_attempts, &n_offline_successes,
214 &sum_offline, &min_offline, &max_offline))
215 torture_online(cpu,
216 &n_online_attempts, &n_online_successes,
217 &sum_online, &min_online, &max_online);
218 schedule_timeout_interruptible(onoff_interval);
219 }
220
221stop:
222 torture_kthread_stopping("torture_onoff");
223 return 0;
224}
225
226#endif
227
228
229
230
231int torture_onoff_init(long ooholdoff, long oointerval)
232{
233 int ret = 0;
234
235#ifdef CONFIG_HOTPLUG_CPU
236 onoff_holdoff = ooholdoff;
237 onoff_interval = oointerval;
238 if (onoff_interval <= 0)
239 return 0;
240 ret = torture_create_kthread(torture_onoff, NULL, onoff_task);
241#endif
242 return ret;
243}
244EXPORT_SYMBOL_GPL(torture_onoff_init);
245
246
247
248
249static void torture_onoff_cleanup(void)
250{
251#ifdef CONFIG_HOTPLUG_CPU
252 if (onoff_task == NULL)
253 return;
254 VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
255 kthread_stop(onoff_task);
256 onoff_task = NULL;
257#endif
258}
259EXPORT_SYMBOL_GPL(torture_onoff_cleanup);
260
261
262
263
264void torture_onoff_stats(void)
265{
266#ifdef CONFIG_HOTPLUG_CPU
267 pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
268 n_online_successes, n_online_attempts,
269 n_offline_successes, n_offline_attempts,
270 min_online, max_online,
271 min_offline, max_offline,
272 sum_online, sum_offline, HZ);
273#endif
274}
275EXPORT_SYMBOL_GPL(torture_onoff_stats);
276
277
278
279
280bool torture_onoff_failures(void)
281{
282#ifdef CONFIG_HOTPLUG_CPU
283 return n_online_successes != n_online_attempts ||
284 n_offline_successes != n_offline_attempts;
285#else
286 return false;
287#endif
288}
289EXPORT_SYMBOL_GPL(torture_onoff_failures);
290
291#define TORTURE_RANDOM_MULT 39916801
292#define TORTURE_RANDOM_ADD 479001701
293#define TORTURE_RANDOM_REFRESH 10000
294
295
296
297
298
299unsigned long
300torture_random(struct torture_random_state *trsp)
301{
302 if (--trsp->trs_count < 0) {
303 trsp->trs_state += (unsigned long)local_clock();
304 trsp->trs_count = TORTURE_RANDOM_REFRESH;
305 }
306 trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
307 TORTURE_RANDOM_ADD;
308 return swahw32(trsp->trs_state);
309}
310EXPORT_SYMBOL_GPL(torture_random);
311
312
313
314
315
316
317struct shuffle_task {
318 struct list_head st_l;
319 struct task_struct *st_t;
320};
321
322static long shuffle_interval;
323static struct task_struct *shuffler_task;
324static cpumask_var_t shuffle_tmp_mask;
325static int shuffle_idle_cpu;
326static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
327static DEFINE_MUTEX(shuffle_task_mutex);
328
329
330
331
332
333void torture_shuffle_task_register(struct task_struct *tp)
334{
335 struct shuffle_task *stp;
336
337 if (WARN_ON_ONCE(tp == NULL))
338 return;
339 stp = kmalloc(sizeof(*stp), GFP_KERNEL);
340 if (WARN_ON_ONCE(stp == NULL))
341 return;
342 stp->st_t = tp;
343 mutex_lock(&shuffle_task_mutex);
344 list_add(&stp->st_l, &shuffle_task_list);
345 mutex_unlock(&shuffle_task_mutex);
346}
347EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
348
349
350
351
352static void torture_shuffle_task_unregister_all(void)
353{
354 struct shuffle_task *stp;
355 struct shuffle_task *p;
356
357 mutex_lock(&shuffle_task_mutex);
358 list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
359 list_del(&stp->st_l);
360 kfree(stp);
361 }
362 mutex_unlock(&shuffle_task_mutex);
363}
364
365
366
367
368
369static void torture_shuffle_tasks(void)
370{
371 struct shuffle_task *stp;
372
373 cpumask_setall(shuffle_tmp_mask);
374 get_online_cpus();
375
376
377 if (num_online_cpus() == 1) {
378 put_online_cpus();
379 return;
380 }
381
382
383 shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
384 if (shuffle_idle_cpu >= nr_cpu_ids)
385 shuffle_idle_cpu = -1;
386 else
387 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
388
389 mutex_lock(&shuffle_task_mutex);
390 list_for_each_entry(stp, &shuffle_task_list, st_l)
391 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
392 mutex_unlock(&shuffle_task_mutex);
393
394 put_online_cpus();
395}
396
397
398
399
400
401static int torture_shuffle(void *arg)
402{
403 VERBOSE_TOROUT_STRING("torture_shuffle task started");
404 do {
405 schedule_timeout_interruptible(shuffle_interval);
406 torture_shuffle_tasks();
407 torture_shutdown_absorb("torture_shuffle");
408 } while (!torture_must_stop());
409 torture_kthread_stopping("torture_shuffle");
410 return 0;
411}
412
413
414
415
416int torture_shuffle_init(long shuffint)
417{
418 shuffle_interval = shuffint;
419
420 shuffle_idle_cpu = -1;
421
422 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
423 VERBOSE_TOROUT_ERRSTRING("Failed to alloc mask");
424 return -ENOMEM;
425 }
426
427
428 return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
429}
430EXPORT_SYMBOL_GPL(torture_shuffle_init);
431
432
433
434
435static void torture_shuffle_cleanup(void)
436{
437 torture_shuffle_task_unregister_all();
438 if (shuffler_task) {
439 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
440 kthread_stop(shuffler_task);
441 free_cpumask_var(shuffle_tmp_mask);
442 }
443 shuffler_task = NULL;
444}
445EXPORT_SYMBOL_GPL(torture_shuffle_cleanup);
446
447
448
449
450
451static struct task_struct *shutdown_task;
452static ktime_t shutdown_time;
453static void (*torture_shutdown_hook)(void);
454
455
456
457
458
459void torture_shutdown_absorb(const char *title)
460{
461 while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
462 pr_notice("torture thread %s parking due to system shutdown\n",
463 title);
464 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
465 }
466}
467EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
468
469
470
471
472
473static int torture_shutdown(void *arg)
474{
475 ktime_t ktime_snap;
476
477 VERBOSE_TOROUT_STRING("torture_shutdown task started");
478 ktime_snap = ktime_get();
479 while (ktime_before(ktime_snap, shutdown_time) &&
480 !torture_must_stop()) {
481 if (verbose)
482 pr_alert("%s" TORTURE_FLAG
483 "torture_shutdown task: %llu ms remaining\n",
484 torture_type,
485 ktime_ms_delta(shutdown_time, ktime_snap));
486 set_current_state(TASK_INTERRUPTIBLE);
487 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
488 ktime_snap = ktime_get();
489 }
490 if (torture_must_stop()) {
491 torture_kthread_stopping("torture_shutdown");
492 return 0;
493 }
494
495
496
497 VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
498 shutdown_task = NULL;
499 if (torture_shutdown_hook)
500 torture_shutdown_hook();
501 else
502 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
503 ftrace_dump(DUMP_ALL);
504 kernel_power_off();
505 return 0;
506}
507
508
509
510
511int torture_shutdown_init(int ssecs, void (*cleanup)(void))
512{
513 int ret = 0;
514
515 torture_shutdown_hook = cleanup;
516 if (ssecs > 0) {
517 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
518 ret = torture_create_kthread(torture_shutdown, NULL,
519 shutdown_task);
520 }
521 return ret;
522}
523EXPORT_SYMBOL_GPL(torture_shutdown_init);
524
525
526
527
528static int torture_shutdown_notify(struct notifier_block *unused1,
529 unsigned long unused2, void *unused3)
530{
531 mutex_lock(&fullstop_mutex);
532 if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
533 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
534 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
535 } else {
536 pr_warn("Concurrent rmmod and shutdown illegal!\n");
537 }
538 mutex_unlock(&fullstop_mutex);
539 return NOTIFY_DONE;
540}
541
542static struct notifier_block torture_shutdown_nb = {
543 .notifier_call = torture_shutdown_notify,
544};
545
546
547
548
549
550static void torture_shutdown_cleanup(void)
551{
552 unregister_reboot_notifier(&torture_shutdown_nb);
553 if (shutdown_task != NULL) {
554 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
555 kthread_stop(shutdown_task);
556 }
557 shutdown_task = NULL;
558}
559
560
561
562
563
564
565static struct task_struct *stutter_task;
566static int stutter_pause_test;
567static int stutter;
568
569
570
571
572
573void stutter_wait(const char *title)
574{
575 cond_resched_rcu_qs();
576 while (READ_ONCE(stutter_pause_test) ||
577 (torture_runnable && !READ_ONCE(*torture_runnable))) {
578 if (stutter_pause_test)
579 if (READ_ONCE(stutter_pause_test) == 1)
580 schedule_timeout_interruptible(1);
581 else
582 while (READ_ONCE(stutter_pause_test))
583 cond_resched();
584 else
585 schedule_timeout_interruptible(round_jiffies_relative(HZ));
586 torture_shutdown_absorb(title);
587 }
588}
589EXPORT_SYMBOL_GPL(stutter_wait);
590
591
592
593
594
595static int torture_stutter(void *arg)
596{
597 VERBOSE_TOROUT_STRING("torture_stutter task started");
598 do {
599 if (!torture_must_stop()) {
600 if (stutter > 1) {
601 schedule_timeout_interruptible(stutter - 1);
602 WRITE_ONCE(stutter_pause_test, 2);
603 }
604 schedule_timeout_interruptible(1);
605 WRITE_ONCE(stutter_pause_test, 1);
606 }
607 if (!torture_must_stop())
608 schedule_timeout_interruptible(stutter);
609 WRITE_ONCE(stutter_pause_test, 0);
610 torture_shutdown_absorb("torture_stutter");
611 } while (!torture_must_stop());
612 torture_kthread_stopping("torture_stutter");
613 return 0;
614}
615
616
617
618
619int torture_stutter_init(int s)
620{
621 int ret;
622
623 stutter = s;
624 ret = torture_create_kthread(torture_stutter, NULL, stutter_task);
625 return ret;
626}
627EXPORT_SYMBOL_GPL(torture_stutter_init);
628
629
630
631
632static void torture_stutter_cleanup(void)
633{
634 if (!stutter_task)
635 return;
636 VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
637 kthread_stop(stutter_task);
638 stutter_task = NULL;
639}
640
641
642
643
644
645
646
647
648
649
650bool torture_init_begin(char *ttype, bool v, int *runnable)
651{
652 mutex_lock(&fullstop_mutex);
653 if (torture_type != NULL) {
654 pr_alert("torture_init_begin: Refusing %s init: %s running.\n",
655 ttype, torture_type);
656 pr_alert("torture_init_begin: One torture test at a time!\n");
657 mutex_unlock(&fullstop_mutex);
658 return false;
659 }
660 torture_type = ttype;
661 verbose = v;
662 torture_runnable = runnable;
663 fullstop = FULLSTOP_DONTSTOP;
664 return true;
665}
666EXPORT_SYMBOL_GPL(torture_init_begin);
667
668
669
670
671void torture_init_end(void)
672{
673 mutex_unlock(&fullstop_mutex);
674 register_reboot_notifier(&torture_shutdown_nb);
675}
676EXPORT_SYMBOL_GPL(torture_init_end);
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693bool torture_cleanup_begin(void)
694{
695 mutex_lock(&fullstop_mutex);
696 if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
697 pr_warn("Concurrent rmmod and shutdown illegal!\n");
698 mutex_unlock(&fullstop_mutex);
699 schedule_timeout_uninterruptible(10);
700 return true;
701 }
702 WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
703 mutex_unlock(&fullstop_mutex);
704 torture_shutdown_cleanup();
705 torture_shuffle_cleanup();
706 torture_stutter_cleanup();
707 torture_onoff_cleanup();
708 return false;
709}
710EXPORT_SYMBOL_GPL(torture_cleanup_begin);
711
712void torture_cleanup_end(void)
713{
714 mutex_lock(&fullstop_mutex);
715 torture_type = NULL;
716 mutex_unlock(&fullstop_mutex);
717}
718EXPORT_SYMBOL_GPL(torture_cleanup_end);
719
720
721
722
723bool torture_must_stop(void)
724{
725 return torture_must_stop_irq() || kthread_should_stop();
726}
727EXPORT_SYMBOL_GPL(torture_must_stop);
728
729
730
731
732
733bool torture_must_stop_irq(void)
734{
735 return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
736}
737EXPORT_SYMBOL_GPL(torture_must_stop_irq);
738
739
740
741
742
743
744
745
746void torture_kthread_stopping(char *title)
747{
748 char buf[128];
749
750 snprintf(buf, sizeof(buf), "Stopping %s", title);
751 VERBOSE_TOROUT_STRING(buf);
752 while (!kthread_should_stop()) {
753 torture_shutdown_absorb(title);
754 schedule_timeout_uninterruptible(1);
755 }
756}
757EXPORT_SYMBOL_GPL(torture_kthread_stopping);
758
759
760
761
762
763
764int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
765 char *f, struct task_struct **tp)
766{
767 int ret = 0;
768
769 VERBOSE_TOROUT_STRING(m);
770 *tp = kthread_run(fn, arg, "%s", s);
771 if (IS_ERR(*tp)) {
772 ret = PTR_ERR(*tp);
773 VERBOSE_TOROUT_ERRSTRING(f);
774 *tp = NULL;
775 }
776 torture_shuffle_task_register(*tp);
777 return ret;
778}
779EXPORT_SYMBOL_GPL(_torture_create_kthread);
780
781
782
783
784void _torture_stop_kthread(char *m, struct task_struct **tp)
785{
786 if (*tp == NULL)
787 return;
788 VERBOSE_TOROUT_STRING(m);
789 kthread_stop(*tp);
790 *tp = NULL;
791}
792EXPORT_SYMBOL_GPL(_torture_stop_kthread);
793