1
2
3
4
5
6
7
8
9
10#define pr_fmt(fmt) fmt
11
12#include <linux/atomic.h>
13#include <linux/bitops.h>
14#include <linux/completion.h>
15#include <linux/cpu.h>
16#include <linux/delay.h>
17#include <linux/err.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/kthread.h>
21#include <linux/kernel.h>
22#include <linux/mm.h>
23#include <linux/module.h>
24#include <linux/moduleparam.h>
25#include <linux/notifier.h>
26#include <linux/percpu.h>
27#include <linux/rcupdate.h>
28#include <linux/rcupdate_trace.h>
29#include <linux/reboot.h>
30#include <linux/sched.h>
31#include <linux/spinlock.h>
32#include <linux/smp.h>
33#include <linux/stat.h>
34#include <linux/srcu.h>
35#include <linux/slab.h>
36#include <linux/torture.h>
37#include <linux/types.h>
38
39#include "rcu.h"
40
41#define SCALE_FLAG "-ref-scale: "
42
43#define SCALEOUT(s, x...) \
44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
45
46#define VERBOSE_SCALEOUT(s, x...) \
47 do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0)
48
49static atomic_t verbose_batch_ctr;
50
51#define VERBOSE_SCALEOUT_BATCH(s, x...) \
52do { \
53 if (verbose && \
54 (verbose_batched <= 0 || \
55 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
56 schedule_timeout_uninterruptible(1); \
57 pr_alert("%s" SCALE_FLAG s, scale_type, ## x); \
58 } \
59} while (0)
60
61#define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \
62 do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0)
63
64MODULE_LICENSE("GPL");
65MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
66
67static char *scale_type = "rcu";
68module_param(scale_type, charp, 0444);
69MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
70
71torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
72torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
73
74
75torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
76 "Holdoff time before test start (s)");
77
78torture_param(long, loops, 10000, "Number of loops per experiment.");
79
80torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
81
82torture_param(int, nruns, 30, "Number of experiments to run.");
83
84torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
85
86#ifdef MODULE
87# define REFSCALE_SHUTDOWN 0
88#else
89# define REFSCALE_SHUTDOWN 1
90#endif
91
92torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
93 "Shutdown at end of scalability tests.");
94
95struct reader_task {
96 struct task_struct *task;
97 int start_reader;
98 wait_queue_head_t wq;
99 u64 last_duration_ns;
100};
101
102static struct task_struct *shutdown_task;
103static wait_queue_head_t shutdown_wq;
104
105static struct task_struct *main_task;
106static wait_queue_head_t main_wq;
107static int shutdown_start;
108
109static struct reader_task *reader_tasks;
110
111
112static atomic_t nreaders_exp;
113
114
115static atomic_t n_init;
116static atomic_t n_started;
117static atomic_t n_warmedup;
118static atomic_t n_cooleddown;
119
120
121static int exp_idx;
122
123
124struct ref_scale_ops {
125 void (*init)(void);
126 void (*cleanup)(void);
127 void (*readsection)(const int nloops);
128 void (*delaysection)(const int nloops, const int udl, const int ndl);
129 const char *name;
130};
131
132static struct ref_scale_ops *cur_ops;
133
134static void un_delay(const int udl, const int ndl)
135{
136 if (udl)
137 udelay(udl);
138 if (ndl)
139 ndelay(ndl);
140}
141
142static void ref_rcu_read_section(const int nloops)
143{
144 int i;
145
146 for (i = nloops; i >= 0; i--) {
147 rcu_read_lock();
148 rcu_read_unlock();
149 }
150}
151
152static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
153{
154 int i;
155
156 for (i = nloops; i >= 0; i--) {
157 rcu_read_lock();
158 un_delay(udl, ndl);
159 rcu_read_unlock();
160 }
161}
162
163static void rcu_sync_scale_init(void)
164{
165}
166
167static struct ref_scale_ops rcu_ops = {
168 .init = rcu_sync_scale_init,
169 .readsection = ref_rcu_read_section,
170 .delaysection = ref_rcu_delay_section,
171 .name = "rcu"
172};
173
174
175DEFINE_STATIC_SRCU(srcu_refctl_scale);
176static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
177
178static void srcu_ref_scale_read_section(const int nloops)
179{
180 int i;
181 int idx;
182
183 for (i = nloops; i >= 0; i--) {
184 idx = srcu_read_lock(srcu_ctlp);
185 srcu_read_unlock(srcu_ctlp, idx);
186 }
187}
188
189static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
190{
191 int i;
192 int idx;
193
194 for (i = nloops; i >= 0; i--) {
195 idx = srcu_read_lock(srcu_ctlp);
196 un_delay(udl, ndl);
197 srcu_read_unlock(srcu_ctlp, idx);
198 }
199}
200
201static struct ref_scale_ops srcu_ops = {
202 .init = rcu_sync_scale_init,
203 .readsection = srcu_ref_scale_read_section,
204 .delaysection = srcu_ref_scale_delay_section,
205 .name = "srcu"
206};
207
208
209
210static void rcu_tasks_ref_scale_read_section(const int nloops)
211{
212 int i;
213
214 for (i = nloops; i >= 0; i--)
215 continue;
216}
217
218static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
219{
220 int i;
221
222 for (i = nloops; i >= 0; i--)
223 un_delay(udl, ndl);
224}
225
226static struct ref_scale_ops rcu_tasks_ops = {
227 .init = rcu_sync_scale_init,
228 .readsection = rcu_tasks_ref_scale_read_section,
229 .delaysection = rcu_tasks_ref_scale_delay_section,
230 .name = "rcu-tasks"
231};
232
233
234static void rcu_trace_ref_scale_read_section(const int nloops)
235{
236 int i;
237
238 for (i = nloops; i >= 0; i--) {
239 rcu_read_lock_trace();
240 rcu_read_unlock_trace();
241 }
242}
243
244static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
245{
246 int i;
247
248 for (i = nloops; i >= 0; i--) {
249 rcu_read_lock_trace();
250 un_delay(udl, ndl);
251 rcu_read_unlock_trace();
252 }
253}
254
255static struct ref_scale_ops rcu_trace_ops = {
256 .init = rcu_sync_scale_init,
257 .readsection = rcu_trace_ref_scale_read_section,
258 .delaysection = rcu_trace_ref_scale_delay_section,
259 .name = "rcu-trace"
260};
261
262
263static atomic_t refcnt;
264
265static void ref_refcnt_section(const int nloops)
266{
267 int i;
268
269 for (i = nloops; i >= 0; i--) {
270 atomic_inc(&refcnt);
271 atomic_dec(&refcnt);
272 }
273}
274
275static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
276{
277 int i;
278
279 for (i = nloops; i >= 0; i--) {
280 atomic_inc(&refcnt);
281 un_delay(udl, ndl);
282 atomic_dec(&refcnt);
283 }
284}
285
286static struct ref_scale_ops refcnt_ops = {
287 .init = rcu_sync_scale_init,
288 .readsection = ref_refcnt_section,
289 .delaysection = ref_refcnt_delay_section,
290 .name = "refcnt"
291};
292
293
294static rwlock_t test_rwlock;
295
296static void ref_rwlock_init(void)
297{
298 rwlock_init(&test_rwlock);
299}
300
301static void ref_rwlock_section(const int nloops)
302{
303 int i;
304
305 for (i = nloops; i >= 0; i--) {
306 read_lock(&test_rwlock);
307 read_unlock(&test_rwlock);
308 }
309}
310
311static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
312{
313 int i;
314
315 for (i = nloops; i >= 0; i--) {
316 read_lock(&test_rwlock);
317 un_delay(udl, ndl);
318 read_unlock(&test_rwlock);
319 }
320}
321
322static struct ref_scale_ops rwlock_ops = {
323 .init = ref_rwlock_init,
324 .readsection = ref_rwlock_section,
325 .delaysection = ref_rwlock_delay_section,
326 .name = "rwlock"
327};
328
329
330static struct rw_semaphore test_rwsem;
331
332static void ref_rwsem_init(void)
333{
334 init_rwsem(&test_rwsem);
335}
336
337static void ref_rwsem_section(const int nloops)
338{
339 int i;
340
341 for (i = nloops; i >= 0; i--) {
342 down_read(&test_rwsem);
343 up_read(&test_rwsem);
344 }
345}
346
347static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
348{
349 int i;
350
351 for (i = nloops; i >= 0; i--) {
352 down_read(&test_rwsem);
353 un_delay(udl, ndl);
354 up_read(&test_rwsem);
355 }
356}
357
358static struct ref_scale_ops rwsem_ops = {
359 .init = ref_rwsem_init,
360 .readsection = ref_rwsem_section,
361 .delaysection = ref_rwsem_delay_section,
362 .name = "rwsem"
363};
364
365
366static DEFINE_SPINLOCK(test_lock);
367
368static void ref_lock_section(const int nloops)
369{
370 int i;
371
372 preempt_disable();
373 for (i = nloops; i >= 0; i--) {
374 spin_lock(&test_lock);
375 spin_unlock(&test_lock);
376 }
377 preempt_enable();
378}
379
380static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
381{
382 int i;
383
384 preempt_disable();
385 for (i = nloops; i >= 0; i--) {
386 spin_lock(&test_lock);
387 un_delay(udl, ndl);
388 spin_unlock(&test_lock);
389 }
390 preempt_enable();
391}
392
393static struct ref_scale_ops lock_ops = {
394 .readsection = ref_lock_section,
395 .delaysection = ref_lock_delay_section,
396 .name = "lock"
397};
398
399
400
401static void ref_lock_irq_section(const int nloops)
402{
403 unsigned long flags;
404 int i;
405
406 preempt_disable();
407 for (i = nloops; i >= 0; i--) {
408 spin_lock_irqsave(&test_lock, flags);
409 spin_unlock_irqrestore(&test_lock, flags);
410 }
411 preempt_enable();
412}
413
414static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
415{
416 unsigned long flags;
417 int i;
418
419 preempt_disable();
420 for (i = nloops; i >= 0; i--) {
421 spin_lock_irqsave(&test_lock, flags);
422 un_delay(udl, ndl);
423 spin_unlock_irqrestore(&test_lock, flags);
424 }
425 preempt_enable();
426}
427
428static struct ref_scale_ops lock_irq_ops = {
429 .readsection = ref_lock_irq_section,
430 .delaysection = ref_lock_irq_delay_section,
431 .name = "lock-irq"
432};
433
434
435static DEFINE_PER_CPU(unsigned long, test_acqrel);
436
437static void ref_acqrel_section(const int nloops)
438{
439 unsigned long x;
440 int i;
441
442 preempt_disable();
443 for (i = nloops; i >= 0; i--) {
444 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
445 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
446 }
447 preempt_enable();
448}
449
450static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
451{
452 unsigned long x;
453 int i;
454
455 preempt_disable();
456 for (i = nloops; i >= 0; i--) {
457 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
458 un_delay(udl, ndl);
459 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
460 }
461 preempt_enable();
462}
463
464static struct ref_scale_ops acqrel_ops = {
465 .readsection = ref_acqrel_section,
466 .delaysection = ref_acqrel_delay_section,
467 .name = "acqrel"
468};
469
470static volatile u64 stopopts;
471
472static void ref_clock_section(const int nloops)
473{
474 u64 x = 0;
475 int i;
476
477 preempt_disable();
478 for (i = nloops; i >= 0; i--)
479 x += ktime_get_real_fast_ns();
480 preempt_enable();
481 stopopts = x;
482}
483
484static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
485{
486 u64 x = 0;
487 int i;
488
489 preempt_disable();
490 for (i = nloops; i >= 0; i--) {
491 x += ktime_get_real_fast_ns();
492 un_delay(udl, ndl);
493 }
494 preempt_enable();
495 stopopts = x;
496}
497
498static struct ref_scale_ops clock_ops = {
499 .readsection = ref_clock_section,
500 .delaysection = ref_clock_delay_section,
501 .name = "clock"
502};
503
504static void rcu_scale_one_reader(void)
505{
506 if (readdelay <= 0)
507 cur_ops->readsection(loops);
508 else
509 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
510}
511
512
513
514static int
515ref_scale_reader(void *arg)
516{
517 unsigned long flags;
518 long me = (long)arg;
519 struct reader_task *rt = &(reader_tasks[me]);
520 u64 start;
521 s64 duration;
522
523 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
524 WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
525 set_user_nice(current, MAX_NICE);
526 atomic_inc(&n_init);
527 if (holdoff)
528 schedule_timeout_interruptible(holdoff * HZ);
529repeat:
530 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
531
532
533 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
534 torture_must_stop());
535
536 if (torture_must_stop())
537 goto end;
538
539
540 WARN_ON_ONCE(raw_smp_processor_id() != me);
541
542 WRITE_ONCE(rt->start_reader, 0);
543 if (!atomic_dec_return(&n_started))
544 while (atomic_read_acquire(&n_started))
545 cpu_relax();
546
547 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
548
549
550
551
552 rcu_scale_one_reader();
553 if (!atomic_dec_return(&n_warmedup))
554 while (atomic_read_acquire(&n_warmedup))
555 rcu_scale_one_reader();
556
557
558 local_irq_save(flags);
559 start = ktime_get_mono_fast_ns();
560
561 rcu_scale_one_reader();
562
563 duration = ktime_get_mono_fast_ns() - start;
564 local_irq_restore(flags);
565
566 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
567
568
569 if (!atomic_dec_return(&n_cooleddown))
570 while (atomic_read_acquire(&n_cooleddown))
571 rcu_scale_one_reader();
572
573 if (atomic_dec_and_test(&nreaders_exp))
574 wake_up(&main_wq);
575
576 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
577 me, exp_idx, atomic_read(&nreaders_exp));
578
579 if (!torture_must_stop())
580 goto repeat;
581end:
582 torture_kthread_stopping("ref_scale_reader");
583 return 0;
584}
585
586static void reset_readers(void)
587{
588 int i;
589 struct reader_task *rt;
590
591 for (i = 0; i < nreaders; i++) {
592 rt = &(reader_tasks[i]);
593
594 rt->last_duration_ns = 0;
595 }
596}
597
598
599static u64 process_durations(int n)
600{
601 int i;
602 struct reader_task *rt;
603 char buf1[64];
604 char *buf;
605 u64 sum = 0;
606
607 buf = kmalloc(128 + nreaders * 32, GFP_KERNEL);
608 if (!buf)
609 return 0;
610 buf[0] = 0;
611 sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
612 exp_idx);
613
614 for (i = 0; i < n && !torture_must_stop(); i++) {
615 rt = &(reader_tasks[i]);
616 sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);
617
618 if (i % 5 == 0)
619 strcat(buf, "\n");
620 strcat(buf, buf1);
621
622 sum += rt->last_duration_ns;
623 }
624 strcat(buf, "\n");
625
626 SCALEOUT("%s\n", buf);
627
628 kfree(buf);
629 return sum;
630}
631
632
633
634
635
636
637
638static int main_func(void *arg)
639{
640 bool errexit = false;
641 int exp, r;
642 char buf1[64];
643 char *buf;
644 u64 *result_avg;
645
646 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
647 set_user_nice(current, MAX_NICE);
648
649 VERBOSE_SCALEOUT("main_func task started");
650 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
651 buf = kzalloc(64 + nruns * 32, GFP_KERNEL);
652 if (!result_avg || !buf) {
653 VERBOSE_SCALEOUT_ERRSTRING("out of memory");
654 errexit = true;
655 }
656 if (holdoff)
657 schedule_timeout_interruptible(holdoff * HZ);
658
659
660 atomic_inc(&n_init);
661 while (atomic_read(&n_init) < nreaders + 1)
662 schedule_timeout_uninterruptible(1);
663
664
665 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
666 if (errexit)
667 break;
668 if (torture_must_stop())
669 goto end;
670
671 reset_readers();
672 atomic_set(&nreaders_exp, nreaders);
673 atomic_set(&n_started, nreaders);
674 atomic_set(&n_warmedup, nreaders);
675 atomic_set(&n_cooleddown, nreaders);
676
677 exp_idx = exp;
678
679 for (r = 0; r < nreaders; r++) {
680 smp_store_release(&reader_tasks[r].start_reader, 1);
681 wake_up(&reader_tasks[r].wq);
682 }
683
684 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
685 nreaders);
686
687 wait_event(main_wq,
688 !atomic_read(&nreaders_exp) || torture_must_stop());
689
690 VERBOSE_SCALEOUT("main_func: experiment ended");
691
692 if (torture_must_stop())
693 goto end;
694
695 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
696 }
697
698
699 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
700
701 if (!errexit) {
702 buf[0] = 0;
703 strcat(buf, "\n");
704 strcat(buf, "Runs\tTime(ns)\n");
705 }
706
707 for (exp = 0; exp < nruns; exp++) {
708 u64 avg;
709 u32 rem;
710
711 if (errexit)
712 break;
713 avg = div_u64_rem(result_avg[exp], 1000, &rem);
714 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
715 strcat(buf, buf1);
716 }
717
718 if (!errexit)
719 SCALEOUT("%s", buf);
720
721
722 if (shutdown) {
723 shutdown_start = 1;
724 wake_up(&shutdown_wq);
725 }
726
727
728 while (!torture_must_stop())
729 schedule_timeout_uninterruptible(1);
730
731end:
732 torture_kthread_stopping("main_func");
733 kfree(result_avg);
734 kfree(buf);
735 return 0;
736}
737
738static void
739ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
740{
741 pr_alert("%s" SCALE_FLAG
742 "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
743 verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay);
744}
745
746static void
747ref_scale_cleanup(void)
748{
749 int i;
750
751 if (torture_cleanup_begin())
752 return;
753
754 if (!cur_ops) {
755 torture_cleanup_end();
756 return;
757 }
758
759 if (reader_tasks) {
760 for (i = 0; i < nreaders; i++)
761 torture_stop_kthread("ref_scale_reader",
762 reader_tasks[i].task);
763 }
764 kfree(reader_tasks);
765
766 torture_stop_kthread("main_task", main_task);
767 kfree(main_task);
768
769
770 if (cur_ops->cleanup != NULL)
771 cur_ops->cleanup();
772
773 torture_cleanup_end();
774}
775
776
777static int
778ref_scale_shutdown(void *arg)
779{
780 wait_event(shutdown_wq, shutdown_start);
781
782 smp_mb();
783 ref_scale_cleanup();
784 kernel_power_off();
785
786 return -EINVAL;
787}
788
789static int __init
790ref_scale_init(void)
791{
792 long i;
793 int firsterr = 0;
794 static struct ref_scale_ops *scale_ops[] = {
795 &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops, &refcnt_ops, &rwlock_ops,
796 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops,
797 };
798
799 if (!torture_init_begin(scale_type, verbose))
800 return -EBUSY;
801
802 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
803 cur_ops = scale_ops[i];
804 if (strcmp(scale_type, cur_ops->name) == 0)
805 break;
806 }
807 if (i == ARRAY_SIZE(scale_ops)) {
808 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
809 pr_alert("rcu-scale types:");
810 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
811 pr_cont(" %s", scale_ops[i]->name);
812 pr_cont("\n");
813 firsterr = -EINVAL;
814 cur_ops = NULL;
815 goto unwind;
816 }
817 if (cur_ops->init)
818 cur_ops->init();
819
820 ref_scale_print_module_parms(cur_ops, "Start of test");
821
822
823 if (shutdown) {
824 init_waitqueue_head(&shutdown_wq);
825 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
826 shutdown_task);
827 if (firsterr)
828 goto unwind;
829 schedule_timeout_uninterruptible(1);
830 }
831
832
833 if (nreaders < 0)
834 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
835 if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
836 loops = 1;
837 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
838 nreaders = 1;
839 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
840 nruns = 1;
841 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
842 GFP_KERNEL);
843 if (!reader_tasks) {
844 VERBOSE_SCALEOUT_ERRSTRING("out of memory");
845 firsterr = -ENOMEM;
846 goto unwind;
847 }
848
849 VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders);
850
851 for (i = 0; i < nreaders; i++) {
852 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
853 reader_tasks[i].task);
854 if (firsterr)
855 goto unwind;
856
857 init_waitqueue_head(&(reader_tasks[i].wq));
858 }
859
860
861 init_waitqueue_head(&main_wq);
862 firsterr = torture_create_kthread(main_func, NULL, main_task);
863 if (firsterr)
864 goto unwind;
865
866 torture_init_end();
867 return 0;
868
869unwind:
870 torture_init_end();
871 ref_scale_cleanup();
872 if (shutdown) {
873 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
874 kernel_power_off();
875 }
876 return firsterr;
877}
878
879module_init(ref_scale_init);
880module_exit(ref_scale_cleanup);
881