1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/ioprio.h>
19#include <linux/kdev_t.h>
20#include <linux/module.h>
21#include <linux/sched/signal.h>
22#include <linux/err.h>
23#include <linux/blkdev.h>
24#include <linux/backing-dev.h>
25#include <linux/slab.h>
26#include <linux/genhd.h>
27#include <linux/delay.h>
28#include <linux/atomic.h>
29#include <linux/ctype.h>
30#include <linux/blk-cgroup.h>
31#include <linux/tracehook.h>
32#include <linux/psi.h>
33#include "blk.h"
34
35#define MAX_KEY_LEN 100
36
37
38
39
40
41
42
43
44static DEFINE_MUTEX(blkcg_pol_register_mutex);
45static DEFINE_MUTEX(blkcg_pol_mutex);
46
47struct blkcg blkcg_root;
48EXPORT_SYMBOL_GPL(blkcg_root);
49
50struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
51EXPORT_SYMBOL_GPL(blkcg_root_css);
52
53static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
54
55static LIST_HEAD(all_blkcgs);
56
57bool blkcg_debug_stats = false;
58static struct workqueue_struct *blkcg_punt_bio_wq;
59
60static bool blkcg_policy_enabled(struct request_queue *q,
61 const struct blkcg_policy *pol)
62{
63 return pol && test_bit(pol->plid, q->blkcg_pols);
64}
65
66
67
68
69
70
71
72static void blkg_free(struct blkcg_gq *blkg)
73{
74 int i;
75
76 if (!blkg)
77 return;
78
79 for (i = 0; i < BLKCG_MAX_POLS; i++)
80 if (blkg->pd[i])
81 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
82
83 blkg_rwstat_exit(&blkg->stat_ios);
84 blkg_rwstat_exit(&blkg->stat_bytes);
85 percpu_ref_exit(&blkg->refcnt);
86 kfree(blkg);
87}
88
89static void __blkg_release(struct rcu_head *rcu)
90{
91 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
92
93 WARN_ON(!bio_list_empty(&blkg->async_bios));
94
95
96 css_put(&blkg->blkcg->css);
97 if (blkg->parent)
98 blkg_put(blkg->parent);
99
100 wb_congested_put(blkg->wb_congested);
101
102 blkg_free(blkg);
103}
104
105
106
107
108
109
110
111
112
113static void blkg_release(struct percpu_ref *ref)
114{
115 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
116
117 call_rcu(&blkg->rcu_head, __blkg_release);
118}
119
120static void blkg_async_bio_workfn(struct work_struct *work)
121{
122 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
123 async_bio_work);
124 struct bio_list bios = BIO_EMPTY_LIST;
125 struct bio *bio;
126
127
128 spin_lock_bh(&blkg->async_bio_lock);
129 bio_list_merge(&bios, &blkg->async_bios);
130 bio_list_init(&blkg->async_bios);
131 spin_unlock_bh(&blkg->async_bio_lock);
132
133 while ((bio = bio_list_pop(&bios)))
134 submit_bio(bio);
135}
136
137
138
139
140
141
142
143
144
145static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
146 gfp_t gfp_mask)
147{
148 struct blkcg_gq *blkg;
149 int i;
150
151
152 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
153 if (!blkg)
154 return NULL;
155
156 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
157 goto err_free;
158
159 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
160 blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
161 goto err_free;
162
163 blkg->q = q;
164 INIT_LIST_HEAD(&blkg->q_node);
165 spin_lock_init(&blkg->async_bio_lock);
166 bio_list_init(&blkg->async_bios);
167 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
168 blkg->blkcg = blkcg;
169
170 for (i = 0; i < BLKCG_MAX_POLS; i++) {
171 struct blkcg_policy *pol = blkcg_policy[i];
172 struct blkg_policy_data *pd;
173
174 if (!blkcg_policy_enabled(q, pol))
175 continue;
176
177
178 pd = pol->pd_alloc_fn(gfp_mask, q->node);
179 if (!pd)
180 goto err_free;
181
182 blkg->pd[i] = pd;
183 pd->blkg = blkg;
184 pd->plid = i;
185 }
186
187 return blkg;
188
189err_free:
190 blkg_free(blkg);
191 return NULL;
192}
193
194struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
195 struct request_queue *q, bool update_hint)
196{
197 struct blkcg_gq *blkg;
198
199
200
201
202
203
204
205 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
206 if (blkg && blkg->q == q) {
207 if (update_hint) {
208 lockdep_assert_held(&q->queue_lock);
209 rcu_assign_pointer(blkcg->blkg_hint, blkg);
210 }
211 return blkg;
212 }
213
214 return NULL;
215}
216EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
217
218
219
220
221
222static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
223 struct request_queue *q,
224 struct blkcg_gq *new_blkg)
225{
226 struct blkcg_gq *blkg;
227 struct bdi_writeback_congested *wb_congested;
228 int i, ret;
229
230 WARN_ON_ONCE(!rcu_read_lock_held());
231 lockdep_assert_held(&q->queue_lock);
232
233
234 if (blk_queue_dying(q)) {
235 ret = -ENODEV;
236 goto err_free_blkg;
237 }
238
239
240 if (!css_tryget_online(&blkcg->css)) {
241 ret = -ENODEV;
242 goto err_free_blkg;
243 }
244
245 wb_congested = wb_congested_get_create(q->backing_dev_info,
246 blkcg->css.id,
247 GFP_NOWAIT | __GFP_NOWARN);
248 if (!wb_congested) {
249 ret = -ENOMEM;
250 goto err_put_css;
251 }
252
253
254 if (!new_blkg) {
255 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
256 if (unlikely(!new_blkg)) {
257 ret = -ENOMEM;
258 goto err_put_congested;
259 }
260 }
261 blkg = new_blkg;
262 blkg->wb_congested = wb_congested;
263
264
265 if (blkcg_parent(blkcg)) {
266 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
267 if (WARN_ON_ONCE(!blkg->parent)) {
268 ret = -ENODEV;
269 goto err_put_congested;
270 }
271 blkg_get(blkg->parent);
272 }
273
274
275 for (i = 0; i < BLKCG_MAX_POLS; i++) {
276 struct blkcg_policy *pol = blkcg_policy[i];
277
278 if (blkg->pd[i] && pol->pd_init_fn)
279 pol->pd_init_fn(blkg->pd[i]);
280 }
281
282
283 spin_lock(&blkcg->lock);
284 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
285 if (likely(!ret)) {
286 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
287 list_add(&blkg->q_node, &q->blkg_list);
288
289 for (i = 0; i < BLKCG_MAX_POLS; i++) {
290 struct blkcg_policy *pol = blkcg_policy[i];
291
292 if (blkg->pd[i] && pol->pd_online_fn)
293 pol->pd_online_fn(blkg->pd[i]);
294 }
295 }
296 blkg->online = true;
297 spin_unlock(&blkcg->lock);
298
299 if (!ret)
300 return blkg;
301
302
303 blkg_put(blkg);
304 return ERR_PTR(ret);
305
306err_put_congested:
307 wb_congested_put(wb_congested);
308err_put_css:
309 css_put(&blkcg->css);
310err_free_blkg:
311 blkg_free(new_blkg);
312 return ERR_PTR(ret);
313}
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
329 struct request_queue *q)
330{
331 struct blkcg_gq *blkg;
332
333 WARN_ON_ONCE(!rcu_read_lock_held());
334 lockdep_assert_held(&q->queue_lock);
335
336 blkg = __blkg_lookup(blkcg, q, true);
337 if (blkg)
338 return blkg;
339
340
341
342
343
344
345 while (true) {
346 struct blkcg *pos = blkcg;
347 struct blkcg *parent = blkcg_parent(blkcg);
348 struct blkcg_gq *ret_blkg = q->root_blkg;
349
350 while (parent) {
351 blkg = __blkg_lookup(parent, q, false);
352 if (blkg) {
353
354 ret_blkg = blkg;
355 break;
356 }
357 pos = parent;
358 parent = blkcg_parent(parent);
359 }
360
361 blkg = blkg_create(pos, q, NULL);
362 if (IS_ERR(blkg))
363 return ret_blkg;
364 if (pos == blkcg)
365 return blkg;
366 }
367}
368
369
370
371
372
373
374
375
376
377struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
378 struct request_queue *q)
379{
380 struct blkcg_gq *blkg = blkg_lookup(blkcg, q);
381
382 if (unlikely(!blkg)) {
383 unsigned long flags;
384
385 spin_lock_irqsave(&q->queue_lock, flags);
386 blkg = __blkg_lookup_create(blkcg, q);
387 spin_unlock_irqrestore(&q->queue_lock, flags);
388 }
389
390 return blkg;
391}
392
393static void blkg_destroy(struct blkcg_gq *blkg)
394{
395 struct blkcg *blkcg = blkg->blkcg;
396 struct blkcg_gq *parent = blkg->parent;
397 int i;
398
399 lockdep_assert_held(&blkg->q->queue_lock);
400 lockdep_assert_held(&blkcg->lock);
401
402
403 WARN_ON_ONCE(list_empty(&blkg->q_node));
404 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
405
406 for (i = 0; i < BLKCG_MAX_POLS; i++) {
407 struct blkcg_policy *pol = blkcg_policy[i];
408
409 if (blkg->pd[i] && pol->pd_offline_fn)
410 pol->pd_offline_fn(blkg->pd[i]);
411 }
412
413 if (parent) {
414 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
415 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
416 }
417
418 blkg->online = false;
419
420 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
421 list_del_init(&blkg->q_node);
422 hlist_del_init_rcu(&blkg->blkcg_node);
423
424
425
426
427
428
429 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
430 rcu_assign_pointer(blkcg->blkg_hint, NULL);
431
432
433
434
435
436 percpu_ref_kill(&blkg->refcnt);
437}
438
439
440
441
442
443
444
445static void blkg_destroy_all(struct request_queue *q)
446{
447 struct blkcg_gq *blkg, *n;
448
449 spin_lock_irq(&q->queue_lock);
450 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
451 struct blkcg *blkcg = blkg->blkcg;
452
453 spin_lock(&blkcg->lock);
454 blkg_destroy(blkg);
455 spin_unlock(&blkcg->lock);
456 }
457
458 q->root_blkg = NULL;
459 spin_unlock_irq(&q->queue_lock);
460}
461
462static int blkcg_reset_stats(struct cgroup_subsys_state *css,
463 struct cftype *cftype, u64 val)
464{
465 struct blkcg *blkcg = css_to_blkcg(css);
466 struct blkcg_gq *blkg;
467 int i;
468
469 mutex_lock(&blkcg_pol_mutex);
470 spin_lock_irq(&blkcg->lock);
471
472
473
474
475
476
477 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
478 blkg_rwstat_reset(&blkg->stat_bytes);
479 blkg_rwstat_reset(&blkg->stat_ios);
480
481 for (i = 0; i < BLKCG_MAX_POLS; i++) {
482 struct blkcg_policy *pol = blkcg_policy[i];
483
484 if (blkg->pd[i] && pol->pd_reset_stats_fn)
485 pol->pd_reset_stats_fn(blkg->pd[i]);
486 }
487 }
488
489 spin_unlock_irq(&blkcg->lock);
490 mutex_unlock(&blkcg_pol_mutex);
491 return 0;
492}
493
494const char *blkg_dev_name(struct blkcg_gq *blkg)
495{
496
497 if (blkg->q->backing_dev_info->dev)
498 return dev_name(blkg->q->backing_dev_info->dev);
499 return NULL;
500}
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
521 u64 (*prfill)(struct seq_file *,
522 struct blkg_policy_data *, int),
523 const struct blkcg_policy *pol, int data,
524 bool show_total)
525{
526 struct blkcg_gq *blkg;
527 u64 total = 0;
528
529 rcu_read_lock();
530 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
531 spin_lock_irq(&blkg->q->queue_lock);
532 if (blkcg_policy_enabled(blkg->q, pol))
533 total += prfill(sf, blkg->pd[pol->plid], data);
534 spin_unlock_irq(&blkg->q->queue_lock);
535 }
536 rcu_read_unlock();
537
538 if (show_total)
539 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
540}
541EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
542
543
544
545
546
547
548
549
550
551u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
552{
553 const char *dname = blkg_dev_name(pd->blkg);
554
555 if (!dname)
556 return 0;
557
558 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
559 return v;
560}
561EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
562
563
564
565
566
567
568
569
570
571u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
572 const struct blkg_rwstat_sample *rwstat)
573{
574 static const char *rwstr[] = {
575 [BLKG_RWSTAT_READ] = "Read",
576 [BLKG_RWSTAT_WRITE] = "Write",
577 [BLKG_RWSTAT_SYNC] = "Sync",
578 [BLKG_RWSTAT_ASYNC] = "Async",
579 [BLKG_RWSTAT_DISCARD] = "Discard",
580 };
581 const char *dname = blkg_dev_name(pd->blkg);
582 u64 v;
583 int i;
584
585 if (!dname)
586 return 0;
587
588 for (i = 0; i < BLKG_RWSTAT_NR; i++)
589 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
590 rwstat->cnt[i]);
591
592 v = rwstat->cnt[BLKG_RWSTAT_READ] +
593 rwstat->cnt[BLKG_RWSTAT_WRITE] +
594 rwstat->cnt[BLKG_RWSTAT_DISCARD];
595 seq_printf(sf, "%s Total %llu\n", dname, v);
596 return v;
597}
598EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
599
600
601
602
603
604
605
606
607
608u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
609 int off)
610{
611 struct blkg_rwstat_sample rwstat = { };
612
613 blkg_rwstat_read((void *)pd + off, &rwstat);
614 return __blkg_prfill_rwstat(sf, pd, &rwstat);
615}
616EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
617
618static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
619 struct blkg_policy_data *pd, int off)
620{
621 struct blkg_rwstat_sample rwstat = { };
622
623 blkg_rwstat_read((void *)pd->blkg + off, &rwstat);
624 return __blkg_prfill_rwstat(sf, pd, &rwstat);
625}
626
627
628
629
630
631
632
633
634
635int blkg_print_stat_bytes(struct seq_file *sf, void *v)
636{
637 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
638 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
639 offsetof(struct blkcg_gq, stat_bytes), true);
640 return 0;
641}
642EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);
643
644
645
646
647
648
649
650
651
652int blkg_print_stat_ios(struct seq_file *sf, void *v)
653{
654 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
655 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
656 offsetof(struct blkcg_gq, stat_ios), true);
657 return 0;
658}
659EXPORT_SYMBOL_GPL(blkg_print_stat_ios);
660
661static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
662 struct blkg_policy_data *pd,
663 int off)
664{
665 struct blkg_rwstat_sample rwstat;
666
667 blkg_rwstat_recursive_sum(pd->blkg, NULL, off, &rwstat);
668 return __blkg_prfill_rwstat(sf, pd, &rwstat);
669}
670
671
672
673
674
675
676int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
677{
678 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
679 blkg_prfill_rwstat_field_recursive,
680 (void *)seq_cft(sf)->private,
681 offsetof(struct blkcg_gq, stat_bytes), true);
682 return 0;
683}
684EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);
685
686
687
688
689
690
691int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
692{
693 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
694 blkg_prfill_rwstat_field_recursive,
695 (void *)seq_cft(sf)->private,
696 offsetof(struct blkcg_gq, stat_ios), true);
697 return 0;
698}
699EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
716 int off, struct blkg_rwstat_sample *sum)
717{
718 struct blkcg_gq *pos_blkg;
719 struct cgroup_subsys_state *pos_css;
720 unsigned int i;
721
722 lockdep_assert_held(&blkg->q->queue_lock);
723
724 rcu_read_lock();
725 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
726 struct blkg_rwstat *rwstat;
727
728 if (!pos_blkg->online)
729 continue;
730
731 if (pol)
732 rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
733 else
734 rwstat = (void *)pos_blkg + off;
735
736 for (i = 0; i < BLKG_RWSTAT_NR; i++)
737 sum->cnt[i] = blkg_rwstat_read_counter(rwstat, i);
738 }
739 rcu_read_unlock();
740}
741EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
742
743
744static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
745 const struct blkcg_policy *pol,
746 struct request_queue *q)
747{
748 WARN_ON_ONCE(!rcu_read_lock_held());
749 lockdep_assert_held(&q->queue_lock);
750
751 if (!blkcg_policy_enabled(q, pol))
752 return ERR_PTR(-EOPNOTSUPP);
753 return __blkg_lookup(blkcg, q, true );
754}
755
756
757
758
759
760
761
762
763
764
765
766
767
768int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
769 char *input, struct blkg_conf_ctx *ctx)
770 __acquires(rcu) __acquires(&disk->queue->queue_lock)
771{
772 struct gendisk *disk;
773 struct request_queue *q;
774 struct blkcg_gq *blkg;
775 unsigned int major, minor;
776 int key_len, part, ret;
777 char *body;
778
779 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
780 return -EINVAL;
781
782 body = input + key_len;
783 if (!isspace(*body))
784 return -EINVAL;
785 body = skip_spaces(body);
786
787 disk = get_gendisk(MKDEV(major, minor), &part);
788 if (!disk)
789 return -ENODEV;
790 if (part) {
791 ret = -ENODEV;
792 goto fail;
793 }
794
795 q = disk->queue;
796
797 rcu_read_lock();
798 spin_lock_irq(&q->queue_lock);
799
800 blkg = blkg_lookup_check(blkcg, pol, q);
801 if (IS_ERR(blkg)) {
802 ret = PTR_ERR(blkg);
803 goto fail_unlock;
804 }
805
806 if (blkg)
807 goto success;
808
809
810
811
812
813 while (true) {
814 struct blkcg *pos = blkcg;
815 struct blkcg *parent;
816 struct blkcg_gq *new_blkg;
817
818 parent = blkcg_parent(blkcg);
819 while (parent && !__blkg_lookup(parent, q, false)) {
820 pos = parent;
821 parent = blkcg_parent(parent);
822 }
823
824
825 spin_unlock_irq(&q->queue_lock);
826 rcu_read_unlock();
827
828 new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
829 if (unlikely(!new_blkg)) {
830 ret = -ENOMEM;
831 goto fail;
832 }
833
834 rcu_read_lock();
835 spin_lock_irq(&q->queue_lock);
836
837 blkg = blkg_lookup_check(pos, pol, q);
838 if (IS_ERR(blkg)) {
839 ret = PTR_ERR(blkg);
840 goto fail_unlock;
841 }
842
843 if (blkg) {
844 blkg_free(new_blkg);
845 } else {
846 blkg = blkg_create(pos, q, new_blkg);
847 if (IS_ERR(blkg)) {
848 ret = PTR_ERR(blkg);
849 goto fail_unlock;
850 }
851 }
852
853 if (pos == blkcg)
854 goto success;
855 }
856success:
857 ctx->disk = disk;
858 ctx->blkg = blkg;
859 ctx->body = body;
860 return 0;
861
862fail_unlock:
863 spin_unlock_irq(&q->queue_lock);
864 rcu_read_unlock();
865fail:
866 put_disk_and_module(disk);
867
868
869
870
871
872
873 if (ret == -EBUSY) {
874 msleep(10);
875 ret = restart_syscall();
876 }
877 return ret;
878}
879
880
881
882
883
884
885
886
887void blkg_conf_finish(struct blkg_conf_ctx *ctx)
888 __releases(&ctx->disk->queue->queue_lock) __releases(rcu)
889{
890 spin_unlock_irq(&ctx->disk->queue->queue_lock);
891 rcu_read_unlock();
892 put_disk_and_module(ctx->disk);
893}
894
895static int blkcg_print_stat(struct seq_file *sf, void *v)
896{
897 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
898 struct blkcg_gq *blkg;
899
900 rcu_read_lock();
901
902 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
903 const char *dname;
904 char *buf;
905 struct blkg_rwstat_sample rwstat;
906 u64 rbytes, wbytes, rios, wios, dbytes, dios;
907 size_t size = seq_get_buf(sf, &buf), off = 0;
908 int i;
909 bool has_stats = false;
910
911 dname = blkg_dev_name(blkg);
912 if (!dname)
913 continue;
914
915
916
917
918
919
920
921 off += scnprintf(buf+off, size-off, "%s ", dname);
922
923 spin_lock_irq(&blkg->q->queue_lock);
924
925 blkg_rwstat_recursive_sum(blkg, NULL,
926 offsetof(struct blkcg_gq, stat_bytes), &rwstat);
927 rbytes = rwstat.cnt[BLKG_RWSTAT_READ];
928 wbytes = rwstat.cnt[BLKG_RWSTAT_WRITE];
929 dbytes = rwstat.cnt[BLKG_RWSTAT_DISCARD];
930
931 blkg_rwstat_recursive_sum(blkg, NULL,
932 offsetof(struct blkcg_gq, stat_ios), &rwstat);
933 rios = rwstat.cnt[BLKG_RWSTAT_READ];
934 wios = rwstat.cnt[BLKG_RWSTAT_WRITE];
935 dios = rwstat.cnt[BLKG_RWSTAT_DISCARD];
936
937 spin_unlock_irq(&blkg->q->queue_lock);
938
939 if (rbytes || wbytes || rios || wios) {
940 has_stats = true;
941 off += scnprintf(buf+off, size-off,
942 "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
943 rbytes, wbytes, rios, wios,
944 dbytes, dios);
945 }
946
947 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
948 has_stats = true;
949 off += scnprintf(buf+off, size-off,
950 " use_delay=%d delay_nsec=%llu",
951 atomic_read(&blkg->use_delay),
952 (unsigned long long)atomic64_read(&blkg->delay_nsec));
953 }
954
955 for (i = 0; i < BLKCG_MAX_POLS; i++) {
956 struct blkcg_policy *pol = blkcg_policy[i];
957 size_t written;
958
959 if (!blkg->pd[i] || !pol->pd_stat_fn)
960 continue;
961
962 written = pol->pd_stat_fn(blkg->pd[i], buf+off, size-off);
963 if (written)
964 has_stats = true;
965 off += written;
966 }
967
968 if (has_stats) {
969 if (off < size - 1) {
970 off += scnprintf(buf+off, size-off, "\n");
971 seq_commit(sf, off);
972 } else {
973 seq_commit(sf, -1);
974 }
975 }
976 }
977
978 rcu_read_unlock();
979 return 0;
980}
981
982static struct cftype blkcg_files[] = {
983 {
984 .name = "stat",
985 .flags = CFTYPE_NOT_ON_ROOT,
986 .seq_show = blkcg_print_stat,
987 },
988 { }
989};
990
991static struct cftype blkcg_legacy_files[] = {
992 {
993 .name = "reset_stats",
994 .write_u64 = blkcg_reset_stats,
995 },
996 { }
997};
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028static void blkcg_css_offline(struct cgroup_subsys_state *css)
1029{
1030 struct blkcg *blkcg = css_to_blkcg(css);
1031
1032
1033 wb_blkcg_offline(blkcg);
1034
1035
1036 blkcg_cgwb_put(blkcg);
1037}
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050void blkcg_destroy_blkgs(struct blkcg *blkcg)
1051{
1052 spin_lock_irq(&blkcg->lock);
1053
1054 while (!hlist_empty(&blkcg->blkg_list)) {
1055 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1056 struct blkcg_gq, blkcg_node);
1057 struct request_queue *q = blkg->q;
1058
1059 if (spin_trylock(&q->queue_lock)) {
1060 blkg_destroy(blkg);
1061 spin_unlock(&q->queue_lock);
1062 } else {
1063 spin_unlock_irq(&blkcg->lock);
1064 cpu_relax();
1065 spin_lock_irq(&blkcg->lock);
1066 }
1067 }
1068
1069 spin_unlock_irq(&blkcg->lock);
1070}
1071
1072static void blkcg_css_free(struct cgroup_subsys_state *css)
1073{
1074 struct blkcg *blkcg = css_to_blkcg(css);
1075 int i;
1076
1077 mutex_lock(&blkcg_pol_mutex);
1078
1079 list_del(&blkcg->all_blkcgs_node);
1080
1081 for (i = 0; i < BLKCG_MAX_POLS; i++)
1082 if (blkcg->cpd[i])
1083 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1084
1085 mutex_unlock(&blkcg_pol_mutex);
1086
1087 kfree(blkcg);
1088}
1089
1090static struct cgroup_subsys_state *
1091blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1092{
1093 struct blkcg *blkcg;
1094 struct cgroup_subsys_state *ret;
1095 int i;
1096
1097 mutex_lock(&blkcg_pol_mutex);
1098
1099 if (!parent_css) {
1100 blkcg = &blkcg_root;
1101 } else {
1102 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1103 if (!blkcg) {
1104 ret = ERR_PTR(-ENOMEM);
1105 goto unlock;
1106 }
1107 }
1108
1109 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1110 struct blkcg_policy *pol = blkcg_policy[i];
1111 struct blkcg_policy_data *cpd;
1112
1113
1114
1115
1116
1117
1118
1119 if (!pol || !pol->cpd_alloc_fn)
1120 continue;
1121
1122 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1123 if (!cpd) {
1124 ret = ERR_PTR(-ENOMEM);
1125 goto free_pd_blkcg;
1126 }
1127 blkcg->cpd[i] = cpd;
1128 cpd->blkcg = blkcg;
1129 cpd->plid = i;
1130 if (pol->cpd_init_fn)
1131 pol->cpd_init_fn(cpd);
1132 }
1133
1134 spin_lock_init(&blkcg->lock);
1135 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
1136 INIT_HLIST_HEAD(&blkcg->blkg_list);
1137#ifdef CONFIG_CGROUP_WRITEBACK
1138 INIT_LIST_HEAD(&blkcg->cgwb_list);
1139 refcount_set(&blkcg->cgwb_refcnt, 1);
1140#endif
1141 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1142
1143 mutex_unlock(&blkcg_pol_mutex);
1144 return &blkcg->css;
1145
1146free_pd_blkcg:
1147 for (i--; i >= 0; i--)
1148 if (blkcg->cpd[i])
1149 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1150
1151 if (blkcg != &blkcg_root)
1152 kfree(blkcg);
1153unlock:
1154 mutex_unlock(&blkcg_pol_mutex);
1155 return ret;
1156}
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168int blkcg_init_queue(struct request_queue *q)
1169{
1170 struct blkcg_gq *new_blkg, *blkg;
1171 bool preloaded;
1172 int ret;
1173
1174 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1175 if (!new_blkg)
1176 return -ENOMEM;
1177
1178 preloaded = !radix_tree_preload(GFP_KERNEL);
1179
1180
1181 rcu_read_lock();
1182 spin_lock_irq(&q->queue_lock);
1183 blkg = blkg_create(&blkcg_root, q, new_blkg);
1184 if (IS_ERR(blkg))
1185 goto err_unlock;
1186 q->root_blkg = blkg;
1187 spin_unlock_irq(&q->queue_lock);
1188 rcu_read_unlock();
1189
1190 if (preloaded)
1191 radix_tree_preload_end();
1192
1193 ret = blk_iolatency_init(q);
1194 if (ret)
1195 goto err_destroy_all;
1196
1197 ret = blk_throtl_init(q);
1198 if (ret)
1199 goto err_destroy_all;
1200 return 0;
1201
1202err_destroy_all:
1203 blkg_destroy_all(q);
1204 return ret;
1205err_unlock:
1206 spin_unlock_irq(&q->queue_lock);
1207 rcu_read_unlock();
1208 if (preloaded)
1209 radix_tree_preload_end();
1210 return PTR_ERR(blkg);
1211}
1212
1213
1214
1215
1216
1217
1218
1219void blkcg_drain_queue(struct request_queue *q)
1220{
1221 lockdep_assert_held(&q->queue_lock);
1222
1223
1224
1225
1226
1227 if (!q->root_blkg)
1228 return;
1229
1230 blk_throtl_drain(q);
1231}
1232
1233
1234
1235
1236
1237
1238
1239void blkcg_exit_queue(struct request_queue *q)
1240{
1241 blkg_destroy_all(q);
1242 blk_throtl_exit(q);
1243}
1244
1245
1246
1247
1248
1249
1250
1251static int blkcg_can_attach(struct cgroup_taskset *tset)
1252{
1253 struct task_struct *task;
1254 struct cgroup_subsys_state *dst_css;
1255 struct io_context *ioc;
1256 int ret = 0;
1257
1258
1259 cgroup_taskset_for_each(task, dst_css, tset) {
1260 task_lock(task);
1261 ioc = task->io_context;
1262 if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1263 ret = -EINVAL;
1264 task_unlock(task);
1265 if (ret)
1266 break;
1267 }
1268 return ret;
1269}
1270
1271static void blkcg_bind(struct cgroup_subsys_state *root_css)
1272{
1273 int i;
1274
1275 mutex_lock(&blkcg_pol_mutex);
1276
1277 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1278 struct blkcg_policy *pol = blkcg_policy[i];
1279 struct blkcg *blkcg;
1280
1281 if (!pol || !pol->cpd_bind_fn)
1282 continue;
1283
1284 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1285 if (blkcg->cpd[pol->plid])
1286 pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1287 }
1288 mutex_unlock(&blkcg_pol_mutex);
1289}
1290
1291static void blkcg_exit(struct task_struct *tsk)
1292{
1293 if (tsk->throttle_queue)
1294 blk_put_queue(tsk->throttle_queue);
1295 tsk->throttle_queue = NULL;
1296}
1297
1298struct cgroup_subsys io_cgrp_subsys = {
1299 .css_alloc = blkcg_css_alloc,
1300 .css_offline = blkcg_css_offline,
1301 .css_free = blkcg_css_free,
1302 .can_attach = blkcg_can_attach,
1303 .bind = blkcg_bind,
1304 .dfl_cftypes = blkcg_files,
1305 .legacy_cftypes = blkcg_legacy_files,
1306 .legacy_name = "blkio",
1307 .exit = blkcg_exit,
1308#ifdef CONFIG_MEMCG
1309
1310
1311
1312
1313
1314 .depends_on = 1 << memory_cgrp_id,
1315#endif
1316};
1317EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335int blkcg_activate_policy(struct request_queue *q,
1336 const struct blkcg_policy *pol)
1337{
1338 struct blkg_policy_data *pd_prealloc = NULL;
1339 struct blkcg_gq *blkg;
1340 int ret;
1341
1342 if (blkcg_policy_enabled(q, pol))
1343 return 0;
1344
1345 if (queue_is_mq(q))
1346 blk_mq_freeze_queue(q);
1347pd_prealloc:
1348 if (!pd_prealloc) {
1349 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
1350 if (!pd_prealloc) {
1351 ret = -ENOMEM;
1352 goto out_bypass_end;
1353 }
1354 }
1355
1356 spin_lock_irq(&q->queue_lock);
1357
1358
1359 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
1360 struct blkg_policy_data *pd;
1361
1362 if (blkg->pd[pol->plid])
1363 continue;
1364
1365 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q->node);
1366 if (!pd)
1367 swap(pd, pd_prealloc);
1368 if (!pd) {
1369 spin_unlock_irq(&q->queue_lock);
1370 goto pd_prealloc;
1371 }
1372
1373 blkg->pd[pol->plid] = pd;
1374 pd->blkg = blkg;
1375 pd->plid = pol->plid;
1376 if (pol->pd_init_fn)
1377 pol->pd_init_fn(pd);
1378 }
1379
1380 __set_bit(pol->plid, q->blkcg_pols);
1381 ret = 0;
1382
1383 spin_unlock_irq(&q->queue_lock);
1384out_bypass_end:
1385 if (queue_is_mq(q))
1386 blk_mq_unfreeze_queue(q);
1387 if (pd_prealloc)
1388 pol->pd_free_fn(pd_prealloc);
1389 return ret;
1390}
1391EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401void blkcg_deactivate_policy(struct request_queue *q,
1402 const struct blkcg_policy *pol)
1403{
1404 struct blkcg_gq *blkg;
1405
1406 if (!blkcg_policy_enabled(q, pol))
1407 return;
1408
1409 if (queue_is_mq(q))
1410 blk_mq_freeze_queue(q);
1411
1412 spin_lock_irq(&q->queue_lock);
1413
1414 __clear_bit(pol->plid, q->blkcg_pols);
1415
1416 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1417 if (blkg->pd[pol->plid]) {
1418 if (pol->pd_offline_fn)
1419 pol->pd_offline_fn(blkg->pd[pol->plid]);
1420 pol->pd_free_fn(blkg->pd[pol->plid]);
1421 blkg->pd[pol->plid] = NULL;
1422 }
1423 }
1424
1425 spin_unlock_irq(&q->queue_lock);
1426
1427 if (queue_is_mq(q))
1428 blk_mq_unfreeze_queue(q);
1429}
1430EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1431
1432
1433
1434
1435
1436
1437
1438
1439int blkcg_policy_register(struct blkcg_policy *pol)
1440{
1441 struct blkcg *blkcg;
1442 int i, ret;
1443
1444 mutex_lock(&blkcg_pol_register_mutex);
1445 mutex_lock(&blkcg_pol_mutex);
1446
1447
1448 ret = -ENOSPC;
1449 for (i = 0; i < BLKCG_MAX_POLS; i++)
1450 if (!blkcg_policy[i])
1451 break;
1452 if (i >= BLKCG_MAX_POLS) {
1453 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
1454 goto err_unlock;
1455 }
1456
1457
1458 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1459 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1460 goto err_unlock;
1461
1462
1463 pol->plid = i;
1464 blkcg_policy[pol->plid] = pol;
1465
1466
1467 if (pol->cpd_alloc_fn) {
1468 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1469 struct blkcg_policy_data *cpd;
1470
1471 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1472 if (!cpd)
1473 goto err_free_cpds;
1474
1475 blkcg->cpd[pol->plid] = cpd;
1476 cpd->blkcg = blkcg;
1477 cpd->plid = pol->plid;
1478 pol->cpd_init_fn(cpd);
1479 }
1480 }
1481
1482 mutex_unlock(&blkcg_pol_mutex);
1483
1484
1485 if (pol->dfl_cftypes)
1486 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1487 pol->dfl_cftypes));
1488 if (pol->legacy_cftypes)
1489 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1490 pol->legacy_cftypes));
1491 mutex_unlock(&blkcg_pol_register_mutex);
1492 return 0;
1493
1494err_free_cpds:
1495 if (pol->cpd_free_fn) {
1496 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1497 if (blkcg->cpd[pol->plid]) {
1498 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1499 blkcg->cpd[pol->plid] = NULL;
1500 }
1501 }
1502 }
1503 blkcg_policy[pol->plid] = NULL;
1504err_unlock:
1505 mutex_unlock(&blkcg_pol_mutex);
1506 mutex_unlock(&blkcg_pol_register_mutex);
1507 return ret;
1508}
1509EXPORT_SYMBOL_GPL(blkcg_policy_register);
1510
1511
1512
1513
1514
1515
1516
1517void blkcg_policy_unregister(struct blkcg_policy *pol)
1518{
1519 struct blkcg *blkcg;
1520
1521 mutex_lock(&blkcg_pol_register_mutex);
1522
1523 if (WARN_ON(blkcg_policy[pol->plid] != pol))
1524 goto out_unlock;
1525
1526
1527 if (pol->dfl_cftypes)
1528 cgroup_rm_cftypes(pol->dfl_cftypes);
1529 if (pol->legacy_cftypes)
1530 cgroup_rm_cftypes(pol->legacy_cftypes);
1531
1532
1533 mutex_lock(&blkcg_pol_mutex);
1534
1535 if (pol->cpd_free_fn) {
1536 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1537 if (blkcg->cpd[pol->plid]) {
1538 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1539 blkcg->cpd[pol->plid] = NULL;
1540 }
1541 }
1542 }
1543 blkcg_policy[pol->plid] = NULL;
1544
1545 mutex_unlock(&blkcg_pol_mutex);
1546out_unlock:
1547 mutex_unlock(&blkcg_pol_register_mutex);
1548}
1549EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
1550
1551bool __blkcg_punt_bio_submit(struct bio *bio)
1552{
1553 struct blkcg_gq *blkg = bio->bi_blkg;
1554
1555
1556 bio->bi_opf &= ~REQ_CGROUP_PUNT;
1557
1558
1559 if (!blkg->parent)
1560 return false;
1561
1562 spin_lock_bh(&blkg->async_bio_lock);
1563 bio_list_add(&blkg->async_bios, bio);
1564 spin_unlock_bh(&blkg->async_bio_lock);
1565
1566 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
1567 return true;
1568}
1569
1570
1571
1572
1573
1574
1575
1576static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
1577{
1578 u64 old = atomic64_read(&blkg->delay_start);
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593 if (time_before64(old + NSEC_PER_SEC, now) &&
1594 atomic64_cmpxchg(&blkg->delay_start, old, now) == old) {
1595 u64 cur = atomic64_read(&blkg->delay_nsec);
1596 u64 sub = min_t(u64, blkg->last_delay, now - old);
1597 int cur_use = atomic_read(&blkg->use_delay);
1598
1599
1600
1601
1602
1603 if (cur_use < blkg->last_use)
1604 sub = max_t(u64, sub, blkg->last_delay >> 1);
1605
1606
1607
1608
1609
1610
1611
1612 if (unlikely(cur < sub)) {
1613 atomic64_set(&blkg->delay_nsec, 0);
1614 blkg->last_delay = 0;
1615 } else {
1616 atomic64_sub(sub, &blkg->delay_nsec);
1617 blkg->last_delay = cur - sub;
1618 }
1619 blkg->last_use = cur_use;
1620 }
1621}
1622
1623
1624
1625
1626
1627
1628
1629static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
1630{
1631 unsigned long pflags;
1632 u64 now = ktime_to_ns(ktime_get());
1633 u64 exp;
1634 u64 delay_nsec = 0;
1635 int tok;
1636
1637 while (blkg->parent) {
1638 if (atomic_read(&blkg->use_delay)) {
1639 blkcg_scale_delay(blkg, now);
1640 delay_nsec = max_t(u64, delay_nsec,
1641 atomic64_read(&blkg->delay_nsec));
1642 }
1643 blkg = blkg->parent;
1644 }
1645
1646 if (!delay_nsec)
1647 return;
1648
1649
1650
1651
1652
1653
1654
1655
1656 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
1657
1658 if (use_memdelay)
1659 psi_memstall_enter(&pflags);
1660
1661 exp = ktime_add_ns(now, delay_nsec);
1662 tok = io_schedule_prepare();
1663 do {
1664 __set_current_state(TASK_KILLABLE);
1665 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
1666 break;
1667 } while (!fatal_signal_pending(current));
1668 io_schedule_finish(tok);
1669
1670 if (use_memdelay)
1671 psi_memstall_leave(&pflags);
1672}
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684void blkcg_maybe_throttle_current(void)
1685{
1686 struct request_queue *q = current->throttle_queue;
1687 struct cgroup_subsys_state *css;
1688 struct blkcg *blkcg;
1689 struct blkcg_gq *blkg;
1690 bool use_memdelay = current->use_memdelay;
1691
1692 if (!q)
1693 return;
1694
1695 current->throttle_queue = NULL;
1696 current->use_memdelay = false;
1697
1698 rcu_read_lock();
1699 css = kthread_blkcg();
1700 if (css)
1701 blkcg = css_to_blkcg(css);
1702 else
1703 blkcg = css_to_blkcg(task_css(current, io_cgrp_id));
1704
1705 if (!blkcg)
1706 goto out;
1707 blkg = blkg_lookup(blkcg, q);
1708 if (!blkg)
1709 goto out;
1710 if (!blkg_tryget(blkg))
1711 goto out;
1712 rcu_read_unlock();
1713
1714 blkcg_maybe_throttle_blkg(blkg, use_memdelay);
1715 blkg_put(blkg);
1716 blk_put_queue(q);
1717 return;
1718out:
1719 rcu_read_unlock();
1720 blk_put_queue(q);
1721}
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay)
1741{
1742 if (unlikely(current->flags & PF_KTHREAD))
1743 return;
1744
1745 if (!blk_get_queue(q))
1746 return;
1747
1748 if (current->throttle_queue)
1749 blk_put_queue(current->throttle_queue);
1750 current->throttle_queue = q;
1751 if (use_memdelay)
1752 current->use_memdelay = use_memdelay;
1753 set_notify_resume(current);
1754}
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
1766{
1767 blkcg_scale_delay(blkg, now);
1768 atomic64_add(delta, &blkg->delay_nsec);
1769}
1770
1771static int __init blkcg_init(void)
1772{
1773 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
1774 WQ_MEM_RECLAIM | WQ_FREEZABLE |
1775 WQ_UNBOUND | WQ_SYSFS, 0);
1776 if (!blkcg_punt_bio_wq)
1777 return -ENOMEM;
1778 return 0;
1779}
1780subsys_initcall(blkcg_init);
1781
1782module_param(blkcg_debug_stats, bool, 0644);
1783MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");
1784