1
2#ifndef _BLK_CGROUP_H
3#define _BLK_CGROUP_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/cgroup.h>
18#include <linux/percpu_counter.h>
19#include <linux/seq_file.h>
20#include <linux/radix-tree.h>
21#include <linux/blkdev.h>
22#include <linux/atomic.h>
23#include <linux/kthread.h>
24
25
26#define BLKG_STAT_CPU_BATCH (INT_MAX / 2)
27
28
29#define THROTL_IOPS_MAX UINT_MAX
30
31#ifdef CONFIG_BLK_CGROUP
32
33enum blkg_rwstat_type {
34 BLKG_RWSTAT_READ,
35 BLKG_RWSTAT_WRITE,
36 BLKG_RWSTAT_SYNC,
37 BLKG_RWSTAT_ASYNC,
38
39 BLKG_RWSTAT_NR,
40 BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
41};
42
43struct blkcg_gq;
44
45struct blkcg {
46 struct cgroup_subsys_state css;
47 spinlock_t lock;
48
49 struct radix_tree_root blkg_tree;
50 struct blkcg_gq __rcu *blkg_hint;
51 struct hlist_head blkg_list;
52
53 struct blkcg_policy_data *cpd[BLKCG_MAX_POLS];
54
55 struct list_head all_blkcgs_node;
56#ifdef CONFIG_CGROUP_WRITEBACK
57 struct list_head cgwb_list;
58#endif
59};
60
61
62
63
64
65
66struct blkg_stat {
67 struct percpu_counter cpu_cnt;
68 atomic64_t aux_cnt;
69};
70
71struct blkg_rwstat {
72 struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
73 atomic64_t aux_cnt[BLKG_RWSTAT_NR];
74};
75
76
77
78
79
80
81
82
83
84
85
86
87struct blkg_policy_data {
88
89 struct blkcg_gq *blkg;
90 int plid;
91};
92
93
94
95
96
97
98
99
100struct blkcg_policy_data {
101
102 struct blkcg *blkcg;
103 int plid;
104};
105
106
107struct blkcg_gq {
108
109 struct request_queue *q;
110 struct list_head q_node;
111 struct hlist_node blkcg_node;
112 struct blkcg *blkcg;
113
114
115
116
117
118 struct bdi_writeback_congested *wb_congested;
119
120
121 struct blkcg_gq *parent;
122
123
124 struct request_list rl;
125
126
127 atomic_t refcnt;
128
129
130 bool online;
131
132 struct blkg_rwstat stat_bytes;
133 struct blkg_rwstat stat_ios;
134
135 struct blkg_policy_data *pd[BLKCG_MAX_POLS];
136
137 struct rcu_head rcu_head;
138};
139
140typedef struct blkcg_policy_data *(blkcg_pol_alloc_cpd_fn)(gfp_t gfp);
141typedef void (blkcg_pol_init_cpd_fn)(struct blkcg_policy_data *cpd);
142typedef void (blkcg_pol_free_cpd_fn)(struct blkcg_policy_data *cpd);
143typedef void (blkcg_pol_bind_cpd_fn)(struct blkcg_policy_data *cpd);
144typedef struct blkg_policy_data *(blkcg_pol_alloc_pd_fn)(gfp_t gfp, int node);
145typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
146typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
147typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
148typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
149typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
150
151struct blkcg_policy {
152 int plid;
153
154 struct cftype *dfl_cftypes;
155 struct cftype *legacy_cftypes;
156
157
158 blkcg_pol_alloc_cpd_fn *cpd_alloc_fn;
159 blkcg_pol_init_cpd_fn *cpd_init_fn;
160 blkcg_pol_free_cpd_fn *cpd_free_fn;
161 blkcg_pol_bind_cpd_fn *cpd_bind_fn;
162
163 blkcg_pol_alloc_pd_fn *pd_alloc_fn;
164 blkcg_pol_init_pd_fn *pd_init_fn;
165 blkcg_pol_online_pd_fn *pd_online_fn;
166 blkcg_pol_offline_pd_fn *pd_offline_fn;
167 blkcg_pol_free_pd_fn *pd_free_fn;
168 blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn;
169};
170
171extern struct blkcg blkcg_root;
172extern struct cgroup_subsys_state * const blkcg_root_css;
173
174struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
175 struct request_queue *q, bool update_hint);
176struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
177 struct request_queue *q);
178int blkcg_init_queue(struct request_queue *q);
179void blkcg_drain_queue(struct request_queue *q);
180void blkcg_exit_queue(struct request_queue *q);
181
182
183int blkcg_policy_register(struct blkcg_policy *pol);
184void blkcg_policy_unregister(struct blkcg_policy *pol);
185int blkcg_activate_policy(struct request_queue *q,
186 const struct blkcg_policy *pol);
187void blkcg_deactivate_policy(struct request_queue *q,
188 const struct blkcg_policy *pol);
189
190const char *blkg_dev_name(struct blkcg_gq *blkg);
191void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
192 u64 (*prfill)(struct seq_file *,
193 struct blkg_policy_data *, int),
194 const struct blkcg_policy *pol, int data,
195 bool show_total);
196u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
197u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
198 const struct blkg_rwstat *rwstat);
199u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
200u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
201 int off);
202int blkg_print_stat_bytes(struct seq_file *sf, void *v);
203int blkg_print_stat_ios(struct seq_file *sf, void *v);
204int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v);
205int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v);
206
207u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
208 struct blkcg_policy *pol, int off);
209struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
210 struct blkcg_policy *pol, int off);
211
212struct blkg_conf_ctx {
213 struct gendisk *disk;
214 struct blkcg_gq *blkg;
215 char *body;
216};
217
218int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
219 char *input, struct blkg_conf_ctx *ctx);
220void blkg_conf_finish(struct blkg_conf_ctx *ctx);
221
222
223static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css)
224{
225 return css ? container_of(css, struct blkcg, css) : NULL;
226}
227
228static inline struct blkcg *bio_blkcg(struct bio *bio)
229{
230 struct cgroup_subsys_state *css;
231
232 if (bio && bio->bi_css)
233 return css_to_blkcg(bio->bi_css);
234 css = kthread_blkcg();
235 if (css)
236 return css_to_blkcg(css);
237 return css_to_blkcg(task_css(current, io_cgrp_id));
238}
239
240
241
242
243
244
245
246static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
247{
248 return css_to_blkcg(blkcg->css.parent);
249}
250
251
252
253
254
255
256
257
258
259
260
261
262static inline struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
263 struct request_queue *q,
264 bool update_hint)
265{
266 struct blkcg_gq *blkg;
267
268 if (blkcg == &blkcg_root)
269 return q->root_blkg;
270
271 blkg = rcu_dereference(blkcg->blkg_hint);
272 if (blkg && blkg->q == q)
273 return blkg;
274
275 return blkg_lookup_slowpath(blkcg, q, update_hint);
276}
277
278
279
280
281
282
283
284
285
286
287static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg,
288 struct request_queue *q)
289{
290 WARN_ON_ONCE(!rcu_read_lock_held());
291
292 if (unlikely(blk_queue_bypass(q)))
293 return NULL;
294 return __blkg_lookup(blkcg, q, false);
295}
296
297
298
299
300
301
302
303
304static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
305 struct blkcg_policy *pol)
306{
307 return blkg ? blkg->pd[pol->plid] : NULL;
308}
309
310static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
311 struct blkcg_policy *pol)
312{
313 return blkcg ? blkcg->cpd[pol->plid] : NULL;
314}
315
316
317
318
319
320
321
322static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
323{
324 return pd ? pd->blkg : NULL;
325}
326
327static inline struct blkcg *cpd_to_blkcg(struct blkcg_policy_data *cpd)
328{
329 return cpd ? cpd->blkcg : NULL;
330}
331
332
333
334
335
336
337
338
339
340static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
341{
342 return cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
343}
344
345
346
347
348
349
350
351static inline void blkg_get(struct blkcg_gq *blkg)
352{
353 WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
354 atomic_inc(&blkg->refcnt);
355}
356
357void __blkg_release_rcu(struct rcu_head *rcu);
358
359
360
361
362
363static inline void blkg_put(struct blkcg_gq *blkg)
364{
365 WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
366 if (atomic_dec_and_test(&blkg->refcnt))
367 call_rcu(&blkg->rcu_head, __blkg_release_rcu);
368}
369
370
371
372
373
374
375
376
377
378
379
380
381
382#define blkg_for_each_descendant_pre(d_blkg, pos_css, p_blkg) \
383 css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css) \
384 if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \
385 (p_blkg)->q, false)))
386
387
388
389
390
391
392
393
394
395
396
397#define blkg_for_each_descendant_post(d_blkg, pos_css, p_blkg) \
398 css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css) \
399 if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \
400 (p_blkg)->q, false)))
401
402
403
404
405
406
407
408
409
410
411
412static inline struct request_list *blk_get_rl(struct request_queue *q,
413 struct bio *bio)
414{
415 struct blkcg *blkcg;
416 struct blkcg_gq *blkg;
417
418 rcu_read_lock();
419
420 blkcg = bio_blkcg(bio);
421
422
423 if (blkcg == &blkcg_root)
424 goto root_rl;
425
426
427
428
429
430
431 blkg = blkg_lookup(blkcg, q);
432 if (unlikely(!blkg))
433 goto root_rl;
434
435 blkg_get(blkg);
436 rcu_read_unlock();
437 return &blkg->rl;
438root_rl:
439 rcu_read_unlock();
440 return &q->root_rl;
441}
442
443
444
445
446
447
448
449
450static inline void blk_put_rl(struct request_list *rl)
451{
452 if (rl->blkg->blkcg != &blkcg_root)
453 blkg_put(rl->blkg);
454}
455
456
457
458
459
460
461
462
463
464static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl)
465{
466 rq->rl = rl;
467}
468
469
470
471
472
473
474
475static inline struct request_list *blk_rq_rl(struct request *rq)
476{
477 return rq->rl;
478}
479
480struct request_list *__blk_queue_next_rl(struct request_list *rl,
481 struct request_queue *q);
482
483
484
485
486
487#define blk_queue_for_each_rl(rl, q) \
488 for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q)))
489
490static inline int blkg_stat_init(struct blkg_stat *stat, gfp_t gfp)
491{
492 int ret;
493
494 ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
495 if (ret)
496 return ret;
497
498 atomic64_set(&stat->aux_cnt, 0);
499 return 0;
500}
501
502static inline void blkg_stat_exit(struct blkg_stat *stat)
503{
504 percpu_counter_destroy(&stat->cpu_cnt);
505}
506
507
508
509
510
511
512
513
514
515static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
516{
517 percpu_counter_add_batch(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
518}
519
520
521
522
523
524static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
525{
526 return percpu_counter_sum_positive(&stat->cpu_cnt);
527}
528
529
530
531
532
533static inline void blkg_stat_reset(struct blkg_stat *stat)
534{
535 percpu_counter_set(&stat->cpu_cnt, 0);
536 atomic64_set(&stat->aux_cnt, 0);
537}
538
539
540
541
542
543
544
545
546static inline void blkg_stat_add_aux(struct blkg_stat *to,
547 struct blkg_stat *from)
548{
549 atomic64_add(blkg_stat_read(from) + atomic64_read(&from->aux_cnt),
550 &to->aux_cnt);
551}
552
553static inline int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp)
554{
555 int i, ret;
556
557 for (i = 0; i < BLKG_RWSTAT_NR; i++) {
558 ret = percpu_counter_init(&rwstat->cpu_cnt[i], 0, gfp);
559 if (ret) {
560 while (--i >= 0)
561 percpu_counter_destroy(&rwstat->cpu_cnt[i]);
562 return ret;
563 }
564 atomic64_set(&rwstat->aux_cnt[i], 0);
565 }
566 return 0;
567}
568
569static inline void blkg_rwstat_exit(struct blkg_rwstat *rwstat)
570{
571 int i;
572
573 for (i = 0; i < BLKG_RWSTAT_NR; i++)
574 percpu_counter_destroy(&rwstat->cpu_cnt[i]);
575}
576
577
578
579
580
581
582
583
584
585
586static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
587 unsigned int op, uint64_t val)
588{
589 struct percpu_counter *cnt;
590
591 if (op_is_write(op))
592 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
593 else
594 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
595
596 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
597
598 if (op_is_sync(op))
599 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
600 else
601 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
602
603 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
604}
605
606
607
608
609
610
611
612static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
613{
614 struct blkg_rwstat result;
615 int i;
616
617 for (i = 0; i < BLKG_RWSTAT_NR; i++)
618 atomic64_set(&result.aux_cnt[i],
619 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]));
620 return result;
621}
622
623
624
625
626
627
628
629
630
631static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
632{
633 struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
634
635 return atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) +
636 atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]);
637}
638
639
640
641
642
643static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
644{
645 int i;
646
647 for (i = 0; i < BLKG_RWSTAT_NR; i++) {
648 percpu_counter_set(&rwstat->cpu_cnt[i], 0);
649 atomic64_set(&rwstat->aux_cnt[i], 0);
650 }
651}
652
653
654
655
656
657
658
659
660static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
661 struct blkg_rwstat *from)
662{
663 struct blkg_rwstat v = blkg_rwstat_read(from);
664 int i;
665
666 for (i = 0; i < BLKG_RWSTAT_NR; i++)
667 atomic64_add(atomic64_read(&v.aux_cnt[i]) +
668 atomic64_read(&from->aux_cnt[i]),
669 &to->aux_cnt[i]);
670}
671
672#ifdef CONFIG_BLK_DEV_THROTTLING
673extern bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
674 struct bio *bio);
675#else
676static inline bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
677 struct bio *bio) { return false; }
678#endif
679
680static inline bool blkcg_bio_issue_check(struct request_queue *q,
681 struct bio *bio)
682{
683 struct blkcg *blkcg;
684 struct blkcg_gq *blkg;
685 bool throtl = false;
686
687 rcu_read_lock();
688 blkcg = bio_blkcg(bio);
689
690
691 bio_associate_blkcg(bio, &blkcg->css);
692
693 blkg = blkg_lookup(blkcg, q);
694 if (unlikely(!blkg)) {
695 spin_lock_irq(q->queue_lock);
696 blkg = blkg_lookup_create(blkcg, q);
697 if (IS_ERR(blkg))
698 blkg = NULL;
699 spin_unlock_irq(q->queue_lock);
700 }
701
702 throtl = blk_throtl_bio(q, blkg, bio);
703
704 if (!throtl) {
705 blkg = blkg ?: q->root_blkg;
706 blkg_rwstat_add(&blkg->stat_bytes, bio->bi_opf,
707 bio->bi_iter.bi_size);
708 blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1);
709 }
710
711 rcu_read_unlock();
712 return !throtl;
713}
714
715#else
716
717struct blkcg {
718};
719
720struct blkg_policy_data {
721};
722
723struct blkcg_policy_data {
724};
725
726struct blkcg_gq {
727};
728
729struct blkcg_policy {
730};
731
732#define blkcg_root_css ((struct cgroup_subsys_state *)ERR_PTR(-EINVAL))
733
734#ifdef CONFIG_BLOCK
735
736static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
737static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
738static inline void blkcg_drain_queue(struct request_queue *q) { }
739static inline void blkcg_exit_queue(struct request_queue *q) { }
740static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
741static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
742static inline int blkcg_activate_policy(struct request_queue *q,
743 const struct blkcg_policy *pol) { return 0; }
744static inline void blkcg_deactivate_policy(struct request_queue *q,
745 const struct blkcg_policy *pol) { }
746
747static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
748
749static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
750 struct blkcg_policy *pol) { return NULL; }
751static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
752static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
753static inline void blkg_get(struct blkcg_gq *blkg) { }
754static inline void blkg_put(struct blkcg_gq *blkg) { }
755
756static inline struct request_list *blk_get_rl(struct request_queue *q,
757 struct bio *bio) { return &q->root_rl; }
758static inline void blk_put_rl(struct request_list *rl) { }
759static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { }
760static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; }
761
762static inline bool blkcg_bio_issue_check(struct request_queue *q,
763 struct bio *bio) { return true; }
764
765#define blk_queue_for_each_rl(rl, q) \
766 for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
767
768#endif
769#endif
770#endif
771