1#ifndef _BLK_CGROUP_H
2#define _BLK_CGROUP_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/cgroup.h>
17#include <linux/percpu_counter.h>
18#include <linux/seq_file.h>
19#include <linux/radix-tree.h>
20#include <linux/blkdev.h>
21#include <linux/atomic.h>
22
23
24#define BLKG_STAT_CPU_BATCH (INT_MAX / 2)
25
26
27#define THROTL_IOPS_MAX UINT_MAX
28
29#ifdef CONFIG_BLK_CGROUP
30
31enum blkg_rwstat_type {
32 BLKG_RWSTAT_READ,
33 BLKG_RWSTAT_WRITE,
34 BLKG_RWSTAT_SYNC,
35 BLKG_RWSTAT_ASYNC,
36
37 BLKG_RWSTAT_NR,
38 BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
39};
40
41struct blkcg_gq;
42
43struct blkcg {
44 struct cgroup_subsys_state css;
45 spinlock_t lock;
46
47 struct radix_tree_root blkg_tree;
48 struct blkcg_gq *blkg_hint;
49 struct hlist_head blkg_list;
50
51 struct blkcg_policy_data *cpd[BLKCG_MAX_POLS];
52
53 struct list_head all_blkcgs_node;
54#ifdef CONFIG_CGROUP_WRITEBACK
55 struct list_head cgwb_list;
56#endif
57};
58
59
60
61
62
63
64struct blkg_stat {
65 struct percpu_counter cpu_cnt;
66 atomic64_t aux_cnt;
67};
68
69struct blkg_rwstat {
70 struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
71 atomic64_t aux_cnt[BLKG_RWSTAT_NR];
72};
73
74
75
76
77
78
79
80
81
82
83
84
85struct blkg_policy_data {
86
87 struct blkcg_gq *blkg;
88 int plid;
89};
90
91
92
93
94
95
96
97
98struct blkcg_policy_data {
99
100 struct blkcg *blkcg;
101 int plid;
102};
103
104
105struct blkcg_gq {
106
107 struct request_queue *q;
108 struct list_head q_node;
109 struct hlist_node blkcg_node;
110 struct blkcg *blkcg;
111
112
113
114
115
116 struct bdi_writeback_congested *wb_congested;
117
118
119 struct blkcg_gq *parent;
120
121
122 struct request_list rl;
123
124
125 atomic_t refcnt;
126
127
128 bool online;
129
130 struct blkg_rwstat stat_bytes;
131 struct blkg_rwstat stat_ios;
132
133 struct blkg_policy_data *pd[BLKCG_MAX_POLS];
134
135 struct rcu_head rcu_head;
136};
137
138typedef struct blkcg_policy_data *(blkcg_pol_alloc_cpd_fn)(gfp_t gfp);
139typedef void (blkcg_pol_init_cpd_fn)(struct blkcg_policy_data *cpd);
140typedef void (blkcg_pol_free_cpd_fn)(struct blkcg_policy_data *cpd);
141typedef void (blkcg_pol_bind_cpd_fn)(struct blkcg_policy_data *cpd);
142typedef struct blkg_policy_data *(blkcg_pol_alloc_pd_fn)(gfp_t gfp, int node);
143typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
144typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
145typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
146typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
147typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
148
149struct blkcg_policy {
150 int plid;
151
152 struct cftype *dfl_cftypes;
153 struct cftype *legacy_cftypes;
154
155
156 blkcg_pol_alloc_cpd_fn *cpd_alloc_fn;
157 blkcg_pol_init_cpd_fn *cpd_init_fn;
158 blkcg_pol_free_cpd_fn *cpd_free_fn;
159 blkcg_pol_bind_cpd_fn *cpd_bind_fn;
160
161 blkcg_pol_alloc_pd_fn *pd_alloc_fn;
162 blkcg_pol_init_pd_fn *pd_init_fn;
163 blkcg_pol_online_pd_fn *pd_online_fn;
164 blkcg_pol_offline_pd_fn *pd_offline_fn;
165 blkcg_pol_free_pd_fn *pd_free_fn;
166 blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn;
167};
168
169extern struct blkcg blkcg_root;
170extern struct cgroup_subsys_state * const blkcg_root_css;
171
172struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
173 struct request_queue *q, bool update_hint);
174struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
175 struct request_queue *q);
176int blkcg_init_queue(struct request_queue *q);
177void blkcg_drain_queue(struct request_queue *q);
178void blkcg_exit_queue(struct request_queue *q);
179
180
181int blkcg_policy_register(struct blkcg_policy *pol);
182void blkcg_policy_unregister(struct blkcg_policy *pol);
183int blkcg_activate_policy(struct request_queue *q,
184 const struct blkcg_policy *pol);
185void blkcg_deactivate_policy(struct request_queue *q,
186 const struct blkcg_policy *pol);
187
188const char *blkg_dev_name(struct blkcg_gq *blkg);
189void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
190 u64 (*prfill)(struct seq_file *,
191 struct blkg_policy_data *, int),
192 const struct blkcg_policy *pol, int data,
193 bool show_total);
194u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
195u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
196 const struct blkg_rwstat *rwstat);
197u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
198u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
199 int off);
200int blkg_print_stat_bytes(struct seq_file *sf, void *v);
201int blkg_print_stat_ios(struct seq_file *sf, void *v);
202int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v);
203int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v);
204
205u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
206 struct blkcg_policy *pol, int off);
207struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
208 struct blkcg_policy *pol, int off);
209
210struct blkg_conf_ctx {
211 struct gendisk *disk;
212 struct blkcg_gq *blkg;
213 char *body;
214};
215
216int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
217 char *input, struct blkg_conf_ctx *ctx);
218void blkg_conf_finish(struct blkg_conf_ctx *ctx);
219
220
221static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css)
222{
223 return css ? container_of(css, struct blkcg, css) : NULL;
224}
225
226static inline struct blkcg *task_blkcg(struct task_struct *tsk)
227{
228 return css_to_blkcg(task_css(tsk, io_cgrp_id));
229}
230
231static inline struct blkcg *bio_blkcg(struct bio *bio)
232{
233 if (bio && bio->bi_css)
234 return css_to_blkcg(bio->bi_css);
235 return task_blkcg(current);
236}
237
238static inline struct cgroup_subsys_state *
239task_get_blkcg_css(struct task_struct *task)
240{
241 return task_get_css(task, io_cgrp_id);
242}
243
244
245
246
247
248
249
250static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
251{
252 return css_to_blkcg(blkcg->css.parent);
253}
254
255
256
257
258
259
260
261
262
263
264
265
266static inline struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
267 struct request_queue *q,
268 bool update_hint)
269{
270 struct blkcg_gq *blkg;
271
272 if (blkcg == &blkcg_root)
273 return q->root_blkg;
274
275 blkg = rcu_dereference(blkcg->blkg_hint);
276 if (blkg && blkg->q == q)
277 return blkg;
278
279 return blkg_lookup_slowpath(blkcg, q, update_hint);
280}
281
282
283
284
285
286
287
288
289
290
291static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg,
292 struct request_queue *q)
293{
294 WARN_ON_ONCE(!rcu_read_lock_held());
295
296 if (unlikely(blk_queue_bypass(q)))
297 return NULL;
298 return __blkg_lookup(blkcg, q, false);
299}
300
301
302
303
304
305
306
307
308static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
309 struct blkcg_policy *pol)
310{
311 return blkg ? blkg->pd[pol->plid] : NULL;
312}
313
314static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
315 struct blkcg_policy *pol)
316{
317 return blkcg ? blkcg->cpd[pol->plid] : NULL;
318}
319
320
321
322
323
324
325
326static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
327{
328 return pd ? pd->blkg : NULL;
329}
330
331static inline struct blkcg *cpd_to_blkcg(struct blkcg_policy_data *cpd)
332{
333 return cpd ? cpd->blkcg : NULL;
334}
335
336
337
338
339
340
341
342
343
344static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
345{
346 char *p;
347
348 p = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
349 if (!p) {
350 strncpy(buf, "<unavailable>", buflen);
351 return -ENAMETOOLONG;
352 }
353
354 memmove(buf, p, buf + buflen - p);
355 return 0;
356}
357
358
359
360
361
362
363
364static inline void blkg_get(struct blkcg_gq *blkg)
365{
366 WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
367 atomic_inc(&blkg->refcnt);
368}
369
370void __blkg_release_rcu(struct rcu_head *rcu);
371
372
373
374
375
376static inline void blkg_put(struct blkcg_gq *blkg)
377{
378 WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
379 if (atomic_dec_and_test(&blkg->refcnt))
380 call_rcu(&blkg->rcu_head, __blkg_release_rcu);
381}
382
383
384
385
386
387
388
389
390
391
392
393
394
395#define blkg_for_each_descendant_pre(d_blkg, pos_css, p_blkg) \
396 css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css) \
397 if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \
398 (p_blkg)->q, false)))
399
400
401
402
403
404
405
406
407
408
409
410#define blkg_for_each_descendant_post(d_blkg, pos_css, p_blkg) \
411 css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css) \
412 if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \
413 (p_blkg)->q, false)))
414
415
416
417
418
419
420
421
422
423
424
425static inline struct request_list *blk_get_rl(struct request_queue *q,
426 struct bio *bio)
427{
428 struct blkcg *blkcg;
429 struct blkcg_gq *blkg;
430
431 rcu_read_lock();
432
433 blkcg = bio_blkcg(bio);
434
435
436 if (blkcg == &blkcg_root)
437 goto root_rl;
438
439
440
441
442
443
444 blkg = blkg_lookup(blkcg, q);
445 if (unlikely(!blkg))
446 goto root_rl;
447
448 blkg_get(blkg);
449 rcu_read_unlock();
450 return &blkg->rl;
451root_rl:
452 rcu_read_unlock();
453 return &q->root_rl;
454}
455
456
457
458
459
460
461
462
463static inline void blk_put_rl(struct request_list *rl)
464{
465 if (rl->blkg->blkcg != &blkcg_root)
466 blkg_put(rl->blkg);
467}
468
469
470
471
472
473
474
475
476
477static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl)
478{
479 rq->rl = rl;
480}
481
482
483
484
485
486
487
488static inline struct request_list *blk_rq_rl(struct request *rq)
489{
490 return rq->rl;
491}
492
493struct request_list *__blk_queue_next_rl(struct request_list *rl,
494 struct request_queue *q);
495
496
497
498
499
500#define blk_queue_for_each_rl(rl, q) \
501 for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q)))
502
503static inline int blkg_stat_init(struct blkg_stat *stat, gfp_t gfp)
504{
505 int ret;
506
507 ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
508 if (ret)
509 return ret;
510
511 atomic64_set(&stat->aux_cnt, 0);
512 return 0;
513}
514
515static inline void blkg_stat_exit(struct blkg_stat *stat)
516{
517 percpu_counter_destroy(&stat->cpu_cnt);
518}
519
520
521
522
523
524
525
526
527
528static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
529{
530 __percpu_counter_add(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
531}
532
533
534
535
536
537static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
538{
539 return percpu_counter_sum_positive(&stat->cpu_cnt);
540}
541
542
543
544
545
546static inline void blkg_stat_reset(struct blkg_stat *stat)
547{
548 percpu_counter_set(&stat->cpu_cnt, 0);
549 atomic64_set(&stat->aux_cnt, 0);
550}
551
552
553
554
555
556
557
558
559static inline void blkg_stat_add_aux(struct blkg_stat *to,
560 struct blkg_stat *from)
561{
562 atomic64_add(blkg_stat_read(from) + atomic64_read(&from->aux_cnt),
563 &to->aux_cnt);
564}
565
566static inline int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp)
567{
568 int i, ret;
569
570 for (i = 0; i < BLKG_RWSTAT_NR; i++) {
571 ret = percpu_counter_init(&rwstat->cpu_cnt[i], 0, gfp);
572 if (ret) {
573 while (--i >= 0)
574 percpu_counter_destroy(&rwstat->cpu_cnt[i]);
575 return ret;
576 }
577 atomic64_set(&rwstat->aux_cnt[i], 0);
578 }
579 return 0;
580}
581
582static inline void blkg_rwstat_exit(struct blkg_rwstat *rwstat)
583{
584 int i;
585
586 for (i = 0; i < BLKG_RWSTAT_NR; i++)
587 percpu_counter_destroy(&rwstat->cpu_cnt[i]);
588}
589
590
591
592
593
594
595
596
597
598
599static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
600 int rw, uint64_t val)
601{
602 struct percpu_counter *cnt;
603
604 if (rw & REQ_WRITE)
605 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
606 else
607 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
608
609 __percpu_counter_add(cnt, val, BLKG_STAT_CPU_BATCH);
610
611 if (rw & REQ_SYNC)
612 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
613 else
614 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
615
616 __percpu_counter_add(cnt, val, BLKG_STAT_CPU_BATCH);
617}
618
619
620
621
622
623
624
625static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
626{
627 struct blkg_rwstat result;
628 int i;
629
630 for (i = 0; i < BLKG_RWSTAT_NR; i++)
631 atomic64_set(&result.aux_cnt[i],
632 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]));
633 return result;
634}
635
636
637
638
639
640
641
642
643
644static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
645{
646 struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
647
648 return atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) +
649 atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]);
650}
651
652
653
654
655
656static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
657{
658 int i;
659
660 for (i = 0; i < BLKG_RWSTAT_NR; i++) {
661 percpu_counter_set(&rwstat->cpu_cnt[i], 0);
662 atomic64_set(&rwstat->aux_cnt[i], 0);
663 }
664}
665
666
667
668
669
670
671
672
673static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
674 struct blkg_rwstat *from)
675{
676 struct blkg_rwstat v = blkg_rwstat_read(from);
677 int i;
678
679 for (i = 0; i < BLKG_RWSTAT_NR; i++)
680 atomic64_add(atomic64_read(&v.aux_cnt[i]) +
681 atomic64_read(&from->aux_cnt[i]),
682 &to->aux_cnt[i]);
683}
684
685#ifdef CONFIG_BLK_DEV_THROTTLING
686extern bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
687 struct bio *bio);
688#else
689static inline bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
690 struct bio *bio) { return false; }
691#endif
692
693static inline bool blkcg_bio_issue_check(struct request_queue *q,
694 struct bio *bio)
695{
696 struct blkcg *blkcg;
697 struct blkcg_gq *blkg;
698 bool throtl = false;
699
700 rcu_read_lock();
701 blkcg = bio_blkcg(bio);
702
703 blkg = blkg_lookup(blkcg, q);
704 if (unlikely(!blkg)) {
705 spin_lock_irq(q->queue_lock);
706 blkg = blkg_lookup_create(blkcg, q);
707 if (IS_ERR(blkg))
708 blkg = NULL;
709 spin_unlock_irq(q->queue_lock);
710 }
711
712 throtl = blk_throtl_bio(q, blkg, bio);
713
714 if (!throtl) {
715 blkg = blkg ?: q->root_blkg;
716 blkg_rwstat_add(&blkg->stat_bytes, bio->bi_rw,
717 bio->bi_iter.bi_size);
718 blkg_rwstat_add(&blkg->stat_ios, bio->bi_rw, 1);
719 }
720
721 rcu_read_unlock();
722 return !throtl;
723}
724
725#else
726
727struct blkcg {
728};
729
730struct blkg_policy_data {
731};
732
733struct blkcg_policy_data {
734};
735
736struct blkcg_gq {
737};
738
739struct blkcg_policy {
740};
741
742#define blkcg_root_css ((struct cgroup_subsys_state *)ERR_PTR(-EINVAL))
743
744static inline struct cgroup_subsys_state *
745task_get_blkcg_css(struct task_struct *task)
746{
747 return NULL;
748}
749
750#ifdef CONFIG_BLOCK
751
752static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
753static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
754static inline void blkcg_drain_queue(struct request_queue *q) { }
755static inline void blkcg_exit_queue(struct request_queue *q) { }
756static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
757static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
758static inline int blkcg_activate_policy(struct request_queue *q,
759 const struct blkcg_policy *pol) { return 0; }
760static inline void blkcg_deactivate_policy(struct request_queue *q,
761 const struct blkcg_policy *pol) { }
762
763static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
764
765static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
766 struct blkcg_policy *pol) { return NULL; }
767static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
768static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
769static inline void blkg_get(struct blkcg_gq *blkg) { }
770static inline void blkg_put(struct blkcg_gq *blkg) { }
771
772static inline struct request_list *blk_get_rl(struct request_queue *q,
773 struct bio *bio) { return &q->root_rl; }
774static inline void blk_put_rl(struct request_list *rl) { }
775static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { }
776static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; }
777
778static inline bool blkcg_bio_issue_check(struct request_queue *q,
779 struct bio *bio) { return true; }
780
781#define blk_queue_for_each_rl(rl, q) \
782 for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
783
784#endif
785#endif
786#endif
787