1
2#ifndef _BLK_CGROUP_H
3#define _BLK_CGROUP_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/cgroup.h>
18#include <linux/percpu.h>
19#include <linux/percpu_counter.h>
20#include <linux/u64_stats_sync.h>
21#include <linux/seq_file.h>
22#include <linux/radix-tree.h>
23#include <linux/blkdev.h>
24#include <linux/atomic.h>
25#include <linux/kthread.h>
26#include <linux/fs.h>
27
28
29#define BLKG_STAT_CPU_BATCH (INT_MAX / 2)
30
31
32#define THROTL_IOPS_MAX UINT_MAX
33
34#ifdef CONFIG_BLK_CGROUP
35
36enum blkg_iostat_type {
37 BLKG_IOSTAT_READ,
38 BLKG_IOSTAT_WRITE,
39 BLKG_IOSTAT_DISCARD,
40
41 BLKG_IOSTAT_NR,
42};
43
44struct blkcg_gq;
45
46struct blkcg {
47 struct cgroup_subsys_state css;
48 spinlock_t lock;
49 refcount_t online_pin;
50
51 struct radix_tree_root blkg_tree;
52 struct blkcg_gq __rcu *blkg_hint;
53 struct hlist_head blkg_list;
54
55 struct blkcg_policy_data *cpd[BLKCG_MAX_POLS];
56
57 struct list_head all_blkcgs_node;
58#ifdef CONFIG_CGROUP_WRITEBACK
59 struct list_head cgwb_list;
60#endif
61};
62
63struct blkg_iostat {
64 u64 bytes[BLKG_IOSTAT_NR];
65 u64 ios[BLKG_IOSTAT_NR];
66};
67
68struct blkg_iostat_set {
69 struct u64_stats_sync sync;
70 struct blkg_iostat cur;
71 struct blkg_iostat last;
72};
73
74
75
76
77
78
79
80
81
82
83
84
85struct blkg_policy_data {
86
87 struct blkcg_gq *blkg;
88 int plid;
89};
90
91
92
93
94
95
96
97
98struct blkcg_policy_data {
99
100 struct blkcg *blkcg;
101 int plid;
102};
103
104
105struct blkcg_gq {
106
107 struct request_queue *q;
108 struct list_head q_node;
109 struct hlist_node blkcg_node;
110 struct blkcg *blkcg;
111
112
113 struct blkcg_gq *parent;
114
115
116 struct percpu_ref refcnt;
117
118
119 bool online;
120
121 struct blkg_iostat_set __percpu *iostat_cpu;
122 struct blkg_iostat_set iostat;
123
124 struct blkg_policy_data *pd[BLKCG_MAX_POLS];
125
126 spinlock_t async_bio_lock;
127 struct bio_list async_bios;
128 struct work_struct async_bio_work;
129
130 atomic_t use_delay;
131 atomic64_t delay_nsec;
132 atomic64_t delay_start;
133 u64 last_delay;
134 int last_use;
135
136 struct rcu_head rcu_head;
137};
138
139typedef struct blkcg_policy_data *(blkcg_pol_alloc_cpd_fn)(gfp_t gfp);
140typedef void (blkcg_pol_init_cpd_fn)(struct blkcg_policy_data *cpd);
141typedef void (blkcg_pol_free_cpd_fn)(struct blkcg_policy_data *cpd);
142typedef void (blkcg_pol_bind_cpd_fn)(struct blkcg_policy_data *cpd);
143typedef struct blkg_policy_data *(blkcg_pol_alloc_pd_fn)(gfp_t gfp,
144 struct request_queue *q, struct blkcg *blkcg);
145typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
146typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
147typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
148typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
149typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
150typedef size_t (blkcg_pol_stat_pd_fn)(struct blkg_policy_data *pd, char *buf,
151 size_t size);
152
153struct blkcg_policy {
154 int plid;
155
156 struct cftype *dfl_cftypes;
157 struct cftype *legacy_cftypes;
158
159
160 blkcg_pol_alloc_cpd_fn *cpd_alloc_fn;
161 blkcg_pol_init_cpd_fn *cpd_init_fn;
162 blkcg_pol_free_cpd_fn *cpd_free_fn;
163 blkcg_pol_bind_cpd_fn *cpd_bind_fn;
164
165 blkcg_pol_alloc_pd_fn *pd_alloc_fn;
166 blkcg_pol_init_pd_fn *pd_init_fn;
167 blkcg_pol_online_pd_fn *pd_online_fn;
168 blkcg_pol_offline_pd_fn *pd_offline_fn;
169 blkcg_pol_free_pd_fn *pd_free_fn;
170 blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn;
171 blkcg_pol_stat_pd_fn *pd_stat_fn;
172};
173
174extern struct blkcg blkcg_root;
175extern struct cgroup_subsys_state * const blkcg_root_css;
176extern bool blkcg_debug_stats;
177
178struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
179 struct request_queue *q, bool update_hint);
180int blkcg_init_queue(struct request_queue *q);
181void blkcg_exit_queue(struct request_queue *q);
182
183
184int blkcg_policy_register(struct blkcg_policy *pol);
185void blkcg_policy_unregister(struct blkcg_policy *pol);
186int blkcg_activate_policy(struct request_queue *q,
187 const struct blkcg_policy *pol);
188void blkcg_deactivate_policy(struct request_queue *q,
189 const struct blkcg_policy *pol);
190
191const char *blkg_dev_name(struct blkcg_gq *blkg);
192void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
193 u64 (*prfill)(struct seq_file *,
194 struct blkg_policy_data *, int),
195 const struct blkcg_policy *pol, int data,
196 bool show_total);
197u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
198
199struct blkg_conf_ctx {
200 struct gendisk *disk;
201 struct blkcg_gq *blkg;
202 char *body;
203};
204
205struct gendisk *blkcg_conf_get_disk(char **inputp);
206int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
207 char *input, struct blkg_conf_ctx *ctx);
208void blkg_conf_finish(struct blkg_conf_ctx *ctx);
209
210
211
212
213
214
215
216
217static inline struct cgroup_subsys_state *blkcg_css(void)
218{
219 struct cgroup_subsys_state *css;
220
221 css = kthread_blkcg();
222 if (css)
223 return css;
224 return task_css(current, io_cgrp_id);
225}
226
227static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css)
228{
229 return css ? container_of(css, struct blkcg, css) : NULL;
230}
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246static inline struct blkcg *__bio_blkcg(struct bio *bio)
247{
248 if (bio && bio->bi_blkg)
249 return bio->bi_blkg->blkcg;
250 return css_to_blkcg(blkcg_css());
251}
252
253
254
255
256
257
258
259
260
261static inline struct blkcg *bio_blkcg(struct bio *bio)
262{
263 if (bio && bio->bi_blkg)
264 return bio->bi_blkg->blkcg;
265 return NULL;
266}
267
268static inline bool blk_cgroup_congested(void)
269{
270 struct cgroup_subsys_state *css;
271 bool ret = false;
272
273 rcu_read_lock();
274 css = kthread_blkcg();
275 if (!css)
276 css = task_css(current, io_cgrp_id);
277 while (css) {
278 if (atomic_read(&css->cgroup->congestion_count)) {
279 ret = true;
280 break;
281 }
282 css = css->parent;
283 }
284 rcu_read_unlock();
285 return ret;
286}
287
288
289
290
291
292
293
294
295
296
297
298
299static inline bool bio_issue_as_root_blkg(struct bio *bio)
300{
301 return (bio->bi_opf & (REQ_META | REQ_SWAP)) != 0;
302}
303
304
305
306
307
308
309
310static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
311{
312 return css_to_blkcg(blkcg->css.parent);
313}
314
315
316
317
318
319
320
321
322
323
324
325
326static inline struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
327 struct request_queue *q,
328 bool update_hint)
329{
330 struct blkcg_gq *blkg;
331
332 if (blkcg == &blkcg_root)
333 return q->root_blkg;
334
335 blkg = rcu_dereference(blkcg->blkg_hint);
336 if (blkg && blkg->q == q)
337 return blkg;
338
339 return blkg_lookup_slowpath(blkcg, q, update_hint);
340}
341
342
343
344
345
346
347
348
349
350static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg,
351 struct request_queue *q)
352{
353 WARN_ON_ONCE(!rcu_read_lock_held());
354 return __blkg_lookup(blkcg, q, false);
355}
356
357
358
359
360
361
362
363static inline struct blkcg_gq *blk_queue_root_blkg(struct request_queue *q)
364{
365 return q->root_blkg;
366}
367
368
369
370
371
372
373
374
375static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
376 struct blkcg_policy *pol)
377{
378 return blkg ? blkg->pd[pol->plid] : NULL;
379}
380
381static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
382 struct blkcg_policy *pol)
383{
384 return blkcg ? blkcg->cpd[pol->plid] : NULL;
385}
386
387
388
389
390
391
392
393static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
394{
395 return pd ? pd->blkg : NULL;
396}
397
398static inline struct blkcg *cpd_to_blkcg(struct blkcg_policy_data *cpd)
399{
400 return cpd ? cpd->blkcg : NULL;
401}
402
403extern void blkcg_destroy_blkgs(struct blkcg *blkcg);
404
405
406
407
408
409
410
411
412
413static inline void blkcg_pin_online(struct blkcg *blkcg)
414{
415 refcount_inc(&blkcg->online_pin);
416}
417
418
419
420
421
422
423
424
425
426
427static inline void blkcg_unpin_online(struct blkcg *blkcg)
428{
429 do {
430 if (!refcount_dec_and_test(&blkcg->online_pin))
431 break;
432 blkcg_destroy_blkgs(blkcg);
433 blkcg = blkcg_parent(blkcg);
434 } while (blkcg);
435}
436
437
438
439
440
441
442
443
444
445static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
446{
447 return cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
448}
449
450
451
452
453
454
455
456static inline void blkg_get(struct blkcg_gq *blkg)
457{
458 percpu_ref_get(&blkg->refcnt);
459}
460
461
462
463
464
465
466
467
468static inline bool blkg_tryget(struct blkcg_gq *blkg)
469{
470 return blkg && percpu_ref_tryget(&blkg->refcnt);
471}
472
473
474
475
476
477static inline void blkg_put(struct blkcg_gq *blkg)
478{
479 percpu_ref_put(&blkg->refcnt);
480}
481
482
483
484
485
486
487
488
489
490
491
492
493
494#define blkg_for_each_descendant_pre(d_blkg, pos_css, p_blkg) \
495 css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css) \
496 if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \
497 (p_blkg)->q, false)))
498
499
500
501
502
503
504
505
506
507
508
509#define blkg_for_each_descendant_post(d_blkg, pos_css, p_blkg) \
510 css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css) \
511 if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \
512 (p_blkg)->q, false)))
513
514bool __blkcg_punt_bio_submit(struct bio *bio);
515
516static inline bool blkcg_punt_bio_submit(struct bio *bio)
517{
518 if (bio->bi_opf & REQ_CGROUP_PUNT)
519 return __blkcg_punt_bio_submit(bio);
520 else
521 return false;
522}
523
524static inline void blkcg_bio_issue_init(struct bio *bio)
525{
526 bio_issue_init(&bio->bi_issue, bio_sectors(bio));
527}
528
529static inline void blkcg_use_delay(struct blkcg_gq *blkg)
530{
531 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
532 return;
533 if (atomic_add_return(1, &blkg->use_delay) == 1)
534 atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);
535}
536
537static inline int blkcg_unuse_delay(struct blkcg_gq *blkg)
538{
539 int old = atomic_read(&blkg->use_delay);
540
541 if (WARN_ON_ONCE(old < 0))
542 return 0;
543 if (old == 0)
544 return 0;
545
546
547
548
549
550
551
552
553 while (old) {
554 int cur = atomic_cmpxchg(&blkg->use_delay, old, old - 1);
555 if (cur == old)
556 break;
557 old = cur;
558 }
559
560 if (old == 0)
561 return 0;
562 if (old == 1)
563 atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
564 return 1;
565}
566
567
568
569
570
571
572
573
574
575
576static inline void blkcg_set_delay(struct blkcg_gq *blkg, u64 delay)
577{
578 int old = atomic_read(&blkg->use_delay);
579
580
581 if (!old && atomic_cmpxchg(&blkg->use_delay, old, -1) == old)
582 atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);
583
584 atomic64_set(&blkg->delay_nsec, delay);
585}
586
587
588
589
590
591
592
593static inline void blkcg_clear_delay(struct blkcg_gq *blkg)
594{
595 int old = atomic_read(&blkg->use_delay);
596
597
598 if (old && atomic_cmpxchg(&blkg->use_delay, old, 0) == old)
599 atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
600}
601
602void blk_cgroup_bio_start(struct bio *bio);
603void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta);
604void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay);
605void blkcg_maybe_throttle_current(void);
606#else
607
608struct blkcg {
609};
610
611struct blkg_policy_data {
612};
613
614struct blkcg_policy_data {
615};
616
617struct blkcg_gq {
618};
619
620struct blkcg_policy {
621};
622
623#define blkcg_root_css ((struct cgroup_subsys_state *)ERR_PTR(-EINVAL))
624
625static inline void blkcg_maybe_throttle_current(void) { }
626static inline bool blk_cgroup_congested(void) { return false; }
627
628#ifdef CONFIG_BLOCK
629
630static inline void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay) { }
631
632static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
633static inline struct blkcg_gq *blk_queue_root_blkg(struct request_queue *q)
634{ return NULL; }
635static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
636static inline void blkcg_exit_queue(struct request_queue *q) { }
637static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
638static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
639static inline int blkcg_activate_policy(struct request_queue *q,
640 const struct blkcg_policy *pol) { return 0; }
641static inline void blkcg_deactivate_policy(struct request_queue *q,
642 const struct blkcg_policy *pol) { }
643
644static inline struct blkcg *__bio_blkcg(struct bio *bio) { return NULL; }
645static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
646
647static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
648 struct blkcg_policy *pol) { return NULL; }
649static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
650static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
651static inline void blkg_get(struct blkcg_gq *blkg) { }
652static inline void blkg_put(struct blkcg_gq *blkg) { }
653
654static inline bool blkcg_punt_bio_submit(struct bio *bio) { return false; }
655static inline void blkcg_bio_issue_init(struct bio *bio) { }
656static inline void blk_cgroup_bio_start(struct bio *bio) { }
657
658#define blk_queue_for_each_rl(rl, q) \
659 for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
660
661#endif
662#endif
663#endif
664