1
2
3
4
5#include <linux/module.h>
6#include <linux/slab.h>
7#include <linux/blkdev.h>
8#include <linux/cgroup.h>
9#include <linux/elevator.h>
10#include <linux/ktime.h>
11#include <linux/rbtree.h>
12#include <linux/ioprio.h>
13#include <linux/sbitmap.h>
14#include <linux/delay.h>
15
16#include "bfq-iosched.h"
17
18#ifdef CONFIG_BFQ_CGROUP_DEBUG
19static int bfq_stat_init(struct bfq_stat *stat, gfp_t gfp)
20{
21 int ret;
22
23 ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
24 if (ret)
25 return ret;
26
27 atomic64_set(&stat->aux_cnt, 0);
28 return 0;
29}
30
31static void bfq_stat_exit(struct bfq_stat *stat)
32{
33 percpu_counter_destroy(&stat->cpu_cnt);
34}
35
36
37
38
39
40
41
42
43
44static inline void bfq_stat_add(struct bfq_stat *stat, uint64_t val)
45{
46 percpu_counter_add_batch(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
47}
48
49
50
51
52
53static inline uint64_t bfq_stat_read(struct bfq_stat *stat)
54{
55 return percpu_counter_sum_positive(&stat->cpu_cnt);
56}
57
58
59
60
61
62static inline void bfq_stat_reset(struct bfq_stat *stat)
63{
64 percpu_counter_set(&stat->cpu_cnt, 0);
65 atomic64_set(&stat->aux_cnt, 0);
66}
67
68
69
70
71
72
73
74
75static inline void bfq_stat_add_aux(struct bfq_stat *to,
76 struct bfq_stat *from)
77{
78 atomic64_add(bfq_stat_read(from) + atomic64_read(&from->aux_cnt),
79 &to->aux_cnt);
80}
81
82
83
84
85
86
87
88
89
90static u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd,
91 int off)
92{
93 return __blkg_prfill_u64(sf, pd, bfq_stat_read((void *)pd + off));
94}
95
96
97enum bfqg_stats_flags {
98 BFQG_stats_waiting = 0,
99 BFQG_stats_idling,
100 BFQG_stats_empty,
101};
102
103#define BFQG_FLAG_FNS(name) \
104static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
105{ \
106 stats->flags |= (1 << BFQG_stats_##name); \
107} \
108static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
109{ \
110 stats->flags &= ~(1 << BFQG_stats_##name); \
111} \
112static int bfqg_stats_##name(struct bfqg_stats *stats) \
113{ \
114 return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
115} \
116
117BFQG_FLAG_FNS(waiting)
118BFQG_FLAG_FNS(idling)
119BFQG_FLAG_FNS(empty)
120#undef BFQG_FLAG_FNS
121
122
123static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
124{
125 u64 now;
126
127 if (!bfqg_stats_waiting(stats))
128 return;
129
130 now = ktime_get_ns();
131 if (now > stats->start_group_wait_time)
132 bfq_stat_add(&stats->group_wait_time,
133 now - stats->start_group_wait_time);
134 bfqg_stats_clear_waiting(stats);
135}
136
137
138static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
139 struct bfq_group *curr_bfqg)
140{
141 struct bfqg_stats *stats = &bfqg->stats;
142
143 if (bfqg_stats_waiting(stats))
144 return;
145 if (bfqg == curr_bfqg)
146 return;
147 stats->start_group_wait_time = ktime_get_ns();
148 bfqg_stats_mark_waiting(stats);
149}
150
151
152static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
153{
154 u64 now;
155
156 if (!bfqg_stats_empty(stats))
157 return;
158
159 now = ktime_get_ns();
160 if (now > stats->start_empty_time)
161 bfq_stat_add(&stats->empty_time,
162 now - stats->start_empty_time);
163 bfqg_stats_clear_empty(stats);
164}
165
166void bfqg_stats_update_dequeue(struct bfq_group *bfqg)
167{
168 bfq_stat_add(&bfqg->stats.dequeue, 1);
169}
170
171void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
172{
173 struct bfqg_stats *stats = &bfqg->stats;
174
175 if (blkg_rwstat_total(&stats->queued))
176 return;
177
178
179
180
181
182
183 if (bfqg_stats_empty(stats))
184 return;
185
186 stats->start_empty_time = ktime_get_ns();
187 bfqg_stats_mark_empty(stats);
188}
189
190void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
191{
192 struct bfqg_stats *stats = &bfqg->stats;
193
194 if (bfqg_stats_idling(stats)) {
195 u64 now = ktime_get_ns();
196
197 if (now > stats->start_idle_time)
198 bfq_stat_add(&stats->idle_time,
199 now - stats->start_idle_time);
200 bfqg_stats_clear_idling(stats);
201 }
202}
203
204void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)
205{
206 struct bfqg_stats *stats = &bfqg->stats;
207
208 stats->start_idle_time = ktime_get_ns();
209 bfqg_stats_mark_idling(stats);
210}
211
212void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)
213{
214 struct bfqg_stats *stats = &bfqg->stats;
215
216 bfq_stat_add(&stats->avg_queue_size_sum,
217 blkg_rwstat_total(&stats->queued));
218 bfq_stat_add(&stats->avg_queue_size_samples, 1);
219 bfqg_stats_update_group_wait_time(stats);
220}
221
222void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
223 unsigned int op)
224{
225 blkg_rwstat_add(&bfqg->stats.queued, op, 1);
226 bfqg_stats_end_empty_time(&bfqg->stats);
227 if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))
228 bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
229}
230
231void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)
232{
233 blkg_rwstat_add(&bfqg->stats.queued, op, -1);
234}
235
236void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)
237{
238 blkg_rwstat_add(&bfqg->stats.merged, op, 1);
239}
240
241void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
242 u64 io_start_time_ns, unsigned int op)
243{
244 struct bfqg_stats *stats = &bfqg->stats;
245 u64 now = ktime_get_ns();
246
247 if (now > io_start_time_ns)
248 blkg_rwstat_add(&stats->service_time, op,
249 now - io_start_time_ns);
250 if (io_start_time_ns > start_time_ns)
251 blkg_rwstat_add(&stats->wait_time, op,
252 io_start_time_ns - start_time_ns);
253}
254
255#else
256
257void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
258 unsigned int op) { }
259void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }
260void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }
261void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
262 u64 io_start_time_ns, unsigned int op) { }
263void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }
264void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }
265void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }
266void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }
267void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { }
268
269#endif
270
271#ifdef CONFIG_BFQ_GROUP_IOSCHED
272
273
274
275
276
277
278
279static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd)
280{
281 return pd ? container_of(pd, struct bfq_group, pd) : NULL;
282}
283
284struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)
285{
286 return pd_to_blkg(&bfqg->pd);
287}
288
289static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)
290{
291 return pd_to_bfqg(blkg_to_pd(blkg, &blkcg_policy_bfq));
292}
293
294
295
296
297
298
299
300
301static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)
302{
303 struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;
304
305 return pblkg ? blkg_to_bfqg(pblkg) : NULL;
306}
307
308struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
309{
310 struct bfq_entity *group_entity = bfqq->entity.parent;
311
312 return group_entity ? container_of(group_entity, struct bfq_group,
313 entity) :
314 bfqq->bfqd->root_group;
315}
316
317
318
319
320
321
322static void bfqg_get(struct bfq_group *bfqg)
323{
324 bfqg->ref++;
325}
326
327static void bfqg_put(struct bfq_group *bfqg)
328{
329 bfqg->ref--;
330
331 if (bfqg->ref == 0)
332 kfree(bfqg);
333}
334
335static void bfqg_and_blkg_get(struct bfq_group *bfqg)
336{
337
338 bfqg_get(bfqg);
339
340 blkg_get(bfqg_to_blkg(bfqg));
341}
342
343void bfqg_and_blkg_put(struct bfq_group *bfqg)
344{
345 blkg_put(bfqg_to_blkg(bfqg));
346
347 bfqg_put(bfqg);
348}
349
350
351static void bfqg_stats_reset(struct bfqg_stats *stats)
352{
353#ifdef CONFIG_BFQ_CGROUP_DEBUG
354
355 blkg_rwstat_reset(&stats->merged);
356 blkg_rwstat_reset(&stats->service_time);
357 blkg_rwstat_reset(&stats->wait_time);
358 bfq_stat_reset(&stats->time);
359 bfq_stat_reset(&stats->avg_queue_size_sum);
360 bfq_stat_reset(&stats->avg_queue_size_samples);
361 bfq_stat_reset(&stats->dequeue);
362 bfq_stat_reset(&stats->group_wait_time);
363 bfq_stat_reset(&stats->idle_time);
364 bfq_stat_reset(&stats->empty_time);
365#endif
366}
367
368
369static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from)
370{
371 if (!to || !from)
372 return;
373
374#ifdef CONFIG_BFQ_CGROUP_DEBUG
375
376 blkg_rwstat_add_aux(&to->merged, &from->merged);
377 blkg_rwstat_add_aux(&to->service_time, &from->service_time);
378 blkg_rwstat_add_aux(&to->wait_time, &from->wait_time);
379 bfq_stat_add_aux(&from->time, &from->time);
380 bfq_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
381 bfq_stat_add_aux(&to->avg_queue_size_samples,
382 &from->avg_queue_size_samples);
383 bfq_stat_add_aux(&to->dequeue, &from->dequeue);
384 bfq_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
385 bfq_stat_add_aux(&to->idle_time, &from->idle_time);
386 bfq_stat_add_aux(&to->empty_time, &from->empty_time);
387#endif
388}
389
390
391
392
393
394
395static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)
396{
397 struct bfq_group *parent;
398
399 if (!bfqg)
400 return;
401
402 parent = bfqg_parent(bfqg);
403
404 lockdep_assert_held(&bfqg_to_blkg(bfqg)->q->queue_lock);
405
406 if (unlikely(!parent))
407 return;
408
409 bfqg_stats_add_aux(&parent->stats, &bfqg->stats);
410 bfqg_stats_reset(&bfqg->stats);
411}
412
413void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
414{
415 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
416
417 entity->weight = entity->new_weight;
418 entity->orig_weight = entity->new_weight;
419 if (bfqq) {
420 bfqq->ioprio = bfqq->new_ioprio;
421 bfqq->ioprio_class = bfqq->new_ioprio_class;
422
423
424
425
426 bfqg_and_blkg_get(bfqg);
427 }
428 entity->parent = bfqg->my_entity;
429 entity->sched_data = &bfqg->sched_data;
430}
431
432static void bfqg_stats_exit(struct bfqg_stats *stats)
433{
434#ifdef CONFIG_BFQ_CGROUP_DEBUG
435 blkg_rwstat_exit(&stats->merged);
436 blkg_rwstat_exit(&stats->service_time);
437 blkg_rwstat_exit(&stats->wait_time);
438 blkg_rwstat_exit(&stats->queued);
439 bfq_stat_exit(&stats->time);
440 bfq_stat_exit(&stats->avg_queue_size_sum);
441 bfq_stat_exit(&stats->avg_queue_size_samples);
442 bfq_stat_exit(&stats->dequeue);
443 bfq_stat_exit(&stats->group_wait_time);
444 bfq_stat_exit(&stats->idle_time);
445 bfq_stat_exit(&stats->empty_time);
446#endif
447}
448
449static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
450{
451#ifdef CONFIG_BFQ_CGROUP_DEBUG
452 if (blkg_rwstat_init(&stats->merged, gfp) ||
453 blkg_rwstat_init(&stats->service_time, gfp) ||
454 blkg_rwstat_init(&stats->wait_time, gfp) ||
455 blkg_rwstat_init(&stats->queued, gfp) ||
456 bfq_stat_init(&stats->time, gfp) ||
457 bfq_stat_init(&stats->avg_queue_size_sum, gfp) ||
458 bfq_stat_init(&stats->avg_queue_size_samples, gfp) ||
459 bfq_stat_init(&stats->dequeue, gfp) ||
460 bfq_stat_init(&stats->group_wait_time, gfp) ||
461 bfq_stat_init(&stats->idle_time, gfp) ||
462 bfq_stat_init(&stats->empty_time, gfp)) {
463 bfqg_stats_exit(stats);
464 return -ENOMEM;
465 }
466#endif
467
468 return 0;
469}
470
471static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd)
472{
473 return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL;
474}
475
476static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
477{
478 return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq));
479}
480
481static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp)
482{
483 struct bfq_group_data *bgd;
484
485 bgd = kzalloc(sizeof(*bgd), gfp);
486 if (!bgd)
487 return NULL;
488 return &bgd->pd;
489}
490
491static void bfq_cpd_init(struct blkcg_policy_data *cpd)
492{
493 struct bfq_group_data *d = cpd_to_bfqgd(cpd);
494
495 d->weight = cgroup_subsys_on_dfl(io_cgrp_subsys) ?
496 CGROUP_WEIGHT_DFL : BFQ_WEIGHT_LEGACY_DFL;
497}
498
499static void bfq_cpd_free(struct blkcg_policy_data *cpd)
500{
501 kfree(cpd_to_bfqgd(cpd));
502}
503
504static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
505{
506 struct bfq_group *bfqg;
507
508 bfqg = kzalloc_node(sizeof(*bfqg), gfp, node);
509 if (!bfqg)
510 return NULL;
511
512 if (bfqg_stats_init(&bfqg->stats, gfp)) {
513 kfree(bfqg);
514 return NULL;
515 }
516
517
518 bfqg_get(bfqg);
519 return &bfqg->pd;
520}
521
522static void bfq_pd_init(struct blkg_policy_data *pd)
523{
524 struct blkcg_gq *blkg = pd_to_blkg(pd);
525 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
526 struct bfq_data *bfqd = blkg->q->elevator->elevator_data;
527 struct bfq_entity *entity = &bfqg->entity;
528 struct bfq_group_data *d = blkcg_to_bfqgd(blkg->blkcg);
529
530 entity->orig_weight = entity->weight = entity->new_weight = d->weight;
531 entity->my_sched_data = &bfqg->sched_data;
532 bfqg->my_entity = entity;
533
534
535
536 bfqg->bfqd = bfqd;
537 bfqg->active_entities = 0;
538 bfqg->rq_pos_tree = RB_ROOT;
539}
540
541static void bfq_pd_free(struct blkg_policy_data *pd)
542{
543 struct bfq_group *bfqg = pd_to_bfqg(pd);
544
545 bfqg_stats_exit(&bfqg->stats);
546 bfqg_put(bfqg);
547}
548
549static void bfq_pd_reset_stats(struct blkg_policy_data *pd)
550{
551 struct bfq_group *bfqg = pd_to_bfqg(pd);
552
553 bfqg_stats_reset(&bfqg->stats);
554}
555
556static void bfq_group_set_parent(struct bfq_group *bfqg,
557 struct bfq_group *parent)
558{
559 struct bfq_entity *entity;
560
561 entity = &bfqg->entity;
562 entity->parent = parent->my_entity;
563 entity->sched_data = &parent->sched_data;
564}
565
566static struct bfq_group *bfq_lookup_bfqg(struct bfq_data *bfqd,
567 struct blkcg *blkcg)
568{
569 struct blkcg_gq *blkg;
570
571 blkg = blkg_lookup(blkcg, bfqd->queue);
572 if (likely(blkg))
573 return blkg_to_bfqg(blkg);
574 return NULL;
575}
576
577struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
578 struct blkcg *blkcg)
579{
580 struct bfq_group *bfqg, *parent;
581 struct bfq_entity *entity;
582
583 bfqg = bfq_lookup_bfqg(bfqd, blkcg);
584
585 if (unlikely(!bfqg))
586 return NULL;
587
588
589
590
591
592
593 entity = &bfqg->entity;
594 for_each_entity(entity) {
595 bfqg = container_of(entity, struct bfq_group, entity);
596 if (bfqg != bfqd->root_group) {
597 parent = bfqg_parent(bfqg);
598 if (!parent)
599 parent = bfqd->root_group;
600 bfq_group_set_parent(bfqg, parent);
601 }
602 }
603
604 return bfqg;
605}
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
622 struct bfq_group *bfqg)
623{
624 struct bfq_entity *entity = &bfqq->entity;
625
626
627
628
629
630
631
632 if (bfqq == bfqd->in_service_queue)
633 bfq_bfqq_expire(bfqd, bfqd->in_service_queue,
634 false, BFQQE_PREEMPTED);
635
636 if (bfq_bfqq_busy(bfqq))
637 bfq_deactivate_bfqq(bfqd, bfqq, false, false);
638 else if (entity->on_st)
639 bfq_put_idle_entity(bfq_entity_service_tree(entity), entity);
640 bfqg_and_blkg_put(bfqq_group(bfqq));
641
642 entity->parent = bfqg->my_entity;
643 entity->sched_data = &bfqg->sched_data;
644
645 bfqg_and_blkg_get(bfqg);
646
647 if (bfq_bfqq_busy(bfqq)) {
648 if (unlikely(!bfqd->nonrot_with_queueing))
649 bfq_pos_tree_add_move(bfqd, bfqq);
650 bfq_activate_bfqq(bfqd, bfqq);
651 }
652
653 if (!bfqd->in_service_queue && !bfqd->rq_in_driver)
654 bfq_schedule_dispatch(bfqd);
655}
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
672 struct bfq_io_cq *bic,
673 struct blkcg *blkcg)
674{
675 struct bfq_queue *async_bfqq = bic_to_bfqq(bic, 0);
676 struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, 1);
677 struct bfq_group *bfqg;
678 struct bfq_entity *entity;
679
680 bfqg = bfq_find_set_group(bfqd, blkcg);
681
682 if (unlikely(!bfqg))
683 bfqg = bfqd->root_group;
684
685 if (async_bfqq) {
686 entity = &async_bfqq->entity;
687
688 if (entity->sched_data != &bfqg->sched_data) {
689 bic_set_bfqq(bic, NULL, 0);
690 bfq_log_bfqq(bfqd, async_bfqq,
691 "bic_change_group: %p %d",
692 async_bfqq, async_bfqq->ref);
693 bfq_put_queue(async_bfqq);
694 }
695 }
696
697 if (sync_bfqq) {
698 entity = &sync_bfqq->entity;
699 if (entity->sched_data != &bfqg->sched_data)
700 bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
701 }
702
703 return bfqg;
704}
705
706void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
707{
708 struct bfq_data *bfqd = bic_to_bfqd(bic);
709 struct bfq_group *bfqg = NULL;
710 uint64_t serial_nr;
711
712 rcu_read_lock();
713 serial_nr = __bio_blkcg(bio)->css.serial_nr;
714
715
716
717
718
719 if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr))
720 goto out;
721
722 bfqg = __bfq_bic_change_cgroup(bfqd, bic, __bio_blkcg(bio));
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773 blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));
774 bic->blkcg_serial_nr = serial_nr;
775out:
776 rcu_read_unlock();
777}
778
779
780
781
782
783static void bfq_flush_idle_tree(struct bfq_service_tree *st)
784{
785 struct bfq_entity *entity = st->first_idle;
786
787 for (; entity ; entity = st->first_idle)
788 __bfq_deactivate_entity(entity, false);
789}
790
791
792
793
794
795
796static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
797 struct bfq_entity *entity)
798{
799 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
800
801 bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);
802}
803
804
805
806
807
808
809
810
811static void bfq_reparent_active_entities(struct bfq_data *bfqd,
812 struct bfq_group *bfqg,
813 struct bfq_service_tree *st)
814{
815 struct rb_root *active = &st->active;
816 struct bfq_entity *entity = NULL;
817
818 if (!RB_EMPTY_ROOT(&st->active))
819 entity = bfq_entity_of(rb_first(active));
820
821 for (; entity ; entity = bfq_entity_of(rb_first(active)))
822 bfq_reparent_leaf_entity(bfqd, entity);
823
824 if (bfqg->sched_data.in_service_entity)
825 bfq_reparent_leaf_entity(bfqd,
826 bfqg->sched_data.in_service_entity);
827}
828
829
830
831
832
833
834
835
836
837static void bfq_pd_offline(struct blkg_policy_data *pd)
838{
839 struct bfq_service_tree *st;
840 struct bfq_group *bfqg = pd_to_bfqg(pd);
841 struct bfq_data *bfqd = bfqg->bfqd;
842 struct bfq_entity *entity = bfqg->my_entity;
843 unsigned long flags;
844 int i;
845
846 spin_lock_irqsave(&bfqd->lock, flags);
847
848 if (!entity)
849 goto put_async_queues;
850
851
852
853
854
855 for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) {
856 st = bfqg->sched_data.service_tree + i;
857
858
859
860
861
862
863 bfq_flush_idle_tree(st);
864
865
866
867
868
869
870
871
872
873
874
875
876
877 bfq_reparent_active_entities(bfqd, bfqg, st);
878 }
879
880 __bfq_deactivate_entity(entity, false);
881
882put_async_queues:
883 bfq_put_async_queues(bfqd, bfqg);
884
885 spin_unlock_irqrestore(&bfqd->lock, flags);
886
887
888
889
890
891
892 bfqg_stats_xfer_dead(bfqg);
893}
894
895void bfq_end_wr_async(struct bfq_data *bfqd)
896{
897 struct blkcg_gq *blkg;
898
899 list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) {
900 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
901
902 bfq_end_wr_async_queues(bfqd, bfqg);
903 }
904 bfq_end_wr_async_queues(bfqd, bfqd->root_group);
905}
906
907static int bfq_io_show_weight(struct seq_file *sf, void *v)
908{
909 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
910 struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
911 unsigned int val = 0;
912
913 if (bfqgd)
914 val = bfqgd->weight;
915
916 seq_printf(sf, "%u\n", val);
917
918 return 0;
919}
920
921static int bfq_io_set_weight_legacy(struct cgroup_subsys_state *css,
922 struct cftype *cftype,
923 u64 val)
924{
925 struct blkcg *blkcg = css_to_blkcg(css);
926 struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
927 struct blkcg_gq *blkg;
928 int ret = -ERANGE;
929
930 if (val < BFQ_MIN_WEIGHT || val > BFQ_MAX_WEIGHT)
931 return ret;
932
933 ret = 0;
934 spin_lock_irq(&blkcg->lock);
935 bfqgd->weight = (unsigned short)val;
936 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
937 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
938
939 if (!bfqg)
940 continue;
941
942
943
944
945
946
947 if ((unsigned short)val != bfqg->entity.new_weight) {
948 bfqg->entity.new_weight = (unsigned short)val;
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964 smp_wmb();
965 bfqg->entity.prio_changed = 1;
966 }
967 }
968 spin_unlock_irq(&blkcg->lock);
969
970 return ret;
971}
972
973static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,
974 char *buf, size_t nbytes,
975 loff_t off)
976{
977 u64 weight;
978
979 int ret = kstrtoull(strim(buf), 0, &weight);
980
981 if (ret)
982 return ret;
983
984 ret = bfq_io_set_weight_legacy(of_css(of), NULL, weight);
985 return ret ?: nbytes;
986}
987
988#ifdef CONFIG_BFQ_CGROUP_DEBUG
989static int bfqg_print_stat(struct seq_file *sf, void *v)
990{
991 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
992 &blkcg_policy_bfq, seq_cft(sf)->private, false);
993 return 0;
994}
995
996static int bfqg_print_rwstat(struct seq_file *sf, void *v)
997{
998 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,
999 &blkcg_policy_bfq, seq_cft(sf)->private, true);
1000 return 0;
1001}
1002
1003static u64 bfqg_prfill_stat_recursive(struct seq_file *sf,
1004 struct blkg_policy_data *pd, int off)
1005{
1006 struct blkcg_gq *blkg = pd_to_blkg(pd);
1007 struct blkcg_gq *pos_blkg;
1008 struct cgroup_subsys_state *pos_css;
1009 u64 sum = 0;
1010
1011 lockdep_assert_held(&blkg->q->queue_lock);
1012
1013 rcu_read_lock();
1014 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
1015 struct bfq_stat *stat;
1016
1017 if (!pos_blkg->online)
1018 continue;
1019
1020 stat = (void *)blkg_to_pd(pos_blkg, &blkcg_policy_bfq) + off;
1021 sum += bfq_stat_read(stat) + atomic64_read(&stat->aux_cnt);
1022 }
1023 rcu_read_unlock();
1024
1025 return __blkg_prfill_u64(sf, pd, sum);
1026}
1027
1028static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf,
1029 struct blkg_policy_data *pd, int off)
1030{
1031 struct blkg_rwstat_sample sum;
1032
1033 blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off, &sum);
1034 return __blkg_prfill_rwstat(sf, pd, &sum);
1035}
1036
1037static int bfqg_print_stat_recursive(struct seq_file *sf, void *v)
1038{
1039 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1040 bfqg_prfill_stat_recursive, &blkcg_policy_bfq,
1041 seq_cft(sf)->private, false);
1042 return 0;
1043}
1044
1045static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)
1046{
1047 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1048 bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq,
1049 seq_cft(sf)->private, true);
1050 return 0;
1051}
1052
1053static u64 bfqg_prfill_sectors(struct seq_file *sf, struct blkg_policy_data *pd,
1054 int off)
1055{
1056 u64 sum = blkg_rwstat_total(&pd->blkg->stat_bytes);
1057
1058 return __blkg_prfill_u64(sf, pd, sum >> 9);
1059}
1060
1061static int bfqg_print_stat_sectors(struct seq_file *sf, void *v)
1062{
1063 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1064 bfqg_prfill_sectors, &blkcg_policy_bfq, 0, false);
1065 return 0;
1066}
1067
1068static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf,
1069 struct blkg_policy_data *pd, int off)
1070{
1071 struct blkg_rwstat_sample tmp;
1072
1073 blkg_rwstat_recursive_sum(pd->blkg, NULL,
1074 offsetof(struct blkcg_gq, stat_bytes), &tmp);
1075
1076 return __blkg_prfill_u64(sf, pd,
1077 (tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]) >> 9);
1078}
1079
1080static int bfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v)
1081{
1082 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1083 bfqg_prfill_sectors_recursive, &blkcg_policy_bfq, 0,
1084 false);
1085 return 0;
1086}
1087
1088static u64 bfqg_prfill_avg_queue_size(struct seq_file *sf,
1089 struct blkg_policy_data *pd, int off)
1090{
1091 struct bfq_group *bfqg = pd_to_bfqg(pd);
1092 u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);
1093 u64 v = 0;
1094
1095 if (samples) {
1096 v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);
1097 v = div64_u64(v, samples);
1098 }
1099 __blkg_prfill_u64(sf, pd, v);
1100 return 0;
1101}
1102
1103
1104static int bfqg_print_avg_queue_size(struct seq_file *sf, void *v)
1105{
1106 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1107 bfqg_prfill_avg_queue_size, &blkcg_policy_bfq,
1108 0, false);
1109 return 0;
1110}
1111#endif
1112
1113struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
1114{
1115 int ret;
1116
1117 ret = blkcg_activate_policy(bfqd->queue, &blkcg_policy_bfq);
1118 if (ret)
1119 return NULL;
1120
1121 return blkg_to_bfqg(bfqd->queue->root_blkg);
1122}
1123
1124struct blkcg_policy blkcg_policy_bfq = {
1125 .dfl_cftypes = bfq_blkg_files,
1126 .legacy_cftypes = bfq_blkcg_legacy_files,
1127
1128 .cpd_alloc_fn = bfq_cpd_alloc,
1129 .cpd_init_fn = bfq_cpd_init,
1130 .cpd_bind_fn = bfq_cpd_init,
1131 .cpd_free_fn = bfq_cpd_free,
1132
1133 .pd_alloc_fn = bfq_pd_alloc,
1134 .pd_init_fn = bfq_pd_init,
1135 .pd_offline_fn = bfq_pd_offline,
1136 .pd_free_fn = bfq_pd_free,
1137 .pd_reset_stats_fn = bfq_pd_reset_stats,
1138};
1139
1140struct cftype bfq_blkcg_legacy_files[] = {
1141 {
1142 .name = "bfq.weight",
1143 .flags = CFTYPE_NOT_ON_ROOT,
1144 .seq_show = bfq_io_show_weight,
1145 .write_u64 = bfq_io_set_weight_legacy,
1146 },
1147
1148
1149 {
1150 .name = "bfq.io_service_bytes",
1151 .private = (unsigned long)&blkcg_policy_bfq,
1152 .seq_show = blkg_print_stat_bytes,
1153 },
1154 {
1155 .name = "bfq.io_serviced",
1156 .private = (unsigned long)&blkcg_policy_bfq,
1157 .seq_show = blkg_print_stat_ios,
1158 },
1159#ifdef CONFIG_BFQ_CGROUP_DEBUG
1160 {
1161 .name = "bfq.time",
1162 .private = offsetof(struct bfq_group, stats.time),
1163 .seq_show = bfqg_print_stat,
1164 },
1165 {
1166 .name = "bfq.sectors",
1167 .seq_show = bfqg_print_stat_sectors,
1168 },
1169 {
1170 .name = "bfq.io_service_time",
1171 .private = offsetof(struct bfq_group, stats.service_time),
1172 .seq_show = bfqg_print_rwstat,
1173 },
1174 {
1175 .name = "bfq.io_wait_time",
1176 .private = offsetof(struct bfq_group, stats.wait_time),
1177 .seq_show = bfqg_print_rwstat,
1178 },
1179 {
1180 .name = "bfq.io_merged",
1181 .private = offsetof(struct bfq_group, stats.merged),
1182 .seq_show = bfqg_print_rwstat,
1183 },
1184 {
1185 .name = "bfq.io_queued",
1186 .private = offsetof(struct bfq_group, stats.queued),
1187 .seq_show = bfqg_print_rwstat,
1188 },
1189#endif
1190
1191
1192 {
1193 .name = "bfq.io_service_bytes_recursive",
1194 .private = (unsigned long)&blkcg_policy_bfq,
1195 .seq_show = blkg_print_stat_bytes_recursive,
1196 },
1197 {
1198 .name = "bfq.io_serviced_recursive",
1199 .private = (unsigned long)&blkcg_policy_bfq,
1200 .seq_show = blkg_print_stat_ios_recursive,
1201 },
1202#ifdef CONFIG_BFQ_CGROUP_DEBUG
1203 {
1204 .name = "bfq.time_recursive",
1205 .private = offsetof(struct bfq_group, stats.time),
1206 .seq_show = bfqg_print_stat_recursive,
1207 },
1208 {
1209 .name = "bfq.sectors_recursive",
1210 .seq_show = bfqg_print_stat_sectors_recursive,
1211 },
1212 {
1213 .name = "bfq.io_service_time_recursive",
1214 .private = offsetof(struct bfq_group, stats.service_time),
1215 .seq_show = bfqg_print_rwstat_recursive,
1216 },
1217 {
1218 .name = "bfq.io_wait_time_recursive",
1219 .private = offsetof(struct bfq_group, stats.wait_time),
1220 .seq_show = bfqg_print_rwstat_recursive,
1221 },
1222 {
1223 .name = "bfq.io_merged_recursive",
1224 .private = offsetof(struct bfq_group, stats.merged),
1225 .seq_show = bfqg_print_rwstat_recursive,
1226 },
1227 {
1228 .name = "bfq.io_queued_recursive",
1229 .private = offsetof(struct bfq_group, stats.queued),
1230 .seq_show = bfqg_print_rwstat_recursive,
1231 },
1232 {
1233 .name = "bfq.avg_queue_size",
1234 .seq_show = bfqg_print_avg_queue_size,
1235 },
1236 {
1237 .name = "bfq.group_wait_time",
1238 .private = offsetof(struct bfq_group, stats.group_wait_time),
1239 .seq_show = bfqg_print_stat,
1240 },
1241 {
1242 .name = "bfq.idle_time",
1243 .private = offsetof(struct bfq_group, stats.idle_time),
1244 .seq_show = bfqg_print_stat,
1245 },
1246 {
1247 .name = "bfq.empty_time",
1248 .private = offsetof(struct bfq_group, stats.empty_time),
1249 .seq_show = bfqg_print_stat,
1250 },
1251 {
1252 .name = "bfq.dequeue",
1253 .private = offsetof(struct bfq_group, stats.dequeue),
1254 .seq_show = bfqg_print_stat,
1255 },
1256#endif
1257 { }
1258};
1259
1260struct cftype bfq_blkg_files[] = {
1261 {
1262 .name = "bfq.weight",
1263 .flags = CFTYPE_NOT_ON_ROOT,
1264 .seq_show = bfq_io_show_weight,
1265 .write = bfq_io_set_weight,
1266 },
1267 {}
1268};
1269
1270#else
1271
1272void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
1273 struct bfq_group *bfqg) {}
1274
1275void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
1276{
1277 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
1278
1279 entity->weight = entity->new_weight;
1280 entity->orig_weight = entity->new_weight;
1281 if (bfqq) {
1282 bfqq->ioprio = bfqq->new_ioprio;
1283 bfqq->ioprio_class = bfqq->new_ioprio_class;
1284 }
1285 entity->sched_data = &bfqg->sched_data;
1286}
1287
1288void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) {}
1289
1290void bfq_end_wr_async(struct bfq_data *bfqd)
1291{
1292 bfq_end_wr_async_queues(bfqd, bfqd->root_group);
1293}
1294
1295struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, struct blkcg *blkcg)
1296{
1297 return bfqd->root_group;
1298}
1299
1300struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
1301{
1302 return bfqq->bfqd->root_group;
1303}
1304
1305struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
1306{
1307 struct bfq_group *bfqg;
1308 int i;
1309
1310 bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);
1311 if (!bfqg)
1312 return NULL;
1313
1314 for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
1315 bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
1316
1317 return bfqg;
1318}
1319#endif
1320