1
2
3
4
5
6#include "sched.h"
7
8#include "pelt.h"
9
10int sched_rr_timeslice = RR_TIMESLICE;
11int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;
12
13static const u64 max_rt_runtime = MAX_BW;
14
15static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
16
17struct rt_bandwidth def_rt_bandwidth;
18
19static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
20{
21 struct rt_bandwidth *rt_b =
22 container_of(timer, struct rt_bandwidth, rt_period_timer);
23 int idle = 0;
24 int overrun;
25
26 raw_spin_lock(&rt_b->rt_runtime_lock);
27 for (;;) {
28 overrun = hrtimer_forward_now(timer, rt_b->rt_period);
29 if (!overrun)
30 break;
31
32 raw_spin_unlock(&rt_b->rt_runtime_lock);
33 idle = do_sched_rt_period_timer(rt_b, overrun);
34 raw_spin_lock(&rt_b->rt_runtime_lock);
35 }
36 if (idle)
37 rt_b->rt_period_active = 0;
38 raw_spin_unlock(&rt_b->rt_runtime_lock);
39
40 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
41}
42
43void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
44{
45 rt_b->rt_period = ns_to_ktime(period);
46 rt_b->rt_runtime = runtime;
47
48 raw_spin_lock_init(&rt_b->rt_runtime_lock);
49
50 hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC,
51 HRTIMER_MODE_REL_HARD);
52 rt_b->rt_period_timer.function = sched_rt_period_timer;
53}
54
55static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
56{
57 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
58 return;
59
60 raw_spin_lock(&rt_b->rt_runtime_lock);
61 if (!rt_b->rt_period_active) {
62 rt_b->rt_period_active = 1;
63
64
65
66
67
68
69
70
71 hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
72 hrtimer_start_expires(&rt_b->rt_period_timer,
73 HRTIMER_MODE_ABS_PINNED_HARD);
74 }
75 raw_spin_unlock(&rt_b->rt_runtime_lock);
76}
77
78void init_rt_rq(struct rt_rq *rt_rq)
79{
80 struct rt_prio_array *array;
81 int i;
82
83 array = &rt_rq->active;
84 for (i = 0; i < MAX_RT_PRIO; i++) {
85 INIT_LIST_HEAD(array->queue + i);
86 __clear_bit(i, array->bitmap);
87 }
88
89 __set_bit(MAX_RT_PRIO, array->bitmap);
90
91#if defined CONFIG_SMP
92 rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
93 rt_rq->highest_prio.next = MAX_RT_PRIO-1;
94 rt_rq->rt_nr_migratory = 0;
95 rt_rq->overloaded = 0;
96 plist_head_init(&rt_rq->pushable_tasks);
97#endif
98
99 rt_rq->rt_queued = 0;
100
101 rt_rq->rt_time = 0;
102 rt_rq->rt_throttled = 0;
103 rt_rq->rt_runtime = 0;
104 raw_spin_lock_init(&rt_rq->rt_runtime_lock);
105}
106
107#ifdef CONFIG_RT_GROUP_SCHED
108static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
109{
110 hrtimer_cancel(&rt_b->rt_period_timer);
111}
112
113#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
114
115static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
116{
117#ifdef CONFIG_SCHED_DEBUG
118 WARN_ON_ONCE(!rt_entity_is_task(rt_se));
119#endif
120 return container_of(rt_se, struct task_struct, rt);
121}
122
123static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
124{
125 return rt_rq->rq;
126}
127
128static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
129{
130 return rt_se->rt_rq;
131}
132
133static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
134{
135 struct rt_rq *rt_rq = rt_se->rt_rq;
136
137 return rt_rq->rq;
138}
139
140void free_rt_sched_group(struct task_group *tg)
141{
142 int i;
143
144 if (tg->rt_se)
145 destroy_rt_bandwidth(&tg->rt_bandwidth);
146
147 for_each_possible_cpu(i) {
148 if (tg->rt_rq)
149 kfree(tg->rt_rq[i]);
150 if (tg->rt_se)
151 kfree(tg->rt_se[i]);
152 }
153
154 kfree(tg->rt_rq);
155 kfree(tg->rt_se);
156}
157
158void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
159 struct sched_rt_entity *rt_se, int cpu,
160 struct sched_rt_entity *parent)
161{
162 struct rq *rq = cpu_rq(cpu);
163
164 rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
165 rt_rq->rt_nr_boosted = 0;
166 rt_rq->rq = rq;
167 rt_rq->tg = tg;
168
169 tg->rt_rq[cpu] = rt_rq;
170 tg->rt_se[cpu] = rt_se;
171
172 if (!rt_se)
173 return;
174
175 if (!parent)
176 rt_se->rt_rq = &rq->rt;
177 else
178 rt_se->rt_rq = parent->my_q;
179
180 rt_se->my_q = rt_rq;
181 rt_se->parent = parent;
182 INIT_LIST_HEAD(&rt_se->run_list);
183}
184
185int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
186{
187 struct rt_rq *rt_rq;
188 struct sched_rt_entity *rt_se;
189 int i;
190
191 tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);
192 if (!tg->rt_rq)
193 goto err;
194 tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);
195 if (!tg->rt_se)
196 goto err;
197
198 init_rt_bandwidth(&tg->rt_bandwidth,
199 ktime_to_ns(def_rt_bandwidth.rt_period), 0);
200
201 for_each_possible_cpu(i) {
202 rt_rq = kzalloc_node(sizeof(struct rt_rq),
203 GFP_KERNEL, cpu_to_node(i));
204 if (!rt_rq)
205 goto err;
206
207 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
208 GFP_KERNEL, cpu_to_node(i));
209 if (!rt_se)
210 goto err_free_rq;
211
212 init_rt_rq(rt_rq);
213 rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
214 init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
215 }
216
217 return 1;
218
219err_free_rq:
220 kfree(rt_rq);
221err:
222 return 0;
223}
224
225#else
226
227#define rt_entity_is_task(rt_se) (1)
228
229static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
230{
231 return container_of(rt_se, struct task_struct, rt);
232}
233
234static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
235{
236 return container_of(rt_rq, struct rq, rt);
237}
238
239static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
240{
241 struct task_struct *p = rt_task_of(rt_se);
242
243 return task_rq(p);
244}
245
246static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
247{
248 struct rq *rq = rq_of_rt_se(rt_se);
249
250 return &rq->rt;
251}
252
253void free_rt_sched_group(struct task_group *tg) { }
254
255int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
256{
257 return 1;
258}
259#endif
260
261#ifdef CONFIG_SMP
262
263static void pull_rt_task(struct rq *this_rq);
264
265static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
266{
267
268 return rq->online && rq->rt.highest_prio.curr > prev->prio;
269}
270
271static inline int rt_overloaded(struct rq *rq)
272{
273 return atomic_read(&rq->rd->rto_count);
274}
275
276static inline void rt_set_overload(struct rq *rq)
277{
278 if (!rq->online)
279 return;
280
281 cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
282
283
284
285
286
287
288
289
290
291 smp_wmb();
292 atomic_inc(&rq->rd->rto_count);
293}
294
295static inline void rt_clear_overload(struct rq *rq)
296{
297 if (!rq->online)
298 return;
299
300
301 atomic_dec(&rq->rd->rto_count);
302 cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
303}
304
305static void update_rt_migration(struct rt_rq *rt_rq)
306{
307 if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
308 if (!rt_rq->overloaded) {
309 rt_set_overload(rq_of_rt_rq(rt_rq));
310 rt_rq->overloaded = 1;
311 }
312 } else if (rt_rq->overloaded) {
313 rt_clear_overload(rq_of_rt_rq(rt_rq));
314 rt_rq->overloaded = 0;
315 }
316}
317
318static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
319{
320 struct task_struct *p;
321
322 if (!rt_entity_is_task(rt_se))
323 return;
324
325 p = rt_task_of(rt_se);
326 rt_rq = &rq_of_rt_rq(rt_rq)->rt;
327
328 rt_rq->rt_nr_total++;
329 if (p->nr_cpus_allowed > 1)
330 rt_rq->rt_nr_migratory++;
331
332 update_rt_migration(rt_rq);
333}
334
335static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
336{
337 struct task_struct *p;
338
339 if (!rt_entity_is_task(rt_se))
340 return;
341
342 p = rt_task_of(rt_se);
343 rt_rq = &rq_of_rt_rq(rt_rq)->rt;
344
345 rt_rq->rt_nr_total--;
346 if (p->nr_cpus_allowed > 1)
347 rt_rq->rt_nr_migratory--;
348
349 update_rt_migration(rt_rq);
350}
351
352static inline int has_pushable_tasks(struct rq *rq)
353{
354 return !plist_head_empty(&rq->rt.pushable_tasks);
355}
356
357static DEFINE_PER_CPU(struct callback_head, rt_push_head);
358static DEFINE_PER_CPU(struct callback_head, rt_pull_head);
359
360static void push_rt_tasks(struct rq *);
361static void pull_rt_task(struct rq *);
362
363static inline void rt_queue_push_tasks(struct rq *rq)
364{
365 if (!has_pushable_tasks(rq))
366 return;
367
368 queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);
369}
370
371static inline void rt_queue_pull_task(struct rq *rq)
372{
373 queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);
374}
375
376static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
377{
378 plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
379 plist_node_init(&p->pushable_tasks, p->prio);
380 plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
381
382
383 if (p->prio < rq->rt.highest_prio.next)
384 rq->rt.highest_prio.next = p->prio;
385}
386
387static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
388{
389 plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
390
391
392 if (has_pushable_tasks(rq)) {
393 p = plist_first_entry(&rq->rt.pushable_tasks,
394 struct task_struct, pushable_tasks);
395 rq->rt.highest_prio.next = p->prio;
396 } else {
397 rq->rt.highest_prio.next = MAX_RT_PRIO-1;
398 }
399}
400
401#else
402
403static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
404{
405}
406
407static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
408{
409}
410
411static inline
412void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
413{
414}
415
416static inline
417void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
418{
419}
420
421static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
422{
423 return false;
424}
425
426static inline void pull_rt_task(struct rq *this_rq)
427{
428}
429
430static inline void rt_queue_push_tasks(struct rq *rq)
431{
432}
433#endif
434
435static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
436static void dequeue_top_rt_rq(struct rt_rq *rt_rq);
437
438static inline int on_rt_rq(struct sched_rt_entity *rt_se)
439{
440 return rt_se->on_rq;
441}
442
443#ifdef CONFIG_UCLAMP_TASK
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
459{
460 unsigned int min_cap;
461 unsigned int max_cap;
462 unsigned int cpu_cap;
463
464
465 if (!static_branch_unlikely(&sched_asym_cpucapacity))
466 return true;
467
468 min_cap = uclamp_eff_value(p, UCLAMP_MIN);
469 max_cap = uclamp_eff_value(p, UCLAMP_MAX);
470
471 cpu_cap = capacity_orig_of(cpu);
472
473 return cpu_cap >= min(min_cap, max_cap);
474}
475#else
476static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
477{
478 return true;
479}
480#endif
481
482#ifdef CONFIG_RT_GROUP_SCHED
483
484static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
485{
486 if (!rt_rq->tg)
487 return RUNTIME_INF;
488
489 return rt_rq->rt_runtime;
490}
491
492static inline u64 sched_rt_period(struct rt_rq *rt_rq)
493{
494 return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
495}
496
497typedef struct task_group *rt_rq_iter_t;
498
499static inline struct task_group *next_task_group(struct task_group *tg)
500{
501 do {
502 tg = list_entry_rcu(tg->list.next,
503 typeof(struct task_group), list);
504 } while (&tg->list != &task_groups && task_group_is_autogroup(tg));
505
506 if (&tg->list == &task_groups)
507 tg = NULL;
508
509 return tg;
510}
511
512#define for_each_rt_rq(rt_rq, iter, rq) \
513 for (iter = container_of(&task_groups, typeof(*iter), list); \
514 (iter = next_task_group(iter)) && \
515 (rt_rq = iter->rt_rq[cpu_of(rq)]);)
516
517#define for_each_sched_rt_entity(rt_se) \
518 for (; rt_se; rt_se = rt_se->parent)
519
520static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
521{
522 return rt_se->my_q;
523}
524
525static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
526static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
527
528static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
529{
530 struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
531 struct rq *rq = rq_of_rt_rq(rt_rq);
532 struct sched_rt_entity *rt_se;
533
534 int cpu = cpu_of(rq);
535
536 rt_se = rt_rq->tg->rt_se[cpu];
537
538 if (rt_rq->rt_nr_running) {
539 if (!rt_se)
540 enqueue_top_rt_rq(rt_rq);
541 else if (!on_rt_rq(rt_se))
542 enqueue_rt_entity(rt_se, 0);
543
544 if (rt_rq->highest_prio.curr < curr->prio)
545 resched_curr(rq);
546 }
547}
548
549static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
550{
551 struct sched_rt_entity *rt_se;
552 int cpu = cpu_of(rq_of_rt_rq(rt_rq));
553
554 rt_se = rt_rq->tg->rt_se[cpu];
555
556 if (!rt_se) {
557 dequeue_top_rt_rq(rt_rq);
558
559 cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);
560 }
561 else if (on_rt_rq(rt_se))
562 dequeue_rt_entity(rt_se, 0);
563}
564
565static inline int rt_rq_throttled(struct rt_rq *rt_rq)
566{
567 return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
568}
569
570static int rt_se_boosted(struct sched_rt_entity *rt_se)
571{
572 struct rt_rq *rt_rq = group_rt_rq(rt_se);
573 struct task_struct *p;
574
575 if (rt_rq)
576 return !!rt_rq->rt_nr_boosted;
577
578 p = rt_task_of(rt_se);
579 return p->prio != p->normal_prio;
580}
581
582#ifdef CONFIG_SMP
583static inline const struct cpumask *sched_rt_period_mask(void)
584{
585 return this_rq()->rd->span;
586}
587#else
588static inline const struct cpumask *sched_rt_period_mask(void)
589{
590 return cpu_online_mask;
591}
592#endif
593
594static inline
595struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
596{
597 return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
598}
599
600static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
601{
602 return &rt_rq->tg->rt_bandwidth;
603}
604
605#else
606
607static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
608{
609 return rt_rq->rt_runtime;
610}
611
612static inline u64 sched_rt_period(struct rt_rq *rt_rq)
613{
614 return ktime_to_ns(def_rt_bandwidth.rt_period);
615}
616
617typedef struct rt_rq *rt_rq_iter_t;
618
619#define for_each_rt_rq(rt_rq, iter, rq) \
620 for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
621
622#define for_each_sched_rt_entity(rt_se) \
623 for (; rt_se; rt_se = NULL)
624
625static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
626{
627 return NULL;
628}
629
630static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
631{
632 struct rq *rq = rq_of_rt_rq(rt_rq);
633
634 if (!rt_rq->rt_nr_running)
635 return;
636
637 enqueue_top_rt_rq(rt_rq);
638 resched_curr(rq);
639}
640
641static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
642{
643 dequeue_top_rt_rq(rt_rq);
644}
645
646static inline int rt_rq_throttled(struct rt_rq *rt_rq)
647{
648 return rt_rq->rt_throttled;
649}
650
651static inline const struct cpumask *sched_rt_period_mask(void)
652{
653 return cpu_online_mask;
654}
655
656static inline
657struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
658{
659 return &cpu_rq(cpu)->rt;
660}
661
662static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
663{
664 return &def_rt_bandwidth;
665}
666
667#endif
668
669bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
670{
671 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
672
673 return (hrtimer_active(&rt_b->rt_period_timer) ||
674 rt_rq->rt_time < rt_b->rt_runtime);
675}
676
677#ifdef CONFIG_SMP
678
679
680
681static void do_balance_runtime(struct rt_rq *rt_rq)
682{
683 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
684 struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
685 int i, weight;
686 u64 rt_period;
687
688 weight = cpumask_weight(rd->span);
689
690 raw_spin_lock(&rt_b->rt_runtime_lock);
691 rt_period = ktime_to_ns(rt_b->rt_period);
692 for_each_cpu(i, rd->span) {
693 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
694 s64 diff;
695
696 if (iter == rt_rq)
697 continue;
698
699 raw_spin_lock(&iter->rt_runtime_lock);
700
701
702
703
704
705 if (iter->rt_runtime == RUNTIME_INF)
706 goto next;
707
708
709
710
711
712 diff = iter->rt_runtime - iter->rt_time;
713 if (diff > 0) {
714 diff = div_u64((u64)diff, weight);
715 if (rt_rq->rt_runtime + diff > rt_period)
716 diff = rt_period - rt_rq->rt_runtime;
717 iter->rt_runtime -= diff;
718 rt_rq->rt_runtime += diff;
719 if (rt_rq->rt_runtime == rt_period) {
720 raw_spin_unlock(&iter->rt_runtime_lock);
721 break;
722 }
723 }
724next:
725 raw_spin_unlock(&iter->rt_runtime_lock);
726 }
727 raw_spin_unlock(&rt_b->rt_runtime_lock);
728}
729
730
731
732
733static void __disable_runtime(struct rq *rq)
734{
735 struct root_domain *rd = rq->rd;
736 rt_rq_iter_t iter;
737 struct rt_rq *rt_rq;
738
739 if (unlikely(!scheduler_running))
740 return;
741
742 for_each_rt_rq(rt_rq, iter, rq) {
743 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
744 s64 want;
745 int i;
746
747 raw_spin_lock(&rt_b->rt_runtime_lock);
748 raw_spin_lock(&rt_rq->rt_runtime_lock);
749
750
751
752
753
754 if (rt_rq->rt_runtime == RUNTIME_INF ||
755 rt_rq->rt_runtime == rt_b->rt_runtime)
756 goto balanced;
757 raw_spin_unlock(&rt_rq->rt_runtime_lock);
758
759
760
761
762
763
764 want = rt_b->rt_runtime - rt_rq->rt_runtime;
765
766
767
768
769 for_each_cpu(i, rd->span) {
770 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
771 s64 diff;
772
773
774
775
776 if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
777 continue;
778
779 raw_spin_lock(&iter->rt_runtime_lock);
780 if (want > 0) {
781 diff = min_t(s64, iter->rt_runtime, want);
782 iter->rt_runtime -= diff;
783 want -= diff;
784 } else {
785 iter->rt_runtime -= want;
786 want -= want;
787 }
788 raw_spin_unlock(&iter->rt_runtime_lock);
789
790 if (!want)
791 break;
792 }
793
794 raw_spin_lock(&rt_rq->rt_runtime_lock);
795
796
797
798
799 BUG_ON(want);
800balanced:
801
802
803
804
805 rt_rq->rt_runtime = RUNTIME_INF;
806 rt_rq->rt_throttled = 0;
807 raw_spin_unlock(&rt_rq->rt_runtime_lock);
808 raw_spin_unlock(&rt_b->rt_runtime_lock);
809
810
811 sched_rt_rq_enqueue(rt_rq);
812 }
813}
814
815static void __enable_runtime(struct rq *rq)
816{
817 rt_rq_iter_t iter;
818 struct rt_rq *rt_rq;
819
820 if (unlikely(!scheduler_running))
821 return;
822
823
824
825
826 for_each_rt_rq(rt_rq, iter, rq) {
827 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
828
829 raw_spin_lock(&rt_b->rt_runtime_lock);
830 raw_spin_lock(&rt_rq->rt_runtime_lock);
831 rt_rq->rt_runtime = rt_b->rt_runtime;
832 rt_rq->rt_time = 0;
833 rt_rq->rt_throttled = 0;
834 raw_spin_unlock(&rt_rq->rt_runtime_lock);
835 raw_spin_unlock(&rt_b->rt_runtime_lock);
836 }
837}
838
839static void balance_runtime(struct rt_rq *rt_rq)
840{
841 if (!sched_feat(RT_RUNTIME_SHARE))
842 return;
843
844 if (rt_rq->rt_time > rt_rq->rt_runtime) {
845 raw_spin_unlock(&rt_rq->rt_runtime_lock);
846 do_balance_runtime(rt_rq);
847 raw_spin_lock(&rt_rq->rt_runtime_lock);
848 }
849}
850#else
851static inline void balance_runtime(struct rt_rq *rt_rq) {}
852#endif
853
854static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
855{
856 int i, idle = 1, throttled = 0;
857 const struct cpumask *span;
858
859 span = sched_rt_period_mask();
860#ifdef CONFIG_RT_GROUP_SCHED
861
862
863
864
865
866
867
868
869
870 if (rt_b == &root_task_group.rt_bandwidth)
871 span = cpu_online_mask;
872#endif
873 for_each_cpu(i, span) {
874 int enqueue = 0;
875 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
876 struct rq *rq = rq_of_rt_rq(rt_rq);
877 int skip;
878
879
880
881
882
883 raw_spin_lock(&rt_rq->rt_runtime_lock);
884 if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF)
885 rt_rq->rt_runtime = rt_b->rt_runtime;
886 skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;
887 raw_spin_unlock(&rt_rq->rt_runtime_lock);
888 if (skip)
889 continue;
890
891 raw_spin_rq_lock(rq);
892 update_rq_clock(rq);
893
894 if (rt_rq->rt_time) {
895 u64 runtime;
896
897 raw_spin_lock(&rt_rq->rt_runtime_lock);
898 if (rt_rq->rt_throttled)
899 balance_runtime(rt_rq);
900 runtime = rt_rq->rt_runtime;
901 rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
902 if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
903 rt_rq->rt_throttled = 0;
904 enqueue = 1;
905
906
907
908
909
910
911
912
913 if (rt_rq->rt_nr_running && rq->curr == rq->idle)
914 rq_clock_cancel_skipupdate(rq);
915 }
916 if (rt_rq->rt_time || rt_rq->rt_nr_running)
917 idle = 0;
918 raw_spin_unlock(&rt_rq->rt_runtime_lock);
919 } else if (rt_rq->rt_nr_running) {
920 idle = 0;
921 if (!rt_rq_throttled(rt_rq))
922 enqueue = 1;
923 }
924 if (rt_rq->rt_throttled)
925 throttled = 1;
926
927 if (enqueue)
928 sched_rt_rq_enqueue(rt_rq);
929 raw_spin_rq_unlock(rq);
930 }
931
932 if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
933 return 1;
934
935 return idle;
936}
937
938static inline int rt_se_prio(struct sched_rt_entity *rt_se)
939{
940#ifdef CONFIG_RT_GROUP_SCHED
941 struct rt_rq *rt_rq = group_rt_rq(rt_se);
942
943 if (rt_rq)
944 return rt_rq->highest_prio.curr;
945#endif
946
947 return rt_task_of(rt_se)->prio;
948}
949
950static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
951{
952 u64 runtime = sched_rt_runtime(rt_rq);
953
954 if (rt_rq->rt_throttled)
955 return rt_rq_throttled(rt_rq);
956
957 if (runtime >= sched_rt_period(rt_rq))
958 return 0;
959
960 balance_runtime(rt_rq);
961 runtime = sched_rt_runtime(rt_rq);
962 if (runtime == RUNTIME_INF)
963 return 0;
964
965 if (rt_rq->rt_time > runtime) {
966 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
967
968
969
970
971
972 if (likely(rt_b->rt_runtime)) {
973 rt_rq->rt_throttled = 1;
974 printk_deferred_once("sched: RT throttling activated\n");
975 } else {
976
977
978
979
980
981 rt_rq->rt_time = 0;
982 }
983
984 if (rt_rq_throttled(rt_rq)) {
985 sched_rt_rq_dequeue(rt_rq);
986 return 1;
987 }
988 }
989
990 return 0;
991}
992
993
994
995
996
997static void update_curr_rt(struct rq *rq)
998{
999 struct task_struct *curr = rq->curr;
1000 struct sched_rt_entity *rt_se = &curr->rt;
1001 u64 delta_exec;
1002 u64 now;
1003
1004 if (curr->sched_class != &rt_sched_class)
1005 return;
1006
1007 now = rq_clock_task(rq);
1008 delta_exec = now - curr->se.exec_start;
1009 if (unlikely((s64)delta_exec <= 0))
1010 return;
1011
1012 schedstat_set(curr->se.statistics.exec_max,
1013 max(curr->se.statistics.exec_max, delta_exec));
1014
1015 curr->se.sum_exec_runtime += delta_exec;
1016 account_group_exec_runtime(curr, delta_exec);
1017
1018 curr->se.exec_start = now;
1019 cgroup_account_cputime(curr, delta_exec);
1020
1021 if (!rt_bandwidth_enabled())
1022 return;
1023
1024 for_each_sched_rt_entity(rt_se) {
1025 struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1026
1027 if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
1028 raw_spin_lock(&rt_rq->rt_runtime_lock);
1029 rt_rq->rt_time += delta_exec;
1030 if (sched_rt_runtime_exceeded(rt_rq))
1031 resched_curr(rq);
1032 raw_spin_unlock(&rt_rq->rt_runtime_lock);
1033 }
1034 }
1035}
1036
1037static void
1038dequeue_top_rt_rq(struct rt_rq *rt_rq)
1039{
1040 struct rq *rq = rq_of_rt_rq(rt_rq);
1041
1042 BUG_ON(&rq->rt != rt_rq);
1043
1044 if (!rt_rq->rt_queued)
1045 return;
1046
1047 BUG_ON(!rq->nr_running);
1048
1049 sub_nr_running(rq, rt_rq->rt_nr_running);
1050 rt_rq->rt_queued = 0;
1051
1052}
1053
1054static void
1055enqueue_top_rt_rq(struct rt_rq *rt_rq)
1056{
1057 struct rq *rq = rq_of_rt_rq(rt_rq);
1058
1059 BUG_ON(&rq->rt != rt_rq);
1060
1061 if (rt_rq->rt_queued)
1062 return;
1063
1064 if (rt_rq_throttled(rt_rq))
1065 return;
1066
1067 if (rt_rq->rt_nr_running) {
1068 add_nr_running(rq, rt_rq->rt_nr_running);
1069 rt_rq->rt_queued = 1;
1070 }
1071
1072
1073 cpufreq_update_util(rq, 0);
1074}
1075
1076#if defined CONFIG_SMP
1077
1078static void
1079inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1080{
1081 struct rq *rq = rq_of_rt_rq(rt_rq);
1082
1083#ifdef CONFIG_RT_GROUP_SCHED
1084
1085
1086
1087 if (&rq->rt != rt_rq)
1088 return;
1089#endif
1090 if (rq->online && prio < prev_prio)
1091 cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
1092}
1093
1094static void
1095dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1096{
1097 struct rq *rq = rq_of_rt_rq(rt_rq);
1098
1099#ifdef CONFIG_RT_GROUP_SCHED
1100
1101
1102
1103 if (&rq->rt != rt_rq)
1104 return;
1105#endif
1106 if (rq->online && rt_rq->highest_prio.curr != prev_prio)
1107 cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
1108}
1109
1110#else
1111
1112static inline
1113void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1114static inline
1115void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1116
1117#endif
1118
1119#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
1120static void
1121inc_rt_prio(struct rt_rq *rt_rq, int prio)
1122{
1123 int prev_prio = rt_rq->highest_prio.curr;
1124
1125 if (prio < prev_prio)
1126 rt_rq->highest_prio.curr = prio;
1127
1128 inc_rt_prio_smp(rt_rq, prio, prev_prio);
1129}
1130
1131static void
1132dec_rt_prio(struct rt_rq *rt_rq, int prio)
1133{
1134 int prev_prio = rt_rq->highest_prio.curr;
1135
1136 if (rt_rq->rt_nr_running) {
1137
1138 WARN_ON(prio < prev_prio);
1139
1140
1141
1142
1143
1144 if (prio == prev_prio) {
1145 struct rt_prio_array *array = &rt_rq->active;
1146
1147 rt_rq->highest_prio.curr =
1148 sched_find_first_bit(array->bitmap);
1149 }
1150
1151 } else {
1152 rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
1153 }
1154
1155 dec_rt_prio_smp(rt_rq, prio, prev_prio);
1156}
1157
1158#else
1159
1160static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}
1161static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}
1162
1163#endif
1164
1165#ifdef CONFIG_RT_GROUP_SCHED
1166
1167static void
1168inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1169{
1170 if (rt_se_boosted(rt_se))
1171 rt_rq->rt_nr_boosted++;
1172
1173 if (rt_rq->tg)
1174 start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
1175}
1176
1177static void
1178dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1179{
1180 if (rt_se_boosted(rt_se))
1181 rt_rq->rt_nr_boosted--;
1182
1183 WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
1184}
1185
1186#else
1187
1188static void
1189inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1190{
1191 start_rt_bandwidth(&def_rt_bandwidth);
1192}
1193
1194static inline
1195void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
1196
1197#endif
1198
1199static inline
1200unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
1201{
1202 struct rt_rq *group_rq = group_rt_rq(rt_se);
1203
1204 if (group_rq)
1205 return group_rq->rt_nr_running;
1206 else
1207 return 1;
1208}
1209
1210static inline
1211unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
1212{
1213 struct rt_rq *group_rq = group_rt_rq(rt_se);
1214 struct task_struct *tsk;
1215
1216 if (group_rq)
1217 return group_rq->rr_nr_running;
1218
1219 tsk = rt_task_of(rt_se);
1220
1221 return (tsk->policy == SCHED_RR) ? 1 : 0;
1222}
1223
1224static inline
1225void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1226{
1227 int prio = rt_se_prio(rt_se);
1228
1229 WARN_ON(!rt_prio(prio));
1230 rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
1231 rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);
1232
1233 inc_rt_prio(rt_rq, prio);
1234 inc_rt_migration(rt_se, rt_rq);
1235 inc_rt_group(rt_se, rt_rq);
1236}
1237
1238static inline
1239void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1240{
1241 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
1242 WARN_ON(!rt_rq->rt_nr_running);
1243 rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
1244 rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);
1245
1246 dec_rt_prio(rt_rq, rt_se_prio(rt_se));
1247 dec_rt_migration(rt_se, rt_rq);
1248 dec_rt_group(rt_se, rt_rq);
1249}
1250
1251
1252
1253
1254
1255
1256static inline bool move_entity(unsigned int flags)
1257{
1258 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE)
1259 return false;
1260
1261 return true;
1262}
1263
1264static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)
1265{
1266 list_del_init(&rt_se->run_list);
1267
1268 if (list_empty(array->queue + rt_se_prio(rt_se)))
1269 __clear_bit(rt_se_prio(rt_se), array->bitmap);
1270
1271 rt_se->on_list = 0;
1272}
1273
1274static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1275{
1276 struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1277 struct rt_prio_array *array = &rt_rq->active;
1278 struct rt_rq *group_rq = group_rt_rq(rt_se);
1279 struct list_head *queue = array->queue + rt_se_prio(rt_se);
1280
1281
1282
1283
1284
1285
1286
1287 if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) {
1288 if (rt_se->on_list)
1289 __delist_rt_entity(rt_se, array);
1290 return;
1291 }
1292
1293 if (move_entity(flags)) {
1294 WARN_ON_ONCE(rt_se->on_list);
1295 if (flags & ENQUEUE_HEAD)
1296 list_add(&rt_se->run_list, queue);
1297 else
1298 list_add_tail(&rt_se->run_list, queue);
1299
1300 __set_bit(rt_se_prio(rt_se), array->bitmap);
1301 rt_se->on_list = 1;
1302 }
1303 rt_se->on_rq = 1;
1304
1305 inc_rt_tasks(rt_se, rt_rq);
1306}
1307
1308static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1309{
1310 struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1311 struct rt_prio_array *array = &rt_rq->active;
1312
1313 if (move_entity(flags)) {
1314 WARN_ON_ONCE(!rt_se->on_list);
1315 __delist_rt_entity(rt_se, array);
1316 }
1317 rt_se->on_rq = 0;
1318
1319 dec_rt_tasks(rt_se, rt_rq);
1320}
1321
1322
1323
1324
1325
1326static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
1327{
1328 struct sched_rt_entity *back = NULL;
1329
1330 for_each_sched_rt_entity(rt_se) {
1331 rt_se->back = back;
1332 back = rt_se;
1333 }
1334
1335 dequeue_top_rt_rq(rt_rq_of_se(back));
1336
1337 for (rt_se = back; rt_se; rt_se = rt_se->back) {
1338 if (on_rt_rq(rt_se))
1339 __dequeue_rt_entity(rt_se, flags);
1340 }
1341}
1342
1343static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1344{
1345 struct rq *rq = rq_of_rt_se(rt_se);
1346
1347 dequeue_rt_stack(rt_se, flags);
1348 for_each_sched_rt_entity(rt_se)
1349 __enqueue_rt_entity(rt_se, flags);
1350 enqueue_top_rt_rq(&rq->rt);
1351}
1352
1353static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1354{
1355 struct rq *rq = rq_of_rt_se(rt_se);
1356
1357 dequeue_rt_stack(rt_se, flags);
1358
1359 for_each_sched_rt_entity(rt_se) {
1360 struct rt_rq *rt_rq = group_rt_rq(rt_se);
1361
1362 if (rt_rq && rt_rq->rt_nr_running)
1363 __enqueue_rt_entity(rt_se, flags);
1364 }
1365 enqueue_top_rt_rq(&rq->rt);
1366}
1367
1368
1369
1370
1371static void
1372enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
1373{
1374 struct sched_rt_entity *rt_se = &p->rt;
1375
1376 if (flags & ENQUEUE_WAKEUP)
1377 rt_se->timeout = 0;
1378
1379 enqueue_rt_entity(rt_se, flags);
1380
1381 if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
1382 enqueue_pushable_task(rq, p);
1383}
1384
1385static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
1386{
1387 struct sched_rt_entity *rt_se = &p->rt;
1388
1389 update_curr_rt(rq);
1390 dequeue_rt_entity(rt_se, flags);
1391
1392 dequeue_pushable_task(rq, p);
1393}
1394
1395
1396
1397
1398
1399static void
1400requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
1401{
1402 if (on_rt_rq(rt_se)) {
1403 struct rt_prio_array *array = &rt_rq->active;
1404 struct list_head *queue = array->queue + rt_se_prio(rt_se);
1405
1406 if (head)
1407 list_move(&rt_se->run_list, queue);
1408 else
1409 list_move_tail(&rt_se->run_list, queue);
1410 }
1411}
1412
1413static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
1414{
1415 struct sched_rt_entity *rt_se = &p->rt;
1416 struct rt_rq *rt_rq;
1417
1418 for_each_sched_rt_entity(rt_se) {
1419 rt_rq = rt_rq_of_se(rt_se);
1420 requeue_rt_entity(rt_rq, rt_se, head);
1421 }
1422}
1423
1424static void yield_task_rt(struct rq *rq)
1425{
1426 requeue_task_rt(rq, rq->curr, 0);
1427}
1428
1429#ifdef CONFIG_SMP
1430static int find_lowest_rq(struct task_struct *task);
1431
1432static int
1433select_task_rq_rt(struct task_struct *p, int cpu, int flags)
1434{
1435 struct task_struct *curr;
1436 struct rq *rq;
1437 bool test;
1438
1439
1440 if (!(flags & (WF_TTWU | WF_FORK)))
1441 goto out;
1442
1443 rq = cpu_rq(cpu);
1444
1445 rcu_read_lock();
1446 curr = READ_ONCE(rq->curr);
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474 test = curr &&
1475 unlikely(rt_task(curr)) &&
1476 (curr->nr_cpus_allowed < 2 || curr->prio <= p->prio);
1477
1478 if (test || !rt_task_fits_capacity(p, cpu)) {
1479 int target = find_lowest_rq(p);
1480
1481
1482
1483
1484
1485 if (!test && target != -1 && !rt_task_fits_capacity(p, target))
1486 goto out_unlock;
1487
1488
1489
1490
1491
1492 if (target != -1 &&
1493 p->prio < cpu_rq(target)->rt.highest_prio.curr)
1494 cpu = target;
1495 }
1496
1497out_unlock:
1498 rcu_read_unlock();
1499
1500out:
1501 return cpu;
1502}
1503
1504static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
1505{
1506
1507
1508
1509
1510 if (rq->curr->nr_cpus_allowed == 1 ||
1511 !cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
1512 return;
1513
1514
1515
1516
1517
1518 if (p->nr_cpus_allowed != 1 &&
1519 cpupri_find(&rq->rd->cpupri, p, NULL))
1520 return;
1521
1522
1523
1524
1525
1526
1527 requeue_task_rt(rq, p, 1);
1528 resched_curr(rq);
1529}
1530
1531static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
1532{
1533 if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
1534
1535
1536
1537
1538
1539
1540 rq_unpin_lock(rq, rf);
1541 pull_rt_task(rq);
1542 rq_repin_lock(rq, rf);
1543 }
1544
1545 return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq);
1546}
1547#endif
1548
1549
1550
1551
1552static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
1553{
1554 if (p->prio < rq->curr->prio) {
1555 resched_curr(rq);
1556 return;
1557 }
1558
1559#ifdef CONFIG_SMP
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572 if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr))
1573 check_preempt_equal_prio(rq, p);
1574#endif
1575}
1576
1577static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
1578{
1579 p->se.exec_start = rq_clock_task(rq);
1580
1581
1582 dequeue_pushable_task(rq, p);
1583
1584 if (!first)
1585 return;
1586
1587
1588
1589
1590
1591
1592 if (rq->curr->sched_class != &rt_sched_class)
1593 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
1594
1595 rt_queue_push_tasks(rq);
1596}
1597
1598static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
1599 struct rt_rq *rt_rq)
1600{
1601 struct rt_prio_array *array = &rt_rq->active;
1602 struct sched_rt_entity *next = NULL;
1603 struct list_head *queue;
1604 int idx;
1605
1606 idx = sched_find_first_bit(array->bitmap);
1607 BUG_ON(idx >= MAX_RT_PRIO);
1608
1609 queue = array->queue + idx;
1610 next = list_entry(queue->next, struct sched_rt_entity, run_list);
1611
1612 return next;
1613}
1614
1615static struct task_struct *_pick_next_task_rt(struct rq *rq)
1616{
1617 struct sched_rt_entity *rt_se;
1618 struct rt_rq *rt_rq = &rq->rt;
1619
1620 do {
1621 rt_se = pick_next_rt_entity(rq, rt_rq);
1622 BUG_ON(!rt_se);
1623 rt_rq = group_rt_rq(rt_se);
1624 } while (rt_rq);
1625
1626 return rt_task_of(rt_se);
1627}
1628
1629static struct task_struct *pick_task_rt(struct rq *rq)
1630{
1631 struct task_struct *p;
1632
1633 if (!sched_rt_runnable(rq))
1634 return NULL;
1635
1636 p = _pick_next_task_rt(rq);
1637
1638 return p;
1639}
1640
1641static struct task_struct *pick_next_task_rt(struct rq *rq)
1642{
1643 struct task_struct *p = pick_task_rt(rq);
1644
1645 if (p)
1646 set_next_task_rt(rq, p, true);
1647
1648 return p;
1649}
1650
1651static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
1652{
1653 update_curr_rt(rq);
1654
1655 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
1656
1657
1658
1659
1660
1661 if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
1662 enqueue_pushable_task(rq, p);
1663}
1664
1665#ifdef CONFIG_SMP
1666
1667
1668#define RT_MAX_TRIES 3
1669
1670static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
1671{
1672 if (!task_running(rq, p) &&
1673 cpumask_test_cpu(cpu, &p->cpus_mask))
1674 return 1;
1675
1676 return 0;
1677}
1678
1679
1680
1681
1682
1683static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
1684{
1685 struct plist_head *head = &rq->rt.pushable_tasks;
1686 struct task_struct *p;
1687
1688 if (!has_pushable_tasks(rq))
1689 return NULL;
1690
1691 plist_for_each_entry(p, head, pushable_tasks) {
1692 if (pick_rt_task(rq, p, cpu))
1693 return p;
1694 }
1695
1696 return NULL;
1697}
1698
1699static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
1700
1701static int find_lowest_rq(struct task_struct *task)
1702{
1703 struct sched_domain *sd;
1704 struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
1705 int this_cpu = smp_processor_id();
1706 int cpu = task_cpu(task);
1707 int ret;
1708
1709
1710 if (unlikely(!lowest_mask))
1711 return -1;
1712
1713 if (task->nr_cpus_allowed == 1)
1714 return -1;
1715
1716
1717
1718
1719
1720 if (static_branch_unlikely(&sched_asym_cpucapacity)) {
1721
1722 ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri,
1723 task, lowest_mask,
1724 rt_task_fits_capacity);
1725 } else {
1726
1727 ret = cpupri_find(&task_rq(task)->rd->cpupri,
1728 task, lowest_mask);
1729 }
1730
1731 if (!ret)
1732 return -1;
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742 if (cpumask_test_cpu(cpu, lowest_mask))
1743 return cpu;
1744
1745
1746
1747
1748
1749 if (!cpumask_test_cpu(this_cpu, lowest_mask))
1750 this_cpu = -1;
1751
1752 rcu_read_lock();
1753 for_each_domain(cpu, sd) {
1754 if (sd->flags & SD_WAKE_AFFINE) {
1755 int best_cpu;
1756
1757
1758
1759
1760
1761 if (this_cpu != -1 &&
1762 cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
1763 rcu_read_unlock();
1764 return this_cpu;
1765 }
1766
1767 best_cpu = cpumask_any_and_distribute(lowest_mask,
1768 sched_domain_span(sd));
1769 if (best_cpu < nr_cpu_ids) {
1770 rcu_read_unlock();
1771 return best_cpu;
1772 }
1773 }
1774 }
1775 rcu_read_unlock();
1776
1777
1778
1779
1780
1781
1782 if (this_cpu != -1)
1783 return this_cpu;
1784
1785 cpu = cpumask_any_distribute(lowest_mask);
1786 if (cpu < nr_cpu_ids)
1787 return cpu;
1788
1789 return -1;
1790}
1791
1792
1793static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
1794{
1795 struct rq *lowest_rq = NULL;
1796 int tries;
1797 int cpu;
1798
1799 for (tries = 0; tries < RT_MAX_TRIES; tries++) {
1800 cpu = find_lowest_rq(task);
1801
1802 if ((cpu == -1) || (cpu == rq->cpu))
1803 break;
1804
1805 lowest_rq = cpu_rq(cpu);
1806
1807 if (lowest_rq->rt.highest_prio.curr <= task->prio) {
1808
1809
1810
1811
1812
1813 lowest_rq = NULL;
1814 break;
1815 }
1816
1817
1818 if (double_lock_balance(rq, lowest_rq)) {
1819
1820
1821
1822
1823
1824
1825 if (unlikely(task_rq(task) != rq ||
1826 !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) ||
1827 task_running(rq, task) ||
1828 !rt_task(task) ||
1829 !task_on_rq_queued(task))) {
1830
1831 double_unlock_balance(rq, lowest_rq);
1832 lowest_rq = NULL;
1833 break;
1834 }
1835 }
1836
1837
1838 if (lowest_rq->rt.highest_prio.curr > task->prio)
1839 break;
1840
1841
1842 double_unlock_balance(rq, lowest_rq);
1843 lowest_rq = NULL;
1844 }
1845
1846 return lowest_rq;
1847}
1848
1849static struct task_struct *pick_next_pushable_task(struct rq *rq)
1850{
1851 struct task_struct *p;
1852
1853 if (!has_pushable_tasks(rq))
1854 return NULL;
1855
1856 p = plist_first_entry(&rq->rt.pushable_tasks,
1857 struct task_struct, pushable_tasks);
1858
1859 BUG_ON(rq->cpu != task_cpu(p));
1860 BUG_ON(task_current(rq, p));
1861 BUG_ON(p->nr_cpus_allowed <= 1);
1862
1863 BUG_ON(!task_on_rq_queued(p));
1864 BUG_ON(!rt_task(p));
1865
1866 return p;
1867}
1868
1869
1870
1871
1872
1873
1874static int push_rt_task(struct rq *rq, bool pull)
1875{
1876 struct task_struct *next_task;
1877 struct rq *lowest_rq;
1878 int ret = 0;
1879
1880 if (!rq->rt.overloaded)
1881 return 0;
1882
1883 next_task = pick_next_pushable_task(rq);
1884 if (!next_task)
1885 return 0;
1886
1887retry:
1888 if (is_migration_disabled(next_task)) {
1889 struct task_struct *push_task = NULL;
1890 int cpu;
1891
1892 if (!pull || rq->push_busy)
1893 return 0;
1894
1895 cpu = find_lowest_rq(rq->curr);
1896 if (cpu == -1 || cpu == rq->cpu)
1897 return 0;
1898
1899
1900
1901
1902
1903
1904
1905 push_task = get_push_task(rq);
1906 if (push_task) {
1907 raw_spin_rq_unlock(rq);
1908 stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
1909 push_task, &rq->push_work);
1910 raw_spin_rq_lock(rq);
1911 }
1912
1913 return 0;
1914 }
1915
1916 if (WARN_ON(next_task == rq->curr))
1917 return 0;
1918
1919
1920
1921
1922
1923
1924 if (unlikely(next_task->prio < rq->curr->prio)) {
1925 resched_curr(rq);
1926 return 0;
1927 }
1928
1929
1930 get_task_struct(next_task);
1931
1932
1933 lowest_rq = find_lock_lowest_rq(next_task, rq);
1934 if (!lowest_rq) {
1935 struct task_struct *task;
1936
1937
1938
1939
1940
1941
1942
1943
1944 task = pick_next_pushable_task(rq);
1945 if (task == next_task) {
1946
1947
1948
1949
1950
1951
1952 goto out;
1953 }
1954
1955 if (!task)
1956
1957 goto out;
1958
1959
1960
1961
1962 put_task_struct(next_task);
1963 next_task = task;
1964 goto retry;
1965 }
1966
1967 deactivate_task(rq, next_task, 0);
1968 set_task_cpu(next_task, lowest_rq->cpu);
1969 activate_task(lowest_rq, next_task, 0);
1970 resched_curr(lowest_rq);
1971 ret = 1;
1972
1973 double_unlock_balance(rq, lowest_rq);
1974out:
1975 put_task_struct(next_task);
1976
1977 return ret;
1978}
1979
1980static void push_rt_tasks(struct rq *rq)
1981{
1982
1983 while (push_rt_task(rq, false))
1984 ;
1985}
1986
1987#ifdef HAVE_RT_PUSH_IPI
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030static int rto_next_cpu(struct root_domain *rd)
2031{
2032 int next;
2033 int cpu;
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048 for (;;) {
2049
2050
2051 cpu = cpumask_next(rd->rto_cpu, rd->rto_mask);
2052
2053 rd->rto_cpu = cpu;
2054
2055 if (cpu < nr_cpu_ids)
2056 return cpu;
2057
2058 rd->rto_cpu = -1;
2059
2060
2061
2062
2063
2064
2065
2066 next = atomic_read_acquire(&rd->rto_loop_next);
2067
2068 if (rd->rto_loop == next)
2069 break;
2070
2071 rd->rto_loop = next;
2072 }
2073
2074 return -1;
2075}
2076
2077static inline bool rto_start_trylock(atomic_t *v)
2078{
2079 return !atomic_cmpxchg_acquire(v, 0, 1);
2080}
2081
2082static inline void rto_start_unlock(atomic_t *v)
2083{
2084 atomic_set_release(v, 0);
2085}
2086
2087static void tell_cpu_to_push(struct rq *rq)
2088{
2089 int cpu = -1;
2090
2091
2092 atomic_inc(&rq->rd->rto_loop_next);
2093
2094
2095 if (!rto_start_trylock(&rq->rd->rto_loop_start))
2096 return;
2097
2098 raw_spin_lock(&rq->rd->rto_lock);
2099
2100
2101
2102
2103
2104
2105
2106 if (rq->rd->rto_cpu < 0)
2107 cpu = rto_next_cpu(rq->rd);
2108
2109 raw_spin_unlock(&rq->rd->rto_lock);
2110
2111 rto_start_unlock(&rq->rd->rto_loop_start);
2112
2113 if (cpu >= 0) {
2114
2115 sched_get_rd(rq->rd);
2116 irq_work_queue_on(&rq->rd->rto_push_work, cpu);
2117 }
2118}
2119
2120
2121void rto_push_irq_work_func(struct irq_work *work)
2122{
2123 struct root_domain *rd =
2124 container_of(work, struct root_domain, rto_push_work);
2125 struct rq *rq;
2126 int cpu;
2127
2128 rq = this_rq();
2129
2130
2131
2132
2133
2134 if (has_pushable_tasks(rq)) {
2135 raw_spin_rq_lock(rq);
2136 while (push_rt_task(rq, true))
2137 ;
2138 raw_spin_rq_unlock(rq);
2139 }
2140
2141 raw_spin_lock(&rd->rto_lock);
2142
2143
2144 cpu = rto_next_cpu(rd);
2145
2146 raw_spin_unlock(&rd->rto_lock);
2147
2148 if (cpu < 0) {
2149 sched_put_rd(rd);
2150 return;
2151 }
2152
2153
2154 irq_work_queue_on(&rd->rto_push_work, cpu);
2155}
2156#endif
2157
2158static void pull_rt_task(struct rq *this_rq)
2159{
2160 int this_cpu = this_rq->cpu, cpu;
2161 bool resched = false;
2162 struct task_struct *p, *push_task;
2163 struct rq *src_rq;
2164 int rt_overload_count = rt_overloaded(this_rq);
2165
2166 if (likely(!rt_overload_count))
2167 return;
2168
2169
2170
2171
2172
2173 smp_rmb();
2174
2175
2176 if (rt_overload_count == 1 &&
2177 cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask))
2178 return;
2179
2180#ifdef HAVE_RT_PUSH_IPI
2181 if (sched_feat(RT_PUSH_IPI)) {
2182 tell_cpu_to_push(this_rq);
2183 return;
2184 }
2185#endif
2186
2187 for_each_cpu(cpu, this_rq->rd->rto_mask) {
2188 if (this_cpu == cpu)
2189 continue;
2190
2191 src_rq = cpu_rq(cpu);
2192
2193
2194
2195
2196
2197
2198
2199
2200 if (src_rq->rt.highest_prio.next >=
2201 this_rq->rt.highest_prio.curr)
2202 continue;
2203
2204
2205
2206
2207
2208
2209 push_task = NULL;
2210 double_lock_balance(this_rq, src_rq);
2211
2212
2213
2214
2215
2216 p = pick_highest_pushable_task(src_rq, this_cpu);
2217
2218
2219
2220
2221
2222 if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
2223 WARN_ON(p == src_rq->curr);
2224 WARN_ON(!task_on_rq_queued(p));
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234 if (p->prio < src_rq->curr->prio)
2235 goto skip;
2236
2237 if (is_migration_disabled(p)) {
2238 push_task = get_push_task(src_rq);
2239 } else {
2240 deactivate_task(src_rq, p, 0);
2241 set_task_cpu(p, this_cpu);
2242 activate_task(this_rq, p, 0);
2243 resched = true;
2244 }
2245
2246
2247
2248
2249
2250
2251 }
2252skip:
2253 double_unlock_balance(this_rq, src_rq);
2254
2255 if (push_task) {
2256 raw_spin_rq_unlock(this_rq);
2257 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
2258 push_task, &src_rq->push_work);
2259 raw_spin_rq_lock(this_rq);
2260 }
2261 }
2262
2263 if (resched)
2264 resched_curr(this_rq);
2265}
2266
2267
2268
2269
2270
2271static void task_woken_rt(struct rq *rq, struct task_struct *p)
2272{
2273 bool need_to_push = !task_running(rq, p) &&
2274 !test_tsk_need_resched(rq->curr) &&
2275 p->nr_cpus_allowed > 1 &&
2276 (dl_task(rq->curr) || rt_task(rq->curr)) &&
2277 (rq->curr->nr_cpus_allowed < 2 ||
2278 rq->curr->prio <= p->prio);
2279
2280 if (need_to_push)
2281 push_rt_tasks(rq);
2282}
2283
2284
2285static void rq_online_rt(struct rq *rq)
2286{
2287 if (rq->rt.overloaded)
2288 rt_set_overload(rq);
2289
2290 __enable_runtime(rq);
2291
2292 cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
2293}
2294
2295
2296static void rq_offline_rt(struct rq *rq)
2297{
2298 if (rq->rt.overloaded)
2299 rt_clear_overload(rq);
2300
2301 __disable_runtime(rq);
2302
2303 cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
2304}
2305
2306
2307
2308
2309
2310static void switched_from_rt(struct rq *rq, struct task_struct *p)
2311{
2312
2313
2314
2315
2316
2317
2318
2319 if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)
2320 return;
2321
2322 rt_queue_pull_task(rq);
2323}
2324
2325void __init init_sched_rt_class(void)
2326{
2327 unsigned int i;
2328
2329 for_each_possible_cpu(i) {
2330 zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
2331 GFP_KERNEL, cpu_to_node(i));
2332 }
2333}
2334#endif
2335
2336
2337
2338
2339
2340
2341static void switched_to_rt(struct rq *rq, struct task_struct *p)
2342{
2343
2344
2345
2346
2347 if (task_current(rq, p)) {
2348 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
2349 return;
2350 }
2351
2352
2353
2354
2355
2356
2357 if (task_on_rq_queued(p)) {
2358#ifdef CONFIG_SMP
2359 if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
2360 rt_queue_push_tasks(rq);
2361#endif
2362 if (p->prio < rq->curr->prio && cpu_online(cpu_of(rq)))
2363 resched_curr(rq);
2364 }
2365}
2366
2367
2368
2369
2370
2371static void
2372prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
2373{
2374 if (!task_on_rq_queued(p))
2375 return;
2376
2377 if (task_current(rq, p)) {
2378#ifdef CONFIG_SMP
2379
2380
2381
2382
2383 if (oldprio < p->prio)
2384 rt_queue_pull_task(rq);
2385
2386
2387
2388
2389
2390 if (p->prio > rq->rt.highest_prio.curr)
2391 resched_curr(rq);
2392#else
2393
2394 if (oldprio < p->prio)
2395 resched_curr(rq);
2396#endif
2397 } else {
2398
2399
2400
2401
2402
2403 if (p->prio < rq->curr->prio)
2404 resched_curr(rq);
2405 }
2406}
2407
2408#ifdef CONFIG_POSIX_TIMERS
2409static void watchdog(struct rq *rq, struct task_struct *p)
2410{
2411 unsigned long soft, hard;
2412
2413
2414 soft = task_rlimit(p, RLIMIT_RTTIME);
2415 hard = task_rlimit_max(p, RLIMIT_RTTIME);
2416
2417 if (soft != RLIM_INFINITY) {
2418 unsigned long next;
2419
2420 if (p->rt.watchdog_stamp != jiffies) {
2421 p->rt.timeout++;
2422 p->rt.watchdog_stamp = jiffies;
2423 }
2424
2425 next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
2426 if (p->rt.timeout > next) {
2427 posix_cputimers_rt_watchdog(&p->posix_cputimers,
2428 p->se.sum_exec_runtime);
2429 }
2430 }
2431}
2432#else
2433static inline void watchdog(struct rq *rq, struct task_struct *p) { }
2434#endif
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
2445{
2446 struct sched_rt_entity *rt_se = &p->rt;
2447
2448 update_curr_rt(rq);
2449 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
2450
2451 watchdog(rq, p);
2452
2453
2454
2455
2456
2457 if (p->policy != SCHED_RR)
2458 return;
2459
2460 if (--p->rt.time_slice)
2461 return;
2462
2463 p->rt.time_slice = sched_rr_timeslice;
2464
2465
2466
2467
2468
2469 for_each_sched_rt_entity(rt_se) {
2470 if (rt_se->run_list.prev != rt_se->run_list.next) {
2471 requeue_task_rt(rq, p, 0);
2472 resched_curr(rq);
2473 return;
2474 }
2475 }
2476}
2477
2478static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
2479{
2480
2481
2482
2483 if (task->policy == SCHED_RR)
2484 return sched_rr_timeslice;
2485 else
2486 return 0;
2487}
2488
2489DEFINE_SCHED_CLASS(rt) = {
2490
2491 .enqueue_task = enqueue_task_rt,
2492 .dequeue_task = dequeue_task_rt,
2493 .yield_task = yield_task_rt,
2494
2495 .check_preempt_curr = check_preempt_curr_rt,
2496
2497 .pick_next_task = pick_next_task_rt,
2498 .put_prev_task = put_prev_task_rt,
2499 .set_next_task = set_next_task_rt,
2500
2501#ifdef CONFIG_SMP
2502 .balance = balance_rt,
2503 .pick_task = pick_task_rt,
2504 .select_task_rq = select_task_rq_rt,
2505 .set_cpus_allowed = set_cpus_allowed_common,
2506 .rq_online = rq_online_rt,
2507 .rq_offline = rq_offline_rt,
2508 .task_woken = task_woken_rt,
2509 .switched_from = switched_from_rt,
2510 .find_lock_rq = find_lock_lowest_rq,
2511#endif
2512
2513 .task_tick = task_tick_rt,
2514
2515 .get_rr_interval = get_rr_interval_rt,
2516
2517 .prio_changed = prio_changed_rt,
2518 .switched_to = switched_to_rt,
2519
2520 .update_curr = update_curr_rt,
2521
2522#ifdef CONFIG_UCLAMP_TASK
2523 .uclamp_enabled = 1,
2524#endif
2525};
2526
2527#ifdef CONFIG_RT_GROUP_SCHED
2528
2529
2530
2531static DEFINE_MUTEX(rt_constraints_mutex);
2532
2533static inline int tg_has_rt_tasks(struct task_group *tg)
2534{
2535 struct task_struct *task;
2536 struct css_task_iter it;
2537 int ret = 0;
2538
2539
2540
2541
2542 if (task_group_is_autogroup(tg))
2543 return 0;
2544
2545 css_task_iter_start(&tg->css, 0, &it);
2546 while (!ret && (task = css_task_iter_next(&it)))
2547 ret |= rt_task(task);
2548 css_task_iter_end(&it);
2549
2550 return ret;
2551}
2552
2553struct rt_schedulable_data {
2554 struct task_group *tg;
2555 u64 rt_period;
2556 u64 rt_runtime;
2557};
2558
2559static int tg_rt_schedulable(struct task_group *tg, void *data)
2560{
2561 struct rt_schedulable_data *d = data;
2562 struct task_group *child;
2563 unsigned long total, sum = 0;
2564 u64 period, runtime;
2565
2566 period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2567 runtime = tg->rt_bandwidth.rt_runtime;
2568
2569 if (tg == d->tg) {
2570 period = d->rt_period;
2571 runtime = d->rt_runtime;
2572 }
2573
2574
2575
2576
2577 if (runtime > period && runtime != RUNTIME_INF)
2578 return -EINVAL;
2579
2580
2581
2582
2583 if (rt_bandwidth_enabled() && !runtime &&
2584 tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg))
2585 return -EBUSY;
2586
2587 total = to_ratio(period, runtime);
2588
2589
2590
2591
2592 if (total > to_ratio(global_rt_period(), global_rt_runtime()))
2593 return -EINVAL;
2594
2595
2596
2597
2598 list_for_each_entry_rcu(child, &tg->children, siblings) {
2599 period = ktime_to_ns(child->rt_bandwidth.rt_period);
2600 runtime = child->rt_bandwidth.rt_runtime;
2601
2602 if (child == d->tg) {
2603 period = d->rt_period;
2604 runtime = d->rt_runtime;
2605 }
2606
2607 sum += to_ratio(period, runtime);
2608 }
2609
2610 if (sum > total)
2611 return -EINVAL;
2612
2613 return 0;
2614}
2615
2616static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
2617{
2618 int ret;
2619
2620 struct rt_schedulable_data data = {
2621 .tg = tg,
2622 .rt_period = period,
2623 .rt_runtime = runtime,
2624 };
2625
2626 rcu_read_lock();
2627 ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
2628 rcu_read_unlock();
2629
2630 return ret;
2631}
2632
2633static int tg_set_rt_bandwidth(struct task_group *tg,
2634 u64 rt_period, u64 rt_runtime)
2635{
2636 int i, err = 0;
2637
2638
2639
2640
2641
2642 if (tg == &root_task_group && rt_runtime == 0)
2643 return -EINVAL;
2644
2645
2646 if (rt_period == 0)
2647 return -EINVAL;
2648
2649
2650
2651
2652 if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime)
2653 return -EINVAL;
2654
2655 mutex_lock(&rt_constraints_mutex);
2656 err = __rt_schedulable(tg, rt_period, rt_runtime);
2657 if (err)
2658 goto unlock;
2659
2660 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2661 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
2662 tg->rt_bandwidth.rt_runtime = rt_runtime;
2663
2664 for_each_possible_cpu(i) {
2665 struct rt_rq *rt_rq = tg->rt_rq[i];
2666
2667 raw_spin_lock(&rt_rq->rt_runtime_lock);
2668 rt_rq->rt_runtime = rt_runtime;
2669 raw_spin_unlock(&rt_rq->rt_runtime_lock);
2670 }
2671 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2672unlock:
2673 mutex_unlock(&rt_constraints_mutex);
2674
2675 return err;
2676}
2677
2678int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
2679{
2680 u64 rt_runtime, rt_period;
2681
2682 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2683 rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
2684 if (rt_runtime_us < 0)
2685 rt_runtime = RUNTIME_INF;
2686 else if ((u64)rt_runtime_us > U64_MAX / NSEC_PER_USEC)
2687 return -EINVAL;
2688
2689 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2690}
2691
2692long sched_group_rt_runtime(struct task_group *tg)
2693{
2694 u64 rt_runtime_us;
2695
2696 if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
2697 return -1;
2698
2699 rt_runtime_us = tg->rt_bandwidth.rt_runtime;
2700 do_div(rt_runtime_us, NSEC_PER_USEC);
2701 return rt_runtime_us;
2702}
2703
2704int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
2705{
2706 u64 rt_runtime, rt_period;
2707
2708 if (rt_period_us > U64_MAX / NSEC_PER_USEC)
2709 return -EINVAL;
2710
2711 rt_period = rt_period_us * NSEC_PER_USEC;
2712 rt_runtime = tg->rt_bandwidth.rt_runtime;
2713
2714 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2715}
2716
2717long sched_group_rt_period(struct task_group *tg)
2718{
2719 u64 rt_period_us;
2720
2721 rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
2722 do_div(rt_period_us, NSEC_PER_USEC);
2723 return rt_period_us;
2724}
2725
2726static int sched_rt_global_constraints(void)
2727{
2728 int ret = 0;
2729
2730 mutex_lock(&rt_constraints_mutex);
2731 ret = __rt_schedulable(NULL, 0, 0);
2732 mutex_unlock(&rt_constraints_mutex);
2733
2734 return ret;
2735}
2736
2737int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
2738{
2739
2740 if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
2741 return 0;
2742
2743 return 1;
2744}
2745
2746#else
2747static int sched_rt_global_constraints(void)
2748{
2749 unsigned long flags;
2750 int i;
2751
2752 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
2753 for_each_possible_cpu(i) {
2754 struct rt_rq *rt_rq = &cpu_rq(i)->rt;
2755
2756 raw_spin_lock(&rt_rq->rt_runtime_lock);
2757 rt_rq->rt_runtime = global_rt_runtime();
2758 raw_spin_unlock(&rt_rq->rt_runtime_lock);
2759 }
2760 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
2761
2762 return 0;
2763}
2764#endif
2765
2766static int sched_rt_global_validate(void)
2767{
2768 if (sysctl_sched_rt_period <= 0)
2769 return -EINVAL;
2770
2771 if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
2772 ((sysctl_sched_rt_runtime > sysctl_sched_rt_period) ||
2773 ((u64)sysctl_sched_rt_runtime *
2774 NSEC_PER_USEC > max_rt_runtime)))
2775 return -EINVAL;
2776
2777 return 0;
2778}
2779
2780static void sched_rt_do_global(void)
2781{
2782 def_rt_bandwidth.rt_runtime = global_rt_runtime();
2783 def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
2784}
2785
2786int sched_rt_handler(struct ctl_table *table, int write, void *buffer,
2787 size_t *lenp, loff_t *ppos)
2788{
2789 int old_period, old_runtime;
2790 static DEFINE_MUTEX(mutex);
2791 int ret;
2792
2793 mutex_lock(&mutex);
2794 old_period = sysctl_sched_rt_period;
2795 old_runtime = sysctl_sched_rt_runtime;
2796
2797 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2798
2799 if (!ret && write) {
2800 ret = sched_rt_global_validate();
2801 if (ret)
2802 goto undo;
2803
2804 ret = sched_dl_global_validate();
2805 if (ret)
2806 goto undo;
2807
2808 ret = sched_rt_global_constraints();
2809 if (ret)
2810 goto undo;
2811
2812 sched_rt_do_global();
2813 sched_dl_do_global();
2814 }
2815 if (0) {
2816undo:
2817 sysctl_sched_rt_period = old_period;
2818 sysctl_sched_rt_runtime = old_runtime;
2819 }
2820 mutex_unlock(&mutex);
2821
2822 return ret;
2823}
2824
2825int sched_rr_handler(struct ctl_table *table, int write, void *buffer,
2826 size_t *lenp, loff_t *ppos)
2827{
2828 int ret;
2829 static DEFINE_MUTEX(mutex);
2830
2831 mutex_lock(&mutex);
2832 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2833
2834
2835
2836
2837 if (!ret && write) {
2838 sched_rr_timeslice =
2839 sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE :
2840 msecs_to_jiffies(sysctl_sched_rr_timeslice);
2841 }
2842 mutex_unlock(&mutex);
2843
2844 return ret;
2845}
2846
2847#ifdef CONFIG_SCHED_DEBUG
2848void print_rt_stats(struct seq_file *m, int cpu)
2849{
2850 rt_rq_iter_t iter;
2851 struct rt_rq *rt_rq;
2852
2853 rcu_read_lock();
2854 for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
2855 print_rt_rq(m, cpu, rt_rq);
2856 rcu_read_unlock();
2857}
2858#endif
2859