1#include <linux/export.h>
2#include <linux/sched.h>
3#include <linux/tsacct_kern.h>
4#include <linux/kernel_stat.h>
5#include <linux/static_key.h>
6#include <linux/context_tracking.h>
7#include "sched.h"
8
9
10#ifdef CONFIG_IRQ_TIME_ACCOUNTING
11
12
13
14
15
16
17
18
19
20
21
22
23DEFINE_PER_CPU(u64, cpu_hardirq_time);
24DEFINE_PER_CPU(u64, cpu_softirq_time);
25
26static DEFINE_PER_CPU(u64, irq_start_time);
27static int sched_clock_irqtime;
28
29void enable_sched_clock_irqtime(void)
30{
31 sched_clock_irqtime = 1;
32}
33
34void disable_sched_clock_irqtime(void)
35{
36 sched_clock_irqtime = 0;
37}
38
39#ifndef CONFIG_64BIT
40DEFINE_PER_CPU(seqcount_t, irq_time_seq);
41#endif
42
43
44
45
46
47void irqtime_account_irq(struct task_struct *curr)
48{
49 unsigned long flags;
50 s64 delta;
51 int cpu;
52
53 if (!sched_clock_irqtime)
54 return;
55
56 local_irq_save(flags);
57
58 cpu = smp_processor_id();
59 delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
60 __this_cpu_add(irq_start_time, delta);
61
62 irq_time_write_begin();
63
64
65
66
67
68
69 if (hardirq_count())
70 __this_cpu_add(cpu_hardirq_time, delta);
71 else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
72 __this_cpu_add(cpu_softirq_time, delta);
73
74 irq_time_write_end();
75 local_irq_restore(flags);
76}
77EXPORT_SYMBOL_GPL(irqtime_account_irq);
78
79static int irqtime_account_hi_update(void)
80{
81 u64 *cpustat = kcpustat_this_cpu->cpustat;
82 unsigned long flags;
83 u64 latest_ns;
84 int ret = 0;
85
86 local_irq_save(flags);
87 latest_ns = this_cpu_read(cpu_hardirq_time);
88 if (nsecs_to_cputime64(latest_ns) > cpustat[CPUTIME_IRQ])
89 ret = 1;
90 local_irq_restore(flags);
91 return ret;
92}
93
94static int irqtime_account_si_update(void)
95{
96 u64 *cpustat = kcpustat_this_cpu->cpustat;
97 unsigned long flags;
98 u64 latest_ns;
99 int ret = 0;
100
101 local_irq_save(flags);
102 latest_ns = this_cpu_read(cpu_softirq_time);
103 if (nsecs_to_cputime64(latest_ns) > cpustat[CPUTIME_SOFTIRQ])
104 ret = 1;
105 local_irq_restore(flags);
106 return ret;
107}
108
109#else
110
111#define sched_clock_irqtime (0)
112
113#endif
114
115static inline void task_group_account_field(struct task_struct *p, int index,
116 u64 tmp)
117{
118
119
120
121
122
123
124 __get_cpu_var(kernel_cpustat).cpustat[index] += tmp;
125
126 cpuacct_account_field(p, index, tmp);
127}
128
129
130
131
132
133
134
135void account_user_time(struct task_struct *p, cputime_t cputime,
136 cputime_t cputime_scaled)
137{
138 int index;
139
140
141 p->utime += cputime;
142 p->utimescaled += cputime_scaled;
143 account_group_user_time(p, cputime);
144
145 index = (TASK_NICE(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
146
147
148 task_group_account_field(p, index, (__force u64) cputime);
149
150
151 acct_account_cputime(p);
152}
153
154
155
156
157
158
159
160static void account_guest_time(struct task_struct *p, cputime_t cputime,
161 cputime_t cputime_scaled)
162{
163 u64 *cpustat = kcpustat_this_cpu->cpustat;
164
165
166 p->utime += cputime;
167 p->utimescaled += cputime_scaled;
168 account_group_user_time(p, cputime);
169 p->gtime += cputime;
170
171
172 if (TASK_NICE(p) > 0) {
173 cpustat[CPUTIME_NICE] += (__force u64) cputime;
174 cpustat[CPUTIME_GUEST_NICE] += (__force u64) cputime;
175 } else {
176 cpustat[CPUTIME_USER] += (__force u64) cputime;
177 cpustat[CPUTIME_GUEST] += (__force u64) cputime;
178 }
179}
180
181
182
183
184
185
186
187
188static inline
189void __account_system_time(struct task_struct *p, cputime_t cputime,
190 cputime_t cputime_scaled, int index)
191{
192
193 p->stime += cputime;
194 p->stimescaled += cputime_scaled;
195 account_group_system_time(p, cputime);
196
197
198 task_group_account_field(p, index, (__force u64) cputime);
199
200
201 acct_account_cputime(p);
202}
203
204
205
206
207
208
209
210
211void account_system_time(struct task_struct *p, int hardirq_offset,
212 cputime_t cputime, cputime_t cputime_scaled)
213{
214 int index;
215
216 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
217 account_guest_time(p, cputime, cputime_scaled);
218 return;
219 }
220
221 if (hardirq_count() - hardirq_offset)
222 index = CPUTIME_IRQ;
223 else if (in_serving_softirq())
224 index = CPUTIME_SOFTIRQ;
225 else
226 index = CPUTIME_SYSTEM;
227
228 __account_system_time(p, cputime, cputime_scaled, index);
229}
230
231
232
233
234
235void account_steal_time(cputime_t cputime)
236{
237 u64 *cpustat = kcpustat_this_cpu->cpustat;
238
239 cpustat[CPUTIME_STEAL] += (__force u64) cputime;
240}
241
242
243
244
245
246void account_idle_time(cputime_t cputime)
247{
248 u64 *cpustat = kcpustat_this_cpu->cpustat;
249 struct rq *rq = this_rq();
250
251 if (atomic_read(&rq->nr_iowait) > 0)
252 cpustat[CPUTIME_IOWAIT] += (__force u64) cputime;
253 else
254 cpustat[CPUTIME_IDLE] += (__force u64) cputime;
255}
256
257static __always_inline bool steal_account_process_tick(void)
258{
259#ifdef CONFIG_PARAVIRT
260 if (static_key_false(¶virt_steal_enabled)) {
261 u64 steal, st = 0;
262
263 steal = paravirt_steal_clock(smp_processor_id());
264 steal -= this_rq()->prev_steal_time;
265
266 st = steal_ticks(steal);
267 this_rq()->prev_steal_time += st * TICK_NSEC;
268
269 account_steal_time(st);
270 return st;
271 }
272#endif
273 return false;
274}
275
276
277
278
279
280void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
281{
282 struct signal_struct *sig = tsk->signal;
283 cputime_t utime, stime;
284 struct task_struct *t;
285
286 times->utime = sig->utime;
287 times->stime = sig->stime;
288 times->sum_exec_runtime = sig->sum_sched_runtime;
289
290 rcu_read_lock();
291
292 if (!likely(pid_alive(tsk)))
293 goto out;
294
295 t = tsk;
296 do {
297 task_cputime(t, &utime, &stime);
298 times->utime += utime;
299 times->stime += stime;
300 times->sum_exec_runtime += task_sched_runtime(t);
301 } while_each_thread(tsk, t);
302out:
303 rcu_read_unlock();
304}
305
306#ifdef CONFIG_IRQ_TIME_ACCOUNTING
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
329 struct rq *rq)
330{
331 cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
332 u64 *cpustat = kcpustat_this_cpu->cpustat;
333
334 if (steal_account_process_tick())
335 return;
336
337 if (irqtime_account_hi_update()) {
338 cpustat[CPUTIME_IRQ] += (__force u64) cputime_one_jiffy;
339 } else if (irqtime_account_si_update()) {
340 cpustat[CPUTIME_SOFTIRQ] += (__force u64) cputime_one_jiffy;
341 } else if (this_cpu_ksoftirqd() == p) {
342
343
344
345
346
347 __account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
348 CPUTIME_SOFTIRQ);
349 } else if (user_tick) {
350 account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
351 } else if (p == rq->idle) {
352 account_idle_time(cputime_one_jiffy);
353 } else if (p->flags & PF_VCPU) {
354 account_guest_time(p, cputime_one_jiffy, one_jiffy_scaled);
355 } else {
356 __account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
357 CPUTIME_SYSTEM);
358 }
359}
360
361static void irqtime_account_idle_ticks(int ticks)
362{
363 int i;
364 struct rq *rq = this_rq();
365
366 for (i = 0; i < ticks; i++)
367 irqtime_account_process_tick(current, 0, rq);
368}
369#else
370static inline void irqtime_account_idle_ticks(int ticks) {}
371static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
372 struct rq *rq) {}
373#endif
374
375
376
377
378#ifdef CONFIG_VIRT_CPU_ACCOUNTING
379
380#ifndef __ARCH_HAS_VTIME_TASK_SWITCH
381void vtime_task_switch(struct task_struct *prev)
382{
383 if (!vtime_accounting_enabled())
384 return;
385
386 if (is_idle_task(prev))
387 vtime_account_idle(prev);
388 else
389 vtime_account_system(prev);
390
391#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
392 vtime_account_user(prev);
393#endif
394 arch_vtime_task_switch(prev);
395}
396#endif
397
398
399
400
401
402
403
404
405
406#ifndef __ARCH_HAS_VTIME_ACCOUNT
407void vtime_account_irq_enter(struct task_struct *tsk)
408{
409 if (!vtime_accounting_enabled())
410 return;
411
412 if (!in_interrupt()) {
413
414
415
416
417
418
419 if (context_tracking_in_user()) {
420 vtime_account_user(tsk);
421 return;
422 }
423
424 if (is_idle_task(tsk)) {
425 vtime_account_idle(tsk);
426 return;
427 }
428 }
429 vtime_account_system(tsk);
430}
431EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
432#endif
433#endif
434
435
436#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
437void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
438{
439 *ut = p->utime;
440 *st = p->stime;
441}
442
443void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
444{
445 struct task_cputime cputime;
446
447 thread_group_cputime(p, &cputime);
448
449 *ut = cputime.utime;
450 *st = cputime.stime;
451}
452#else
453
454
455
456
457
458void account_process_tick(struct task_struct *p, int user_tick)
459{
460 cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
461 struct rq *rq = this_rq();
462
463 if (vtime_accounting_enabled())
464 return;
465
466 if (sched_clock_irqtime) {
467 irqtime_account_process_tick(p, user_tick, rq);
468 return;
469 }
470
471 if (steal_account_process_tick())
472 return;
473
474 if (user_tick)
475 account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
476 else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
477 account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
478 one_jiffy_scaled);
479 else
480 account_idle_time(cputime_one_jiffy);
481}
482
483
484
485
486
487
488void account_steal_ticks(unsigned long ticks)
489{
490 account_steal_time(jiffies_to_cputime(ticks));
491}
492
493
494
495
496
497void account_idle_ticks(unsigned long ticks)
498{
499
500 if (sched_clock_irqtime) {
501 irqtime_account_idle_ticks(ticks);
502 return;
503 }
504
505 account_idle_time(jiffies_to_cputime(ticks));
506}
507
508
509
510
511
512static cputime_t scale_stime(u64 stime, u64 rtime, u64 total)
513{
514 u64 scaled;
515
516 for (;;) {
517
518 if (stime > rtime)
519 swap(rtime, stime);
520
521
522 if (total >> 32)
523 goto drop_precision;
524
525
526 if (!(rtime >> 32))
527 break;
528
529
530 if (stime >> 31)
531 goto drop_precision;
532
533
534 stime <<= 1;
535 rtime >>= 1;
536 continue;
537
538drop_precision:
539
540 rtime >>= 1;
541 total >>= 1;
542 }
543
544
545
546
547
548 scaled = div_u64((u64) (u32) stime * (u64) (u32) rtime, (u32)total);
549 return (__force cputime_t) scaled;
550}
551
552
553
554
555
556static void cputime_adjust(struct task_cputime *curr,
557 struct cputime *prev,
558 cputime_t *ut, cputime_t *st)
559{
560 cputime_t rtime, stime, utime, total;
561
562 if (vtime_accounting_enabled()) {
563 *ut = curr->utime;
564 *st = curr->stime;
565 return;
566 }
567
568 stime = curr->stime;
569 total = stime + curr->utime;
570
571
572
573
574
575
576
577
578
579
580
581 rtime = nsecs_to_cputime(curr->sum_exec_runtime);
582
583
584
585
586
587
588 if (prev->stime + prev->utime >= rtime)
589 goto out;
590
591 if (total) {
592 stime = scale_stime((__force u64)stime,
593 (__force u64)rtime, (__force u64)total);
594 utime = rtime - stime;
595 } else {
596 stime = rtime;
597 utime = 0;
598 }
599
600
601
602
603
604
605 prev->stime = max(prev->stime, stime);
606 prev->utime = max(prev->utime, utime);
607
608out:
609 *ut = prev->utime;
610 *st = prev->stime;
611}
612
613void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
614{
615 struct task_cputime cputime = {
616 .sum_exec_runtime = p->se.sum_exec_runtime,
617 };
618
619 task_cputime(p, &cputime.utime, &cputime.stime);
620 cputime_adjust(&cputime, &p->prev_cputime, ut, st);
621}
622
623
624
625
626void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
627{
628 struct task_cputime cputime;
629
630 thread_group_cputime(p, &cputime);
631 cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
632}
633#endif
634
635#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
636static unsigned long long vtime_delta(struct task_struct *tsk)
637{
638 unsigned long long clock;
639
640 clock = local_clock();
641 if (clock < tsk->vtime_snap)
642 return 0;
643
644 return clock - tsk->vtime_snap;
645}
646
647static cputime_t get_vtime_delta(struct task_struct *tsk)
648{
649 unsigned long long delta = vtime_delta(tsk);
650
651 WARN_ON_ONCE(tsk->vtime_snap_whence == VTIME_SLEEPING);
652 tsk->vtime_snap += delta;
653
654
655 return nsecs_to_cputime(delta);
656}
657
658static void __vtime_account_system(struct task_struct *tsk)
659{
660 cputime_t delta_cpu = get_vtime_delta(tsk);
661
662 account_system_time(tsk, irq_count(), delta_cpu, cputime_to_scaled(delta_cpu));
663}
664
665void vtime_account_system(struct task_struct *tsk)
666{
667 if (!vtime_accounting_enabled())
668 return;
669
670 write_seqlock(&tsk->vtime_seqlock);
671 __vtime_account_system(tsk);
672 write_sequnlock(&tsk->vtime_seqlock);
673}
674
675void vtime_account_irq_exit(struct task_struct *tsk)
676{
677 if (!vtime_accounting_enabled())
678 return;
679
680 write_seqlock(&tsk->vtime_seqlock);
681 if (context_tracking_in_user())
682 tsk->vtime_snap_whence = VTIME_USER;
683 __vtime_account_system(tsk);
684 write_sequnlock(&tsk->vtime_seqlock);
685}
686
687void vtime_account_user(struct task_struct *tsk)
688{
689 cputime_t delta_cpu;
690
691 if (!vtime_accounting_enabled())
692 return;
693
694 delta_cpu = get_vtime_delta(tsk);
695
696 write_seqlock(&tsk->vtime_seqlock);
697 tsk->vtime_snap_whence = VTIME_SYS;
698 account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu));
699 write_sequnlock(&tsk->vtime_seqlock);
700}
701
702void vtime_user_enter(struct task_struct *tsk)
703{
704 if (!vtime_accounting_enabled())
705 return;
706
707 write_seqlock(&tsk->vtime_seqlock);
708 tsk->vtime_snap_whence = VTIME_USER;
709 __vtime_account_system(tsk);
710 write_sequnlock(&tsk->vtime_seqlock);
711}
712
713void vtime_guest_enter(struct task_struct *tsk)
714{
715 write_seqlock(&tsk->vtime_seqlock);
716 __vtime_account_system(tsk);
717 current->flags |= PF_VCPU;
718 write_sequnlock(&tsk->vtime_seqlock);
719}
720
721void vtime_guest_exit(struct task_struct *tsk)
722{
723 write_seqlock(&tsk->vtime_seqlock);
724 __vtime_account_system(tsk);
725 current->flags &= ~PF_VCPU;
726 write_sequnlock(&tsk->vtime_seqlock);
727}
728
729void vtime_account_idle(struct task_struct *tsk)
730{
731 cputime_t delta_cpu = get_vtime_delta(tsk);
732
733 account_idle_time(delta_cpu);
734}
735
736bool vtime_accounting_enabled(void)
737{
738 return context_tracking_active();
739}
740
741void arch_vtime_task_switch(struct task_struct *prev)
742{
743 write_seqlock(&prev->vtime_seqlock);
744 prev->vtime_snap_whence = VTIME_SLEEPING;
745 write_sequnlock(&prev->vtime_seqlock);
746
747 write_seqlock(¤t->vtime_seqlock);
748 current->vtime_snap_whence = VTIME_SYS;
749 current->vtime_snap = sched_clock_cpu(smp_processor_id());
750 write_sequnlock(¤t->vtime_seqlock);
751}
752
753void vtime_init_idle(struct task_struct *t, int cpu)
754{
755 unsigned long flags;
756
757 write_seqlock_irqsave(&t->vtime_seqlock, flags);
758 t->vtime_snap_whence = VTIME_SYS;
759 t->vtime_snap = sched_clock_cpu(cpu);
760 write_sequnlock_irqrestore(&t->vtime_seqlock, flags);
761}
762
763cputime_t task_gtime(struct task_struct *t)
764{
765 unsigned int seq;
766 cputime_t gtime;
767
768 do {
769 seq = read_seqbegin(&t->vtime_seqlock);
770
771 gtime = t->gtime;
772 if (t->flags & PF_VCPU)
773 gtime += vtime_delta(t);
774
775 } while (read_seqretry(&t->vtime_seqlock, seq));
776
777 return gtime;
778}
779
780
781
782
783
784
785static void
786fetch_task_cputime(struct task_struct *t,
787 cputime_t *u_dst, cputime_t *s_dst,
788 cputime_t *u_src, cputime_t *s_src,
789 cputime_t *udelta, cputime_t *sdelta)
790{
791 unsigned int seq;
792 unsigned long long delta;
793
794 do {
795 *udelta = 0;
796 *sdelta = 0;
797
798 seq = read_seqbegin(&t->vtime_seqlock);
799
800 if (u_dst)
801 *u_dst = *u_src;
802 if (s_dst)
803 *s_dst = *s_src;
804
805
806 if (t->vtime_snap_whence == VTIME_SLEEPING ||
807 is_idle_task(t))
808 continue;
809
810 delta = vtime_delta(t);
811
812
813
814
815
816 if (t->vtime_snap_whence == VTIME_USER || t->flags & PF_VCPU) {
817 *udelta = delta;
818 } else {
819 if (t->vtime_snap_whence == VTIME_SYS)
820 *sdelta = delta;
821 }
822 } while (read_seqretry(&t->vtime_seqlock, seq));
823}
824
825
826void task_cputime(struct task_struct *t, cputime_t *utime, cputime_t *stime)
827{
828 cputime_t udelta, sdelta;
829
830 fetch_task_cputime(t, utime, stime, &t->utime,
831 &t->stime, &udelta, &sdelta);
832 if (utime)
833 *utime += udelta;
834 if (stime)
835 *stime += sdelta;
836}
837
838void task_cputime_scaled(struct task_struct *t,
839 cputime_t *utimescaled, cputime_t *stimescaled)
840{
841 cputime_t udelta, sdelta;
842
843 fetch_task_cputime(t, utimescaled, stimescaled,
844 &t->utimescaled, &t->stimescaled, &udelta, &sdelta);
845 if (utimescaled)
846 *utimescaled += cputime_to_scaled(udelta);
847 if (stimescaled)
848 *stimescaled += cputime_to_scaled(sdelta);
849}
850#endif
851