1
2
3
4
5#include "sched.h"
6
7#ifdef CONFIG_IRQ_TIME_ACCOUNTING
8
9
10
11
12
13
14
15
16
17
18
19
20DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
21
22static int sched_clock_irqtime;
23
24void enable_sched_clock_irqtime(void)
25{
26 sched_clock_irqtime = 1;
27}
28
29void disable_sched_clock_irqtime(void)
30{
31 sched_clock_irqtime = 0;
32}
33
34static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
35 enum cpu_usage_stat idx)
36{
37 u64 *cpustat = kcpustat_this_cpu->cpustat;
38
39 u64_stats_update_begin(&irqtime->sync);
40 cpustat[idx] += delta;
41 irqtime->total += delta;
42 irqtime->tick_delta += delta;
43 u64_stats_update_end(&irqtime->sync);
44}
45
46
47
48
49
50void irqtime_account_irq(struct task_struct *curr, unsigned int offset)
51{
52 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
53 unsigned int pc;
54 s64 delta;
55 int cpu;
56
57 if (!sched_clock_irqtime)
58 return;
59
60 cpu = smp_processor_id();
61 delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
62 irqtime->irq_start_time += delta;
63 pc = irq_count() - offset;
64
65
66
67
68
69
70
71 if (pc & HARDIRQ_MASK)
72 irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
73 else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd())
74 irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
75}
76
77static u64 irqtime_tick_accounted(u64 maxtime)
78{
79 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
80 u64 delta;
81
82 delta = min(irqtime->tick_delta, maxtime);
83 irqtime->tick_delta -= delta;
84
85 return delta;
86}
87
88#else
89
90#define sched_clock_irqtime (0)
91
92static u64 irqtime_tick_accounted(u64 dummy)
93{
94 return 0;
95}
96
97#endif
98
99static inline void task_group_account_field(struct task_struct *p, int index,
100 u64 tmp)
101{
102
103
104
105
106
107
108 __this_cpu_add(kernel_cpustat.cpustat[index], tmp);
109
110 cgroup_account_cputime_field(p, index, tmp);
111}
112
113
114
115
116
117
118void account_user_time(struct task_struct *p, u64 cputime)
119{
120 int index;
121
122
123 p->utime += cputime;
124 account_group_user_time(p, cputime);
125
126 index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
127
128
129 task_group_account_field(p, index, cputime);
130
131
132 acct_account_cputime(p);
133}
134
135
136
137
138
139
140void account_guest_time(struct task_struct *p, u64 cputime)
141{
142 u64 *cpustat = kcpustat_this_cpu->cpustat;
143
144
145 p->utime += cputime;
146 account_group_user_time(p, cputime);
147 p->gtime += cputime;
148
149
150 if (task_nice(p) > 0) {
151 task_group_account_field(p, CPUTIME_NICE, cputime);
152 cpustat[CPUTIME_GUEST_NICE] += cputime;
153 } else {
154 task_group_account_field(p, CPUTIME_USER, cputime);
155 cpustat[CPUTIME_GUEST] += cputime;
156 }
157}
158
159
160
161
162
163
164
165void account_system_index_time(struct task_struct *p,
166 u64 cputime, enum cpu_usage_stat index)
167{
168
169 p->stime += cputime;
170 account_group_system_time(p, cputime);
171
172
173 task_group_account_field(p, index, cputime);
174
175
176 acct_account_cputime(p);
177}
178
179
180
181
182
183
184
185void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
186{
187 int index;
188
189 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
190 account_guest_time(p, cputime);
191 return;
192 }
193
194 if (hardirq_count() - hardirq_offset)
195 index = CPUTIME_IRQ;
196 else if (in_serving_softirq())
197 index = CPUTIME_SOFTIRQ;
198 else
199 index = CPUTIME_SYSTEM;
200
201 account_system_index_time(p, cputime, index);
202}
203
204
205
206
207
208void account_steal_time(u64 cputime)
209{
210 u64 *cpustat = kcpustat_this_cpu->cpustat;
211
212 cpustat[CPUTIME_STEAL] += cputime;
213}
214
215
216
217
218
219void account_idle_time(u64 cputime)
220{
221 u64 *cpustat = kcpustat_this_cpu->cpustat;
222 struct rq *rq = this_rq();
223
224 if (atomic_read(&rq->nr_iowait) > 0)
225 cpustat[CPUTIME_IOWAIT] += cputime;
226 else
227 cpustat[CPUTIME_IDLE] += cputime;
228}
229
230
231
232
233
234
235static __always_inline u64 steal_account_process_time(u64 maxtime)
236{
237#ifdef CONFIG_PARAVIRT
238 if (static_key_false(¶virt_steal_enabled)) {
239 u64 steal;
240
241 steal = paravirt_steal_clock(smp_processor_id());
242 steal -= this_rq()->prev_steal_time;
243 steal = min(steal, maxtime);
244 account_steal_time(steal);
245 this_rq()->prev_steal_time += steal;
246
247 return steal;
248 }
249#endif
250 return 0;
251}
252
253
254
255
256static inline u64 account_other_time(u64 max)
257{
258 u64 accounted;
259
260 lockdep_assert_irqs_disabled();
261
262 accounted = steal_account_process_time(max);
263
264 if (accounted < max)
265 accounted += irqtime_tick_accounted(max - accounted);
266
267 return accounted;
268}
269
270#ifdef CONFIG_64BIT
271static inline u64 read_sum_exec_runtime(struct task_struct *t)
272{
273 return t->se.sum_exec_runtime;
274}
275#else
276static u64 read_sum_exec_runtime(struct task_struct *t)
277{
278 u64 ns;
279 struct rq_flags rf;
280 struct rq *rq;
281
282 rq = task_rq_lock(t, &rf);
283 ns = t->se.sum_exec_runtime;
284 task_rq_unlock(rq, t, &rf);
285
286 return ns;
287}
288#endif
289
290
291
292
293
294void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
295{
296 struct signal_struct *sig = tsk->signal;
297 u64 utime, stime;
298 struct task_struct *t;
299 unsigned int seq, nextseq;
300 unsigned long flags;
301
302
303
304
305
306
307
308
309
310 if (same_thread_group(current, tsk))
311 (void) task_sched_runtime(current);
312
313 rcu_read_lock();
314
315 nextseq = 0;
316 do {
317 seq = nextseq;
318 flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);
319 times->utime = sig->utime;
320 times->stime = sig->stime;
321 times->sum_exec_runtime = sig->sum_sched_runtime;
322
323 for_each_thread(tsk, t) {
324 task_cputime(t, &utime, &stime);
325 times->utime += utime;
326 times->stime += stime;
327 times->sum_exec_runtime += read_sum_exec_runtime(t);
328 }
329
330 nextseq = 1;
331 } while (need_seqretry(&sig->stats_lock, seq));
332 done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
333 rcu_read_unlock();
334}
335
336#ifdef CONFIG_IRQ_TIME_ACCOUNTING
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
359 int ticks)
360{
361 u64 other, cputime = TICK_NSEC * ticks;
362
363
364
365
366
367
368
369
370 other = account_other_time(ULONG_MAX);
371 if (other >= cputime)
372 return;
373
374 cputime -= other;
375
376 if (this_cpu_ksoftirqd() == p) {
377
378
379
380
381
382 account_system_index_time(p, cputime, CPUTIME_SOFTIRQ);
383 } else if (user_tick) {
384 account_user_time(p, cputime);
385 } else if (p == this_rq()->idle) {
386 account_idle_time(cputime);
387 } else if (p->flags & PF_VCPU) {
388 account_guest_time(p, cputime);
389 } else {
390 account_system_index_time(p, cputime, CPUTIME_SYSTEM);
391 }
392}
393
394static void irqtime_account_idle_ticks(int ticks)
395{
396 irqtime_account_process_tick(current, 0, ticks);
397}
398#else
399static inline void irqtime_account_idle_ticks(int ticks) { }
400static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
401 int nr_ticks) { }
402#endif
403
404
405
406
407#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
408
409# ifndef __ARCH_HAS_VTIME_TASK_SWITCH
410void vtime_task_switch(struct task_struct *prev)
411{
412 if (is_idle_task(prev))
413 vtime_account_idle(prev);
414 else
415 vtime_account_kernel(prev);
416
417 vtime_flush(prev);
418 arch_vtime_task_switch(prev);
419}
420# endif
421
422void vtime_account_irq(struct task_struct *tsk, unsigned int offset)
423{
424 unsigned int pc = irq_count() - offset;
425
426 if (pc & HARDIRQ_OFFSET) {
427 vtime_account_hardirq(tsk);
428 } else if (pc & SOFTIRQ_OFFSET) {
429 vtime_account_softirq(tsk);
430 } else if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
431 is_idle_task(tsk)) {
432 vtime_account_idle(tsk);
433 } else {
434 vtime_account_kernel(tsk);
435 }
436}
437
438void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
439 u64 *ut, u64 *st)
440{
441 *ut = curr->utime;
442 *st = curr->stime;
443}
444
445void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
446{
447 *ut = p->utime;
448 *st = p->stime;
449}
450EXPORT_SYMBOL_GPL(task_cputime_adjusted);
451
452void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
453{
454 struct task_cputime cputime;
455
456 thread_group_cputime(p, &cputime);
457
458 *ut = cputime.utime;
459 *st = cputime.stime;
460}
461
462#else
463
464
465
466
467
468
469void account_process_tick(struct task_struct *p, int user_tick)
470{
471 u64 cputime, steal;
472
473 if (vtime_accounting_enabled_this_cpu())
474 return;
475
476 if (sched_clock_irqtime) {
477 irqtime_account_process_tick(p, user_tick, 1);
478 return;
479 }
480
481 cputime = TICK_NSEC;
482 steal = steal_account_process_time(ULONG_MAX);
483
484 if (steal >= cputime)
485 return;
486
487 cputime -= steal;
488
489 if (user_tick)
490 account_user_time(p, cputime);
491 else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET))
492 account_system_time(p, HARDIRQ_OFFSET, cputime);
493 else
494 account_idle_time(cputime);
495}
496
497
498
499
500
501void account_idle_ticks(unsigned long ticks)
502{
503 u64 cputime, steal;
504
505 if (sched_clock_irqtime) {
506 irqtime_account_idle_ticks(ticks);
507 return;
508 }
509
510 cputime = ticks * TICK_NSEC;
511 steal = steal_account_process_time(ULONG_MAX);
512
513 if (steal >= cputime)
514 return;
515
516 cputime -= steal;
517 account_idle_time(cputime);
518}
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
541 u64 *ut, u64 *st)
542{
543 u64 rtime, stime, utime;
544 unsigned long flags;
545
546
547 raw_spin_lock_irqsave(&prev->lock, flags);
548 rtime = curr->sum_exec_runtime;
549
550
551
552
553
554
555
556
557
558 if (prev->stime + prev->utime >= rtime)
559 goto out;
560
561 stime = curr->stime;
562 utime = curr->utime;
563
564
565
566
567
568
569 if (stime == 0) {
570 utime = rtime;
571 goto update;
572 }
573
574 if (utime == 0) {
575 stime = rtime;
576 goto update;
577 }
578
579 stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
580
581update:
582
583
584
585
586
587
588
589
590
591 if (stime < prev->stime)
592 stime = prev->stime;
593 utime = rtime - stime;
594
595
596
597
598
599 if (utime < prev->utime) {
600 utime = prev->utime;
601 stime = rtime - utime;
602 }
603
604 prev->stime = stime;
605 prev->utime = utime;
606out:
607 *ut = prev->utime;
608 *st = prev->stime;
609 raw_spin_unlock_irqrestore(&prev->lock, flags);
610}
611
612void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
613{
614 struct task_cputime cputime = {
615 .sum_exec_runtime = p->se.sum_exec_runtime,
616 };
617
618 if (task_cputime(p, &cputime.utime, &cputime.stime))
619 cputime.sum_exec_runtime = task_sched_runtime(p);
620 cputime_adjust(&cputime, &p->prev_cputime, ut, st);
621}
622EXPORT_SYMBOL_GPL(task_cputime_adjusted);
623
624void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
625{
626 struct task_cputime cputime;
627
628 thread_group_cputime(p, &cputime);
629 cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
630}
631#endif
632
633#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
634static u64 vtime_delta(struct vtime *vtime)
635{
636 unsigned long long clock;
637
638 clock = sched_clock();
639 if (clock < vtime->starttime)
640 return 0;
641
642 return clock - vtime->starttime;
643}
644
645static u64 get_vtime_delta(struct vtime *vtime)
646{
647 u64 delta = vtime_delta(vtime);
648 u64 other;
649
650
651
652
653
654
655
656
657 other = account_other_time(delta);
658 WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);
659 vtime->starttime += delta;
660
661 return delta - other;
662}
663
664static void vtime_account_system(struct task_struct *tsk,
665 struct vtime *vtime)
666{
667 vtime->stime += get_vtime_delta(vtime);
668 if (vtime->stime >= TICK_NSEC) {
669 account_system_time(tsk, irq_count(), vtime->stime);
670 vtime->stime = 0;
671 }
672}
673
674static void vtime_account_guest(struct task_struct *tsk,
675 struct vtime *vtime)
676{
677 vtime->gtime += get_vtime_delta(vtime);
678 if (vtime->gtime >= TICK_NSEC) {
679 account_guest_time(tsk, vtime->gtime);
680 vtime->gtime = 0;
681 }
682}
683
684static void __vtime_account_kernel(struct task_struct *tsk,
685 struct vtime *vtime)
686{
687
688 if (vtime->state == VTIME_GUEST)
689 vtime_account_guest(tsk, vtime);
690 else
691 vtime_account_system(tsk, vtime);
692}
693
694void vtime_account_kernel(struct task_struct *tsk)
695{
696 struct vtime *vtime = &tsk->vtime;
697
698 if (!vtime_delta(vtime))
699 return;
700
701 write_seqcount_begin(&vtime->seqcount);
702 __vtime_account_kernel(tsk, vtime);
703 write_seqcount_end(&vtime->seqcount);
704}
705
706void vtime_user_enter(struct task_struct *tsk)
707{
708 struct vtime *vtime = &tsk->vtime;
709
710 write_seqcount_begin(&vtime->seqcount);
711 vtime_account_system(tsk, vtime);
712 vtime->state = VTIME_USER;
713 write_seqcount_end(&vtime->seqcount);
714}
715
716void vtime_user_exit(struct task_struct *tsk)
717{
718 struct vtime *vtime = &tsk->vtime;
719
720 write_seqcount_begin(&vtime->seqcount);
721 vtime->utime += get_vtime_delta(vtime);
722 if (vtime->utime >= TICK_NSEC) {
723 account_user_time(tsk, vtime->utime);
724 vtime->utime = 0;
725 }
726 vtime->state = VTIME_SYS;
727 write_seqcount_end(&vtime->seqcount);
728}
729
730void vtime_guest_enter(struct task_struct *tsk)
731{
732 struct vtime *vtime = &tsk->vtime;
733
734
735
736
737
738
739
740 write_seqcount_begin(&vtime->seqcount);
741 vtime_account_system(tsk, vtime);
742 tsk->flags |= PF_VCPU;
743 vtime->state = VTIME_GUEST;
744 write_seqcount_end(&vtime->seqcount);
745}
746EXPORT_SYMBOL_GPL(vtime_guest_enter);
747
748void vtime_guest_exit(struct task_struct *tsk)
749{
750 struct vtime *vtime = &tsk->vtime;
751
752 write_seqcount_begin(&vtime->seqcount);
753 vtime_account_guest(tsk, vtime);
754 tsk->flags &= ~PF_VCPU;
755 vtime->state = VTIME_SYS;
756 write_seqcount_end(&vtime->seqcount);
757}
758EXPORT_SYMBOL_GPL(vtime_guest_exit);
759
760void vtime_account_idle(struct task_struct *tsk)
761{
762 account_idle_time(get_vtime_delta(&tsk->vtime));
763}
764
765void vtime_task_switch_generic(struct task_struct *prev)
766{
767 struct vtime *vtime = &prev->vtime;
768
769 write_seqcount_begin(&vtime->seqcount);
770 if (vtime->state == VTIME_IDLE)
771 vtime_account_idle(prev);
772 else
773 __vtime_account_kernel(prev, vtime);
774 vtime->state = VTIME_INACTIVE;
775 vtime->cpu = -1;
776 write_seqcount_end(&vtime->seqcount);
777
778 vtime = ¤t->vtime;
779
780 write_seqcount_begin(&vtime->seqcount);
781 if (is_idle_task(current))
782 vtime->state = VTIME_IDLE;
783 else if (current->flags & PF_VCPU)
784 vtime->state = VTIME_GUEST;
785 else
786 vtime->state = VTIME_SYS;
787 vtime->starttime = sched_clock();
788 vtime->cpu = smp_processor_id();
789 write_seqcount_end(&vtime->seqcount);
790}
791
792void vtime_init_idle(struct task_struct *t, int cpu)
793{
794 struct vtime *vtime = &t->vtime;
795 unsigned long flags;
796
797 local_irq_save(flags);
798 write_seqcount_begin(&vtime->seqcount);
799 vtime->state = VTIME_IDLE;
800 vtime->starttime = sched_clock();
801 vtime->cpu = cpu;
802 write_seqcount_end(&vtime->seqcount);
803 local_irq_restore(flags);
804}
805
806u64 task_gtime(struct task_struct *t)
807{
808 struct vtime *vtime = &t->vtime;
809 unsigned int seq;
810 u64 gtime;
811
812 if (!vtime_accounting_enabled())
813 return t->gtime;
814
815 do {
816 seq = read_seqcount_begin(&vtime->seqcount);
817
818 gtime = t->gtime;
819 if (vtime->state == VTIME_GUEST)
820 gtime += vtime->gtime + vtime_delta(vtime);
821
822 } while (read_seqcount_retry(&vtime->seqcount, seq));
823
824 return gtime;
825}
826
827
828
829
830
831
832bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
833{
834 struct vtime *vtime = &t->vtime;
835 unsigned int seq;
836 u64 delta;
837 int ret;
838
839 if (!vtime_accounting_enabled()) {
840 *utime = t->utime;
841 *stime = t->stime;
842 return false;
843 }
844
845 do {
846 ret = false;
847 seq = read_seqcount_begin(&vtime->seqcount);
848
849 *utime = t->utime;
850 *stime = t->stime;
851
852
853 if (vtime->state < VTIME_SYS)
854 continue;
855
856 ret = true;
857 delta = vtime_delta(vtime);
858
859
860
861
862
863 if (vtime->state == VTIME_SYS)
864 *stime += vtime->stime + delta;
865 else
866 *utime += vtime->utime + delta;
867 } while (read_seqcount_retry(&vtime->seqcount, seq));
868
869 return ret;
870}
871
872static int vtime_state_fetch(struct vtime *vtime, int cpu)
873{
874 int state = READ_ONCE(vtime->state);
875
876
877
878
879
880 if (vtime->cpu != cpu && vtime->cpu != -1)
881 return -EAGAIN;
882
883
884
885
886
887
888
889
890
891
892 if (state == VTIME_INACTIVE)
893 return -EAGAIN;
894
895 return state;
896}
897
898static u64 kcpustat_user_vtime(struct vtime *vtime)
899{
900 if (vtime->state == VTIME_USER)
901 return vtime->utime + vtime_delta(vtime);
902 else if (vtime->state == VTIME_GUEST)
903 return vtime->gtime + vtime_delta(vtime);
904 return 0;
905}
906
907static int kcpustat_field_vtime(u64 *cpustat,
908 struct task_struct *tsk,
909 enum cpu_usage_stat usage,
910 int cpu, u64 *val)
911{
912 struct vtime *vtime = &tsk->vtime;
913 unsigned int seq;
914
915 do {
916 int state;
917
918 seq = read_seqcount_begin(&vtime->seqcount);
919
920 state = vtime_state_fetch(vtime, cpu);
921 if (state < 0)
922 return state;
923
924 *val = cpustat[usage];
925
926
927
928
929
930
931
932
933 switch (usage) {
934 case CPUTIME_SYSTEM:
935 if (state == VTIME_SYS)
936 *val += vtime->stime + vtime_delta(vtime);
937 break;
938 case CPUTIME_USER:
939 if (task_nice(tsk) <= 0)
940 *val += kcpustat_user_vtime(vtime);
941 break;
942 case CPUTIME_NICE:
943 if (task_nice(tsk) > 0)
944 *val += kcpustat_user_vtime(vtime);
945 break;
946 case CPUTIME_GUEST:
947 if (state == VTIME_GUEST && task_nice(tsk) <= 0)
948 *val += vtime->gtime + vtime_delta(vtime);
949 break;
950 case CPUTIME_GUEST_NICE:
951 if (state == VTIME_GUEST && task_nice(tsk) > 0)
952 *val += vtime->gtime + vtime_delta(vtime);
953 break;
954 default:
955 break;
956 }
957 } while (read_seqcount_retry(&vtime->seqcount, seq));
958
959 return 0;
960}
961
962u64 kcpustat_field(struct kernel_cpustat *kcpustat,
963 enum cpu_usage_stat usage, int cpu)
964{
965 u64 *cpustat = kcpustat->cpustat;
966 u64 val = cpustat[usage];
967 struct rq *rq;
968 int err;
969
970 if (!vtime_accounting_enabled_cpu(cpu))
971 return val;
972
973 rq = cpu_rq(cpu);
974
975 for (;;) {
976 struct task_struct *curr;
977
978 rcu_read_lock();
979 curr = rcu_dereference(rq->curr);
980 if (WARN_ON_ONCE(!curr)) {
981 rcu_read_unlock();
982 return cpustat[usage];
983 }
984
985 err = kcpustat_field_vtime(cpustat, curr, usage, cpu, &val);
986 rcu_read_unlock();
987
988 if (!err)
989 return val;
990
991 cpu_relax();
992 }
993}
994EXPORT_SYMBOL_GPL(kcpustat_field);
995
996static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
997 const struct kernel_cpustat *src,
998 struct task_struct *tsk, int cpu)
999{
1000 struct vtime *vtime = &tsk->vtime;
1001 unsigned int seq;
1002
1003 do {
1004 u64 *cpustat;
1005 u64 delta;
1006 int state;
1007
1008 seq = read_seqcount_begin(&vtime->seqcount);
1009
1010 state = vtime_state_fetch(vtime, cpu);
1011 if (state < 0)
1012 return state;
1013
1014 *dst = *src;
1015 cpustat = dst->cpustat;
1016
1017
1018 if (state < VTIME_SYS)
1019 continue;
1020
1021 delta = vtime_delta(vtime);
1022
1023
1024
1025
1026
1027 if (state == VTIME_SYS) {
1028 cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;
1029 } else if (state == VTIME_USER) {
1030 if (task_nice(tsk) > 0)
1031 cpustat[CPUTIME_NICE] += vtime->utime + delta;
1032 else
1033 cpustat[CPUTIME_USER] += vtime->utime + delta;
1034 } else {
1035 WARN_ON_ONCE(state != VTIME_GUEST);
1036 if (task_nice(tsk) > 0) {
1037 cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;
1038 cpustat[CPUTIME_NICE] += vtime->gtime + delta;
1039 } else {
1040 cpustat[CPUTIME_GUEST] += vtime->gtime + delta;
1041 cpustat[CPUTIME_USER] += vtime->gtime + delta;
1042 }
1043 }
1044 } while (read_seqcount_retry(&vtime->seqcount, seq));
1045
1046 return 0;
1047}
1048
1049void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
1050{
1051 const struct kernel_cpustat *src = &kcpustat_cpu(cpu);
1052 struct rq *rq;
1053 int err;
1054
1055 if (!vtime_accounting_enabled_cpu(cpu)) {
1056 *dst = *src;
1057 return;
1058 }
1059
1060 rq = cpu_rq(cpu);
1061
1062 for (;;) {
1063 struct task_struct *curr;
1064
1065 rcu_read_lock();
1066 curr = rcu_dereference(rq->curr);
1067 if (WARN_ON_ONCE(!curr)) {
1068 rcu_read_unlock();
1069 *dst = *src;
1070 return;
1071 }
1072
1073 err = kcpustat_cpu_fetch_vtime(dst, src, curr, cpu);
1074 rcu_read_unlock();
1075
1076 if (!err)
1077 return;
1078
1079 cpu_relax();
1080 }
1081}
1082EXPORT_SYMBOL_GPL(kcpustat_cpu_fetch);
1083
1084#endif
1085