1
2
3
4
5
6
7
8
9
10#include <linux/module.h>
11#include <linux/kernel.h>
12#include <linux/time.h>
13#include <linux/delay.h>
14#include <linux/init.h>
15#include <linux/smp.h>
16#include <linux/types.h>
17#include <linux/timex.h>
18#include <linux/notifier.h>
19#include <linux/kernel_stat.h>
20#include <linux/rcupdate.h>
21#include <linux/posix-timers.h>
22
23#include <asm/s390_ext.h>
24#include <asm/timer.h>
25#include <asm/irq_regs.h>
26#include <asm/cputime.h>
27
28static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
29
30DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
31
32static inline __u64 get_vtimer(void)
33{
34 __u64 timer;
35
36 asm volatile("STPT %0" : "=m" (timer));
37 return timer;
38}
39
40static inline void set_vtimer(__u64 expires)
41{
42 __u64 timer;
43
44 asm volatile (" STPT %0\n"
45 " SPT %1"
46 : "=m" (timer) : "m" (expires) );
47 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
48 S390_lowcore.last_update_timer = expires;
49}
50
51
52
53
54
55static void do_account_vtime(struct task_struct *tsk, int hardirq_offset)
56{
57 struct thread_info *ti = task_thread_info(tsk);
58 __u64 timer, clock, user, system, steal;
59
60 timer = S390_lowcore.last_update_timer;
61 clock = S390_lowcore.last_update_clock;
62 asm volatile (" STPT %0\n"
63 " STCK %1"
64 : "=m" (S390_lowcore.last_update_timer),
65 "=m" (S390_lowcore.last_update_clock) );
66 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
67 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
68
69 user = S390_lowcore.user_timer - ti->user_timer;
70 S390_lowcore.steal_timer -= user;
71 ti->user_timer = S390_lowcore.user_timer;
72 account_user_time(tsk, user, user);
73
74 system = S390_lowcore.system_timer - ti->system_timer;
75 S390_lowcore.steal_timer -= system;
76 ti->system_timer = S390_lowcore.system_timer;
77 account_system_time(tsk, hardirq_offset, system, system);
78
79 steal = S390_lowcore.steal_timer;
80 if ((s64) steal > 0) {
81 S390_lowcore.steal_timer = 0;
82 account_steal_time(steal);
83 }
84}
85
86void account_vtime(struct task_struct *prev, struct task_struct *next)
87{
88 struct thread_info *ti;
89
90 do_account_vtime(prev, 0);
91 ti = task_thread_info(prev);
92 ti->user_timer = S390_lowcore.user_timer;
93 ti->system_timer = S390_lowcore.system_timer;
94 ti = task_thread_info(next);
95 S390_lowcore.user_timer = ti->user_timer;
96 S390_lowcore.system_timer = ti->system_timer;
97}
98
99void account_process_tick(struct task_struct *tsk, int user_tick)
100{
101 do_account_vtime(tsk, HARDIRQ_OFFSET);
102}
103
104
105
106
107
108void account_system_vtime(struct task_struct *tsk)
109{
110 struct thread_info *ti = task_thread_info(tsk);
111 __u64 timer, system;
112
113 timer = S390_lowcore.last_update_timer;
114 S390_lowcore.last_update_timer = get_vtimer();
115 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
116
117 system = S390_lowcore.system_timer - ti->system_timer;
118 S390_lowcore.steal_timer -= system;
119 ti->system_timer = S390_lowcore.system_timer;
120 account_system_time(tsk, 0, system, system);
121}
122EXPORT_SYMBOL_GPL(account_system_vtime);
123
124void vtime_start_cpu(void)
125{
126 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
127 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
128 __u64 idle_time, expires;
129
130
131 idle_time = S390_lowcore.int_clock - idle->idle_enter;
132 account_idle_time(idle_time);
133 S390_lowcore.steal_timer +=
134 idle->idle_enter - S390_lowcore.last_update_clock;
135 S390_lowcore.last_update_clock = S390_lowcore.int_clock;
136
137
138 S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle;
139 S390_lowcore.last_update_timer = S390_lowcore.async_enter_timer;
140
141
142 if (vq->do_spt) {
143
144 expires = vq->idle - S390_lowcore.async_enter_timer;
145 expires += get_vtimer();
146 set_vtimer(expires);
147 } else {
148
149 vq->elapsed -= vq->idle - S390_lowcore.async_enter_timer;
150 }
151
152 idle->sequence++;
153 smp_wmb();
154 idle->idle_time += idle_time;
155 idle->idle_enter = 0ULL;
156 idle->idle_count++;
157 smp_wmb();
158 idle->sequence++;
159}
160
161void vtime_stop_cpu(void)
162{
163 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
164 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
165 psw_t psw;
166
167
168 psw.mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT;
169
170
171 if (vq->do_spt) {
172 __u64 vmax = VTIMER_MAX_SLICE;
173
174
175
176
177
178
179
180
181
182
183
184 asm volatile(
185#ifndef CONFIG_64BIT
186 " basr 1,0\n"
187 "0: ahi 1,1f-0b\n"
188 " st 1,4(%2)\n"
189#else
190 " larl 1,1f\n"
191 " stg 1,8(%2)\n"
192#endif
193 " stpt 0(%4)\n"
194 " spt 0(%5)\n"
195 " stck 0(%3)\n"
196#ifndef CONFIG_64BIT
197 " lpsw 0(%2)\n"
198#else
199 " lpswe 0(%2)\n"
200#endif
201 "1:"
202 : "=m" (idle->idle_enter), "=m" (vq->idle)
203 : "a" (&psw), "a" (&idle->idle_enter),
204 "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw)
205 : "memory", "cc", "1");
206 } else {
207
208
209
210
211
212
213
214
215
216
217 asm volatile(
218#ifndef CONFIG_64BIT
219 " basr 1,0\n"
220 "0: ahi 1,1f-0b\n"
221 " st 1,4(%2)\n"
222#else
223 " larl 1,1f\n"
224 " stg 1,8(%2)\n"
225#endif
226 " stpt 0(%4)\n"
227 " stck 0(%3)\n"
228#ifndef CONFIG_64BIT
229 " lpsw 0(%2)\n"
230#else
231 " lpswe 0(%2)\n"
232#endif
233 "1:"
234 : "=m" (idle->idle_enter), "=m" (vq->idle)
235 : "a" (&psw), "a" (&idle->idle_enter),
236 "a" (&vq->idle), "m" (psw)
237 : "memory", "cc", "1");
238 }
239}
240
241cputime64_t s390_get_idle_time(int cpu)
242{
243 struct s390_idle_data *idle;
244 unsigned long long now, idle_time, idle_enter;
245 unsigned int sequence;
246
247 idle = &per_cpu(s390_idle, cpu);
248
249 now = get_clock();
250repeat:
251 sequence = idle->sequence;
252 smp_rmb();
253 if (sequence & 1)
254 goto repeat;
255 idle_time = 0;
256 idle_enter = idle->idle_enter;
257 if (idle_enter != 0ULL && idle_enter < now)
258 idle_time = now - idle_enter;
259 smp_rmb();
260 if (idle->sequence != sequence)
261 goto repeat;
262 return idle_time;
263}
264
265
266
267
268
269static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
270{
271 struct vtimer_list *event;
272
273 list_for_each_entry(event, head, entry) {
274 if (event->expires > timer->expires) {
275 list_add_tail(&timer->entry, &event->entry);
276 return;
277 }
278 }
279 list_add_tail(&timer->entry, head);
280}
281
282
283
284
285
286static void do_callbacks(struct list_head *cb_list)
287{
288 struct vtimer_queue *vq;
289 struct vtimer_list *event, *tmp;
290
291 if (list_empty(cb_list))
292 return;
293
294 vq = &__get_cpu_var(virt_cpu_timer);
295
296 list_for_each_entry_safe(event, tmp, cb_list, entry) {
297 list_del_init(&event->entry);
298 (event->function)(event->data);
299 if (event->interval) {
300
301 event->expires = event->interval + vq->elapsed;
302 spin_lock(&vq->lock);
303 list_add_sorted(event, &vq->list);
304 spin_unlock(&vq->lock);
305 }
306 }
307}
308
309
310
311
312static void do_cpu_timer_interrupt(__u16 error_code)
313{
314 struct vtimer_queue *vq;
315 struct vtimer_list *event, *tmp;
316 struct list_head cb_list;
317 __u64 elapsed, next;
318
319 INIT_LIST_HEAD(&cb_list);
320 vq = &__get_cpu_var(virt_cpu_timer);
321
322
323 spin_lock(&vq->lock);
324
325 elapsed = vq->elapsed + (vq->timer - S390_lowcore.async_enter_timer);
326 BUG_ON((s64) elapsed < 0);
327 vq->elapsed = 0;
328 list_for_each_entry_safe(event, tmp, &vq->list, entry) {
329 if (event->expires < elapsed)
330
331 list_move_tail(&event->entry, &cb_list);
332 else
333 event->expires -= elapsed;
334 }
335 spin_unlock(&vq->lock);
336
337 vq->do_spt = list_empty(&cb_list);
338 do_callbacks(&cb_list);
339
340
341 next = VTIMER_MAX_SLICE;
342 spin_lock(&vq->lock);
343 if (!list_empty(&vq->list)) {
344 event = list_first_entry(&vq->list, struct vtimer_list, entry);
345 next = event->expires;
346 } else
347 vq->do_spt = 0;
348 spin_unlock(&vq->lock);
349
350
351
352
353
354
355 elapsed = S390_lowcore.async_enter_timer - get_vtimer();
356 set_vtimer(next - elapsed);
357 vq->timer = next - elapsed;
358 vq->elapsed = elapsed;
359}
360
361void init_virt_timer(struct vtimer_list *timer)
362{
363 timer->function = NULL;
364 INIT_LIST_HEAD(&timer->entry);
365}
366EXPORT_SYMBOL(init_virt_timer);
367
368static inline int vtimer_pending(struct vtimer_list *timer)
369{
370 return (!list_empty(&timer->entry));
371}
372
373
374
375
376static void internal_add_vtimer(struct vtimer_list *timer)
377{
378 struct vtimer_queue *vq;
379 unsigned long flags;
380 __u64 left, expires;
381
382 vq = &per_cpu(virt_cpu_timer, timer->cpu);
383 spin_lock_irqsave(&vq->lock, flags);
384
385 BUG_ON(timer->cpu != smp_processor_id());
386
387 if (list_empty(&vq->list)) {
388
389 list_add(&timer->entry, &vq->list);
390 set_vtimer(timer->expires);
391 vq->timer = timer->expires;
392 vq->elapsed = 0;
393 } else {
394
395 expires = timer->expires;
396 left = get_vtimer();
397 if (likely((s64) expires < (s64) left)) {
398
399 set_vtimer(expires);
400 vq->elapsed += vq->timer - left;
401 vq->timer = expires;
402 } else {
403 vq->elapsed += vq->timer - left;
404 vq->timer = left;
405 }
406
407 timer->expires += vq->elapsed;
408 list_add_sorted(timer, &vq->list);
409 }
410
411 spin_unlock_irqrestore(&vq->lock, flags);
412
413 put_cpu();
414}
415
416static inline void prepare_vtimer(struct vtimer_list *timer)
417{
418 BUG_ON(!timer->function);
419 BUG_ON(!timer->expires || timer->expires > VTIMER_MAX_SLICE);
420 BUG_ON(vtimer_pending(timer));
421 timer->cpu = get_cpu();
422}
423
424
425
426
427void add_virt_timer(void *new)
428{
429 struct vtimer_list *timer;
430
431 timer = (struct vtimer_list *)new;
432 prepare_vtimer(timer);
433 timer->interval = 0;
434 internal_add_vtimer(timer);
435}
436EXPORT_SYMBOL(add_virt_timer);
437
438
439
440
441void add_virt_timer_periodic(void *new)
442{
443 struct vtimer_list *timer;
444
445 timer = (struct vtimer_list *)new;
446 prepare_vtimer(timer);
447 timer->interval = timer->expires;
448 internal_add_vtimer(timer);
449}
450EXPORT_SYMBOL(add_virt_timer_periodic);
451
452int __mod_vtimer(struct vtimer_list *timer, __u64 expires, int periodic)
453{
454 struct vtimer_queue *vq;
455 unsigned long flags;
456 int cpu;
457
458 BUG_ON(!timer->function);
459 BUG_ON(!expires || expires > VTIMER_MAX_SLICE);
460
461 if (timer->expires == expires && vtimer_pending(timer))
462 return 1;
463
464 cpu = get_cpu();
465 vq = &per_cpu(virt_cpu_timer, cpu);
466
467
468 spin_lock_irqsave(&vq->lock, flags);
469
470
471 if (!vtimer_pending(timer)) {
472 spin_unlock_irqrestore(&vq->lock, flags);
473
474 if (periodic)
475 timer->interval = expires;
476 else
477 timer->interval = 0;
478 timer->expires = expires;
479 timer->cpu = cpu;
480 internal_add_vtimer(timer);
481 return 0;
482 }
483
484
485 BUG_ON(timer->cpu != cpu);
486
487 list_del_init(&timer->entry);
488 timer->expires = expires;
489 if (periodic)
490 timer->interval = expires;
491
492
493 spin_unlock_irqrestore(&vq->lock, flags);
494 internal_add_vtimer(timer);
495 return 1;
496}
497
498
499
500
501
502
503
504int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
505{
506 return __mod_vtimer(timer, expires, 0);
507}
508EXPORT_SYMBOL(mod_virt_timer);
509
510
511
512
513
514
515
516int mod_virt_timer_periodic(struct vtimer_list *timer, __u64 expires)
517{
518 return __mod_vtimer(timer, expires, 1);
519}
520EXPORT_SYMBOL(mod_virt_timer_periodic);
521
522
523
524
525
526
527int del_virt_timer(struct vtimer_list *timer)
528{
529 unsigned long flags;
530 struct vtimer_queue *vq;
531
532
533 if (!vtimer_pending(timer))
534 return 0;
535
536 vq = &per_cpu(virt_cpu_timer, timer->cpu);
537 spin_lock_irqsave(&vq->lock, flags);
538
539
540 list_del_init(&timer->entry);
541
542 spin_unlock_irqrestore(&vq->lock, flags);
543 return 1;
544}
545EXPORT_SYMBOL(del_virt_timer);
546
547
548
549
550void init_cpu_vtimer(void)
551{
552 struct vtimer_queue *vq;
553
554
555 vq = &__get_cpu_var(virt_cpu_timer);
556 INIT_LIST_HEAD(&vq->list);
557 spin_lock_init(&vq->lock);
558
559
560 __ctl_set_bit(0,10);
561}
562
563void __init vtime_init(void)
564{
565
566 if (register_external_interrupt(0x1005, do_cpu_timer_interrupt))
567 panic("Couldn't request external interrupt 0x1005");
568
569
570 init_cpu_vtimer();
571}
572
573