LXR linux/arch/s390/kernel/vtime.c

   1/*
   2 *    Virtual cpu timer based timer functions.
   3 *
   4 *    Copyright IBM Corp. 2004, 2012
   5 *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
   6 */
   7
   8#include <linux/kernel_stat.h>
   9#include <linux/notifier.h>
  10#include <linux/kprobes.h>
  11#include <linux/export.h>
  12#include <linux/kernel.h>
  13#include <linux/timex.h>
  14#include <linux/types.h>
  15#include <linux/time.h>
  16#include <linux/cpu.h>
  17#include <linux/smp.h>
  18
  19#include <asm/irq_regs.h>
  20#include <asm/cputime.h>
  21#include <asm/vtimer.h>
  22#include <asm/irq.h>
  23#include "entry.h"
  24
  25static void virt_timer_expire(void);
  26
  27DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
  28
  29static LIST_HEAD(virt_timer_list);
  30static DEFINE_SPINLOCK(virt_timer_lock);
  31static atomic64_t virt_timer_current;
  32static atomic64_t virt_timer_elapsed;
  33
  34static inline u64 get_vtimer(void)
  35{
  36        u64 timer;
  37
  38        asm volatile("stpt %0" : "=m" (timer));
  39        return timer;
  40}
  41
  42static inline void set_vtimer(u64 expires)
  43{
  44        u64 timer;
  45
  46        asm volatile(
  47                "       stpt    %0\n"   /* Store current cpu timer value */
  48                "       spt     %1"     /* Set new value imm. afterwards */
  49                : "=m" (timer) : "m" (expires));
  50        S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
  51        S390_lowcore.last_update_timer = expires;
  52}
  53
  54static inline int virt_timer_forward(u64 elapsed)
  55{
  56        BUG_ON(!irqs_disabled());
  57
  58        if (list_empty(&virt_timer_list))
  59                return 0;
  60        elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
  61        return elapsed >= atomic64_read(&virt_timer_current);
  62}
  63
  64/*
  65 * Update process times based on virtual cpu times stored by entry.S
  66 * to the lowcore fields user_timer, system_timer & steal_clock.
  67 */
  68static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
  69{
  70        struct thread_info *ti = task_thread_info(tsk);
  71        u64 timer, clock, user, system, steal;
  72
  73        timer = S390_lowcore.last_update_timer;
  74        clock = S390_lowcore.last_update_clock;
  75        asm volatile(
  76                "       stpt    %0\n"   /* Store current cpu timer value */
  77                "       stck    %1"     /* Store current tod clock value */
  78                : "=m" (S390_lowcore.last_update_timer),
  79                  "=m" (S390_lowcore.last_update_clock));
  80        S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
  81        S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
  82
  83        user = S390_lowcore.user_timer - ti->user_timer;
  84        S390_lowcore.steal_timer -= user;
  85        ti->user_timer = S390_lowcore.user_timer;
  86        account_user_time(tsk, user, user);
  87
  88        system = S390_lowcore.system_timer - ti->system_timer;
  89        S390_lowcore.steal_timer -= system;
  90        ti->system_timer = S390_lowcore.system_timer;
  91        account_system_time(tsk, hardirq_offset, system, system);
  92
  93        steal = S390_lowcore.steal_timer;
  94        if ((s64) steal > 0) {
  95                S390_lowcore.steal_timer = 0;
  96                account_steal_time(steal);
  97        }
  98
  99        return virt_timer_forward(user + system);
 100}
 101
 102void vtime_task_switch(struct task_struct *prev)
 103{
 104        struct thread_info *ti;
 105
 106        do_account_vtime(prev, 0);
 107        ti = task_thread_info(prev);
 108        ti->user_timer = S390_lowcore.user_timer;
 109        ti->system_timer = S390_lowcore.system_timer;
 110        ti = task_thread_info(current);
 111        S390_lowcore.user_timer = ti->user_timer;
 112        S390_lowcore.system_timer = ti->system_timer;
 113}
 114
 115/*
 116 * In s390, accounting pending user time also implies
 117 * accounting system time in order to correctly compute
 118 * the stolen time accounting.
 119 */
 120void vtime_account_user(struct task_struct *tsk)
 121{
 122        if (do_account_vtime(tsk, HARDIRQ_OFFSET))
 123                virt_timer_expire();
 124}
 125
 126/*
 127 * Update process times based on virtual cpu times stored by entry.S
 128 * to the lowcore fields user_timer, system_timer & steal_clock.
 129 */
 130void vtime_account_irq_enter(struct task_struct *tsk)
 131{
 132        struct thread_info *ti = task_thread_info(tsk);
 133        u64 timer, system;
 134
 135        WARN_ON_ONCE(!irqs_disabled());
 136
 137        timer = S390_lowcore.last_update_timer;
 138        S390_lowcore.last_update_timer = get_vtimer();
 139        S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
 140
 141        system = S390_lowcore.system_timer - ti->system_timer;
 142        S390_lowcore.steal_timer -= system;
 143        ti->system_timer = S390_lowcore.system_timer;
 144        account_system_time(tsk, 0, system, system);
 145
 146        virt_timer_forward(system);
 147}
 148EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
 149
 150void vtime_account_system(struct task_struct *tsk)
 151__attribute__((alias("vtime_account_irq_enter")));
 152EXPORT_SYMBOL_GPL(vtime_account_system);
 153
 154void __kprobes vtime_stop_cpu(void)
 155{
 156        struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
 157        unsigned long long idle_time;
 158        unsigned long psw_mask;
 159
 160        trace_hardirqs_on();
 161
 162        /* Wait for external, I/O or machine check interrupt. */
 163        psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT |
 164                PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
 165        idle->nohz_delay = 0;
 166
 167        /* Call the assembler magic in entry.S */
 168        psw_idle(idle, psw_mask);
 169
 170        /* Account time spent with enabled wait psw loaded as idle time. */
 171        idle->sequence++;
 172        smp_wmb();
 173        idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
 174        idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
 175        idle->idle_time += idle_time;
 176        idle->idle_count++;
 177        account_idle_time(idle_time);
 178        smp_wmb();
 179        idle->sequence++;
 180}
 181
 182cputime64_t s390_get_idle_time(int cpu)
 183{
 184        struct s390_idle_data *idle = &per_cpu(s390_idle, cpu);
 185        unsigned long long now, idle_enter, idle_exit;
 186        unsigned int sequence;
 187
 188        do {
 189                now = get_tod_clock();
 190                sequence = ACCESS_ONCE(idle->sequence);
 191                idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
 192                idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
 193        } while ((sequence & 1) || (idle->sequence != sequence));
 194        return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0;
 195}
 196
 197/*
 198 * Sorted add to a list. List is linear searched until first bigger
 199 * element is found.
 200 */
 201static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
 202{
 203        struct vtimer_list *tmp;
 204
 205        list_for_each_entry(tmp, head, entry) {
 206                if (tmp->expires > timer->expires) {
 207                        list_add_tail(&timer->entry, &tmp->entry);
 208                        return;
 209                }
 210        }
 211        list_add_tail(&timer->entry, head);
 212}
 213
 214/*
 215 * Handler for expired virtual CPU timer.
 216 */
 217static void virt_timer_expire(void)
 218{
 219        struct vtimer_list *timer, *tmp;
 220        unsigned long elapsed;
 221        LIST_HEAD(cb_list);
 222
 223        /* walk timer list, fire all expired timers */
 224        spin_lock(&virt_timer_lock);
 225        elapsed = atomic64_read(&virt_timer_elapsed);
 226        list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
 227                if (timer->expires < elapsed)
 228                        /* move expired timer to the callback queue */
 229                        list_move_tail(&timer->entry, &cb_list);
 230                else
 231                        timer->expires -= elapsed;
 232        }
 233        if (!list_empty(&virt_timer_list)) {
 234                timer = list_first_entry(&virt_timer_list,
 235                                         struct vtimer_list, entry);
 236                atomic64_set(&virt_timer_current, timer->expires);
 237        }
 238        atomic64_sub(elapsed, &virt_timer_elapsed);
 239        spin_unlock(&virt_timer_lock);
 240
 241        /* Do callbacks and recharge periodic timers */
 242        list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
 243                list_del_init(&timer->entry);
 244                timer->function(timer->data);
 245                if (timer->interval) {
 246                        /* Recharge interval timer */
 247                        timer->expires = timer->interval +
 248                                atomic64_read(&virt_timer_elapsed);
 249                        spin_lock(&virt_timer_lock);
 250                        list_add_sorted(timer, &virt_timer_list);
 251                        spin_unlock(&virt_timer_lock);
 252                }
 253        }
 254}
 255
 256void init_virt_timer(struct vtimer_list *timer)
 257{
 258        timer->function = NULL;
 259        INIT_LIST_HEAD(&timer->entry);
 260}
 261EXPORT_SYMBOL(init_virt_timer);
 262
 263static inline int vtimer_pending(struct vtimer_list *timer)
 264{
 265        return !list_empty(&timer->entry);
 266}
 267
 268static void internal_add_vtimer(struct vtimer_list *timer)
 269{
 270        if (list_empty(&virt_timer_list)) {
 271                /* First timer, just program it. */
 272                atomic64_set(&virt_timer_current, timer->expires);
 273                atomic64_set(&virt_timer_elapsed, 0);
 274                list_add(&timer->entry, &virt_timer_list);
 275        } else {
 276                /* Update timer against current base. */
 277                timer->expires += atomic64_read(&virt_timer_elapsed);
 278                if (likely((s64) timer->expires <
 279                           (s64) atomic64_read(&virt_timer_current)))
 280                        /* The new timer expires before the current timer. */
 281                        atomic64_set(&virt_timer_current, timer->expires);
 282                /* Insert new timer into the list. */
 283                list_add_sorted(timer, &virt_timer_list);
 284        }
 285}
 286
 287static void __add_vtimer(struct vtimer_list *timer, int periodic)
 288{
 289        unsigned long flags;
 290
 291        timer->interval = periodic ? timer->expires : 0;
 292        spin_lock_irqsave(&virt_timer_lock, flags);
 293        internal_add_vtimer(timer);
 294        spin_unlock_irqrestore(&virt_timer_lock, flags);
 295}
 296
 297/*
 298 * add_virt_timer - add an oneshot virtual CPU timer
 299 */
 300void add_virt_timer(struct vtimer_list *timer)
 301{
 302        __add_vtimer(timer, 0);
 303}
 304EXPORT_SYMBOL(add_virt_timer);
 305
 306/*
 307 * add_virt_timer_int - add an interval virtual CPU timer
 308 */
 309void add_virt_timer_periodic(struct vtimer_list *timer)
 310{
 311        __add_vtimer(timer, 1);
 312}
 313EXPORT_SYMBOL(add_virt_timer_periodic);
 314
 315static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
 316{
 317        unsigned long flags;
 318        int rc;
 319
 320        BUG_ON(!timer->function);
 321
 322        if (timer->expires == expires && vtimer_pending(timer))
 323                return 1;
 324        spin_lock_irqsave(&virt_timer_lock, flags);
 325        rc = vtimer_pending(timer);
 326        if (rc)
 327                list_del_init(&timer->entry);
 328        timer->interval = periodic ? expires : 0;
 329        timer->expires = expires;
 330        internal_add_vtimer(timer);
 331        spin_unlock_irqrestore(&virt_timer_lock, flags);
 332        return rc;
 333}
 334
 335/*
 336 * returns whether it has modified a pending timer (1) or not (0)
 337 */
 338int mod_virt_timer(struct vtimer_list *timer, u64 expires)
 339{
 340        return __mod_vtimer(timer, expires, 0);
 341}
 342EXPORT_SYMBOL(mod_virt_timer);
 343
 344/*
 345 * returns whether it has modified a pending timer (1) or not (0)
 346 */
 347int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
 348{
 349        return __mod_vtimer(timer, expires, 1);
 350}
 351EXPORT_SYMBOL(mod_virt_timer_periodic);
 352
 353/*
 354 * Delete a virtual timer.
 355 *
 356 * returns whether the deleted timer was pending (1) or not (0)
 357 */
 358int del_virt_timer(struct vtimer_list *timer)
 359{
 360        unsigned long flags;
 361
 362        if (!vtimer_pending(timer))
 363                return 0;
 364        spin_lock_irqsave(&virt_timer_lock, flags);
 365        list_del_init(&timer->entry);
 366        spin_unlock_irqrestore(&virt_timer_lock, flags);
 367        return 1;
 368}
 369EXPORT_SYMBOL(del_virt_timer);
 370
 371/*
 372 * Start the virtual CPU timer on the current CPU.
 373 */
 374void __cpuinit init_cpu_vtimer(void)
 375{
 376        /* set initial cpu timer */
 377        set_vtimer(VTIMER_MAX_SLICE);
 378}
 379
 380static int __cpuinit s390_nohz_notify(struct notifier_block *self,
 381                                      unsigned long action, void *hcpu)
 382{
 383        struct s390_idle_data *idle;
 384        long cpu = (long) hcpu;
 385
 386        idle = &per_cpu(s390_idle, cpu);
 387        switch (action & ~CPU_TASKS_FROZEN) {
 388        case CPU_DYING:
 389                idle->nohz_delay = 0;
 390        default:
 391                break;
 392        }
 393        return NOTIFY_OK;
 394}
 395
 396void __init vtime_init(void)
 397{
 398        /* Enable cpu timer interrupts on the boot cpu. */
 399        init_cpu_vtimer();
 400        cpu_notifier(s390_nohz_notify, 0);
 401}
 402