linux/arch/x86/xen/time.c
<<
>>
Prefs
   1/*
   2 * Xen time implementation.
   3 *
   4 * This is implemented in terms of a clocksource driver which uses
   5 * the hypervisor clock as a nanosecond timebase, and a clockevent
   6 * driver which uses the hypervisor's timer mechanism.
   7 *
   8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
   9 */
  10#include <linux/kernel.h>
  11#include <linux/interrupt.h>
  12#include <linux/clocksource.h>
  13#include <linux/clockchips.h>
  14#include <linux/kernel_stat.h>
  15#include <linux/math64.h>
  16#include <linux/gfp.h>
  17
  18#include <asm/pvclock.h>
  19#include <asm/xen/hypervisor.h>
  20#include <asm/xen/hypercall.h>
  21
  22#include <xen/events.h>
  23#include <xen/features.h>
  24#include <xen/interface/xen.h>
  25#include <xen/interface/vcpu.h>
  26
  27#include "xen-ops.h"
  28
  29/* Xen may fire a timer up to this many ns early */
  30#define TIMER_SLOP      100000
  31#define NS_PER_TICK     (1000000000LL / HZ)
  32
  33/* runstate info updated by Xen */
  34static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
  35
  36/* snapshots of runstate info */
  37static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);
  38
  39/* unused ns of stolen and blocked time */
  40static DEFINE_PER_CPU(u64, xen_residual_stolen);
  41static DEFINE_PER_CPU(u64, xen_residual_blocked);
  42
  43/* return an consistent snapshot of 64-bit time/counter value */
  44static u64 get64(const u64 *p)
  45{
  46        u64 ret;
  47
  48        if (BITS_PER_LONG < 64) {
  49                u32 *p32 = (u32 *)p;
  50                u32 h, l;
  51
  52                /*
  53                 * Read high then low, and then make sure high is
  54                 * still the same; this will only loop if low wraps
  55                 * and carries into high.
  56                 * XXX some clean way to make this endian-proof?
  57                 */
  58                do {
  59                        h = p32[1];
  60                        barrier();
  61                        l = p32[0];
  62                        barrier();
  63                } while (p32[1] != h);
  64
  65                ret = (((u64)h) << 32) | l;
  66        } else
  67                ret = *p;
  68
  69        return ret;
  70}
  71
  72/*
  73 * Runstate accounting
  74 */
  75static void get_runstate_snapshot(struct vcpu_runstate_info *res)
  76{
  77        u64 state_time;
  78        struct vcpu_runstate_info *state;
  79
  80        BUG_ON(preemptible());
  81
  82        state = &__get_cpu_var(xen_runstate);
  83
  84        /*
  85         * The runstate info is always updated by the hypervisor on
  86         * the current CPU, so there's no need to use anything
  87         * stronger than a compiler barrier when fetching it.
  88         */
  89        do {
  90                state_time = get64(&state->state_entry_time);
  91                barrier();
  92                *res = *state;
  93                barrier();
  94        } while (get64(&state->state_entry_time) != state_time);
  95}
  96
  97/* return true when a vcpu could run but has no real cpu to run on */
  98bool xen_vcpu_stolen(int vcpu)
  99{
 100        return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
 101}
 102
 103void xen_setup_runstate_info(int cpu)
 104{
 105        struct vcpu_register_runstate_memory_area area;
 106
 107        area.addr.v = &per_cpu(xen_runstate, cpu);
 108
 109        if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
 110                               cpu, &area))
 111                BUG();
 112}
 113
 114static void do_stolen_accounting(void)
 115{
 116        struct vcpu_runstate_info state;
 117        struct vcpu_runstate_info *snap;
 118        s64 blocked, runnable, offline, stolen;
 119        cputime_t ticks;
 120
 121        get_runstate_snapshot(&state);
 122
 123        WARN_ON(state.state != RUNSTATE_running);
 124
 125        snap = &__get_cpu_var(xen_runstate_snapshot);
 126
 127        /* work out how much time the VCPU has not been runn*ing*  */
 128        blocked = state.time[RUNSTATE_blocked] - snap->time[RUNSTATE_blocked];
 129        runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
 130        offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
 131
 132        *snap = state;
 133
 134        /* Add the appropriate number of ticks of stolen time,
 135           including any left-overs from last time. */
 136        stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);
 137
 138        if (stolen < 0)
 139                stolen = 0;
 140
 141        ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
 142        __this_cpu_write(xen_residual_stolen, stolen);
 143        account_steal_ticks(ticks);
 144
 145        /* Add the appropriate number of ticks of blocked time,
 146           including any left-overs from last time. */
 147        blocked += __this_cpu_read(xen_residual_blocked);
 148
 149        if (blocked < 0)
 150                blocked = 0;
 151
 152        ticks = iter_div_u64_rem(blocked, NS_PER_TICK, &blocked);
 153        __this_cpu_write(xen_residual_blocked, blocked);
 154        account_idle_ticks(ticks);
 155}
 156
 157/* Get the TSC speed from Xen */
 158static unsigned long xen_tsc_khz(void)
 159{
 160        struct pvclock_vcpu_time_info *info =
 161                &HYPERVISOR_shared_info->vcpu_info[0].time;
 162
 163        return pvclock_tsc_khz(info);
 164}
 165
 166cycle_t xen_clocksource_read(void)
 167{
 168        struct pvclock_vcpu_time_info *src;
 169        cycle_t ret;
 170
 171        preempt_disable_notrace();
 172        src = &__get_cpu_var(xen_vcpu)->time;
 173        ret = pvclock_clocksource_read(src);
 174        preempt_enable_notrace();
 175        return ret;
 176}
 177
 178static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
 179{
 180        return xen_clocksource_read();
 181}
 182
 183static void xen_read_wallclock(struct timespec *ts)
 184{
 185        struct shared_info *s = HYPERVISOR_shared_info;
 186        struct pvclock_wall_clock *wall_clock = &(s->wc);
 187        struct pvclock_vcpu_time_info *vcpu_time;
 188
 189        vcpu_time = &get_cpu_var(xen_vcpu)->time;
 190        pvclock_read_wallclock(wall_clock, vcpu_time, ts);
 191        put_cpu_var(xen_vcpu);
 192}
 193
 194static unsigned long xen_get_wallclock(void)
 195{
 196        struct timespec ts;
 197
 198        xen_read_wallclock(&ts);
 199        return ts.tv_sec;
 200}
 201
 202static int xen_set_wallclock(unsigned long now)
 203{
 204        struct xen_platform_op op;
 205        int rc;
 206
 207        /* do nothing for domU */
 208        if (!xen_initial_domain())
 209                return -1;
 210
 211        op.cmd = XENPF_settime;
 212        op.u.settime.secs = now;
 213        op.u.settime.nsecs = 0;
 214        op.u.settime.system_time = xen_clocksource_read();
 215
 216        rc = HYPERVISOR_dom0_op(&op);
 217        WARN(rc != 0, "XENPF_settime failed: now=%ld\n", now);
 218
 219        return rc;
 220}
 221
 222static struct clocksource xen_clocksource __read_mostly = {
 223        .name = "xen",
 224        .rating = 400,
 225        .read = xen_clocksource_get_cycles,
 226        .mask = ~0,
 227        .flags = CLOCK_SOURCE_IS_CONTINUOUS,
 228};
 229
 230/*
 231   Xen clockevent implementation
 232
 233   Xen has two clockevent implementations:
 234
 235   The old timer_op one works with all released versions of Xen prior
 236   to version 3.0.4.  This version of the hypervisor provides a
 237   single-shot timer with nanosecond resolution.  However, sharing the
 238   same event channel is a 100Hz tick which is delivered while the
 239   vcpu is running.  We don't care about or use this tick, but it will
 240   cause the core time code to think the timer fired too soon, and
 241   will end up resetting it each time.  It could be filtered, but
 242   doing so has complications when the ktime clocksource is not yet
 243   the xen clocksource (ie, at boot time).
 244
 245   The new vcpu_op-based timer interface allows the tick timer period
 246   to be changed or turned off.  The tick timer is not useful as a
 247   periodic timer because events are only delivered to running vcpus.
 248   The one-shot timer can report when a timeout is in the past, so
 249   set_next_event is capable of returning -ETIME when appropriate.
 250   This interface is used when available.
 251*/
 252
 253
 254/*
 255  Get a hypervisor absolute time.  In theory we could maintain an
 256  offset between the kernel's time and the hypervisor's time, and
 257  apply that to a kernel's absolute timeout.  Unfortunately the
 258  hypervisor and kernel times can drift even if the kernel is using
 259  the Xen clocksource, because ntp can warp the kernel's clocksource.
 260*/
 261static s64 get_abs_timeout(unsigned long delta)
 262{
 263        return xen_clocksource_read() + delta;
 264}
 265
 266static void xen_timerop_set_mode(enum clock_event_mode mode,
 267                                 struct clock_event_device *evt)
 268{
 269        switch (mode) {
 270        case CLOCK_EVT_MODE_PERIODIC:
 271                /* unsupported */
 272                WARN_ON(1);
 273                break;
 274
 275        case CLOCK_EVT_MODE_ONESHOT:
 276        case CLOCK_EVT_MODE_RESUME:
 277                break;
 278
 279        case CLOCK_EVT_MODE_UNUSED:
 280        case CLOCK_EVT_MODE_SHUTDOWN:
 281                HYPERVISOR_set_timer_op(0);  /* cancel timeout */
 282                break;
 283        }
 284}
 285
 286static int xen_timerop_set_next_event(unsigned long delta,
 287                                      struct clock_event_device *evt)
 288{
 289        WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
 290
 291        if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
 292                BUG();
 293
 294        /* We may have missed the deadline, but there's no real way of
 295           knowing for sure.  If the event was in the past, then we'll
 296           get an immediate interrupt. */
 297
 298        return 0;
 299}
 300
 301static const struct clock_event_device xen_timerop_clockevent = {
 302        .name = "xen",
 303        .features = CLOCK_EVT_FEAT_ONESHOT,
 304
 305        .max_delta_ns = 0xffffffff,
 306        .min_delta_ns = TIMER_SLOP,
 307
 308        .mult = 1,
 309        .shift = 0,
 310        .rating = 500,
 311
 312        .set_mode = xen_timerop_set_mode,
 313        .set_next_event = xen_timerop_set_next_event,
 314};
 315
 316
 317
 318static void xen_vcpuop_set_mode(enum clock_event_mode mode,
 319                                struct clock_event_device *evt)
 320{
 321        int cpu = smp_processor_id();
 322
 323        switch (mode) {
 324        case CLOCK_EVT_MODE_PERIODIC:
 325                WARN_ON(1);     /* unsupported */
 326                break;
 327
 328        case CLOCK_EVT_MODE_ONESHOT:
 329                if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
 330                        BUG();
 331                break;
 332
 333        case CLOCK_EVT_MODE_UNUSED:
 334        case CLOCK_EVT_MODE_SHUTDOWN:
 335                if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
 336                    HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
 337                        BUG();
 338                break;
 339        case CLOCK_EVT_MODE_RESUME:
 340                break;
 341        }
 342}
 343
 344static int xen_vcpuop_set_next_event(unsigned long delta,
 345                                     struct clock_event_device *evt)
 346{
 347        int cpu = smp_processor_id();
 348        struct vcpu_set_singleshot_timer single;
 349        int ret;
 350
 351        WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
 352
 353        single.timeout_abs_ns = get_abs_timeout(delta);
 354        single.flags = VCPU_SSHOTTMR_future;
 355
 356        ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
 357
 358        BUG_ON(ret != 0 && ret != -ETIME);
 359
 360        return ret;
 361}
 362
 363static const struct clock_event_device xen_vcpuop_clockevent = {
 364        .name = "xen",
 365        .features = CLOCK_EVT_FEAT_ONESHOT,
 366
 367        .max_delta_ns = 0xffffffff,
 368        .min_delta_ns = TIMER_SLOP,
 369
 370        .mult = 1,
 371        .shift = 0,
 372        .rating = 500,
 373
 374        .set_mode = xen_vcpuop_set_mode,
 375        .set_next_event = xen_vcpuop_set_next_event,
 376};
 377
 378static const struct clock_event_device *xen_clockevent =
 379        &xen_timerop_clockevent;
 380static DEFINE_PER_CPU(struct clock_event_device, xen_clock_events);
 381
 382static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
 383{
 384        struct clock_event_device *evt = &__get_cpu_var(xen_clock_events);
 385        irqreturn_t ret;
 386
 387        ret = IRQ_NONE;
 388        if (evt->event_handler) {
 389                evt->event_handler(evt);
 390                ret = IRQ_HANDLED;
 391        }
 392
 393        do_stolen_accounting();
 394
 395        return ret;
 396}
 397
 398void xen_setup_timer(int cpu)
 399{
 400        const char *name;
 401        struct clock_event_device *evt;
 402        int irq;
 403
 404        printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
 405
 406        name = kasprintf(GFP_KERNEL, "timer%d", cpu);
 407        if (!name)
 408                name = "<timer kasprintf failed>";
 409
 410        irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
 411                                      IRQF_DISABLED|IRQF_PERCPU|
 412                                      IRQF_NOBALANCING|IRQF_TIMER|
 413                                      IRQF_FORCE_RESUME,
 414                                      name, NULL);
 415
 416        evt = &per_cpu(xen_clock_events, cpu);
 417        memcpy(evt, xen_clockevent, sizeof(*evt));
 418
 419        evt->cpumask = cpumask_of(cpu);
 420        evt->irq = irq;
 421}
 422
 423void xen_teardown_timer(int cpu)
 424{
 425        struct clock_event_device *evt;
 426        BUG_ON(cpu == 0);
 427        evt = &per_cpu(xen_clock_events, cpu);
 428        unbind_from_irqhandler(evt->irq, NULL);
 429}
 430
 431void xen_setup_cpu_clockevents(void)
 432{
 433        BUG_ON(preemptible());
 434
 435        clockevents_register_device(&__get_cpu_var(xen_clock_events));
 436}
 437
 438void xen_timer_resume(void)
 439{
 440        int cpu;
 441
 442        pvclock_resume();
 443
 444        if (xen_clockevent != &xen_vcpuop_clockevent)
 445                return;
 446
 447        for_each_online_cpu(cpu) {
 448                if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
 449                        BUG();
 450        }
 451}
 452
 453static const struct pv_time_ops xen_time_ops __initconst = {
 454        .sched_clock = xen_clocksource_read,
 455};
 456
 457static void __init xen_time_init(void)
 458{
 459        int cpu = smp_processor_id();
 460        struct timespec tp;
 461
 462        clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
 463
 464        if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
 465                /* Successfully turned off 100Hz tick, so we have the
 466                   vcpuop-based timer interface */
 467                printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
 468                xen_clockevent = &xen_vcpuop_clockevent;
 469        }
 470
 471        /* Set initial system time with full resolution */
 472        xen_read_wallclock(&tp);
 473        do_settimeofday(&tp);
 474
 475        setup_force_cpu_cap(X86_FEATURE_TSC);
 476
 477        xen_setup_runstate_info(cpu);
 478        xen_setup_timer(cpu);
 479        xen_setup_cpu_clockevents();
 480}
 481
 482void __init xen_init_time_ops(void)
 483{
 484        pv_time_ops = xen_time_ops;
 485
 486        x86_init.timers.timer_init = xen_time_init;
 487        x86_init.timers.setup_percpu_clockev = x86_init_noop;
 488        x86_cpuinit.setup_percpu_clockev = x86_init_noop;
 489
 490        x86_platform.calibrate_tsc = xen_tsc_khz;
 491        x86_platform.get_wallclock = xen_get_wallclock;
 492        x86_platform.set_wallclock = xen_set_wallclock;
 493}
 494
 495#ifdef CONFIG_XEN_PVHVM
 496static void xen_hvm_setup_cpu_clockevents(void)
 497{
 498        int cpu = smp_processor_id();
 499        xen_setup_runstate_info(cpu);
 500        xen_setup_timer(cpu);
 501        xen_setup_cpu_clockevents();
 502}
 503
 504void __init xen_hvm_init_time_ops(void)
 505{
 506        /* vector callback is needed otherwise we cannot receive interrupts
 507         * on cpu > 0 and at this point we don't know how many cpus are
 508         * available */
 509        if (!xen_have_vector_callback)
 510                return;
 511        if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
 512                printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
 513                                "disable pv timer\n");
 514                return;
 515        }
 516
 517        pv_time_ops = xen_time_ops;
 518        x86_init.timers.setup_percpu_clockev = xen_time_init;
 519        x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
 520
 521        x86_platform.calibrate_tsc = xen_tsc_khz;
 522        x86_platform.get_wallclock = xen_get_wallclock;
 523        x86_platform.set_wallclock = xen_set_wallclock;
 524}
 525#endif
 526